seeing_is_believing 3.0.0.beta.4 → 3.0.0.beta.5

data/lib/seeing_is_believing/binary/data_structures.rb

@@ -0,0 +1,57 @@
+require 'seeing_is_believing/hash_struct'
+
+class SeeingIsBelieving
+  module Binary
+    class Markers < HashStruct
+      attribute(:value)     { Marker.new prefix: '# => ', regex: '^#\s*=>\s*' }
+      attribute(:exception) { Marker.new prefix: '# ~> ', regex: '^#\s*~>\s*' }
+      attribute(:stdout)    { Marker.new prefix: '# >> ', regex: '^#\s*>>\s*' }
+      attribute(:stderr)    { Marker.new prefix: '# !> ', regex: '^#\s*!>\s*' }
+    end
+
+
+    class Marker < HashStruct
+      def self.to_regex(string)
+        return string if string.kind_of? Regexp
+        flag_to_bit = {
+          'i' => 0b001,
+          'x' => 0b010,
+          'm' => 0b100
+        }
+        string =~ %r{\A/(.*)/([mxi]*)\Z}
+        body  = $1 || string
+        flags = ($2 || "").each_char.inject(0) { |bits, flag| bits | flag_to_bit[flag] }
+        Regexp.new body, flags
+      end
+
+      attribute :prefix  # e.g. "# => "
+      attribute :regex   # identify prefix in a comment, e.g. /^# => /
+
+      def []=(key, value)
+        value = Marker.to_regex(value) if key == :regex
+        super key, value
+      end
+    end
+
+
+    class AnnotatorOptions < HashStruct
+      attribute(:alignment_strategy) { AlignChunk }
+      attribute(:markers)            { Markers.new }
+      attribute(:max_line_length)    { Float::INFINITY }
+      attribute(:max_result_length)  { Float::INFINITY }
+    end
+
+
+    class ErrorMessage < HashStruct.for(:explanation)
+      def to_s
+        "Error: #{explanation}"
+      end
+    end
+
+    class SyntaxErrorMessage < ErrorMessage.for(:line_number, :filename)
+      def to_s
+        "Syntax Error: #{filename}:#{line_number}\n#{explanation}\n"
+      end
+    end
+  end
+end

data/lib/seeing_is_believing/binary/engine.rb

@@ -0,0 +1,104 @@
+require 'seeing_is_believing/code'
+require 'seeing_is_believing/binary/data_structures'
+require 'seeing_is_believing/binary/remove_annotations'
+require 'seeing_is_believing/event_stream/handlers/record_exitstatus'
+
+class SeeingIsBelieving
+  module Binary
+    class MustEvaluateFirst < SeeingIsBelievingError
+      def initialize(method)
+        super "Cannot call #{method} on engine until it has evaluated the program."
+      end
+    end
+
+    class Engine
+      def initialize(config)
+        self.config = config
+      end
+
+      def cleaned_body
+        @cleaned_body ||= if missing_newline?
+                            normalized_cleaned_body.chomp!
+                          else
+                            normalized_cleaned_body
+                          end
+      end
+
+      def syntax_error?
+        code.syntax.invalid?
+      end
+
+      def syntax_error
+        return unless syntax_error?
+        SyntaxErrorMessage.new line_number: code.syntax.line_number,
+                               explanation: code.syntax.error_message,
+                               filename:    config.lib_options.filename
+      end
+
+      def evaluate!
+        @evaluated || begin
+          SeeingIsBelieving.call normalized_cleaned_body,
+                                 config.lib_options.merge(event_handler: record_exitstatus)
+          @timed_out = false
+          @evaluated = true
+        end
+      rescue Timeout::Error
+        @timed_out = true
+        @evaluated = true
+      ensure return self unless $! # idk, maybe too tricky, but was really annoying putting it in three places
+      end
+
+      def timed_out?
+        @evaluated || raise(MustEvaluateFirst.new __method__)
+        @timed_out
+      end
+
+      def result
+        @evaluated || raise(MustEvaluateFirst.new __method__)
+        config.lib_options.event_handler.result # The stream handler will not have a result (implies this was used wrong)
+      end
+
+      def annotated_body
+        @annotated_body ||= begin
+          @evaluated || raise(MustEvaluateFirst.new __method__)
+          annotated = config.annotator.call normalized_cleaned_body,
+                                            result,
+                                            config.annotator_options.to_h
+          annotated.chomp! if missing_newline?
+          annotated
+        end
+      end
+
+      def exitstatus
+        @evaluated || raise(MustEvaluateFirst.new __method__)
+        record_exitstatus.exitstatus
+      end
+
+      private
+
+      attr_accessor :config
+
+      def missing_newline?
+        @missing_newline ||= !config.body.end_with?("\n")
+      end
+
+      def code
+        @code ||= Code.new(normalized_cleaned_body, config.filename)
+      end
+
+      def record_exitstatus
+        @record_exitstatus ||= SeeingIsBelieving::EventStream::Handlers::RecordExitStatus.new config.lib_options.event_handler
+      end
+
+      def normalized_cleaned_body
+        @normalized_cleaned_body ||= begin
+          body_with_nl = config.body
+          body_with_nl += "\n" if missing_newline?
+          RemoveAnnotations.call body_with_nl,
+                                 config.remove_value_prefixes,
+                                 config.markers
+        end
+      end
+    end
+  end
+end

data/lib/seeing_is_believing/binary/{comment_formatter.rb → format_comment.rb}

@@ -6,17 +6,17 @@ class SeeingIsBelieving
     # line_length is the length of the line this comment is being appended to
     #
     # For examples of what the options are, and how they all fit together, see
-    # spec/binary/comment_formatter_spec.rb
-    class CommentFormatter
+    # spec/binary/format_comment.rb
+    class FormatComment
       def self.call(*args)
         new(*args).call
       end
 
       def initialize(line_length, separator, result, options)
-       self.line_length = line_length
-       self.separator   = separator
-       self.result      = result
-       self.options     = options
+        self.line_length = line_length
+        self.separator   = separator
+        self.result      = result
+        self.options     = options
       end
 
       def call

data/lib/seeing_is_believing/binary/remove_annotations.rb

@@ -1,21 +1,18 @@
-require 'seeing_is_believing/binary'         # Defines the regexes to locate the markers
-require 'seeing_is_believing/parser_helpers' # We have to parse the file to find the comments
+require 'seeing_is_believing/binary' # Defines the regexes to locate the markers
+require 'seeing_is_believing/code'   # We have to parse the file to find the comments
 
 class SeeingIsBelieving
   module Binary
-    # TODO: might be here that we hit the issue where
-    # you sometimes have to run it 2x to get it to correctly reset whitespace
-    # should wipe out the full_range rather than just the comment_range
     class RemoveAnnotations
-      def self.call(raw_code, should_clean_values, markers)
-        new(raw_code, should_clean_values, markers).call
+      def self.call(raw_code, remove_value_prefixes, markers)
+        new(raw_code, remove_value_prefixes, markers).call
       end
 
-      def initialize(raw_code, should_clean_values, markers)
-        self.should_clean_values = should_clean_values
-        self.raw_code            = raw_code
-        self.markers             = markers # TECHNICALLY THESE ARE REGEXES RIGHT NOW
-        self.code                = Code.new(raw_code, 'strip_comments')
+      def initialize(raw_code, remove_value_prefixes, markers)
+        self.remove_value_prefixes = remove_value_prefixes
+        self.raw_code              = raw_code
+        self.markers               = markers
+        self.code                  = Code.new(raw_code, 'strip_comments')
       end
 
       def call
@@ -27,9 +24,13 @@ class SeeingIsBelieving
 
           case comment.text
           when value_regex
-            next unless should_clean_values
-            code.rewriter.remove comment.comment_range
-            remove_whitespace_before comment.comment_range.begin_pos, code.buffer, code.rewriter, false
+            if remove_value_prefixes
+              code.rewriter.remove comment.comment_range
+              remove_whitespace_before comment.comment_range.begin_pos, code.buffer, code.rewriter, false
+            else
+              prefix = comment.text[value_regex].rstrip
+              code.rewriter.replace comment.comment_range, prefix
+            end
           when exception_regex
             code.rewriter.remove comment.comment_range
             remove_whitespace_before comment.comment_range.begin_pos, code.buffer, code.rewriter, true
@@ -49,7 +50,7 @@ class SeeingIsBelieving
 
       private
 
-      attr_accessor :raw_code, :should_clean_values, :markers, :code
+      attr_accessor :raw_code, :remove_value_prefixes, :markers, :code
 
       # any whitespace before the index (on the same line) will be removed
       # if the preceding whitespace is at the beginning of the line, the newline will be removed
@@ -57,11 +58,11 @@ class SeeingIsBelieving
       def remove_whitespace_before(index, buffer, rewriter, remove_preceding_newline)
         end_pos   = index
         begin_pos = end_pos - 1
-        begin_pos -= 1 while raw_code[begin_pos] =~ /\s/ && raw_code[begin_pos] != "\n"
-        begin_pos -= 1 if raw_code[begin_pos] == "\n"
-        begin_pos -= 1 if raw_code[begin_pos] == "\n" && remove_preceding_newline
+        begin_pos -= 1 while 0 <= begin_pos && raw_code[begin_pos] =~ /\s/ && raw_code[begin_pos] != "\n"
+        begin_pos -= 1 if 0 <= begin_pos && raw_code[begin_pos] == "\n"
+        begin_pos -= 1 if remove_preceding_newline && 0 <= begin_pos && raw_code[begin_pos] == "\n"
         return if begin_pos.next == end_pos
-        rewriter.remove Parser::Source::Range.new(buffer, begin_pos.next, end_pos)
+        rewriter.remove code.range_for(begin_pos.next, end_pos)
       end
 
       def annotation_chunks_in(code)
@@ -90,24 +91,24 @@ class SeeingIsBelieving
           }
       end
 
+      def value_prefix
+        @value_prefix ||= markers.fetch(:value).fetch(:prefix)
+      end
+
       def value_regex
-        markers.fetch(:value)
+        @value_regex ||= markers.fetch(:value).fetch(:regex)
       end
 
       def exception_regex
-        markers.fetch(:exception)
+        @exception_regex ||= markers.fetch(:exception).fetch(:regex)
       end
 
       def stdout_regex
-        markers.fetch(:stdout)
+        @stdout_regex ||= markers.fetch(:stdout).fetch(:regex)
       end
 
       def stderr_regex
-        markers.fetch(:stderr)
-      end
-
-      def nextline_regex
-        markers.fetch(:nextline)
+        @stderr_regex ||= markers.fetch(:stderr).fetch(:regex)
       end
     end
   end

data/lib/seeing_is_believing/binary/rewrite_comments.rb

@@ -1,63 +1,62 @@
 require 'seeing_is_believing/code'
+require 'seeing_is_believing/binary/commentable_lines'
 
 class SeeingIsBelieving
   module Binary
+    # can this be joined into CommentLines?
+    # that one yields every commentable line, this one just lines which have comments
+    # what they yield is a little different, too, but their algorithms and domain are very similar
     module RewriteComments
-      def self.call(code, options={}, &mapping)
-        code     = Code.new(code)
-        comments = code.inline_comments
-        buffer   = code.buffer
+      Options = HashStruct.anon do
+        attribute(:include_lines) { [] }
+      end
+
+      def self.call(raw_code, options={}, &mapping)
+        code        = Code.new(raw_code)
+        comments    = code.inline_comments
+        extra_lines = Options.new(options).include_lines
 
+        # update existing comments
         comments.each do |comment|
           new_whitespace, new_comment = mapping.call comment
           code.rewriter.replace comment.whitespace_range, new_whitespace
           code.rewriter.replace comment.comment_range,    new_comment
         end
 
-        line_begins = line_begins_for(buffer.source)
-        options.fetch(:always_rewrite, []).each { |line_number|
-          next if comments.any? { |c| c.line_number == line_number }
-
-          # TODO: can this move down into Code?
-          _, next_line_index = (line_begins.find { |ln, index| ln == line_number } || [nil, buffer.source.size.next])
-          col  = 0
-          col += 1 until col == next_line_index || buffer.source[next_line_index-2-col] == "\n"
-
-          index   = next_line_index - 1
-          range   = Parser::Source::Range.new buffer, index, index
-
-          comment = Code::InlineComment.new line_number, # line_number,
-                                            col,         # preceding_whitespace_range.column,
-                                            "",          # preceding_whitespace,
-                                            col,         # comment.location.column,
-                                            "",          # comment.text,
-                                            range,       # range_for(first_char, comment.location.expression.end_pos),
-                                            range,       # preceding_whitespace_range,
-                                            range        # comment.location.expression
+        # remove extra lines that are handled / uncommentable
+        comments.each { |c| extra_lines.delete c.line_number }
+        commentable_linenums = CommentableLines.call(code.raw).map { |linenum, *| linenum }
+        extra_lines.select! { |linenum| commentable_linenums.include? linenum }
+
+        # add additional comments
+        extra_lines.each do |line_number|
+          line_begin_col     = code.linenum_to_index(line_number)
+          nextline_begin_col = code.linenum_to_index(line_number.next)
+          nextline_begin_col -= 1 if code.raw[nextline_begin_col-1] == "\n"
+          whitespace_col     = nextline_begin_col-1
+          whitespace_col     -= 1 while line_begin_col < whitespace_col &&
+                                        code.raw[whitespace_col] =~ /\s/
+          whitespace_col += 1
+          whitespace_range = code.range_for(whitespace_col, nextline_begin_col)
+          comment_range = code.range_for(nextline_begin_col, nextline_begin_col)
+
+          comment = Code::InlineComment.new \
+            line_number:      line_number,
+            whitespace_col:   whitespace_col-line_begin_col,
+            whitespace:       code.raw[whitespace_col...nextline_begin_col]||"",
+            text_col:         nextline_begin_col-line_begin_col,
+            text:             "",
+            full_range:       whitespace_range,
+            whitespace_range: whitespace_range,
+            comment_range:    comment_range
 
           whitespace, body = mapping.call comment
-          code.rewriter.insert_before range, "#{whitespace}#{body}"
-        }
+          code.rewriter.replace whitespace_range, "#{whitespace}#{body}"
+        end
 
+        # perform the rewrite
         code.rewriter.process
       end
-
-      # TODO: Move down into the Code obj?
-      # returns: [[lineno, index], ...]
-      def self.line_begins_for(raw_code)
-        # Copied from here https://github.com/whitequark/parser/blob/34c40479293bb9b5ba217039cf349111466d1f9a/lib/parser/source/buffer.rb#L213-227
-        # I figured it's better to copy it than to violate encapsulation since this is private
-        line_begins, index = [ [ 0, 0 ] ], 1
-
-        raw_code.each_char do |char|
-          if char == "\n"
-            line_begins.unshift [ line_begins.length, index ]
-          end
-
-          index += 1
-        end
-        line_begins
-      end
     end
   end
 end

data/lib/seeing_is_believing/code.rb

@@ -1,99 +1,155 @@
-require 'parser/current'
+# With new versioning, there's lots of small versions
+# we don't need Parser to complain that we're on 2.1.1 and its parsing 2.1.5
+# https://github.com/whitequark/parser/blob/e2249d7051b1adb6979139928e14a81bc62f566e/lib/parser/current.rb#L3
+class << (Parser ||= Module.new)
+  def warn(*) end
+  require 'parser/current'
+  remove_method :warn
+end
+
+require 'seeing_is_believing/hash_struct'
+
 class SeeingIsBelieving
   class Code
-    InlineComment = Struct.new :line_number,
-                               :whitespace_col,
-                               :whitespace,
-                               :text_col,
-                               :text,
-                               :full_range,
-                               :whitespace_range,
-                               :comment_range
-
-    # At prsent, it is expected that the syntax is validated before code arrives here
-    # or that its validity doesn't matter (e.g. extracting comments)
-    def initialize(raw_ruby_code, name="SeeingIsBelieving")
-      self.code = raw_ruby_code
-      self.name = name
+    InlineComment = HashStruct.for :line_number, :whitespace_col, :whitespace, :text_col, :text, :full_range, :whitespace_range, :comment_range
+    Syntax = HashStruct.for error_message: nil, line_number: nil do
+      def valid?()   !error_message end
+      def invalid?() !valid?        end
     end
 
-    def buffer()          @buffer   ||= (parse && @buffer         ) end
-    def parser()          @parser   ||= (parse && @parser         ) end
-    def rewriter()        @rewriter ||= (parse && @rewriter       ) end
-    def inline_comments() @comments ||= (parse && @inline_comments) end
-    def root()            @root     ||= (parse && @root           ) end
+    attr_reader :raw, :buffer, :parser, :rewriter, :inline_comments, :root, :raw_comments, :syntax, :body_range
+
+    def initialize(raw_code, name=nil)
+      raw_code[-1] == "\n" || raise(SyntaxError, "Code must end in a newline for the sake of consistency (sanity)")
+      @raw             = raw_code
+      @buffer          = Parser::Source::Buffer.new(name||"SeeingIsBelieving")
+      @buffer.source   = raw
+      builder          = Parser::Builders::Default.new.tap { |b| b.emit_file_line_as_literals = false }
+      @rewriter        = Parser::Source::Rewriter.new buffer
+      @raw_comments, tokens = comments_and_tokens(builder, buffer)
+      @body_range      = body_range_from_tokens(tokens)
+      @parser          = Parser::CurrentRuby.new builder
+      @inline_comments = raw_comments.select(&:inline?).map { |c| wrap_comment c }
+      begin
+        @root          = @parser.parse(@buffer)
+        @syntax        = Syntax.new
+      rescue Parser::SyntaxError
+        @syntax        = Syntax.new error_message: $!.message, line_number: index_to_linenum($!.diagnostic.location.begin_pos)
+      ensure
+        @root        ||= null_node
+      end
+    end
 
     def range_for(start_index, end_index)
       Parser::Source::Range.new buffer, start_index, end_index
     end
 
-    private
-
-    attr_accessor :code, :name
+    def index_to_linenum(char_index)
+      line_indexes.index { |line_index| char_index < line_index } || line_indexes.size
+    end
 
-    def parse
-      @buffer                             = Parser::Source::Buffer.new(name)
-      @buffer.source                      = code
-      builder                             = Parser::Builders::Default.new
-      builder.emit_file_line_as_literals  = false # should be injectible?
-      @parser                             = Parser::CurrentRuby.new builder
-      @rewriter                           = Parser::Source::Rewriter.new @buffer
+    def linenum_to_index(line_num)
+      return raw.size if line_indexes.size < line_num
+      line_indexes[line_num - 1]
+    end
 
-      can_parse_invalid_code(@parser)
+    def heredoc?(ast)
+      # some strings are fucking weird.
+      # e.g. the "1" in `%w[1]` returns nil for ast.location.begin
+      # and `__FILE__` is a string whose location is a Parser::Source::Map instead of a Parser::Source::Map::Collection,
+      # so it has no #begin
+      ast.kind_of?(Parser::AST::Node)           &&
+        (ast.type == :dstr || ast.type == :str) &&
+        (location = ast.location)               &&
+        (location.kind_of? Parser::Source::Map::Heredoc)
+    end
 
-      @root, all_comments, tokens = parser.tokenize(@buffer)
+    def void_value?(ast)
+      case ast && ast.type
+      when :begin, :kwbegin, :resbody
+        void_value?(ast.children.last)
+      when :rescue, :ensure
+        ast.children.any? { |child| void_value? child }
+      when :if
+        void_value?(ast.children[1]) || void_value?(ast.children[2])
+      when :return, :next, :redo, :retry, :break
+        true
+      else
+        false
+      end
+    end
 
-      @inline_comments = all_comments.select(&:inline?).map { |c| wrap_comment c }
+    def line_indexes
+      @line_indexes ||= [ 0,
+                          *raw.each_char
+                              .with_index(1)
+                              .select { |char, index| char == "\n" }
+                              .map    { |char, index| index },
+                        ].freeze
     end
 
-    def can_parse_invalid_code(parser)
-      # THIS IS SO WE CAN EXTRACT COMMENTS FROM INVALID FILES.
+    private
 
+    def comments_and_tokens(builder, buffer)
+      # THIS IS SO WE CAN EXTRACT COMMENTS FROM INVALID FILES.
       # We do it by telling Parser's diagnostic to not blow up.
       #   https://github.com/whitequark/parser/blob/2d69a1b5f34ef15b3a8330beb036ac4bf4775e29/lib/parser/diagnostic/engine.rb
-
       # However, this probably implies SiB won't work on Rbx/JRuby
       #   https://github.com/whitequark/parser/blob/2d69a1b5f34ef15b3a8330beb036ac4bf4775e29/lib/parser/base.rb#L129-134
-
       # Ideally we could just do this
       #   parser.diagnostics.all_errors_are_fatal = false
       #   parser.diagnostics.ignore_warnings      = false
-
       # But, the parser will still blow up on "fatal" errors (e.g. unterminated string) So we need to actually change it.
       #   https://github.com/whitequark/parser/blob/2d69a1b5f34ef15b3a8330beb036ac4bf4775e29/lib/parser/diagnostic/engine.rb#L99
-
       # We could make a NullDiagnostics like this:
       #   class NullDiagnostics < Parser::Diagnostic::Engine
       #     def process(*)
       #       # no op
       #     end
       #   end
-
       # But we don't control initialization of the variable, and the value gets passed around, at least into the lexer.
       #   https://github.com/whitequark/parser/blob/2d69a1b5f34ef15b3a8330beb036ac4bf4775e29/lib/parser/base.rb#L139
       #   and since it's all private, it could change at any time (Parser is very state based),
       #   so I think it's just generally safer to mutate that one object, as we do now.
+      parser = Parser::CurrentRuby.new builder
       diagnostics = parser.diagnostics
       def diagnostics.process(*)
         self
       end
+      _, all_comments, tokens = parser.tokenize(@buffer)
+      [all_comments, tokens]
+    end
+
+    def body_range_from_tokens(tokens)
+      return range_for(0, 0) if raw.start_with? "__END__\n"
+      (name, (_, range)) = tokens.max_by { |name, (data, range)| range.end_pos } ||
+                            [nil, [nil, range_for(0, 1)]]
+      end_pos = range.end_pos
+      end_pos += 1 if name == :tCOMMENT || name == :tSTRING_END #
+      range_for 0, end_pos
     end
 
     def wrap_comment(comment)
       last_char  = comment.location.expression.begin_pos
       first_char = last_char
-      first_char -= 1 while first_char > 0 && code[first_char-1] =~ /[ \t]/
+      first_char -= 1 while first_char > 0 && raw[first_char-1] =~ /[ \t]/
       preceding_whitespace        = buffer.source[first_char...last_char]
       preceding_whitespace_range  = range_for first_char, last_char
 
-      InlineComment.new comment.location.line,
-                        preceding_whitespace_range.column,
-                        preceding_whitespace,
-                        comment.location.column,
-                        comment.text,
-                        range_for(first_char, comment.location.expression.end_pos),
-                        preceding_whitespace_range,
-                        comment.location.expression
+      InlineComment.new line_number:      comment.location.line,
+                        whitespace_col:   preceding_whitespace_range.column,
+                        whitespace:       preceding_whitespace,
+                        text_col:         comment.location.column,
+                        text:             comment.text,
+                        full_range:       range_for(first_char, comment.location.expression.end_pos),
+                        whitespace_range: preceding_whitespace_range,
+                        comment_range:    comment.location.expression
     end
+
+    def null_node
+      location = Parser::Source::Map::Collection.new nil, nil, range_for(0, 0)
+      Parser::AST::Node.new :null_node, [], location: location
+    end
+
   end
 end