parser 2.3.0.6 → 2.3.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ff5a785602415af9bb4752bc39eb9315b502f4bf
-  data.tar.gz: 611593226a424ad609a42b72e5cc79a2a3010043
+  metadata.gz: 5a26d867d04edc904373d219c6b5a1e96ef04d4d
+  data.tar.gz: 85ac7d2151c8f28584ff798c7e23e12ae1500afe
 SHA512:
-  metadata.gz: 051c85691ec47584262edb2477862918713d31e87f36c0ce3235a0f1eb7f14bf78c40cba6a8d5993281024231d666e77774becd71874c8c55fc6166d4d39ac20
-  data.tar.gz: af772a8dd0be6622896d24443fc2999e117645f8a67bee0d2d7179c601b4938d2a4055a70450b2771f23a770130aec36c7be339d4fcd740eba5971025567ec51
+  metadata.gz: ffa1d0b020bcb5802d13cb69477427d9f95d303a53de4addc37acdc82b03ac6abd2cc061e5a1a9fbf1cf6c2407a10c1dbb821ec0ce5c45b8ec103f5138d0894d
+  data.tar.gz: e75209c9674253351150671eed73530c052bbfc122cac1f199163f23cbd77420a2cd7ae7cf8268c059ca12c68a5750acdb74e81433328c5472b8bb81db4b2237

data/.travis.yml

@@ -13,6 +13,7 @@ rvm:
 matrix:
  allow_failures:
   - rvm: ruby-head
+  - rvm: rbx-2
 before_install:
  - gem update bundler
  - bundle --version

data/CHANGELOG.md

@@ -1,6 +1,20 @@
 Changelog
 =========
 
+v2.3.0.7 (2016-03-25)
+---------------------
+
+API modifications:
+ * Source::Diagnostic: handle ranges pointing to newlines (#273). (whitequark)
+
+Features implemented:
+ * Parser::Base#tokenize: allow recovery from syntax errors. (whitequark)
+ * lexer.rl: "a=1; a b: 1": allow label after command clashing with local. (whitequark)
+ * lexer.rl: "undef %s(x)": emit %s literals in expr_fname in 2.3 mode. (whitequark)
+
+Bugs fixed:
+ * Builders::Default: reject non-UTF-8 compatible literals. (whitequark)
+
 v2.3.0.6 (2016-02-14)
 ---------------------
 
@@ -29,7 +43,7 @@ Bugs fixed:
  * Add :csend to Parser::Meta::NODE_TYPES (Markus Schirp)
  * lexer/dedenter: "\<\<x\n  y\\n  z\nx": don't dedent after escaped newline. (whitequark)
 
-v2.3.0.6 (2016-01-16)
+v2.3.0.7 (2016-01-16)
 ---------------------
 
 v2.3.0.1 (2016-01-14)

data/README.md

@@ -227,9 +227,9 @@ issue](https://github.com/whitequark/parser/issues/72).
 
 It is unknown whether any gems are affected by this issue.
 
-### Invalid characters inside comments
+### Invalid characters inside comments and literals
 
-Ruby MRI permits arbitrary non-7-bit characters to appear in comments regardless of source encoding.
+Ruby MRI permits arbitrary non-7-bit byte sequences to appear in comments, as well as in string or symbol literals in form of escape sequences, regardless of source encoding. Parser requires all source code, including the expanded escape sequences, to consist of valid byte sequences in the source encoding that are convertible to UTF-8.
 
 As of 2013-07-25, there are about 180 affected gems.
 
@@ -239,12 +239,6 @@ Ruby MRI 1.8 permits to specify a bare `\u` escape sequence in a string; it trea
 
 As of 2013-07-25, affected gems are: activerdf, activerdf_net7, fastreader, gkellog-reddy.
 
-### Invalid Unicode escape sequences
-
-Ruby MRI 1.9+ permits to specify invalid Unicode codepoints in Unicode escape sequences, such as `\u{d800}`.
-
-As of 2013-07-25, affected gems are: aws_cloud_search.
-
 ### Dollar-dash
 
 (This one is so obscure I couldn't even think of a saner name for this issue.) Pre-2.1 Ruby allows

data/lib/parser/base.rb

@@ -167,8 +167,7 @@ module Parser
     end
 
     ##
-    # Parses a source buffer and returns the AST along with the comments of the
-    # Ruby source code.
+    # Parses a source buffer and returns the AST and the source code comments.
     #
     # @see #parse
     # @see Parser::Source::Comment#associate
@@ -183,6 +182,12 @@ module Parser
     end
 
     ##
+    # Parses a source buffer and returns the AST, the source code comments,
+    # and the tokens emitted by the lexer. If `recover` is true and a fatal
+    # {SyntaxError} is encountered, `nil` is returned instead of the AST, and
+    # comments as well as tokens are only returned up to the location of
+    # the error.
+    #
     # Currently, token stream format returned by #tokenize is not documented,
     # but is considered part of a public API and only changed according
     # to Semantic Versioning.
@@ -193,16 +198,23 @@ module Parser
     # `:tSTRING "foo"`; such details must not be relied upon.
     #
     # @param [Parser::Source::Buffer] source_buffer
+    # @param [Boolean] recover If true, recover from syntax errors. False by default.
     # @return [Array]
     #
-    def tokenize(source_buffer)
+    def tokenize(source_buffer, recover=false)
       @lexer.tokens = []
+      @lexer.comments = []
 
-      ast, comments = parse_with_comments(source_buffer)
+      begin
+        ast = parse(source_buffer)
+      rescue Parser::SyntaxError
+        raise if !recover
+      end
 
-      [ ast, comments, @lexer.tokens ]
+      [ ast, @lexer.comments, @lexer.tokens ]
     ensure
       @lexer.tokens = nil
+      @lexer.comments = nil
     end
 
     ##

data/lib/parser/builders/default.rb

@@ -120,12 +120,12 @@ module Parser
     # Strings
 
     def string(string_t)
-      n(:str, [ value(string_t) ],
+      n(:str, [ string_value(string_t) ],
         delimited_string_map(string_t))
     end
 
     def string_internal(string_t)
-      n(:str, [ value(string_t) ],
+      n(:str, [ string_value(string_t) ],
         unquoted_map(string_t))
     end
 
@@ -144,7 +144,7 @@ module Parser
     end
 
     def character(char_t)
-      n(:str, [ value(char_t) ],
+      n(:str, [ string_value(char_t) ],
         prefix_string_map(char_t))
     end
 
@@ -156,12 +156,12 @@ module Parser
     # Symbols
 
     def symbol(symbol_t)
-      n(:sym, [ value(symbol_t).to_sym ],
+      n(:sym, [ string_value(symbol_t).to_sym ],
         prefix_string_map(symbol_t))
     end
 
     def symbol_internal(symbol_t)
-      n(:sym, [ value(symbol_t).to_sym ],
+      n(:sym, [ string_value(symbol_t).to_sym ],
         unquoted_map(symbol_t))
     end
 
@@ -1534,6 +1534,18 @@ module Parser
       token[0]
     end
 
+    if defined?(Encoding)
+      def string_value(token)
+        unless token[0].valid_encoding?
+          diagnostic(:error, :invalid_encoding, nil, token[1])
+        end
+
+        token[0]
+      end
+    else
+      alias string_value value
+    end
+
     def loc(token)
       # Pass through `nil`s and return nil for tNL.
       token[1] if token && token[0]

data/lib/parser/diagnostic.rb

@@ -82,18 +82,19 @@ module Parser
     # @return [Array<String>]
     #
     def render
-      if @location.line != @location.last_line
+      if @location.line == @location.last_line || @location.is?("\n")
+        ["#{@location}: #{@level}: #{message}"] + render_line(@location)
+      else
         # multi-line diagnostic
         first_line = first_line_only(@location)
         last_line  = last_line_only(@location)
+        num_lines  = (@location.last_line - @location.line) + 1
         buffer     = @location.source_buffer
 
         last_lineno, last_column = buffer.decompose_position(@location.end_pos)
         ["#{@location}-#{last_lineno}:#{last_column}: #{@level}: #{message}"] +
-          render_line(first_line, true, false) +
+          render_line(first_line, num_lines > 2, false) +
           render_line(last_line, false, true)
-      else
-        ["#{@location}: #{@level}: #{message}"] + render_line(@location)
       end
     end
 
@@ -104,7 +105,7 @@ module Parser
     #
     # @return [Array<String>]
     #
-    def render_line(range, range_start=false, range_end=false)
+    def render_line(range, ellipsis=false, range_end=false)
       source_line    = range.source_line
       highlight_line = ' ' * source_line.length
 
@@ -115,15 +116,17 @@ module Parser
         end
       end
 
-      if !range_end && range.size >= 1
-        highlight_line[range.column_range] = '^' + '~' * (range.size - 1)
+      if range.is?("\n")
+        highlight_line += "^"
       else
-        highlight_line[range.column_range] = '~' * range.size
+        if !range_end && range.size >= 1
+          highlight_line[range.column_range] = '^' + '~' * (range.size - 1)
+        else
+          highlight_line[range.column_range] = '~' * range.size
+        end
       end
 
-      if range_start
-        highlight_line += '...'
-      end
+      highlight_line += '...' if ellipsis
 
       [source_line, highlight_line].
         map { |line| "#{range.source_buffer.name}:#{range.line}: #{line}" }
@@ -150,7 +153,7 @@ module Parser
     def last_line_only(range)
       if range.line != range.last_line
         Source::Range.new(range.source_buffer,
-                          range.begin_pos + (range.source =~ /[^\n]*\Z/),
+                          range.begin_pos + (range.source =~ /[^\n]*\z/),
                           range.end_pos)
       else
         range

data/lib/parser/lexer.rl

@@ -1197,7 +1197,7 @@ class Parser::Lexer
     emit(:tIDENTIFIER)
 
     if !@static_env.nil? && @static_env.declared?(tok)
-      fnext expr_end; fbreak;
+      fnext expr_endfn; fbreak;
     else
       fnext *arg_or_cmdarg; fbreak;
     end
@@ -1282,6 +1282,17 @@ class Parser::Lexer
       ':'
       => { fhold; fgoto expr_beg; };
 
+      '%s' c_any
+      => {
+        if version?(23)
+          type, delimiter = tok[0..-2], tok[-1].chr
+          fgoto *push_literal(type, delimiter, @ts);
+        else
+          p = @ts - 1
+          fgoto expr_end;
+        end
+      };
+
       w_any;
 
       c_any
@@ -1697,7 +1708,11 @@ class Parser::Lexer
         value = @escape || tok(@ts + 1)
 
         if version?(18)
-          emit(:tINTEGER, value[0].ord)
+          if defined?(Encoding)
+            emit(:tINTEGER, value.dup.force_encoding(Encoding::BINARY)[0].ord)
+          else
+            emit(:tINTEGER, value[0].ord)
+          end
         else
           emit(:tCHARACTER, value)
         end

data/lib/parser/messages.rb

@@ -59,6 +59,9 @@ module Parser
     # Parser warnings
     :useless_else            => 'else without rescue is useless',
 
+    # Parser errors that are not Ruby errors
+    :invalid_encoding        => 'literal contains escape sequences incompatible with UTF-8',
+
     # Rewriter diagnostics
     :invalid_action          => 'cannot %{action}',
     :clobbered               => 'clobbered by: %{action}',

data/lib/parser/source/rewriter.rb

@@ -39,6 +39,9 @@ module Parser
         @queue         = []
         @clobber       = 0
         @insertions    = 0 # clobbered zero-length positions; index 0 is the far left
+
+        @insert_before_multi_order = 0
+        @insert_after_multi_order = 0
       end
 
       ##
@@ -64,6 +67,28 @@ module Parser
         append Rewriter::Action.new(range.begin, content)
       end
 
+      ##
+      # Inserts new code before the given source range by allowing other
+      # insertions at the same position.
+      # Note that an insertion with latter invocation comes _before_ earlier
+      # insertion at the same position in the rewritten source.
+      #
+      # @example Inserting '[('
+      #   rewriter.
+      #     insert_before_multi(range, '(').
+      #     insert_before_multi(range, '[').
+      #     process
+      #
+      # @param [Range] range
+      # @param [String] content
+      # @return [Rewriter] self
+      # @raise [ClobberingError] when clobbering is detected
+      #
+      def insert_before_multi(range, content)
+        @insert_before_multi_order -= 1
+        append Rewriter::Action.new(range.begin, content, true, @insert_before_multi_order)
+      end
+
       ##
       # Inserts new code after the given source range.
       #
@@ -76,6 +101,28 @@ module Parser
         append Rewriter::Action.new(range.end, content)
       end
 
+      ##
+      # Inserts new code after the given source range by allowing other
+      # insertions at the same position.
+      # Note that an insertion with latter invocation comes _after_ earlier
+      # insertion at the same position in the rewritten source.
+      #
+      # @example Inserting ')]'
+      #   rewriter.
+      #     insert_after_multi(range, ')').
+      #     insert_after_multi(range, ']').
+      #     process
+      #
+      # @param [Range] range
+      # @param [String] content
+      # @return [Rewriter] self
+      # @raise [ClobberingError] when clobbering is detected
+      #
+      def insert_after_multi(range, content)
+        @insert_after_multi_order += 1
+        append Rewriter::Action.new(range.end, content, true, @insert_after_multi_order)
+      end
+
       ##
       # Replaces the code of the source range `range` with `content`.
       #
@@ -102,9 +149,7 @@ module Parser
         adjustment = 0
         source     = @source_buffer.source.dup
 
-        sorted_queue = @queue.sort_by { |action| action.range.begin_pos }
-
-        sorted_queue.each do |action|
+        @queue.sort.each do |action|
           begin_pos = action.range.begin_pos + adjustment
           end_pos   = begin_pos + action.range.length
 
@@ -205,7 +250,7 @@ module Parser
           # Replacing nothing with... nothing?
           return self if action.replacement.empty?
 
-          if (conflicting = clobbered_insertion?(range))
+          if !action.allow_multiple_insertions? && (conflicting = clobbered_insertion?(range))
             raise_clobber_error(action, [conflicting])
           end
 
@@ -331,7 +376,7 @@ module Parser
         actions = existing.push(action).sort_by do |a|
           [a.range.begin_pos, a.range.end_pos]
         end
-        range = actions.first.range.join(actions.last.range)
+        range = actions.first.range.join(actions.max_by { |a| a.range.end_pos }.range)
 
         Rewriter::Action.new(range, merge_replacements(actions))
       end

data/lib/parser/source/rewriter/action.rb

@@ -5,14 +5,26 @@ module Parser
     # @api private
     #
     class Rewriter::Action
-      attr_reader :range, :replacement
+      include Comparable
 
-      def initialize(range, replacement='')
-        @range, @replacement = range, replacement
+      attr_reader :range, :replacement, :allow_multiple_insertions, :order
+      alias_method :allow_multiple_insertions?, :allow_multiple_insertions
+
+      def initialize(range, replacement='', allow_multiple_insertions = false, order = 0)
+        @range = range
+        @replacement = replacement
+        @allow_multiple_insertions = allow_multiple_insertions
+        @order = order
 
         freeze
       end
 
+      def <=>(other)
+        result = range.begin_pos <=> other.range.begin_pos
+        return result unless result.zero?
+        order <=> other.order
+      end
+
       def to_s
         if @range.length == 0 && @replacement.empty?
           'do nothing'

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.3.0.6'
+  VERSION = '2.3.0.7'
 end

data/test/test_diagnostic.rb

@@ -69,4 +69,26 @@ class TestDiagnostic < Minitest::Test
       '(string):3: ~~~ ~~~    '
     ], diag.render)
   end
+
+  def test_bug_error_on_newline
+    # regression test; see GitHub issue 273
+    source = <<-CODE
+{
+  foo: ->() # I forgot my brace
+  }
+}
+    CODE
+    @buffer = Parser::Source::Buffer.new('(string)')
+    @buffer.source = source
+
+    location = Parser::Source::Range.new(@buffer, 33, 34)
+    diag = Parser::Diagnostic.new(:error, :unexpected_token, { :token => 'tNL' },
+                                  location)
+
+    assert_equal([
+      '(string):2:32: error: unexpected token tNL',
+      '(string):2:   foo: ->() # I forgot my brace',
+      '(string):2:                                ^'
+    ], diag.render)
+  end
 end

data/test/test_lexer.rb

@@ -2349,6 +2349,26 @@ class TestLexer < Minitest::Test
                    :tSYMBOL,     'b',     [10, 12])
   end
 
+  def test_fname_pct_s__22
+    setup_lexer 22
+    @lex.state = :expr_fname
+    assert_scanned("%s(a)",
+                   :tPERCENT,    '%', [0, 1],
+                   :tIDENTIFIER, 's', [1, 2],
+                   :tLPAREN2,    '(', [2, 3],
+                   :tIDENTIFIER, 'a', [3, 4],
+                   :tRPAREN,     ')', [4, 5])
+  end
+
+  def test_fname_pct_s__23
+    setup_lexer 23
+    @lex.state = :expr_fname
+    assert_scanned("%s(a)",
+                   :tSYMBEG,         '%s(', [0, 3],
+                   :tSTRING_CONTENT, 'a',   [3, 4],
+                   :tSTRING_END,     ')',   [4, 5])
+  end
+
   def test_static_env
     env = Parser::StaticEnvironment.new
     env.declare "a"

data/test/test_parser.rb

@@ -5114,6 +5114,31 @@ class TestParser < Minitest::Test
     end
   end
 
+  def test_tokenize_recover
+    with_versions(ALL_VERSIONS) do |_ver, parser|
+      source_file = Parser::Source::Buffer.new('(tokenize)')
+      source_file.source = "1 + # foo\n "
+
+      range = lambda do |from, to|
+        Parser::Source::Range.new(source_file, from, to)
+      end
+
+      ast, comments, tokens = parser.tokenize(source_file, true)
+
+      assert_equal nil, ast
+
+      assert_equal [
+                     Parser::Source::Comment.new(range.call(4, 9))
+                   ], comments
+
+      assert_equal [
+                     [:tINTEGER, [ 1,       range.call(0, 1) ]],
+                     [:tPLUS,    [ '+',     range.call(2, 3) ]],
+                     [:tCOMMENT, [ '# foo', range.call(4, 9) ]],
+                   ], tokens
+    end
+  end
+
   #
   # Bug-specific tests
   #
@@ -5200,6 +5225,53 @@ class TestParser < Minitest::Test
       %Q{f <<-TABLE do\nTABLE\nend})
   end
 
+  def test_bug_ascii_8bit_in_literal
+    if defined?(Encoding)
+      assert_diagnoses(
+        [:error, :invalid_encoding],
+        %q{".\xc3."},
+        %q{^^^^^^^^ location},
+        ALL_VERSIONS)
+
+      assert_diagnoses(
+        [:error, :invalid_encoding],
+        %q{%W"x .\xc3."},
+        %q{     ^^^^^^ location},
+        ALL_VERSIONS)
+
+      assert_diagnoses(
+        [:error, :invalid_encoding],
+        %q{:".\xc3."},
+        %q{  ^^^^^^ location},
+        ALL_VERSIONS)
+    end
+
+    assert_diagnoses(
+      [:error, :invalid_encoding],
+      %q{%I"x .\xc3."},
+      %q{     ^^^^^^ location},
+      ALL_VERSIONS - %w(1.8 1.9 ios mac))
+
+    assert_parses(
+      s(:int, 0xc3),
+      %q{?\xc3},
+      %q{},
+      %w(1.8))
+
+    assert_diagnoses(
+      [:error, :invalid_encoding],
+      %q{?\xc3},
+      %q{^^^^^ location},
+      ALL_VERSIONS - %w(1.8))
+
+    assert_parses(
+      s(:str, "проверка"),
+      %q{# coding:utf-8
+         "\xD0\xBF\xD1\x80\xD0\xBE\xD0\xB2\xD0\xB5\xD1\x80\xD0\xBA\xD0\xB0"},
+      %q{},
+      ALL_VERSIONS - %w(1.8))
+  end
+
   def test_ruby_bug_9669
     assert_parses(
       s(:def, :a, s(:args, s(:kwarg, :b)), s(:return)),
@@ -5318,6 +5390,21 @@ class TestParser < Minitest::Test
       ALL_VERSIONS - %w(1.8 1.9 2.0 2.1 2.2 ios mac))
   end
 
+  def test_ruby_bug_12073
+    assert_parses(
+      s(:begin,
+        s(:lvasgn, :a,
+          s(:int, 1)),
+        s(:send, nil, :a,
+          s(:hash,
+            s(:pair,
+              s(:sym, :b),
+              s(:int, 1))))),
+      %q{a = 1; a b: 1},
+      %q{},
+      ALL_VERSIONS - %w(1.8))
+  end
+
   def test_parser_bug_198
     assert_parses(
       s(:array,

data/test/test_source_rewriter.rb

@@ -135,6 +135,16 @@ class TestSourceRewriter < Minitest::Test
     end
   end
 
+  def test_intentional_multiple_insertions_at_same_location
+    assert_equal 'foo [(bar)] baz',
+      @rewriter.
+        insert_before_multi(range(4, 0), '(').
+        insert_after_multi(range(7, 0), ')').
+        insert_before_multi(range(4, 0), '[').
+        insert_after_multi(range(7, 0), ']').
+        process
+  end
+
   def test_insertion_within_replace_clobber
     silence_diagnostics
 
@@ -175,6 +185,46 @@ class TestSourceRewriter < Minitest::Test
     end
   end
 
+  def test_multi_insertion_within_replace_clobber
+    silence_diagnostics
+
+    assert_raises Parser::ClobberingError do
+      @rewriter.
+        replace(range(3, 2), '<').
+        insert_after_multi(range(3, 1), '>')
+    end
+  end
+
+  def test_multi_insertion_within_remove_clobber
+    silence_diagnostics
+
+    assert_raises Parser::ClobberingError do
+      @rewriter.
+        remove(range(3, 2)).
+        insert_after_multi(range(3, 1), '>')
+    end
+  end
+
+  def test_replace_overlapping_multi_insertion_clobber
+    silence_diagnostics
+
+    assert_raises Parser::ClobberingError do
+      @rewriter.
+        insert_after_multi(range(3, 1), '>').
+        replace(range(3, 2), '<')
+    end
+  end
+
+  def test_remove_overlapping_multi_insertion_clobber
+    silence_diagnostics
+
+    assert_raises Parser::ClobberingError do
+      @rewriter.
+        insert_after_multi(range(3, 1), '>').
+        remove(range(3, 2))
+    end
+  end
+
   def test_insertion_on_merged_insertion_clobber
     # 2 insertions at the same point clobber each other, even if the 1st one
     # was merged with an adjacent edit, and even if the same text is being
@@ -314,6 +364,17 @@ class TestSourceRewriter < Minitest::Test
     end
   end
 
+  def test_replaced_ranges_merge_when_furthest_right_range_is_not_furthest_left
+    # regression test; previously, when actions were merged, the resulting
+    # replaced range could be too small sometimes
+    assert_equal 'foo_***_***',
+      @rewriter.
+        replace(range(3, 1), '_').
+        replace(range(7, 1), '_').
+        replace(range(4, 7), '***_***').
+        process
+  end
+
   def test_clobber
     diagnostics = []
     @rewriter.diagnostics.consumer = lambda do |diag|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.3.0.6
+  version: 2.3.0.7
 platform: ruby
 authors:
 - whitequark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-14 00:00:00.000000000 Z
+date: 2016-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast