parser 2.6.3.0 → 2.6.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bceee2aee10ca7b86d7fb0e29f985b0b1848425b3d256af84e5dd324046ab2f7
-  data.tar.gz: 9ea970432bb2dc6dc68320baf39d7f24d928c45fc8ca236c14ed4c7c8db4d29e
+  metadata.gz: 9e113f6f42c44591674f288ebda60f8c4f17f5b6aee601d1cc5c2e20c50ef2e6
+  data.tar.gz: '08cf94385ef5cd8f211de395e1c272e5b7950e155463160e327cda2eb047dfad'
 SHA512:
-  metadata.gz: 577db299468896bb4dec4ca23dc14d78e58dd41dd323219de110e70ad2c691d6a9d3563339c1c4dcded9e7aaa69df5227f80ed1565f910d562928cf64c0936b4
-  data.tar.gz: 28c3a2a9e40727a71ac9255f1ad076ff86d2b60652eb4ac81719645c2542c3be0c818862003eae32b13c983397b86e023d9c0177f495dd45299845f1902871d5
+  metadata.gz: 25fd7715c94cf32eb097e8c0049d48f7b8185bef6ea4dafe21f0f7013c13892ff6d929e2e45f18480903f081ed9738aa8f0059287dd528cdb63feb94a47fd42e
+  data.tar.gz: 8dfe6966704a8b5b662f99a2ee15d525f6d4cd3d74b62b0c3946f01933c86f50075572cf32a81de3bfe2fe96a07d8197a55a11906f870b80daa2d81e50987944

data/.travis.yml

@@ -11,14 +11,14 @@ matrix:
     - name: 2.3.8 / Parser tests
       rvm: 2.3.8
       script: bundle exec rake test_cov
-    - name: 2.4.6 / Parser tests
-      rvm: 2.4.6
+    - name: 2.4.7 / Parser tests
+      rvm: 2.4.7
       script: bundle exec rake test_cov
-    - name: 2.5.5 / Parser tests
-      rvm: 2.5.5
+    - name: 2.5.6 / Parser tests
+      rvm: 2.5.6
       script: bundle exec rake test_cov
-    - name: 2.6.3 / Parser tests
-      rvm: 2.6.3
+    - name: 2.6.4 / Parser tests
+      rvm: 2.6.4
       script: bundle exec rake test_cov
     - name: ruby-head / Parser tests
       rvm: ruby-head
@@ -29,11 +29,11 @@ matrix:
     - name: rbx-2 / Parser tests
       rvm: rbx-2
       script: bundle exec rake test_cov
-    - name: 2.5.5 / Rubocop tests
-      rvm: 2.5.5
+    - name: 2.5.6 / Rubocop tests
+      rvm: 2.5.6
       script: ./ci/run_rubocop_specs
-    - name: 2.6.3 / Rubocop tests
-      rvm: 2.6.3
+    - name: 2.6.4 / Rubocop tests
+      rvm: 2.6.4
       script: ./ci/run_rubocop_specs
   allow_failures:
     - rvm: ruby-head

data/CHANGELOG.md

@@ -1,8 +1,30 @@
 Changelog
 =========
 
-Not released (2019-04-28)
--------------------------
+v2.6.4.0 (2019-08-30)
+---------------------
+
+API modifications:
+ * Added specs for heredocs with mixed encoding. (#581) (Ilya Bylich)
+
+Features implemented:
+ * ruby27.y: Revert "pipeline operator" (#601) (Koichi ITO)
+ * ruby27.y: Fix parsing of mutiple assignment with rescue modifier (#600) (Koichi ITO)
+ * ruby27.y: hoisted out f_rest_marg. (#594) (Ilya Bylich)
+ * ruby27.y: added pipeline operator. (#592) (Ilya Bylich)
+ * ruby27.y: reject safe navigator in LHS of mass-assignment. (#586) (Ilya Bylich)
+ * lexer.rl: reject whitespaces in meta and control chars. (#585) (Ilya Bylich)
+ * lexer.rl: Reject numparams as symbol literals. (#582) (Ilya Bylich)
+ * ruby27.y: Added numbered parameters support. (#565) (Ilya Bylich)
+ * lexer.rl: Reject \n and \r in heredoc identifiers starting from 2.7. (#575) (Ilya Bylich)
+
+Bugs fixed:
+ * ruby-parse: print empty string when --emit-json and empty input are given. (#590) (Ilya Bylich)
+ * AST_FORMAT: fixed documentation of the string with interpolation. (#589) (Ilya Bylich)
+ * builder.rb, processor.rb: Changed format of the procarg0 node. (#587) (Ilya Bylich)
+
+v2.6.3.0 (2019-04-28)
+---------------------
 
 Features implemented:
  * ruby27.y: Added beginless ranges support. (#570) (Ilya Bylich)

data/doc/AST_FORMAT.md

@@ -102,10 +102,13 @@ Format:
 Format:
 
 ~~~
-(dstr (str "foo") (lvar bar) (str "baz"))
+(dstr (str "foo") (begin (lvar bar)) (str "baz"))
 '"foo#{bar}baz"'
  ^ begin      ^ end
  ~~~~~~~~~~~~~~ expression
+     ^^ begin (begin)
+          ^ end (begin)
+     ^^^^^^ expression (begin)
 ~~~
 
 #### Here document
@@ -933,10 +936,15 @@ the sole argument (e.g. `|foo,|`).
 Format:
 
 ~~~
-(procarg0 :foo)
+(procarg0 (arg :foo))
 "|foo|"
   ~~~ expression
-  ~~~ name
+
+(procarg0 (arg :foo) (arg :bar))
+"|(foo, bar)|"
+  ~ begin
+           ~ end
+  ~~~~~~~~~~ expression
 ~~~
 
 ### Expression arguments
@@ -1083,6 +1091,40 @@ However, the following code results in a parse error:
 def f(*a: b); end
 ~~~
 
+## Numbered parameters
+
+### Block with numbered parameters
+
+Ruby 2.7 introduced a feature called "numbered parameters".
+Numbered and ordinal parameters are mutually exclusive, so if the block
+has only numbered parameters it also has a different AST node.
+
+Note that the second child represents a total number of numbered parameters.
+
+Format:
+
+~~~
+s(:numblock,
+  s(:send, nil, :proc), 3,
+  s(:send,
+    s(:numparam, 1), :+,
+    s(:numparam, 3)))
+"proc { @1 + @3 }"
+      ~ begin   ~ end
+ ~~~~~~~~~~~~~~~~ expression
+~~~
+
+### Numbered parameter
+
+Format:
+
+~~~
+(numparam 10)
+"@10"
+ ~~~ name
+ ~~~ expression
+~~~
+
 ## Send
 
 ### To self

data/lib/parser.rb

@@ -67,6 +67,7 @@ module Parser
   require 'parser/lexer/literal'
   require 'parser/lexer/stack_state'
   require 'parser/lexer/dedenter'
+  require 'parser/lexer/max_numparam_stack'
 
   module Builders
     require 'parser/builders/default'

data/lib/parser/ast/processor.rb

@@ -124,7 +124,22 @@ module Parser
       alias on_kwarg          process_argument_node
       alias on_kwoptarg       process_argument_node
       alias on_kwrestarg      process_argument_node
-      alias on_procarg0       process_argument_node
+
+      def on_procarg0(node)
+        if node.children[0].is_a?(Symbol)
+          # This branch gets executed when the builder
+          # is not configured to emit and 'arg' inside 'procarg0', i.e. when
+          #   Parser::Builders::Default.emit_arg_inside_procarg0
+          # is set to false.
+          #
+          # If this flag is set to true this branch is unreachable.
+          # s(:procarg0, :a)
+          on_argument(node)
+        else
+          # s(:procarg0, s(:arg, :a), s(:arg, :b))
+          process_regular_node(node)
+        end
+      end
 
       alias on_arg_expr       process_regular_node
       alias on_restarg_expr   process_regular_node
@@ -173,6 +188,14 @@ module Parser
       alias on_block    process_regular_node
       alias on_lambda   process_regular_node
 
+      def on_numblock(node)
+        method_call, max_numparam, body = *node
+
+        node.updated(nil, [
+          process(method_call), max_numparam, process(body)
+        ])
+      end
+
       alias on_while      process_regular_node
       alias on_while_post process_regular_node
       alias on_until      process_regular_node

data/lib/parser/builders/default.rb

@@ -80,6 +80,21 @@ module Parser
       attr_accessor :emit_index
     end
 
+    class << self
+      ##
+      # AST compatibility attribute; causes a single non-mlhs
+      # block argument to be wrapped in s(:procarg0).
+      #
+      # If set to false (the default), block arguments `|a|` are emitted as
+      # `s(:args, s(:procarg0, :a))`
+      #
+      # If set to true, block arguments `|a|` are emitted as
+      # `s(:args, s(:procarg0, s(:arg, :a))`
+      #
+      # @return [Boolean]
+      attr_accessor :emit_arg_inside_procarg0
+    end
+
     @emit_index = false
 
     class << self
@@ -90,6 +105,7 @@ module Parser
         @emit_procarg0 = true
         @emit_encoding = true
         @emit_index = true
+        @emit_arg_inside_procarg0 = true
       end
     end
 
@@ -437,6 +453,11 @@ module Parser
         variable_map(token))
     end
 
+    def numparam(token)
+      n(:numparam, [ value(token).to_i ],
+        variable_map(token))
+    end
+
     def back_ref(token)
       n(:back_ref, [ value(token).to_sym ],
         token_map(token))
@@ -663,6 +684,10 @@ module Parser
         collection_map(begin_t, args, end_t))
     end
 
+    def numargs(max_numparam)
+      n(:numargs, [ max_numparam ], nil)
+    end
+
     def arg(name_t)
       n(:arg, [ value(name_t).to_sym ],
         variable_map(name_t))
@@ -717,7 +742,12 @@ module Parser
 
     def procarg0(arg)
       if self.class.emit_procarg0
-        arg.updated(:procarg0)
+        if arg.type == :arg && self.class.emit_arg_inside_procarg0
+          n(:procarg0, [ arg ],
+            Source::Map::Collection.new(nil, nil, arg.location.expression))
+        else
+          arg.updated(:procarg0)
+        end
       else
         arg
       end
@@ -835,15 +865,23 @@ module Parser
         diagnostic :error, :block_and_blockarg, nil, last_arg.loc.expression, [loc(begin_t)]
       end
 
+
+      if args.type == :numargs
+        block_type = :numblock
+        args = args.children[0]
+      else
+        block_type = :block
+      end
+
       if [:send, :csend, :index, :super, :zsuper, :lambda].include?(method_call.type)
-        n(:block, [ method_call, args, body ],
+        n(block_type, [ method_call, args, body ],
           block_map(method_call.loc.expression, begin_t, end_t))
       else
         # Code like "return foo 1 do end" is reduced in a weird sequence.
         # Here, method_call is actually (return).
         actual_send, = *method_call
         block =
-          n(:block, [ actual_send, args, body ],
+          n(block_type, [ actual_send, args, body ],
             block_map(actual_send.loc.expression, begin_t, end_t))
 
         n(method_call.type, [ block ],
@@ -1215,18 +1253,18 @@ module Parser
         case this_arg.type
         when :arg, :optarg, :restarg, :blockarg,
              :kwarg, :kwoptarg, :kwrestarg,
-             :shadowarg, :procarg0
+             :shadowarg
 
-          this_name, = *this_arg
+          check_duplicate_arg(this_arg, map)
 
-          that_arg   = map[this_name]
-          that_name, = *that_arg
+        when :procarg0
 
-          if that_arg.nil?
-            map[this_name] = this_arg
-          elsif arg_name_collides?(this_name, that_name)
-            diagnostic :error, :duplicate_argument, nil,
-                       this_arg.loc.name, [ that_arg.loc.name ]
+          if this_arg.children[0].is_a?(Symbol)
+            # s(:procarg0, :a)
+            check_duplicate_arg(this_arg, map)
+          else
+            # s(:procarg0, s(:arg, :a), ...)
+            check_duplicate_args(this_arg.children, map)
           end
 
         when :mlhs
@@ -1235,6 +1273,20 @@ module Parser
       end
     end
 
+    def check_duplicate_arg(this_arg, map={})
+      this_name, = *this_arg
+
+      that_arg   = map[this_name]
+      that_name, = *that_arg
+
+      if that_arg.nil?
+        map[this_name] = this_arg
+      elsif arg_name_collides?(this_name, that_name)
+        diagnostic :error, :duplicate_argument, nil,
+                   this_arg.loc.name, [ that_arg.loc.name ]
+      end
+    end
+
     def arg_name_collides?(this_name, that_name)
       case @parser.version
       when 18

data/lib/parser/context.rb

@@ -47,5 +47,13 @@ module Parser
     end
     alias module_definition_allowed? class_definition_allowed?
     alias dynamic_const_definition_allowed? class_definition_allowed?
+
+    def in_block?
+      @stack.last == :block
+    end
+
+    def in_lambda?
+      @stack.last == :lambda
+    end
   end
 end

data/lib/parser/current.rb

@@ -48,7 +48,7 @@ module Parser
     CurrentRuby = Ruby23
 
   when /^2\.4\./
-    current_version = '2.4.6'
+    current_version = '2.4.7'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby24', current_version
     end
@@ -57,7 +57,7 @@ module Parser
     CurrentRuby = Ruby24
 
   when /^2\.5\./
-    current_version = '2.5.5'
+    current_version = '2.5.6'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby25', current_version
     end
@@ -66,7 +66,7 @@ module Parser
     CurrentRuby = Ruby25
 
   when /^2\.6\./
-    current_version = '2.6.3'
+    current_version = '2.6.4'
     if RUBY_VERSION != current_version
       warn_syntax_deviation 'parser/ruby26', current_version
     end

data/lib/parser/lexer.rl

@@ -89,7 +89,10 @@ class Parser::Lexer
 
   REGEXP_META_CHARACTERS = Regexp.union(*"\\$()*+.<>?[]^{|}".chars).freeze
 
+  NUMPARAM_MAX = 100
+
   attr_reader   :source_buffer
+  attr_reader   :max_numparam_stack
 
   attr_accessor :diagnostics
   attr_accessor :static_env
@@ -176,6 +179,9 @@ class Parser::Lexer
 
     # State before =begin / =end block comment
     @cs_before_block_comment = self.class.lex_en_line_begin
+
+    # Maximum numbered parameters stack
+    @max_numparam_stack = MaxNumparamStack.new
   end
 
   def source_buffer=(source_buffer)
@@ -249,6 +255,10 @@ class Parser::Lexer
     @cond = @cond_stack.pop
   end
 
+  def max_numparam
+    @max_numparam_stack.top
+  end
+
   def dedent_level
     # We erase @dedent_level as a precaution to avoid accidentally
     # using a stale value.
@@ -705,6 +715,14 @@ class Parser::Lexer
     diagnostic :fatal, :invalid_escape
   }
 
+  action read_post_meta_or_ctrl_char {
+    @escape = @source_buffer.slice(p - 1).chr
+
+    if @version >= 27 && ((0..8).include?(@escape.ord) || (14..31).include?(@escape.ord))
+      diagnostic :fatal, :invalid_escape
+    end
+  }
+
   action slash_c_char {
     @escape = encode_escape(@escape[0].ord & 0x9f)
   }
@@ -715,13 +733,13 @@ class Parser::Lexer
 
   maybe_escaped_char = (
         '\\' c_any      %unescape_char
-    | ( c_any - [\\] )  % { @escape = @source_buffer.slice(p - 1).chr }
+    | ( c_any - [\\] )  %read_post_meta_or_ctrl_char
   );
 
   maybe_escaped_ctrl_char = ( # why?!
         '\\' c_any      %unescape_char %slash_c_char
     |   '?'             % { @escape = "\x7f" }
-    | ( c_any - [\\?] ) % { @escape = @source_buffer.slice(p - 1).chr } %slash_c_char
+    | ( c_any - [\\?] ) %read_post_meta_or_ctrl_char %slash_c_char
   );
 
   escape = (
@@ -1301,6 +1319,36 @@ class Parser::Lexer
         fnext *stack_pop; fbreak;
       };
 
+      '@' [0-9]+
+      => {
+        if @version < 27
+          diagnostic :error, :ivar_name, { :name => tok }
+        end
+
+        value = tok[1..-1]
+
+        if value[0] == '0'
+          diagnostic :error, :leading_zero_in_numparam, nil, range(@ts, @te)
+        end
+
+        if value.to_i > NUMPARAM_MAX
+          diagnostic :error, :too_large_numparam, nil, range(@ts, @te)
+        end
+
+        if !@context.in_block? && !@context.in_lambda?
+          diagnostic :error, :numparam_outside_block, nil, range(@ts, @te)
+        end
+
+        if !@max_numparam_stack.can_have_numparams?
+          diagnostic :error, :ordinary_param_defined, nil, range(@ts, @te)
+        end
+
+        @max_numparam_stack.register(value.to_i)
+
+        emit(:tNUMPARAM, tok[1..-1])
+        fnext *stack_pop; fbreak;
+      };
+
       instance_var_v
       => {
         if tok =~ /^@[0-9]/
@@ -1733,7 +1781,11 @@ class Parser::Lexer
         type        =  $3.empty? ? '<<"'.freeze : ('<<'.freeze + $3)
         delimiter   =  $4
 
-        if @version >= 24
+        if @version >= 27
+          if delimiter.count("\n") > 0 || delimiter.count("\r") > 0
+            diagnostic :error, :unterminated_heredoc_id, nil, range(@ts, @ts + 1)
+          end
+        elsif @version >= 24
           if delimiter.count("\n") > 0
             if delimiter.end_with?("\n")
               diagnostic :warning, :heredoc_id_ends_with_nl, nil, range(@ts, @ts + 1)
@@ -1756,6 +1808,21 @@ class Parser::Lexer
         end
       };
 
+      # Escaped unterminated heredoc start
+      # <<'END  | <<"END  | <<`END  |
+      # <<-'END | <<-"END | <<-`END |
+      # <<~'END | <<~"END | <<~`END
+      #
+      # If the heredoc is terminated the rule above should handle it
+      '<<' [~\-]?
+        ('"' (any - c_nl - '"')*
+        |"'" (any - c_nl - "'")*
+        |"`" (any - c_nl - "`")
+        )
+      => {
+        diagnostic :error, :unterminated_heredoc_id, nil, range(@ts, @ts + 1)
+      };
+
       #
       # SYMBOL LITERALS
       #
@@ -1796,6 +1863,20 @@ class Parser::Lexer
         fnext expr_end; fbreak;
       };
 
+      ':' ( '@'  %{ tm = p - 1; diag_msg = :ivar_name }
+          | '@@' %{ tm = p - 2; diag_msg = :cvar_name }
+          ) [0-9]*
+      => {
+        if @version >= 27
+          diagnostic :error, diag_msg, { name: tok(tm, @te) }, range(tm, @te)
+        else
+          emit(:tCOLON, tok(@ts, @ts + 1), @ts, @ts + 1)
+          p = @ts
+        end
+
+        fnext expr_end; fbreak;
+      };
+
       #
       # AMBIGUOUS TERNARY OPERATOR
       #