rubocop-ast 0.5.1 → 1.0.0

data/lib/rubocop/ast/node_pattern/parser.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative 'parser.racc'
+
+module RuboCop
+  module AST
+    class NodePattern
+      # Parser for NodePattern
+      # Note: class reopened in `parser.racc`
+      #
+      # Doc on how this fits in the compiling process:
+      #   /doc/modules/ROOT/pages/node_pattern.md
+      class Parser < Racc::Parser
+        extend Forwardable
+
+        Builder = NodePattern::Builder
+        Lexer = NodePattern::Lexer
+
+        def initialize(builder = self.class::Builder.new)
+          super()
+          @builder = builder
+        end
+
+        ##
+        # (Similar API to `parser` gem)
+        # Parses a source and returns the AST.
+        #
+        # @param [Parser::Source::Buffer, String] source_buffer The source buffer to parse.
+        # @return [NodePattern::Node]
+        #
+        def parse(source)
+          @lexer = self.class::Lexer.new(source)
+          do_parse
+        rescue Lexer::Error => e
+          raise NodePattern::Invalid, e.message
+        ensure
+          @lexer = nil # Don't keep references
+        end
+
+        def inspect
+          "<##{self.class}>"
+        end
+
+        private
+
+        def_delegators :@builder, :emit_list, :emit_unary_op, :emit_atom, :emit_capture,
+                       :emit_call, :emit_union
+        def_delegators :@lexer, :next_token
+
+        def enforce_unary(node)
+          return node if node.arity == 1
+
+          detail = node.loc&.expression&.source || node.to_s
+          raise NodePattern::Invalid, 'parse error, expected unary node pattern ' \
+            "but got expression matching multiple elements: #{detail}"
+        end
+
+        # Overrides Racc::Parser's method:
+        def on_error(token, val, _vstack)
+          detail = token_to_str(token) || '?'
+          raise NodePattern::Invalid, "parse error on value #{val.inspect} (#{detail})"
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/ast/node_pattern/parser.y

@@ -0,0 +1,103 @@
+class RuboCop::AST::NodePattern::Parser
+options no_result_var
+token tSYMBOL tNUMBER tSTRING tWILDCARD tPARAM_NAMED tPARAM_CONST tPARAM_NUMBER
+      tFUNCTION_CALL tPREDICATE tNODE_TYPE tARG_LIST tUNIFY tREGEXP
+rule
+  node_pattern                               # @return Node
+    : node_pattern_no_union
+    | union                                  { enforce_unary(val[0]) }
+    ;
+
+  node_pattern_no_union                      # @return Node
+    : '(' variadic_pattern_list ')'          { emit_list :sequence, *val }
+    | '[' node_pattern_list ']'              { emit_list :intersection, *val }
+    | '!' node_pattern                       { emit_unary_op :negation, *val }
+    | '^' node_pattern                       { emit_unary_op :ascend, *val }
+    | '`' node_pattern                       { emit_unary_op :descend, *val }
+    | '$' node_pattern                       { emit_capture(*val) }
+    | tFUNCTION_CALL args                    { emit_call :function_call, *val }
+    | tPREDICATE args                        { emit_call :predicate, *val }
+    | tNODE_TYPE                             { emit_call :node_type, *val }
+    | atom
+    ;
+
+  atom                                       # @return Node
+    : tSYMBOL                                { emit_atom :symbol, *val }
+    | tNUMBER                                { emit_atom :number, *val }
+    | tSTRING                                { emit_atom :string, *val }
+    | tPARAM_CONST                           { emit_atom :const, *val }
+    | tPARAM_NAMED                           { emit_atom :named_parameter, *val }
+    | tPARAM_NUMBER                          { emit_atom :positional_parameter, *val }
+    | tREGEXP                                { emit_atom :regexp, *val }
+    | tWILDCARD                              { emit_atom :wildcard, *val }
+    | tUNIFY                                 { emit_atom :unify, *val }
+    ;
+
+  union                                      # @return Node
+    : '{' separated_variadic_patterns '}'    { emit_union(*val) }
+    ;
+
+  variadic_pattern                           # @return Node
+    : node_pattern_no_union
+    | union
+    | node_pattern repetition
+      {
+        main, repeat_t = val
+        emit_unary_op(:repetition, repeat_t, main, repeat_t)
+      }
+    | opt_capture '<' node_pattern_list opt_rest '>'
+      {
+        opt_capture, bracket, node_pattern_list, opt_rest, close_bracket = val
+        node_pattern_list << opt_rest if opt_rest
+        main = emit_list :any_order, bracket, node_pattern_list, close_bracket
+        emit_capture(opt_capture, main)
+      }
+    | rest
+    ;
+
+  repetition                                 # @return Token
+    : '?'
+    | '*'
+    | '+'
+    ;
+
+  opt_capture                                # @return Token | nil
+    :
+    | '$'
+    ;
+
+  rest                                       # @return Node
+    : opt_capture '...'                      { emit_capture(val[0], emit_atom(:rest, val[1])) }
+    ;
+
+  opt_rest                                   # @return Node | nil
+    :
+    | rest
+    ;
+
+  args                                       # @return [Token, Array<Node>, Token] | nil
+    :
+    | tARG_LIST arg_list ')'                 { val }
+    ;
+
+  arg_list                                   # @return Array<Node>
+    : node_pattern                           { val }
+    | arg_list ',' node_pattern              { val[0] << val[2] }
+    ;
+
+  node_pattern_list                          # @return Array<Node>
+    : node_pattern                           { val }
+    | node_pattern_list node_pattern         { val[0] << val[1] }
+    ;
+
+  variadic_pattern_list                      # @return Array<Node>
+    : variadic_pattern                       { val }
+    | variadic_pattern_list variadic_pattern { val[0] << val[1] }
+    ;
+
+  separated_variadic_patterns                # @return Array<Array<Node>>
+    :                                        { [[]] }
+    | separated_variadic_patterns variadic_pattern { val[0].last << val[1]; val[0] }
+    | separated_variadic_patterns '|'        { val[0] << [] }
+    ;
+end

data/lib/rubocop/ast/node_pattern/sets.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module AST
+    class NodePattern
+      # Utility to assign a set of values to a constant
+      module Sets
+        REGISTRY = Hash.new do |h, set|
+          name = Sets.name(set).freeze
+          Sets.const_set(name, set)
+          h[set] = "::RuboCop::AST::NodePattern::Sets::#{name}"
+        end
+
+        MAX = 4
+        def self.name(set)
+          elements = set
+          elements = set.first(MAX - 1) << :etc if set.size > MAX
+          name = elements.to_a.join('_').upcase.gsub(/[^A-Z0-9_]/, '')
+          uniq("SET_#{name}")
+        end
+
+        def self.uniq(name)
+          return name unless Sets.const_defined?(name)
+
+          (2..Float::INFINITY).each do |i|
+            uniq = "#{name}_#{i}"
+            return uniq unless Sets.const_defined?(uniq)
+          end
+        end
+
+        def self.[](set)
+          REGISTRY[set]
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/ast/node_pattern/with_meta.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+module RuboCop
+  module AST
+    class NodePattern
+      class Parser
+        # Overrides Parser to use `WithMeta` variants and provide additional methods
+        class WithMeta < Parser
+          # Overrides Lexer to token locations and comments
+          class Lexer < NodePattern::Lexer
+            attr_reader :source_buffer
+
+            def initialize(str_or_buffer)
+              @source_buffer = if str_or_buffer.respond_to?(:source)
+                                 str_or_buffer
+                               else
+                                 ::Parser::Source::Buffer.new('(string)', source: str_or_buffer)
+                               end
+              @comments = []
+              super(@source_buffer.source)
+            end
+
+            def token(type, value)
+              super(type, [value, pos])
+            end
+
+            def emit_comment
+              @comments << Comment.new(pos)
+              super
+            end
+
+            # @return [::Parser::Source::Range] last match's position
+            def pos
+              ::Parser::Source::Range.new(source_buffer, ss.pos - ss.matched_size, ss.pos)
+            end
+          end
+
+          # Overrides Builder to emit nodes with locations
+          class Builder < NodePattern::Builder
+            def emit_atom(type, token)
+              value, loc = token
+              begin_l = loc.resize(1)
+              end_l = loc.end.adjust(begin_pos: -1)
+              begin_l = nil if begin_l.source.match?(/\w/)
+              end_l = nil if end_l.source.match?(/\w/)
+              n(type, [value], source_map(token, begin_t: begin_l, end_t: end_l))
+            end
+
+            def emit_unary_op(type, operator_t = nil, *children)
+              children[-1] = children[-1].first if children[-1].is_a?(Array) # token?
+              map = source_map(children.first.loc.expression, operator_t: operator_t)
+              n(type, children, map)
+            end
+
+            def emit_list(type, begin_t, children, end_t)
+              expr = children.first.loc.expression.join(children.last.loc.expression)
+              map = source_map(expr, begin_t: begin_t, end_t: end_t)
+              n(type, children, map)
+            end
+
+            def emit_call(type, selector_t, args = nil)
+              selector, = selector_t
+              begin_t, arg_nodes, end_t = args
+
+              map = source_map(selector_t, begin_t: begin_t, end_t: end_t, selector_t: selector_t)
+              n(type, [selector, *arg_nodes], map)
+            end
+
+            private
+
+            def n(type, children, source_map)
+              super(type, children, { location: source_map })
+            end
+
+            def loc(token_or_range)
+              return token_or_range[1] if token_or_range.is_a?(Array)
+
+              token_or_range
+            end
+
+            def join_exprs(left_expr, right_expr)
+              left_expr.loc.expression
+                       .join(right_expr.loc.expression)
+            end
+
+            def source_map(token_or_range, begin_t: nil, end_t: nil, operator_t: nil, selector_t: nil)
+              expression_l = loc(token_or_range)
+              expression_l = expression_l.expression if expression_l.respond_to?(:expression)
+              locs = [begin_t, end_t, operator_t, selector_t].map { |token| loc(token) }
+              begin_l, end_l, operator_l, selector_l = locs
+
+              expression_l = locs.compact.inject(expression_l, :join)
+
+              ::Parser::Source::Map::Send.new(_dot_l = nil, selector_l, begin_l, end_l, expression_l)
+                                         .with_operator(operator_l)
+            end
+          end
+
+          attr_reader :comments, :tokens
+
+          def do_parse
+            r = super
+            @comments = @lexer.comments
+            @tokens = @lexer.tokens
+            r
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rubocop/ast/processed_source.rb

@@ -9,6 +9,7 @@ module RuboCop
     # and other information such as disabled lines for cops.
     # It also provides a convenient way to access source lines.
     class ProcessedSource
+      # @api private
       STRING_SOURCE_NAME = '(string)'
 
       attr_reader :path, :buffer, :ast, :comments, :tokens, :diagnostics,
@@ -19,6 +20,9 @@ module RuboCop
         new(file, ruby_version, path)
       end
 
+      INVALID_LEVELS = %i[error fatal].freeze
+      private_constant :INVALID_LEVELS
+
       def initialize(source, ruby_version, path = nil)
         # Defaults source encoding to UTF-8, regardless of the encoding it has
         # been read with, which could be non-utf8 depending on the default
@@ -63,7 +67,7 @@ module RuboCop
       def valid_syntax?
         return false if @parser_error
 
-        @diagnostics.none? { |d| %i[error fatal].include?(d.level) }
+        @diagnostics.none? { |d| INVALID_LEVELS.include?(d.level) }
       end
 
       # Raw source checksum for tracking infinite loops.

data/lib/rubocop/ast/traversal.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-# rubocop:disable Metrics/ModuleLength
 module RuboCop
   module AST
     # Provides methods for traversing an AST.
@@ -8,197 +7,175 @@ module RuboCop
     # Override methods to perform custom processing. Remember to call `super`
     # if you want to recursively process descendant nodes.
     module Traversal
+      # Only for debugging.
+      # @api private
+      class DebugError < RuntimeError
+      end
+
+      TYPE_TO_METHOD = Hash.new { |h, type| h[type] = :"on_#{type}" }
+
       def walk(node)
         return if node.nil?
 
-        send(:"on_#{node.type}", node)
+        send(TYPE_TO_METHOD[node.type], node)
         nil
       end
 
-      NO_CHILD_NODES    = %i[true false nil int float complex
-                             rational str sym regopt self lvar
-                             ivar cvar gvar nth_ref back_ref cbase
-                             arg restarg blockarg shadowarg
-                             kwrestarg zsuper redo retry
-                             forward_args forwarded_args
-                             match_var match_nil_pattern empty_else
-                             forward_arg lambda procarg0 __ENCODING__].freeze
-      ONE_CHILD_NODE    = %i[splat kwsplat block_pass not break next
-                             preexe postexe match_current_line defined?
-                             arg_expr pin match_rest if_guard unless_guard
-                             match_with_trailing_comma].freeze
-      MANY_CHILD_NODES  = %i[dstr dsym xstr regexp array hash pair
-                             mlhs masgn or_asgn and_asgn rasgn mrasgn
-                             undef alias args super yield or and
-                             while_post until_post iflipflop eflipflop
-                             match_with_lvasgn begin kwbegin return
-                             in_match match_alt
-                             match_as array_pattern array_pattern_with_tail
-                             hash_pattern const_pattern find_pattern
-                             index indexasgn].freeze
-      SECOND_CHILD_ONLY = %i[lvasgn ivasgn cvasgn gvasgn optarg kwarg
-                             kwoptarg].freeze
-
-      NO_CHILD_NODES.each do |type|
-        module_eval("def on_#{type}(node); end", __FILE__, __LINE__)
-      end
-
-      ONE_CHILD_NODE.each do |type|
-        module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
-          def on_#{type}(node)
-            if (child = node.children[0])
-              send(:"on_\#{child.type}", child)
+      # @api private
+      module CallbackCompiler
+        SEND = 'send(TYPE_TO_METHOD[child.type], child)'
+        assign_code = 'child = node.children[%<index>i]'
+        code = "#{assign_code}\n#{SEND}"
+        TEMPLATE = {
+          skip: '',
+          always: code,
+          nil?: "#{code} if child"
+        }.freeze
+
+        def def_callback(type, *signature,
+                         arity: signature.size..signature.size,
+                         arity_check: ENV['RUBOCOP_DEBUG'] && self.arity_check(arity),
+                         body: self.body(signature, arity_check))
+          type, *aliases = type
+          lineno = caller_locations(1, 1).first.lineno
+          module_eval(<<~RUBY, __FILE__, lineno) # rubocop:disable Style/EvalWithLocation
+            def on_#{type}(node)
+              #{body}
+              nil
             end
+          RUBY
+          aliases.each do |m|
+            alias_method "on_#{m}", "on_#{type}"
           end
-        RUBY
-      end
-
-      MANY_CHILD_NODES.each do |type|
-        module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
-          def on_#{type}(node)
-            node.children.each { |child| send(:"on_\#{child.type}", child) }
-            nil
-          end
-        RUBY
-      end
+        end
 
-      SECOND_CHILD_ONLY.each do |type|
-        # Guard clause is for nodes nested within mlhs
-        module_eval(<<~RUBY, __FILE__, __LINE__ + 1)
-          def on_#{type}(node)
-            if (child = node.children[1])
-              send(:"on_\#{child.type}", child)
+        def body(signature, prelude)
+          signature
+            .map.with_index do |arg, i|
+              TEMPLATE[arg].gsub('%<index>i', i.to_s)
             end
-          end
-        RUBY
-      end
-
-      def on_const(node)
-        return unless (child = node.children[0])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_casgn(node)
-        children = node.children
-        if (child = children[0]) # always const???
-          send(:"on_#{child.type}", child)
+            .unshift(prelude)
+            .join("\n")
         end
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
-      end
 
-      def on_class(node)
-        children = node.children
-        child = children[0] # always const???
-        send(:"on_#{child.type}", child)
-        if (child = children[1])
-          send(:"on_#{child.type}", child)
+        def arity_check(range)
+          <<~RUBY
+            n = node.children.size
+            raise DebugError, [
+              'Expected #{range} children, got',
+              n, 'for', node.inspect
+            ].join(' ') unless (#{range}).cover?(node.children.size)
+          RUBY
         end
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_def(node)
-        children = node.children
-        on_args(children[1])
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
       end
-
-      def on_send(node)
+      private_constant :CallbackCompiler
+      extend CallbackCompiler
+      send_code = CallbackCompiler::SEND
+
+      ### arity == 0
+      no_children = %i[true false nil self cbase zsuper redo retry
+                       forward_args forwarded_args match_nil_pattern
+                       forward_arg lambda empty_else kwnilarg
+                       __FILE__ __LINE__ __ENCODING__]
+
+      ### arity == 0..1
+      opt_symbol_child = %i[restarg kwrestarg]
+      opt_node_child = %i[splat kwsplat match_rest]
+
+      ### arity == 1
+      literal_child = %i[int float complex
+                         rational str sym lvar
+                         ivar cvar gvar nth_ref back_ref
+                         arg blockarg shadowarg
+                         kwarg match_var]
+
+      many_symbol_children = %i[regopt]
+
+      node_child = %i[block_pass not
+                      match_current_line defined?
+                      arg_expr pin if_guard unless_guard
+                      match_with_trailing_comma]
+      node_or_nil_child = %i[preexe postexe]
+
+      NO_CHILD_NODES = (no_children + opt_symbol_child + literal_child).to_set.freeze
+      private_constant :NO_CHILD_NODES # Used by Commissioner
+
+      ### arity > 1
+      symbol_then_opt_node = %i[lvasgn ivasgn cvasgn gvasgn]
+      symbol_then_node_or_nil = %i[optarg kwoptarg]
+      node_then_opt_node = %i[while until module sclass]
+
+      ### variable arity
+      many_node_children = %i[dstr dsym xstr regexp array hash pair
+                              mlhs masgn or_asgn and_asgn rasgn mrasgn
+                              undef alias args super yield or and
+                              while_post until_post iflipflop eflipflop
+                              match_with_lvasgn begin kwbegin return
+                              in_match match_alt break next
+                              match_as array_pattern array_pattern_with_tail
+                              hash_pattern const_pattern find_pattern
+                              index indexasgn procarg0]
+      many_opt_node_children = %i[case rescue resbody ensure for when
+                                  case_match in_pattern irange erange]
+
+      ### Callbacks for above
+      def_callback no_children
+      def_callback opt_symbol_child, :skip, arity: 0..1
+      def_callback opt_node_child, :nil?, arity: 0..1
+
+      def_callback literal_child, :skip
+      def_callback node_child, :always
+      def_callback node_or_nil_child, :nil?
+
+      def_callback symbol_then_opt_node, :skip, :nil?, arity: 1..2
+      def_callback symbol_then_node_or_nil, :skip, :nil?
+      def_callback node_then_opt_node, :always, :nil?
+
+      def_callback many_symbol_children, :skip, arity_check: nil
+      def_callback many_node_children, body: <<~RUBY
+        node.children.each { |child| #{send_code} }
+      RUBY
+      def_callback many_opt_node_children,
+                   body: <<~RUBY
+                     node.children.each { |child| #{send_code} if child }
+                   RUBY
+
+      ### Other particular cases
+      def_callback :const, :nil?, :skip
+      def_callback :casgn, :nil?, :skip, :nil?, arity: 2..3
+      def_callback :class, :always, :nil?, :nil?
+      def_callback :def, :skip, :always, :nil?
+      def_callback :op_asgn, :always, :skip, :always
+      def_callback :if, :always, :nil?, :nil?
+      def_callback :block, :always, :always, :nil?
+      def_callback :numblock, :always, :skip, :nil?
+      def_callback :defs, :always, :skip, :always, :nil?
+
+      def_callback %i[send csend], body: <<~RUBY
         node.children.each_with_index do |child, i|
           next if i == 1
 
-          send(:"on_#{child.type}", child) if child
+          #{send_code} if child
         end
-        nil
-      end
-
-      alias on_csend on_send
-
-      def on_op_asgn(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child)
-        child = children[2]
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_defs(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child)
-        on_args(children[2])
-        return unless (child = children[3])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_if(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child)
-        if (child = children[1])
-          send(:"on_#{child.type}", child)
-        end
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_while(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child)
-        return unless (child = children[1])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      alias on_until  on_while
-      alias on_module on_while
-      alias on_sclass on_while
-
-      def on_block(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child) # can be send, zsuper...
-        on_args(children[1])
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
-      end
-
-      def on_case(node)
+      RUBY
+
+      ### generic processing of any other node (forward compatibility)
+      defined = instance_methods(false)
+                .grep(/^on_/)
+                .map { |s| s.to_s[3..-1].to_sym } # :on_foo => :foo
+
+      to_define = ::Parser::Meta::NODE_TYPES.to_a
+      to_define -= defined
+      to_define -= %i[numargs ident] # transient
+      to_define -= %i[blockarg_expr restarg_expr] # obsolete
+      to_define -= %i[objc_kwarg objc_restarg objc_varargs] # mac_ruby
+      def_callback to_define, body: <<~RUBY
         node.children.each do |child|
-          send(:"on_#{child.type}", child) if child
+          next unless child.class == Node
+          #{send_code}
         end
-        nil
-      end
-
-      alias on_rescue     on_case
-      alias on_resbody    on_case
-      alias on_ensure     on_case
-      alias on_for        on_case
-      alias on_when       on_case
-      alias on_case_match on_case
-      alias on_in_pattern on_case
-      alias on_irange     on_case
-      alias on_erange     on_case
-
-      def on_numblock(node)
-        children = node.children
-        child = children[0]
-        send(:"on_#{child.type}", child)
-        return unless (child = children[2])
-
-        send(:"on_#{child.type}", child)
-      end
+      RUBY
+      MISSING = to_define if ENV['RUBOCOP_DEBUG']
     end
   end
 end
-# rubocop:enable Metrics/ModuleLength