reflexive 0.0.1

data/lib/reflexive/parse_tree_top_down_walker.rb

@@ -0,0 +1,392 @@
+require "zlib"
+require "reflexive/variables_scope"
+
+module Reflexive
+  class ParseTreeTopDownWalker
+    def initialize(events_tree)
+      @events_tree = events_tree
+      @local_variables = []
+      @dynamic_variables = []
+      @variables_scope_id = 1
+      VariablesScope.reset_guid
+      # empty - top level
+      # ["Module", "Class"] - class Class in module Module, class dev level
+      # ["Module", "Class", :instance] - class Class in module Module, instance level
+      @scope = [] # @scope.last - is basically "implicit self"
+      # also used for constant lookup
+    end
+
+    def push_namespace_scope(namespace_name)
+      @scope << namespace_name
+    end
+
+    def push_namespace_instance_scope
+      @scope << :instance
+    end
+
+    def pop_namespace_scope
+      @scope.pop
+    end
+
+    def current_variables_scope
+      (@dynamic_variables.size > 0 ? @dynamic_variables : @local_variables).last
+    end
+
+    def add_local_variable(scanner_event)
+      current_variables_scope.merge!(scanner_event[:ident] => scanner_event)
+      local_variable_assignment(scanner_event)
+    end
+
+    def variables_scope
+      merged_scope = @dynamic_variables.dup.reverse
+      merged_scope << @local_variables.last if @local_variables.size > 0
+      merged_scope
+    end
+
+    # TODO crazy, replace that with something more appropriate
+    def variables_scope_id
+      current_variables_scope.guid
+    end
+
+    def local_variable_defined?(name)
+      variables_scope.any? { |variables| variables.has_key?(name) }
+    end
+
+    def local_variable_scope_id(name)
+      variables_scope.detect { |variables| variables.has_key?(name) }.guid
+    end
+
+    def add_local_variables_from_lhs(lhs_event)
+      # [:var_field, {:ident=>"v1"}]
+      if lhs_event[0] == :var_field
+        if (scanner_event = lhs_event[1]).is_a?(Hash)
+          if scanner_event[:ident]
+            add_local_variable(scanner_event)
+          end
+        end
+      end
+      # raise "don't know how to add local variables from lhs_event : #{ lhs_event }"
+    end
+
+    def add_local_variables_from_mlhs(mlhs_event)
+      # [{:ident=>"a"}, {:ident=>"b"}],
+      if mlhs_event.is_a?(Array)
+        if mlhs_event[0] == :mlhs_add_star
+          add_local_variables_from_mlhs(mlhs_event[1])
+          add_local_variable(mlhs_event[2]) if mlhs_event[2][:ident]
+          add_local_variables_from_mlhs(mlhs_event[3]) if mlhs_event[3].is_a?(Array)
+        else
+          mlhs_event.each do |event|
+            next unless scanner_event?(event)
+            if event[:ident]
+              add_local_variable(event)
+            end
+          end
+        end
+      end
+      # raise "don't know how to add local variables from lhs_event : #{ lhs_event }"
+    end
+
+    def add_local_variables_from_params_event(params_event)
+      return unless params_event
+      params_event = params_event[1] if params_event[0] == :paren # ?
+      found = false
+      for scanner_event in extract_scanner_events_from_tree(params_event)
+        if scanner_event[:ident]
+          found = true
+          add_local_variable(scanner_event)
+        end
+      end
+      # raise "don't know how to add local variables from params_event: #{ params_event }" unless found
+    end
+
+    def extract_scanner_events_from_tree(tree)
+      tree.flatten.select { |e| scanner_event?(e) }
+    end
+
+    def push_local_variables_context
+      @local_variables << VariablesScope.new
+    end
+
+    def pop_local_variables_context
+      @local_variables.pop
+    end
+
+    def push_dynamic_variables_context
+      @dynamic_variables << VariablesScope.new
+    end
+
+    def pop_dynamic_variables_context
+      @dynamic_variables.pop
+    end
+
+    def walk(event = @events_tree)
+      type, *args = event
+      if respond_to?("on_#{ type }")
+        send("on_#{ type }", *args) #rescue r($!, event)
+      else
+        on_default(type, args)
+      end
+    end
+
+    def on_default(type, event_args)
+      return unless event_args # no-args event
+      event_args.each do |arg|
+        if arg == nil
+          # empty arg - pass
+
+          # why the following isn't reported with scanner events?
+        elsif type == :call && [:".", :"::"].include?(arg)
+        elsif type == :var_ref && [:".", :"::"].include?(arg)
+        elsif type == :field && [:".", :"::"].include?(arg)
+        elsif type == :command_call && ([:".", :"::"].include?(arg) || arg == false)
+        elsif type == :args_add_block && arg == false
+        elsif type == :unary && arg.is_a?(Symbol)
+        elsif type == :binary && arg.is_a?(Symbol)
+        elsif scanner_event?(arg)
+          # scanner event - pass
+        elsif (parser_events?(arg) rescue r(type, event_args))
+          arg.each do |event|
+            walk(event)
+          end
+        elsif parser_event?(arg)
+          walk(arg)
+        end
+      end
+    end
+
+    def keep_walking(*args)
+      on_default(nil, args)
+    end
+
+    def on_program(body)
+      push_local_variables_context
+      keep_walking(body)
+      pop_local_variables_context
+    end
+
+    def on_def(name, params, body)
+      push_local_variables_context
+      add_local_variables_from_params_event(params)
+      push_namespace_instance_scope
+      keep_walking(body)
+      pop_namespace_scope
+      pop_local_variables_context
+    end
+
+    def on_defs(target, period, name, params, body)
+      push_local_variables_context
+      add_local_variables_from_params_event(params)
+      keep_walking(body)
+      pop_local_variables_context
+    end
+
+    def on_class(name, ancestor, body)
+      keep_walking(name)
+      push_local_variables_context
+      push_namespace_scope(resolve_constant_ref(name))
+      keep_walking(body)
+      pop_namespace_scope
+      pop_local_variables_context
+    end
+
+    def on_sclass(target, body)
+      push_local_variables_context
+      keep_walking(body)
+      pop_local_variables_context
+    end
+
+    def on_module(name, body)
+      keep_walking(name)
+      push_local_variables_context
+      push_namespace_scope(resolve_constant_ref(name))
+      keep_walking(body)
+      pop_namespace_scope
+      pop_local_variables_context
+    end
+
+    def on_do_block(params, body)
+      push_dynamic_variables_context
+      add_local_variables_from_params_event(params)
+      keep_walking(body)
+      pop_dynamic_variables_context
+    end
+
+    def on_brace_block(params, body)
+      push_dynamic_variables_context
+      add_local_variables_from_params_event(params)
+      keep_walking(body)
+      pop_dynamic_variables_context
+    end
+
+    def on_assign(lhs, rhs)
+      add_local_variables_from_lhs(lhs)
+      keep_walking(rhs)
+    end
+
+    def on_massign(mlhs, mrhs)
+      add_local_variables_from_mlhs(mlhs)
+      keep_walking(mrhs)
+    end
+
+    def on_command(operation, command_args)
+      method_call(operation, nil) if is_ident?(operation)
+      keep_walking(command_args)
+    end
+
+    def on_fcall(operation)
+      method_call(operation, nil) if is_ident?(operation)
+    end
+
+    # primary_value => anything
+    # operation2	: tIDENTIFIER
+    #		| tCONSTANT
+    #		| tFID
+    #		| op
+    #		;
+    # command_args => anything
+    def on_command_call(receiver, dot, method, args)
+      if is_ident?(method) &&
+              (constant = resolve_constant_ref(receiver))
+        method_call(method, [constant])
+      end
+      keep_walking(receiver, args)
+    end
+
+    def resolve_constant_ref(events)
+      if events[0] == :var_ref || events[0] == :const_ref &&
+              scanner_event?(events[1]) &&
+              events[1][:const]
+        events[1][:const]
+      elsif events[0] == :top_const_ref &&
+              scanner_event?(events[1]) &&
+              events[1][:const]
+        "::" + events[1][:const]
+      elsif events[0] == :const_path_ref &&
+              (constant = resolve_constant_ref(events[1]))
+        "#{ constant }::#{ events[2][:const] }"
+      end
+    end
+
+    def resolve_receiver(receiver)
+      resolve_constant_ref(receiver)
+    end
+
+    #  [:call,
+    #             [:var_ref, {:ident=>"subclasses"}]
+    def on_call(receiver, dot, method)
+      if rcv = resolve_receiver(receiver)
+        method_call(method, [rcv])
+      else
+        keep_walking(receiver)
+      end
+    end
+
+    def on_var_ref(ref_event)
+      # [:var_ref, {:kw=>"false"}]
+      # [:var_ref, {:cvar=>"@@subclasses"}]
+      # [:var_ref, {:ident=>"child"}] (sic!)
+      #   [:binary,
+#                 [:var_ref, {:ident=>"nonreloadables"}],
+#                 :<<,
+#                 [:var_ref, {:ident=>"klass"}]
+      #
+      #[:call,
+#                 [:var_ref, {:ident=>"klass"}],
+#                 :".",
+#                 {:ident=>"instance_variables"}],
+      #
+      #
+      if scanner_event?(ref_event)
+        if ref_event[:ident]
+          if local_variable_defined?(ref_event[:ident])
+            local_variable_access(ref_event)
+          else
+            method_call(ref_event, nil)
+          end
+        elsif ref_event[:const]
+          constant_access(ref_event)
+        end
+      end
+    end
+
+    def on_const_ref(const_ref_event)
+      if scanner_event?(const_ref_event)
+        if const_ref_event[:const]
+          constant_access(const_ref_event)
+        end
+      end
+    end
+
+    def on_const_path_ref(primary_value, name)
+      keep_walking(primary_value)
+      if (constant = resolve_constant_ref(primary_value)) &&
+              scanner_event?(name) && name[:const]
+        constant_access(name, "#{ constant }::#{ name[:const] }")
+      end
+    end
+
+    def method_call(scanner_event, receiver, *args)
+      unless receiver
+        # implict self concept
+        receiver = @scope.dup
+      end
+      merge_tags(scanner_event,
+                 {:method_call =>
+                         {:name => scanner_event[:ident],
+                          :receiver => receiver,
+                          :scope => constant_access_scope}})
+    end
+
+    def local_variable_access(scanner_event)
+      existing_variable_id = "#{ local_variable_scope_id(scanner_event[:ident]) }:#{ scanner_event[:ident] }"
+      merge_tags(scanner_event,
+                 :local_variable_access => existing_variable_id )
+    end
+
+    def constant_access_scope
+      scope = @scope.dup
+      scope.pop if scope.last == :instance # class/instance doesn't matter for constant lookup
+      scope
+    end
+
+    def constant_access(scanner_event, name = nil)
+      unless name
+        name = scanner_event[:const]
+      end
+      merge_tags(scanner_event,
+                 :constant_access => { :name => name,
+                                       :scope => constant_access_scope })
+    end
+
+    def local_variable_assignment(scanner_event)
+      merge_tags(scanner_event,
+                 :local_variable_assignment => variable_id(scanner_event))
+    end    
+
+    def variable_id(scanner_event)
+      "#{ variables_scope_id }:#{ scanner_event[:ident] }"
+    end
+
+    def merge_tags(scanner_event, tags)
+      scanner_event[:tags] ||= {}
+      scanner_event[:tags].merge!(tags)
+    end
+
+    def scanner_event?(event)
+      event.is_a?(Hash)
+    end
+
+    def parser_event?(event)
+      event.is_a?(Array) && event[0].is_a?(Symbol)
+    end
+
+    def parser_events?(events)
+      events.all? { |event| parser_event?(event) }
+    end
+
+    def is_ident?(event)
+      scanner_event?(event) && event[:ident]
+    end
+  end
+end

data/lib/reflexive/reflexive_ripper.rb

@@ -0,0 +1,274 @@
+require "ripper"
+require "reflexive/parse_tree_top_down_walker"
+
+module Reflexive
+  # Records all scanner events, injects meta-events when scope
+  # changes (required to implement constant/method look-up) 
+  class ReflexiveRipper < Ripper
+    attr_accessor :scanner_events
+
+    META_EVENT = [ :meta_scope ].freeze
+
+    def initialize(*args)
+      super
+      @scanner_events = []
+      @await_scope_change = false
+      @scope = []
+      @new_scope = nil
+    end
+
+    def parse
+      parse_tree = super
+      require 'pp'
+      # pp parse_tree
+
+      ParseTreeTopDownWalker.new(parse_tree).walk
+      parse_tree
+    end
+
+    # Returns array in format: [event_value, event_name, tags]
+    def self.destruct_scanner_event(scanner_event)
+      tags = scanner_event.delete(:tags)
+      [ scanner_event.values.first, scanner_event.keys.first, tags ]  
+    end
+
+    def self.is_meta_event?(event)
+      META_EVENT.include?(event)
+    end
+
+    Ripper::SCANNER_EVENTS.each do |meth|
+      define_method("on_#{ meth }") do |*args|
+        result = super(*args)
+        @scanner_events << { meth.to_sym => args[0] }
+        stop_await_scope_change
+        @scanner_events[-1]
+      end
+    end
+
+    def stop_await_scope_change
+      # get here when any of 3 possible constant reference ends,
+      # which means we have new scope in effect
+      if @await_scope_change
+        if @new_scope
+          @scope << @new_scope.dup
+          inject_current_scope
+          @new_scope = nil
+        end
+        @await_scope_change = false
+      end
+    end
+
+    def inject_current_scope
+      @scanner_events << { :meta_scope => @scope.dup }
+    end
+
+    PARSER_EVENT_TABLE.each do |event, arity|
+      if /_new\z/ =~ event.to_s and arity == 0
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}
+            []
+          end
+        End
+      elsif /_add\z/ =~ event.to_s
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(list, item)
+            list.push item
+            list
+          end
+        End
+      else
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(*args)
+            [:#{event}, *args]
+          end
+        End
+      end
+    end
+
+    # Two parser events which fire when class/module definition ends
+    def on_class(*args)
+      @scope.pop
+      inject_current_scope
+      
+      [:class, *args]
+    end
+
+    def on_module(*args)
+      @scope.pop
+      inject_current_scope
+
+      [:module, *args]
+    end
+
+    def on_parse_error(*args)
+      raise SyntaxError, "#{ lineno }: #{ args[0] }"
+    end
+
+#    def on_call(*args)
+#      # puts "PARSE: on_call: #{ args.inspect }"
+#      if args.size == 3 &&
+#              (constant = resolve_constant_ref(args[0])) &&
+#              args[1] == :"." &&
+#              args[2][1] == :ident
+#
+#        meth_ident_token = args[2]
+#        index = token_index(meth_ident_token)
+#        tags = { :constant => constant, :method => args[2][0] }
+#        # wrap the method identifier token in :method_call meta tokens
+#        @scanner_events[index .. index] = [
+#          # [ :method_call, :open, tags ],
+#          meth_ident_token,
+#          # [ :method_call, :close ]
+#        ]
+#      end
+#      [:call, *args]
+#    end
+#
+#    def on_var_ref(*args)
+#      # puts "PARSER: var_ref: #{ args.inspect }"
+#      if args.size == 1 &&
+#              args[0].size == 2 &&
+#              args[0][1] == :ident
+#        meth_ident_token = args[0]
+#        index = token_index(meth_ident_token)
+#        tags = { :instance_method => args[0][0] }
+#        @scanner_events[index .. index] = [
+#          [ :method_call, :open, tags ],
+#          meth_ident_token,
+#          [ :method_call, :close ]
+#        ]
+#        # r @scanner_events[177]
+#      end
+#      [:var_ref, *args]
+#    end
+
+    def token_index(token)
+      @scanner_events.index do |t|
+         t.object_id == token.object_id
+      end
+    end
+
+    def resolve_constant_ref(tokens)
+      self.class.resolve_constant_ref(tokens)
+    end
+
+    def self.resolve_constant_ref(tokens)
+      if tokens[0] == :var_ref && tokens[1][1] == :const
+        # [:var_ref, ["B", :const]]
+        tokens[1][0]
+      elsif tokens[0] == :top_const_ref && tokens[1][1] == :const
+        # [:top_const_ref, ["A", :const]]
+        "::" + tokens[1][0]
+        # [:const_path_ref, [:top_const_ref, ["A", :const]], ["B", :const]]
+        # [:const_path_ref, [:var_ref, ["A", :const]], ["B", :const]]
+      elsif tokens[0] == :const_path_ref && (constant = resolve_constant_ref(tokens[1]))
+        "#{ constant }::#{ tokens[2][0] }"
+      end
+    end
+
+    def on_sp(*args)
+      @scanner_events << { :sp => args[0] }
+      # ignore space tokens when waiting for scope changes:
+      # stop_await_scope_change
+      
+      @scanner_events[-1]
+    end
+
+    # parse.y:
+    #
+    #     | k_class cpath superclass
+    #     | k_module cpath
+    #
+    # matches "class"
+    def on_kw(kw)
+      if %w(class module).include?(kw)
+        @await_scope_change = "k_class"
+      else
+        stop_await_scope_change
+      end
+
+      @scanner_events << { :kw => kw }
+      
+      @scanner_events[-1]
+    end
+
+    # matches "::" in two cases
+    #   1. "class ::TopLevel",
+    #   2. "class Nested::Class"
+    def on_op(token)
+      if @await_scope_change == "k_class" &&
+              token == "::"
+        # tCOLON2, tCOLON3
+        #  cpath		: tCOLON3 cname
+        #		    {
+        #		    /*%%%*/
+        #			$$ = NEW_COLON3($2);
+        #		    /*%
+        #			$$ = dispatch1(top_const_ref, $2);
+        #		    %*/
+        #		    }
+        @await_scope_change = "k_class tCOLON3"
+      elsif @await_scope_change == "k_class primary_value tCOLON2" &&
+              token == "::" # tCOLON2, tCOLON3
+        @new_scope << "::" # append to the last defined scope
+      else
+        stop_await_scope_change
+      end
+      
+      @scanner_events << { :op => token }
+      
+      @scanner_events[-1]
+    end
+
+    #  cname		: tIDENTIFIER
+    #          {
+    #          /*%%%*/
+    #        yyerror("class/module name must be CONSTANT");
+    #          /*%
+    #        $$ = dispatch1(class_name_error, $1);
+    #          %*/
+    #          }
+    #      | tCONSTANT
+    #      ;
+    #
+    # "ClassName"
+    # or
+    # "ClassName::NestedClassName"
+    def on_const(const_name)
+      if @await_scope_change == "k_class tCOLON3" # "class ::TopClass"
+        @new_scope = "::#{ const_name }"
+      elsif @await_scope_change == "k_class" # "class NormalClass"
+        #		| cname
+        #		    {
+        #		    /*%%%*/
+        #			$$ = NEW_COLON2(0, $$);
+        #		    /*%
+        #			$$ = dispatch1(const_ref, $1);
+        #		    %*/
+        #		    }
+        @new_scope = "#{ const_name }"
+        @await_scope_change = "k_class primary_value tCOLON2" # "class Class::NestedClass"
+      elsif @await_scope_change == "k_class primary_value tCOLON2"
+        #
+        #		| primary_value tCOLON2 cname
+        #		    {
+        #		    /*%%%*/
+        #			$$ = NEW_COLON2($1, $3);
+        #		    /*%
+        #			$$ = dispatch2(const_path_ref, $1, $3);
+        #		    %*/
+        #		    }
+        #		;
+        #
+        # cname		: tIDENTIFIER
+        @new_scope << "#{ const_name }" # append to the scope
+      else
+        stop_await_scope_change
+      end
+
+      @scanner_events << { :const => const_name }
+      
+      @scanner_events[-1]
+    end
+  end
+end