neolytics 0.1.0 → 0.2.0

data/examples/output/time_spent_by_method.csv

@@ -0,0 +1,51 @@
+line,method,sum,count,average
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/railties-4.2.5/lib/rails/generators/named_base.rb:2,Kernel#require,55422,2,27711
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/rack-1.6.4/lib/rack/utils.rb:5,Kernel#require,27433,2,13716.5
+/Users/brian/.rvm/rubies/ruby-2.2.3/lib/ruby/2.2.0/set.rb:292,Set#add,19018,2343,8.116944088775075
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/core_ext/module/delegation.rb:214,Module#module_eval,17404,160,108.775
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/commands.rb:5,Kernel#require,16623,104,159.83653846153845
+/Users/brian/.rvm/rubies/ruby-2.2.3/lib/ruby/2.2.0/set.rb:283,Hash#each_key,15520,54,287.4074074074074
+/Users/brian/.rvm/rubies/ruby-2.2.3/lib/ruby/2.2.0/forwardable.rb:192,Module#module_eval,11143,108,103.17592592592592
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command.rb:34,#<Class:Pry::Command>#match,8824,810,10.893827160493828
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sprockets-rails-2.3.3/lib/sprockets/railtie.rb:6,Kernel#require,8672,2,4336
+/Users/brian/.rvm/rubies/ruby-2.2.3/lib/ruby/2.2.0/set.rb:94,Array#each,8640,90,96
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/inflector/methods.rb:100,ActiveSupport::Inflector#underscore,7288,237,30.751054852320674
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activemodel-4.2.5/lib/active_model/validations.rb:404,Kernel#require,6561,26,252.34615384615384
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/script/value/color.rb:23,Range#each,6495,298,21.79530201342282
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command.rb:183,#<Class:Pry::Command>#command_regex,6131,510,12.02156862745098
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command.rb:47,#<Class:Pry::Command>#command_options,6111,641,9.533541341653667
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command.rb:192,#<Class:Pry::Command>#convert_to_regex,5683,1020,5.57156862745098
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/neo4j-6.0.1/lib/neo4j.rb:47,Kernel#require,5516,2,2758
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/script.rb:1,Kernel#require,5350,2,2675
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/actionview-4.2.5/lib/action_view/helpers/form_helper.rb:1321,Module#class_eval,5178,34,152.2941176470588
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/bundler-1.11.2/lib/bundler/runtime.rb:77,#<Class:Kernel>#require,4845,30,161.5
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/nokogiri-1.6.7/lib/nokogiri/html/element_description_defaults.rb:650,Array#each,4676,2,2338
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/inflector/inflections.rb:209,ActiveSupport::Inflector#inflections,4633,238,19.46638655462185
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/neo4j-6.0.1/lib/neo4j.rb:38,Kernel#require,4486,2,2243
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/script.rb:34,Kernel#require,4464,2,2232
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/tree/node.rb:35,BasicObject#instance_eval,4387,54,81.24074074074075
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/core_ext/module/attribute_accessors.rb:57,Module#class_eval,4263,66,64.5909090909091
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command_set.rb:318,Array#select,4040,24,168.33333333333334
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/dependencies/autoload.rb:46,ActiveSupport::Autoload#autoload,3681,233,15.798283261802576
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sprockets-3.5.2/lib/sprockets/utils.rb:41,Sprockets::Utils#hash_reassoc1,3467,115,30.14782608695652
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/core_ext/string/output_safety.rb:231,Module#class_eval,3212,38,84.52631578947368
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/neo4j-6.0.1/lib/neo4j.rb:51,Kernel#require,3087,2,1543.5
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/neo4j-6.0.1/lib/neo4j.rb:82,Kernel#require,2914,2,1457
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/temple-0.7.6/lib/temple/mixins/engine_dsl.rb:55,Module#const_get,2849,18,158.27777777777777
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/temple-0.7.6/lib/temple/map.rb:12,Array#any?,2833,290,9.76896551724138
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/actionpack-4.2.5/lib/action_controller/metal/live.rb:1,Kernel#require,2765,2,1382.5
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/script/functions.rb:407,Array#map,2717,180,15.094444444444445
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/nokogiri-1.6.7/lib/nokogiri/xml/parse_options.rb:61,Module#class_eval,2715,44,61.70454545454545
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/activesupport-4.2.5/lib/active_support/core_ext/module/attribute_accessors.rb:123,Module#class_eval,2640,60,44
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/scss.rb:5,Kernel#require,2618,2,1309
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass.rb:99,Kernel#require,2556,2,1278
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/concurrent-ruby-1.0.0/lib/concurrent.rb:26,Kernel#require,2500,2,1250
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/concurrent-ruby-1.0.0/lib/concurrent/configuration.rb:2,Kernel#require,2488,2,1244
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sprockets-3.5.2/lib/sprockets/environment.rb:3,Kernel#require,2458,2,1229
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/temple-0.7.6/lib/temple/map.rb:97,Array#any?,2369,222,10.67117117117117
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/actionpack-4.2.5/lib/action_controller.rb:4,Kernel#require,2344,2,1172
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/slim-3.0.6/lib/slim.rb:2,Kernel#require,2279,2,1139.5
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/pry-0.10.3/lib/pry/command.rb:144,#<Class:Pry::Command>#matches?,2245,498,4.508032128514056
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/scss.rb:4,Kernel#require,2245,2,1122.5
+/Users/brian/github/neo4j-examples/ruby_code_analytics/config/application.rb:18,Kernel#require,2223,2,1111.5
+/Users/brian/.rvm/gems/ruby-2.2.3/gems/sass-3.4.20/lib/sass/script.rb:35,Kernel#require,2220,2,1110

data/lib/neo4apis/neolytics.rb

@@ -9,7 +9,7 @@ module Neo4Apis
     uuid :ASTNode, :uuid
     uuid :File, :path
 
-    batch_size 10000
+    batch_size 6000
 
     IMPORTED_OBJECT_NODES = {}
 
@@ -50,16 +50,25 @@ module Neo4Apis
       object_node
     end
 
-    importer :TracePoint do |tp, execution_time, execution_index, last_tracepoint_node, parent, associated_call|
-      next nil if tp.method_id.blank? && tp.defined_class.blank?
+    importer :TracePoint do |tp, execution_time, total_execution_time, execution_index, last_tracepoint_node, parent, associated_call|
+      next nil if tp.method_id.to_s.strip.empty? && tp.defined_class.to_s.strip.empty?
 
+      # if tp.lineno == 19 && tp.event == :line
+      #   require 'pry'
+      #   binding.pry
+      # end
       trace_point_node = add_node :TracePoint, tp, %i(event lineno method_id) do |node|
         node.uuid = SecureRandom.uuid
         node.defined_class = tp.defined_class.to_s
         node.execution_time = execution_time if execution_time
+        node.total_execution_time = total_execution_time if total_execution_time
         node.execution_index = execution_index
 
-        node.path = Pathname.new(tp.path).realpath.to_s
+        node.path = if tp.path == '(eval)'
+          tp.path
+        else
+          Pathname.new(tp.path).realpath.to_s
+        end
       end
 
 
@@ -82,11 +91,16 @@ module Neo4Apis
           object_node = import :Object, value
           add_relationship :HAS_VARIABLE_VALUE, trace_point_node, object_node, variable_name: var
         end
+
+        # TracePointHelpers.each_referenced_object(tp) do |object|
+        #   object_node = import :Object, object
+        #   add_relationship :REFERENCES_OBJECT, trace_point_node, object_node, variable_name: var
+        # end
       end
 
       if tp.event == :call
-        TracePointHelpers.each_received_arguments(tp) do |argument, value|
-          object_node = import :Object, value
+        TracePointHelpers.each_received_arguments(tp) do |argument, object|
+          object_node = import :Object, object
           add_relationship :RECEIVED_ARGUMENT, trace_point_node, object_node, argument_name: argument
         end
       end

data/lib/neo4apis/neolytics/trace_point_helpers.rb

@@ -14,14 +14,18 @@ module Neo4Apis
             tp.self.class.instance_method(tp.method_id)
           end
           parameter_names = method.parameters.map {|_, name| name }
-          arguments = parameter_names.each_with_object({}) do |name, arguments|
+          arguments = parameter_names.compact.each_with_object({}) do |name, arguments|
             catch :not_found do
+              begin
               arguments[name] = get_trace_point_var(tp, name)
+              rescue Exception => e
+                require 'pry'
+                binding.pry
+              end
             end
           end
           arguments.each do |name, object|
-            value = RubyObject.from_object(object)
-            yield name, value
+            yield name, object
           end
         end
 
@@ -36,7 +40,18 @@ module Neo4Apis
           end
         rescue Parser::SyntaxError
           nil
-        end    
+        end
+
+        def each_referenced_object(tp)
+          line = get_file_line(tp.path, tp.lineno)
+          root = Parser::CurrentRuby.parse(line)
+          extract_object_references(root).each do |inspect_string|
+            value = tp.binding.eval(inspect_string)
+            yield value
+          end
+        rescue Parser::SyntaxError
+          nil
+        end
 
         private
 
@@ -56,7 +71,23 @@ module Neo4Apis
           end
         end
 
+        OBJECT_NODE_TYPES = [:str, :sym, :int, :float, :hash, :array, :true, :false, :nil]
+        def extract_object_references(ast_node)
+          if ast_node.is_a?(Parser::AST::Node)
+            if OBJECT_NODE_TYPES.include?(ast_node.type)
+              [ast_node.children[0].inspect]
+            else
+              ast_node.children.flat_map do |child|
+                extract_object_references(child)
+              end
+            end
+          else
+            []
+          end
+        end
+
         def get_file_line(path, lineno)
+          return '' if path == '(eval)'
           FILE_LINES[path] ||= File.read(path).lines
 
           FILE_LINES[path][lineno - 1]
@@ -64,7 +95,7 @@ module Neo4Apis
 
         def get_trace_point_var(tp, var_name)
           begin
-            tp.binding.local_variable_get(variable)
+            tp.binding.local_variable_get(var_name)
           rescue NameError
             throw :not_found
           end

data/lib/neolytics/recorder.rb

@@ -8,6 +8,21 @@ module Neolytics
     def initialize(neo4j_session)
       @neo4j_session = neo4j_session
       @neo4apis_session = Neo4Apis::Neolytics.new(neo4j_session)
+      create_indexes
+    end
+
+    def create_indexes
+      @neo4j_session.query('CREATE INDEX ON :ASTNode(file_path)')
+      @neo4j_session.query('CREATE INDEX ON :ASTNode(first_line)')
+      @neo4j_session.query('CREATE INDEX ON :ASTNode(type)')
+      @neo4j_session.query('CREATE INDEX ON :ASTNode(name)')
+
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(path)')
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(event)')
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(lineno)')
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(defined_class)')
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(method_id)')
+      @neo4j_session.query('CREATE INDEX ON :TracePoint(execution_index)')
     end
 
     def record(&block)
@@ -20,16 +35,24 @@ module Neolytics
           end
         end
 
-        file_paths = @neo4j_session.query("MATCH (tp) WHERE NOT(tp.path IS NULL) RETURN DISTINCT tp.path AS path").map(&:path)
+        @neo4apis_session.instance_variable_get('@buffer').flush
+        query = <<QUERY
+MATCH (tp:TracePoint)
+WHERE NOT(tp.path IS NULL) AND NOT(tp.path = '(eval)')
+RETURN DISTINCT tp.path AS path
+QUERY
+        file_paths = @neo4j_session.query(query).map(&:path)
         file_paths.each(&method(:record_ast))
 
         link_query = <<QUERY
-MATCH (tp:TracePoint), (node:ASTNode)
+MATCH (tp:TracePoint)
+WITH tp, tp.lineno AS lineno, tp.path AS path, tp.method_id AS method_id
+MATCH (node:ASTNode {type: 'def'})
+USING INDEX node:ASTNode(name)
 WHERE
-  node.file_path = tp.path AND
-  node.first_line = tp.lineno AND
-  node.name = tp.method_id AND
-  node.type = 'def'
+  node.name = method_id AND
+  node.file_path = path AND
+  node.first_line = lineno
 MERGE (tp)-[:HAS_AST_NODE]->(node)
 QUERY
         @neo4j_session.query(link_query)
@@ -43,7 +66,11 @@ QUERY
       code = File.read(full_path)
 
       require 'parser/current'
-      root = Parser::CurrentRuby.parse(code)
+      begin
+        root = Parser::CurrentRuby.parse(code)
+      rescue EncodingError
+        return
+      end
 
       file_node = @neo4apis_session.import :File, full_path, code
       node_node = @neo4apis_session.import :ASTNode, root, file_node
@@ -58,6 +85,7 @@ QUERY
       last_start_time = nil
       ancestor_stack = []
       run_time_stack = []
+      total_run_time_stack = []
 
       last_tracepoint_end_time = nil
       last_run_time = nil
@@ -66,15 +94,23 @@ QUERY
         begin
           last_run_time = 1_000_000.0 * (Time.now - last_tracepoint_end_time) if last_tracepoint_end_time
 
+          start = Time.now
           output << tracepoint_string(tp, indent)
 
           last_method_time = nil
           if [:call, :c_call].include?(tp.event)
             run_time_stack.push(0)
+            total_run_time_stack.push(0)
           elsif [:return, :c_return].include?(tp.event)
             last_method_time = run_time_stack.pop
+            last_method_total_time = total_run_time_stack.pop
           else
-            run_time_stack[-1] += last_run_time if run_time_stack[-1] && last_run_time
+            #puts "total_run_time_stack: #{total_run_time_stack.inspect}"
+            #puts "increment by #{last_run_time}"
+            if run_time_stack[-1] && last_run_time
+              run_time_stack[-1] += last_run_time
+              total_run_time_stack.map! { |i| i + last_run_time }
+            end
           end
 
           associated_call = nil
@@ -87,6 +123,7 @@ QUERY
 
           last_tracepoint_node = @neo4apis_session.import :TracePoint, tp,
                               last_method_time,
+                              last_method_total_time,
                               (execution_index += 1),
                               last_tracepoint_node,
                               ancestor_stack.last,
@@ -96,6 +133,12 @@ QUERY
             ancestor_stack.push(last_tracepoint_node)
           end
 
+          stop = Time.now
+          diff = stop - start
+          if diff > 0.5
+            puts "time: #{diff}"
+            puts "tp: #{tp.inspect}"
+          end
           last_tracepoint_end_time = Time.now
         rescue Exception => e
           puts 'EXCEPTION!!'

data/lib/neolytics/version.rb

@@ -1,3 +1,3 @@
 module Neolytics
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/neolytics.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency 'neo4apis', '~> 0.8.2'
+  spec.add_dependency 'neo4apis', '~> 0.9.0'
   spec.add_dependency 'neo4j-rake_tasks', '~> 0.3.0'
   spec.add_dependency 'parser', '~> 2.2.3.0'
   spec.add_development_dependency "bundler", "~> 1.10"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: neolytics
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Brian Underwood
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-12-28 00:00:00.000000000 Z
+date: 2016-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: neo4apis
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.2
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.8.2
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: neo4j-rake_tasks
   requirement: !ruby/object:Gem::Requirement
@@ -111,6 +111,23 @@ files:
 - Rakefile
 - bin/console
 - bin/setup
+- examples/model/ast.png
+- examples/model/object_relationships.png
+- examples/model/trace_point_ast_nodes.png
+- examples/model/trace_point_flow.png
+- examples/model/trace_point_objects.png
+- examples/output/abc.csv
+- examples/output/all_args_vars_statics.csv
+- examples/output/arguments_and_return_values.csv
+- examples/output/assignments.csv
+- examples/output/ast.csv
+- examples/output/common_ancestor.csv
+- examples/output/common_ancestor.json
+- examples/output/cyclomatic_complexity.csv
+- examples/output/cyclomatic_complexity_and_runtime.csv
+- examples/output/numeric_class_hierarchy.png
+- examples/output/numeric_return_values.csv
+- examples/output/time_spent_by_method.csv
 - lib/neo4apis/neolytics.rb
 - lib/neo4apis/neolytics/trace_point_helpers.rb
 - lib/neolytics.rb