delfos 0.0.1 → 0.0.2.pre.rc2

data/lib/delfos/neo4j/batch/retryable.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "execution"
+
+module Delfos
+  module Neo4j
+    module Batch
+      class Retryable
+        class << self
+          MUTEX = Mutex.new
+
+          def reset!
+            MUTEX.synchronize do
+              Thread.current[:_delfos__retryable_batch] = nil
+            end
+          end
+
+          def ensure_instance(size: nil)
+            MUTEX.synchronize do
+              Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
+            end
+          end
+
+          def execute!(query, params: {}, size: nil)
+            ensure_instance(size: size).execute!(query, params: params)
+          end
+
+          def flush!
+            ensure_instance&.flush!
+            reset!
+          end
+        end
+
+        attr_reader :size, :queries, :execution
+        attr_accessor :retry_count
+
+        def initialize(size:)
+          @size = size
+          reset!
+        end
+
+        def execute!(query, params: {}, retrying: false)
+          queries.push([query, params]) unless retrying
+
+          with_retry(retrying) do
+            flushed = execution.execute!(query, params: params)
+
+            reset! if flushed
+          end
+        end
+
+        def flush!
+          execution.flush!
+          reset!
+        end
+
+        private
+
+        def with_retry(already_retrying)
+          yield
+        rescue QueryExecution::ExpiredTransaction
+          check_retry_limit! if already_retrying
+
+          retry_batch!
+        end
+
+        def check_retry_limit!
+          self.retry_count += 1
+
+          return if self.retry_count <= 5
+
+          self.retry_count = 0
+          Delfos.logger.error "Transaction expired - 5 retries failed aborting"
+          raise
+        end
+
+        def retry_batch!
+          Delfos.logger.error do
+            "Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
+          end
+
+          new_execution
+
+          queries.each { |q, p| execute!(q, params: p, retrying: true) }
+
+          Delfos.logger.error { "Batch retry successful" }
+        end
+
+        def reset!
+          @queries = []
+          new_execution
+          @retry_count = 0
+        end
+
+        def new_execution
+          @execution = Execution.new(size: size)
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/call_site_query.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Delfos
+  module Neo4j
+    class CallSiteQuery
+      def initialize(call_site, stack_uuid, step_number)
+        @call_site        = call_site
+        @container_method = call_site.container_method
+        @called_method    = call_site.called_method
+        @stack_uuid       = stack_uuid
+        @step_number      = step_number
+      end
+
+      def params
+        @params ||= calculate_params
+      end
+
+      def query
+        BODY
+      end
+
+      private
+
+      # rubocop:disable Metrics/MethodLength
+      def calculate_params
+        params = {
+          "step_number"                  => @step_number,
+          "stack_uuid"                   => @stack_uuid,
+
+          "call_site_file"               => @call_site        .file,
+          "call_site_line_number"        => @call_site        .line_number,
+
+          "container_method_klass_name"  => @container_method .klass_name,
+          "container_method_type"        => @container_method .method_type,
+          "container_method_name"        => @container_method .method_name,
+          "container_method_file"        => @container_method .file,
+          "container_method_line_number" => @container_method .line_number || -1,
+
+          "called_method_klass_name"     => @called_method    .klass_name,
+          "called_method_type"           => @called_method    .method_type,
+          "called_method_name"           => @called_method    .method_name,
+          "called_method_file"           => @called_method    .file,
+          "called_method_line_number"    => @called_method    .line_number,
+        }
+
+        params
+      end
+      # rubocop:enable Metrics/MethodLength,Metrics/LineLength,Metrics/AbcSize
+
+      BODY = <<-QUERY
+          MERGE (container_method_klass:Class {name: {container_method_klass_name}})
+          MERGE (called_method_klass:Class {name: {called_method_klass_name}})
+
+          MERGE (container_method_klass) - [:OWNS] ->
+            (container_method:Method
+              {
+                type: {container_method_type},
+                name: {container_method_name},
+                file: {container_method_file},
+                line_number: {container_method_line_number}
+              }
+            )
+          MERGE (container_method) - [:CONTAINS] ->
+            (call_site:CallSite
+              {
+                file: {call_site_file},
+                line_number: {call_site_line_number}
+              }
+            )
+
+          MERGE (called_method_klass) - [:OWNS] ->
+            (called_method:Method
+              {
+                type: {called_method_type},
+                name: {called_method_name},
+                file: {called_method_file},
+                line_number: {called_method_line_number}
+              }
+            )
+
+          MERGE (call_site) - [:CALLS] -> (called_method)
+
+          MERGE (call_stack:CallStack{uuid: {stack_uuid}})
+
+          MERGE (call_stack) - [:STEP {number: {step_number}}] -> (call_site)
+      QUERY
+    end
+  end
+end

data/lib/delfos/neo4j/distance/call_site_fetcher.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Delfos
   module Neo4j
     module Distance

data/lib/delfos/neo4j/distance/update.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
-require "delfos/file_system/distance_calculation"
+
+require "delfos/file_system"
 require_relative "call_site_fetcher"
 
 module Delfos
@@ -11,9 +12,7 @@ module Delfos
           return if results.length.negative?
 
           results.each do |start_file, call_site_id, finish_file, called_id|
-            calc = FileSystem::DistanceCalculation.new(start_file, finish_file)
-
-            update(call_site_id, called_id, calc)
+            handle(start_file, call_site_id, finish_file, called_id)
           end
 
           Neo4j.flush!
@@ -25,7 +24,7 @@ module Delfos
             call_site_id:            call_site_id,
             called_id:               called_id,
             sum_traversals:          calc.sum_traversals,
-            sum_possible_traversals: calc.sum_possible_traversals
+            sum_possible_traversals: calc.sum_possible_traversals,
           )
         end
 
@@ -44,6 +43,24 @@ module Delfos
                -> (called)
           QUERY
         end
+
+        private
+
+        def handle(start_file, call_site_id, finish_file, called_id)
+          calc = FileSystem.distance_calculation(start_file, finish_file)
+
+          update(call_site_id, called_id, calc)
+        rescue FileSystem::Error => e
+          error(e, start_file, call_site_id, finish_file, called_id)
+        end
+
+        def error(e, start_file, call_site_id, finish_file, called_id)
+          Delfos.logger.error([
+            "Whilst updating distance #{start_file}->#{finish_file} ",
+            "call_site_id: #{call_site_id}",
+            "called_id: #{called_id} - #{e.message} #{e.backtrace}",
+          ].join("\n"))
+        end
       end
     end
   end

data/lib/delfos/neo4j/live/call_site_logger.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "delfos/neo4j/call_site_query"
+
+module Delfos
+  module Neo4j
+    module Live
+      class CallSiteLogger
+        def log(call_site, stack_uuid, step_number)
+          q = CallSiteQuery.new(call_site, stack_uuid, step_number)
+
+          Neo4j.execute(q.query, q.params)
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/offline/call_site_logger.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "delfos/neo4j/call_site_query"
+require "json"
+
+module Delfos
+  module Neo4j
+    module Offline
+      class CallSiteLogger
+        attr_writer :count
+
+        def log(call_site, stack_uuid, step_number)
+          query = CallSiteQuery.new(call_site, stack_uuid, step_number)
+
+          file.puts JSON.dump(query.params)
+          self.count += 1
+          file.flush if (count % 100).zero?
+        end
+
+        def finish!
+          return if file.closed?
+
+          file.flush
+          file.close
+        end
+
+        def count
+          @count ||= 0
+        end
+
+        private
+
+        def file
+          @file ||= File.open(Delfos.offline_query_filename, "a")
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/offline/importer.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "json"
+require "delfos/neo4j"
+require "delfos/neo4j/call_site_query"
+require "fileutils"
+
+module Delfos
+  module Neo4j
+    module Offline
+      class Importer
+        attr_reader :filename
+
+        def initialize(filename)
+          @filename = filename
+        end
+
+        def perform
+          each_line do |params, err|
+            params = JSON.parse(params)
+
+            execute(CallSiteQuery::BODY, params, err)
+          end
+
+          Neo4j.flush!
+        end
+
+        private
+
+        def execute(query, params, err)
+          Neo4j.execute_sync(query, params)
+        rescue Delfos::Neo4j::QueryExecution::InvalidQuery => e
+          @no_errors = false
+          Delfos.logger.error e.message.to_s
+          err.puts JSON.dump(params)
+        end
+
+        def each_line
+          with_errors do |err|
+            with_input do |f|
+              f.each_line.lazy.each do |params|
+                yield params, err
+              end
+            end
+          end
+        end
+
+        def with_errors
+          @no_errors = true
+
+          error_filename = "#{filename}.errors"
+
+          File.open(error_filename, "w") do |err|
+            yield err
+          end
+
+          FileUtils.rm_rf error_filename if @no_errors
+        end
+
+        def with_input
+          File.open(filename, "r") do |f|
+            yield f
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/query_execution/errors.rb

@@ -1,25 +1,35 @@
 # frozen_string_literal: true
+
+require "net/http"
+
 module Delfos
   module Neo4j
     module QueryExecution
       HTTP_ERRORS = [
         EOFError,
-        Errno::ECONNRESET,
+        Errno::EAGAIN,
         Errno::ECONNREFUSED,
+        Errno::ECONNRESET,
+        Errno::EHOSTUNREACH,
         Errno::EINVAL,
+        Errno::ENETDOWN,
+        Errno::ENETUNREACH,
+        Errno::ETIMEDOUT,
         Net::HTTPBadResponse,
         Net::HTTPHeaderSyntaxError,
         Net::ProtocolError,
+        Net::ReadTimeout,
+        SocketError,
         Timeout::Error,
       ].freeze
 
       class InvalidQuery < IOError
         def initialize(errors, query, params)
           message = errors.map do |e|
-            e.select { |k, _| %w(code message).include?(k) }.inspect
+            e.select { |k, _| %w[code message].include?(k) }
           end.join("\n")
 
-          super [message, { query: query, params: params }.to_json].join("\n\n")
+          super [message, { query: query.to_s }, { params: params.to_s }].join("\n\n")
         end
       end
 

data/lib/delfos/neo4j/query_execution/http.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require "delfos"
 require "net/http"
 

data/lib/delfos/neo4j/query_execution/http_query.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Delfos
   module Neo4j
     module QueryExecution
@@ -15,19 +16,20 @@ module Delfos
           @uri = uri
         end
 
+        def query_length
+          request_body.length
+        end
+
         private
 
         def request_body
-          {
-            "statements": [{ "statement": strip_whitespace(query), "parameters": params }],
+          @request_body ||= {
+            "statements": [{ "statement": formatted_query, "parameters": params }],
           }.to_json
         end
 
-        def strip_whitespace(s)
-          s.
-            gsub(/^\s+/, "").
-            gsub(/ +/, " ").
-            gsub("\n\n", "\n")
+        def formatted_query
+          strip_whitespace(query)
         end
 
         def json
@@ -49,18 +51,21 @@ module Delfos
         end
 
         def log_query
-          statement = strip_whitespace(query)
+          Delfos.logger.debug do
+            statement = formatted_query
 
-          if statement
-            begin
             params.each { |k, v| statement = statement.gsub("{#{k}}", v.inspect) }
-            rescue
-            byebug
-            end
-            Delfos.logger.debug "sending query: "
-            Delfos.logger.debug statement.gsub(/^/, "    ")
+            statement.gsub(/^/, "    ")
           end
         end
+
+        def strip_whitespace(s)
+          s.
+            gsub(/^\s+/, "").
+            gsub(/ +/, " ").
+            gsub("\n\n", "\n").
+            tr("\n", " ")
+        end
       end
     end
   end