delfos 0.0.2.pre.rc2 → 0.0.2.pre.rc3

data/lib/delfos/neo4j/offline/importer.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 require "json"
-require "delfos/neo4j"
 require "delfos/neo4j/call_site_query"
+require "delfos/neo4j/query_execution/errors"
 require "fileutils"
 
 module Delfos
@@ -17,8 +17,6 @@ module Delfos
 
         def perform
           each_line do |params, err|
-            params = JSON.parse(params)
-
             execute(CallSiteQuery::BODY, params, err)
           end
 
@@ -28,8 +26,9 @@ module Delfos
         private
 
         def execute(query, params, err)
+          params = JSON.parse(params)
           Neo4j.execute_sync(query, params)
-        rescue Delfos::Neo4j::QueryExecution::InvalidQuery => e
+        rescue Delfos::Neo4j::QueryExecution::InvalidQuery, JSON::ParserError => e
           @no_errors = false
           Delfos.logger.error e.message.to_s
           err.puts JSON.dump(params)
@@ -48,13 +47,15 @@ module Delfos
         def with_errors
           @no_errors = true
 
-          error_filename = "#{filename}.errors"
-
           File.open(error_filename, "w") do |err|
             yield err
           end
 
-          FileUtils.rm_rf error_filename if @no_errors
+          FileUtils.rm_rf(error_filename) if @no_errors
+        end
+
+        def error_filename
+          "#{filename}.errors"
         end
 
         def with_input

data/lib/delfos/neo4j/query_execution/batch/execution.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "delfos/neo4j/query_execution/transactional"
+
+module Delfos
+  module Neo4j
+    module QueryExecution
+      module Batch
+        class Execution
+          attr_reader :size, :current_transaction_url, :commit_url, :expires, :query_count
+
+          def initialize(size:, clock: Time)
+            @size               = size
+            @clock              = clock
+            reset!
+          end
+
+          def execute!(query, params: {})
+            check_for_expiry!
+
+            perform_query(query, params)
+            flush_if_required!
+          end
+
+          def flush!
+            return unless query_count.positive?
+            return unless @commit_url
+            QueryExecution::Transactional.commit!(@commit_url)
+
+            reset!
+          end
+
+          private
+
+          def perform_query(query, params)
+            transactional_query = QueryExecution::Transactional.new(query, params, url)
+            @total_query_length += transactional_query.query_length
+            transaction_url, @commit_url, @expires = transactional_query.perform
+            @current_transaction_url ||= transaction_url # the transaction_url is only returned with the first request
+            @query_count += 1
+          end
+
+          def url
+            return @commit_url if @commit_url && batch_full? || expires_soon?
+
+            current_transaction_url || new_transaction_url
+          end
+
+          def new_transaction_url
+            Delfos.config.neo4j.uri_for("/db/data/transaction")
+          end
+
+          def flush_if_required!
+            check_for_expiry!
+
+            if batch_full? || expires_soon? || large_query?
+              flush!
+              return true
+            end
+
+            false
+          end
+
+          def check_for_expiry!
+            return if @expires.nil? || (@clock.now <= @expires)
+
+            raise QueryExecution::ExpiredTransaction.new(@commit_url, "")
+          end
+
+          def batch_full?
+            query_count >= size
+          end
+
+          def expires_soon?
+            @expires && (@clock.now + 10 > @expires)
+          end
+
+          def large_query?
+            @total_query_length >= Delfos.max_query_size
+          end
+
+          def reset!
+            @query_count             = 0
+            @total_query_length      = 0
+            @current_transaction_url = nil
+            @commit_url              = nil
+            @expires                 = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/query_execution/batch/retryable.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "execution"
+
+module Delfos
+  module Neo4j
+    module QueryExecution
+      module Batch
+        class Retryable
+          class << self
+            MUTEX = Mutex.new
+
+            def reset!
+              MUTEX.synchronize do
+                Thread.current[:_delfos__retryable_batch] = nil
+              end
+            end
+
+            def ensure_instance(size: nil)
+              MUTEX.synchronize do
+                Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
+              end
+            end
+
+            def execute!(query, params: {}, size: nil)
+              ensure_instance(size: size).execute!(query, params: params)
+            end
+
+            def flush!
+              ensure_instance&.flush!
+              reset!
+            end
+          end
+
+          attr_reader :size, :queries, :execution
+          attr_accessor :retry_count
+
+          def initialize(size:)
+            @size = size
+            reset!
+          end
+
+          def execute!(query, params: {}, retrying: false)
+            queries.push([query, params]) unless retrying
+
+            with_retry(retrying) do
+              flushed = execution.execute!(query, params: params)
+
+              reset! if flushed
+            end
+          end
+
+          def flush!
+            execution.flush!
+            reset!
+          end
+
+          private
+
+          def with_retry(already_retrying)
+            yield
+          rescue QueryExecution::ExpiredTransaction
+            check_retry_limit! if already_retrying
+
+            retry_batch!
+          end
+
+          def check_retry_limit!
+            self.retry_count += 1
+
+            return if self.retry_count <= 5
+
+            self.retry_count = 0
+            Delfos.logger.error "Transaction expired - 5 retries failed aborting"
+            raise
+          end
+
+          def retry_batch!
+            Delfos.logger.error do
+              "Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
+            end
+
+            new_execution
+
+            queries.each { |q, p| execute!(q, params: p, retrying: true) }
+
+            Delfos.logger.error { "Batch retry successful" }
+          end
+
+          def reset!
+            @queries = []
+            new_execution
+            @retry_count = 0
+          end
+
+          def new_execution
+            @execution = Execution.new(size: size)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/delfos/neo4j/query_execution/errors.rb

@@ -5,6 +5,9 @@ require "net/http"
 module Delfos
   module Neo4j
     module QueryExecution
+      class ConnectionError < IOError
+      end
+
       HTTP_ERRORS = [
         EOFError,
         Errno::EAGAIN,
@@ -21,6 +24,7 @@ module Delfos
         Net::ReadTimeout,
         SocketError,
         Timeout::Error,
+        ConnectionError,
       ].freeze
 
       class InvalidQuery < IOError
@@ -32,9 +36,6 @@ module Delfos
           super [message, { query: query.to_s }, { params: params.to_s }].join("\n\n")
         end
       end
-
-      class ConnectionError < IOError
-      end
     end
   end
 end

data/lib/delfos/neo4j/query_execution/http.rb

@@ -53,7 +53,7 @@ module Delfos
 
         def add_headers(request)
           request.initialize_http_header("Accept" => "application/json; charset=UTF-8")
-          request.basic_auth(Delfos.neo4j.username, Delfos.neo4j.password)
+          request.basic_auth(Delfos.config.neo4j.username, Delfos.config.neo4j.password)
           request.content_type = "application/json"
         end
       end

data/lib/delfos/neo4j/query_execution/sync.rb

@@ -28,7 +28,7 @@ module Delfos
         end
 
         def uri
-          @uri ||= Delfos.neo4j.uri_for("/db/data/transaction/commit")
+          @uri ||= Delfos.config.neo4j.uri_for("/db/data/transaction/commit")
         end
       end
     end

data/lib/delfos/neo4j/query_execution/transactional.rb

@@ -62,7 +62,7 @@ module Delfos
         end
 
         def uri
-          @uri ||= Delfos.neo4j.uri_for("/db/data/transaction")
+          @uri ||= Delfos.config.neo4j.uri_for("/db/data/transaction")
         end
       end
 

data/lib/delfos/neo4j/schema.rb

@@ -52,7 +52,7 @@ module Delfos
       end
 
       def constraints_uri
-        Delfos.neo4j.uri_for("/db/data/schema/constraint")
+        Delfos.config.neo4j.uri_for("/db/data/schema/constraint")
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: delfos
 version: !ruby/object:Gem::Version
-  version: 0.0.2.pre.rc2
+  version: 0.0.2.pre.rc3
 platform: ruby
 authors:
 - Mark Burns
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-06-22 00:00:00.000000000 Z
+date: 2017-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: binding_of_caller
@@ -34,20 +34,23 @@ dependencies:
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec_junit_formatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.3
@@ -55,6 +58,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.2.3
@@ -62,20 +68,23 @@ dependencies:
   name: codeclimate-test-reporter
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.3.2
@@ -83,6 +92,9 @@ dependencies:
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.3.2
@@ -90,14 +102,14 @@ dependencies:
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -146,35 +158,45 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.6.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.6.0
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.5.0
+        version: 3.6.0
 description: Record every method call, call-site, arguments and their types in your
   application code
 email:
 - markthedeveloper@gmail..com
-executables: []
+executables:
+- delfos_import
+- delfos_import_offline_queries
+- delfos_update_distance
 extensions: []
 extra_rdoc_files: []
 files:
+- exe/delfos_import
+- exe/delfos_import_offline_queries
+- exe/delfos_update_distance
 - lib/delfos.rb
 - lib/delfos/call_stack.rb
 - lib/delfos/call_stack/stack.rb
+- lib/delfos/config.rb
+- lib/delfos/config/inclusion.rb
 - lib/delfos/file_system.rb
 - lib/delfos/file_system/app_directories.rb
+- lib/delfos/file_system/app_files.rb
 - lib/delfos/file_system/common_path.rb
 - lib/delfos/file_system/distance_calculation.rb
+- lib/delfos/file_system/file_cache.rb
 - lib/delfos/file_system/path_determination.rb
 - lib/delfos/file_system/relation.rb
 - lib/delfos/method_trace.rb
@@ -186,21 +208,21 @@ files:
 - lib/delfos/method_trace/code_location/filename_helpers.rb
 - lib/delfos/method_trace/code_location/method.rb
 - lib/delfos/neo4j.rb
-- lib/delfos/neo4j/batch/execution.rb
-- lib/delfos/neo4j/batch/retryable.rb
 - lib/delfos/neo4j/call_site_query.rb
 - lib/delfos/neo4j/distance/call_site_fetcher.rb
 - lib/delfos/neo4j/distance/update.rb
 - lib/delfos/neo4j/live/call_site_logger.rb
+- lib/delfos/neo4j/offline.rb
 - lib/delfos/neo4j/offline/call_site_logger.rb
 - lib/delfos/neo4j/offline/importer.rb
+- lib/delfos/neo4j/query_execution/batch/execution.rb
+- lib/delfos/neo4j/query_execution/batch/retryable.rb
 - lib/delfos/neo4j/query_execution/errors.rb
 - lib/delfos/neo4j/query_execution/http.rb
 - lib/delfos/neo4j/query_execution/http_query.rb
 - lib/delfos/neo4j/query_execution/sync.rb
 - lib/delfos/neo4j/query_execution/transactional.rb
 - lib/delfos/neo4j/schema.rb
-- lib/delfos/setup.rb
 homepage: https://github.com/markburns/delfos
 licenses:
 - MIT