delfos 0.0.2.pre.rc2 → 0.0.2.pre.rc3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/exe/delfos_import +38 -0
- data/exe/delfos_import_offline_queries +9 -0
- data/exe/delfos_update_distance +11 -0
- data/lib/delfos.rb +29 -73
- data/lib/delfos/config.rb +90 -0
- data/lib/delfos/config/inclusion.rb +90 -0
- data/lib/delfos/file_system.rb +7 -2
- data/lib/delfos/file_system/app_directories.rb +16 -18
- data/lib/delfos/file_system/app_files.rb +31 -0
- data/lib/delfos/file_system/file_cache.rb +21 -0
- data/lib/delfos/file_system/path_determination.rb +10 -7
- data/lib/delfos/method_trace.rb +18 -6
- data/lib/delfos/method_trace/call_handler.rb +6 -2
- data/lib/delfos/method_trace/code_location.rb +2 -12
- data/lib/delfos/method_trace/code_location/container_method_factory.rb +50 -25
- data/lib/delfos/method_trace/code_location/eval_in_caller.rb +2 -4
- data/lib/delfos/method_trace/code_location/filename_helpers.rb +1 -1
- data/lib/delfos/method_trace/code_location/method.rb +13 -9
- data/lib/delfos/neo4j.rb +12 -3
- data/lib/delfos/neo4j/distance/update.rb +1 -1
- data/lib/delfos/neo4j/live/call_site_logger.rb +27 -0
- data/lib/delfos/neo4j/offline.rb +13 -0
- data/lib/delfos/neo4j/offline/call_site_logger.rb +6 -1
- data/lib/delfos/neo4j/offline/importer.rb +8 -7
- data/lib/delfos/neo4j/query_execution/batch/execution.rb +93 -0
- data/lib/delfos/neo4j/query_execution/batch/retryable.rb +103 -0
- data/lib/delfos/neo4j/query_execution/errors.rb +4 -3
- data/lib/delfos/neo4j/query_execution/http.rb +1 -1
- data/lib/delfos/neo4j/query_execution/sync.rb +1 -1
- data/lib/delfos/neo4j/query_execution/transactional.rb +1 -1
- data/lib/delfos/neo4j/schema.rb +1 -1
- metadata +38 -16
- data/lib/delfos/neo4j/batch/execution.rb +0 -91
- data/lib/delfos/neo4j/batch/retryable.rb +0 -101
- data/lib/delfos/setup.rb +0 -120
@@ -1,91 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "delfos/neo4j/query_execution/transactional"
|
4
|
-
|
5
|
-
module Delfos
|
6
|
-
module Neo4j
|
7
|
-
module Batch
|
8
|
-
class Execution
|
9
|
-
attr_reader :size, :current_transaction_url, :commit_url, :expires, :query_count
|
10
|
-
|
11
|
-
def initialize(size:, clock: Time)
|
12
|
-
@size = size
|
13
|
-
@clock = clock
|
14
|
-
reset!
|
15
|
-
end
|
16
|
-
|
17
|
-
def execute!(query, params: {})
|
18
|
-
check_for_expiry!
|
19
|
-
|
20
|
-
perform_query(query, params)
|
21
|
-
flush_if_required!
|
22
|
-
end
|
23
|
-
|
24
|
-
def flush!
|
25
|
-
return unless query_count.positive?
|
26
|
-
return unless @commit_url
|
27
|
-
QueryExecution::Transactional.commit!(@commit_url)
|
28
|
-
|
29
|
-
reset!
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def perform_query(query, params)
|
35
|
-
transactional_query = QueryExecution::Transactional.new(query, params, url)
|
36
|
-
@total_query_length += transactional_query.query_length
|
37
|
-
transaction_url, @commit_url, @expires = transactional_query.perform
|
38
|
-
@current_transaction_url ||= transaction_url # the transaction_url is only returned with the first request
|
39
|
-
@query_count += 1
|
40
|
-
end
|
41
|
-
|
42
|
-
def url
|
43
|
-
return @commit_url if @commit_url && batch_full? || expires_soon?
|
44
|
-
|
45
|
-
current_transaction_url || new_transaction_url
|
46
|
-
end
|
47
|
-
|
48
|
-
def new_transaction_url
|
49
|
-
Delfos.neo4j.uri_for("/db/data/transaction")
|
50
|
-
end
|
51
|
-
|
52
|
-
def flush_if_required!
|
53
|
-
check_for_expiry!
|
54
|
-
|
55
|
-
if batch_full? || expires_soon? || large_query?
|
56
|
-
flush!
|
57
|
-
return true
|
58
|
-
end
|
59
|
-
|
60
|
-
false
|
61
|
-
end
|
62
|
-
|
63
|
-
def check_for_expiry!
|
64
|
-
return if @expires.nil? || (@clock.now <= @expires)
|
65
|
-
|
66
|
-
raise QueryExecution::ExpiredTransaction.new(@commit_url, "")
|
67
|
-
end
|
68
|
-
|
69
|
-
def batch_full?
|
70
|
-
query_count >= size
|
71
|
-
end
|
72
|
-
|
73
|
-
def expires_soon?
|
74
|
-
@expires && (@clock.now + 10 > @expires)
|
75
|
-
end
|
76
|
-
|
77
|
-
def large_query?
|
78
|
-
@total_query_length >= Delfos.max_query_size
|
79
|
-
end
|
80
|
-
|
81
|
-
def reset!
|
82
|
-
@query_count = 0
|
83
|
-
@total_query_length = 0
|
84
|
-
@current_transaction_url = nil
|
85
|
-
@commit_url = nil
|
86
|
-
@expires = nil
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative "execution"
|
4
|
-
|
5
|
-
module Delfos
|
6
|
-
module Neo4j
|
7
|
-
module Batch
|
8
|
-
class Retryable
|
9
|
-
class << self
|
10
|
-
MUTEX = Mutex.new
|
11
|
-
|
12
|
-
def reset!
|
13
|
-
MUTEX.synchronize do
|
14
|
-
Thread.current[:_delfos__retryable_batch] = nil
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def ensure_instance(size: nil)
|
19
|
-
MUTEX.synchronize do
|
20
|
-
Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def execute!(query, params: {}, size: nil)
|
25
|
-
ensure_instance(size: size).execute!(query, params: params)
|
26
|
-
end
|
27
|
-
|
28
|
-
def flush!
|
29
|
-
ensure_instance&.flush!
|
30
|
-
reset!
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
attr_reader :size, :queries, :execution
|
35
|
-
attr_accessor :retry_count
|
36
|
-
|
37
|
-
def initialize(size:)
|
38
|
-
@size = size
|
39
|
-
reset!
|
40
|
-
end
|
41
|
-
|
42
|
-
def execute!(query, params: {}, retrying: false)
|
43
|
-
queries.push([query, params]) unless retrying
|
44
|
-
|
45
|
-
with_retry(retrying) do
|
46
|
-
flushed = execution.execute!(query, params: params)
|
47
|
-
|
48
|
-
reset! if flushed
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def flush!
|
53
|
-
execution.flush!
|
54
|
-
reset!
|
55
|
-
end
|
56
|
-
|
57
|
-
private
|
58
|
-
|
59
|
-
def with_retry(already_retrying)
|
60
|
-
yield
|
61
|
-
rescue QueryExecution::ExpiredTransaction
|
62
|
-
check_retry_limit! if already_retrying
|
63
|
-
|
64
|
-
retry_batch!
|
65
|
-
end
|
66
|
-
|
67
|
-
def check_retry_limit!
|
68
|
-
self.retry_count += 1
|
69
|
-
|
70
|
-
return if self.retry_count <= 5
|
71
|
-
|
72
|
-
self.retry_count = 0
|
73
|
-
Delfos.logger.error "Transaction expired - 5 retries failed aborting"
|
74
|
-
raise
|
75
|
-
end
|
76
|
-
|
77
|
-
def retry_batch!
|
78
|
-
Delfos.logger.error do
|
79
|
-
"Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
|
80
|
-
end
|
81
|
-
|
82
|
-
new_execution
|
83
|
-
|
84
|
-
queries.each { |q, p| execute!(q, params: p, retrying: true) }
|
85
|
-
|
86
|
-
Delfos.logger.error { "Batch retry successful" }
|
87
|
-
end
|
88
|
-
|
89
|
-
def reset!
|
90
|
-
@queries = []
|
91
|
-
new_execution
|
92
|
-
@retry_count = 0
|
93
|
-
end
|
94
|
-
|
95
|
-
def new_execution
|
96
|
-
@execution = Execution.new(size: size)
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
data/lib/delfos/setup.rb
DELETED
@@ -1,120 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Delfos
|
4
|
-
module Setup
|
5
|
-
extend self
|
6
|
-
attr_accessor :neo4j
|
7
|
-
|
8
|
-
def perform!(call_site_logger: nil,
|
9
|
-
application_directories: nil,
|
10
|
-
offline_query_saving: nil)
|
11
|
-
self.application_directories = application_directories if application_directories
|
12
|
-
self.call_site_logger = call_site_logger if call_site_logger
|
13
|
-
|
14
|
-
assign_offline_attributes(offline_query_saving)
|
15
|
-
|
16
|
-
require "delfos/method_trace"
|
17
|
-
::Delfos::MethodTrace.trace!
|
18
|
-
end
|
19
|
-
|
20
|
-
def application_directories=(dirs)
|
21
|
-
dirs ||= %w[app lib]
|
22
|
-
require "pathname"
|
23
|
-
Delfos.application_directories = Array(dirs).map { |f| Pathname.new(f.to_s).expand_path }
|
24
|
-
end
|
25
|
-
|
26
|
-
def call_site_logger=(call_site_logger)
|
27
|
-
Delfos.call_site_logger = call_site_logger || default_call_site_logger
|
28
|
-
end
|
29
|
-
|
30
|
-
def default_call_site_logger
|
31
|
-
if Delfos.offline_query_saving
|
32
|
-
require "delfos/neo4j/offline/call_site_logger"
|
33
|
-
Delfos:: Neo4j::Offline::CallSiteLogger.new
|
34
|
-
else
|
35
|
-
Delfos.setup_neo4j!
|
36
|
-
|
37
|
-
require "delfos/neo4j/live/call_site_logger"
|
38
|
-
Delfos:: Neo4j::Live::CallSiteLogger.new
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def disable!
|
43
|
-
disable_tracepoint!
|
44
|
-
|
45
|
-
reset_call_stack!
|
46
|
-
reset_batch!
|
47
|
-
|
48
|
-
reset_top_level_variables!
|
49
|
-
reset_app_directories!
|
50
|
-
end
|
51
|
-
|
52
|
-
def disable_tracepoint!
|
53
|
-
require "delfos/method_trace"
|
54
|
-
::Delfos::MethodTrace.disable!
|
55
|
-
end
|
56
|
-
|
57
|
-
def reset_call_stack!
|
58
|
-
ignoring_undefined("Delfos::CallStack") do |k|
|
59
|
-
k.pop_until_top!
|
60
|
-
k.reset!
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def reset_batch!
|
65
|
-
Delfos.batch_size = nil
|
66
|
-
|
67
|
-
ignoring_undefined "Delfos::Neo4j::Batch::Retryable" do |b|
|
68
|
-
begin
|
69
|
-
with_rescue { b.flush! }
|
70
|
-
ensure
|
71
|
-
b.instance = nil
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def reset_app_directories!
|
77
|
-
ignoring_undefined("Delfos::FileSystem::AppDirectories", &:reset!)
|
78
|
-
end
|
79
|
-
|
80
|
-
def reset_top_level_variables!
|
81
|
-
Delfos.offline_query_saving = nil
|
82
|
-
Delfos.offline_query_filename = nil
|
83
|
-
Delfos.neo4j = nil
|
84
|
-
Delfos.logger = nil
|
85
|
-
Delfos.application_directories = nil
|
86
|
-
Delfos.call_site_logger = nil
|
87
|
-
Delfos.max_query_size = nil
|
88
|
-
Delfos.neo4j = nil
|
89
|
-
end
|
90
|
-
|
91
|
-
# This method allows resetting in between every spec. So we avoid load
|
92
|
-
# order issues in cases where we have not run particular specs that require
|
93
|
-
# and define these constants
|
94
|
-
def ignoring_undefined(k)
|
95
|
-
o = Object.const_get(k)
|
96
|
-
yield(o)
|
97
|
-
rescue NameError => e
|
98
|
-
raise unless e.message[k]
|
99
|
-
end
|
100
|
-
|
101
|
-
def with_rescue
|
102
|
-
yield
|
103
|
-
rescue Delfos::Neo4j::QueryExecution::ExpiredTransaction
|
104
|
-
puts # no-op
|
105
|
-
end
|
106
|
-
|
107
|
-
def assign_offline_attributes(offline_query_saving)
|
108
|
-
return unless offline_query_saving
|
109
|
-
Delfos.offline_query_saving = offline_query_saving
|
110
|
-
|
111
|
-
Delfos.offline_query_filename = filename_from(offline_query_saving)
|
112
|
-
end
|
113
|
-
|
114
|
-
def filename_from(offline_query_saving)
|
115
|
-
return offline_query_saving if offline_query_saving.is_a?(String)
|
116
|
-
|
117
|
-
"delfos_query_parameters.json"
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|