delfos 0.0.2.pre.rc2 → 0.0.2.pre.rc3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/exe/delfos_import +38 -0
- data/exe/delfos_import_offline_queries +9 -0
- data/exe/delfos_update_distance +11 -0
- data/lib/delfos.rb +29 -73
- data/lib/delfos/config.rb +90 -0
- data/lib/delfos/config/inclusion.rb +90 -0
- data/lib/delfos/file_system.rb +7 -2
- data/lib/delfos/file_system/app_directories.rb +16 -18
- data/lib/delfos/file_system/app_files.rb +31 -0
- data/lib/delfos/file_system/file_cache.rb +21 -0
- data/lib/delfos/file_system/path_determination.rb +10 -7
- data/lib/delfos/method_trace.rb +18 -6
- data/lib/delfos/method_trace/call_handler.rb +6 -2
- data/lib/delfos/method_trace/code_location.rb +2 -12
- data/lib/delfos/method_trace/code_location/container_method_factory.rb +50 -25
- data/lib/delfos/method_trace/code_location/eval_in_caller.rb +2 -4
- data/lib/delfos/method_trace/code_location/filename_helpers.rb +1 -1
- data/lib/delfos/method_trace/code_location/method.rb +13 -9
- data/lib/delfos/neo4j.rb +12 -3
- data/lib/delfos/neo4j/distance/update.rb +1 -1
- data/lib/delfos/neo4j/live/call_site_logger.rb +27 -0
- data/lib/delfos/neo4j/offline.rb +13 -0
- data/lib/delfos/neo4j/offline/call_site_logger.rb +6 -1
- data/lib/delfos/neo4j/offline/importer.rb +8 -7
- data/lib/delfos/neo4j/query_execution/batch/execution.rb +93 -0
- data/lib/delfos/neo4j/query_execution/batch/retryable.rb +103 -0
- data/lib/delfos/neo4j/query_execution/errors.rb +4 -3
- data/lib/delfos/neo4j/query_execution/http.rb +1 -1
- data/lib/delfos/neo4j/query_execution/sync.rb +1 -1
- data/lib/delfos/neo4j/query_execution/transactional.rb +1 -1
- data/lib/delfos/neo4j/schema.rb +1 -1
- metadata +38 -16
- data/lib/delfos/neo4j/batch/execution.rb +0 -91
- data/lib/delfos/neo4j/batch/retryable.rb +0 -101
- data/lib/delfos/setup.rb +0 -120
@@ -1,91 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "delfos/neo4j/query_execution/transactional"
|
4
|
-
|
5
|
-
module Delfos
|
6
|
-
module Neo4j
|
7
|
-
module Batch
|
8
|
-
class Execution
|
9
|
-
attr_reader :size, :current_transaction_url, :commit_url, :expires, :query_count
|
10
|
-
|
11
|
-
def initialize(size:, clock: Time)
|
12
|
-
@size = size
|
13
|
-
@clock = clock
|
14
|
-
reset!
|
15
|
-
end
|
16
|
-
|
17
|
-
def execute!(query, params: {})
|
18
|
-
check_for_expiry!
|
19
|
-
|
20
|
-
perform_query(query, params)
|
21
|
-
flush_if_required!
|
22
|
-
end
|
23
|
-
|
24
|
-
def flush!
|
25
|
-
return unless query_count.positive?
|
26
|
-
return unless @commit_url
|
27
|
-
QueryExecution::Transactional.commit!(@commit_url)
|
28
|
-
|
29
|
-
reset!
|
30
|
-
end
|
31
|
-
|
32
|
-
private
|
33
|
-
|
34
|
-
def perform_query(query, params)
|
35
|
-
transactional_query = QueryExecution::Transactional.new(query, params, url)
|
36
|
-
@total_query_length += transactional_query.query_length
|
37
|
-
transaction_url, @commit_url, @expires = transactional_query.perform
|
38
|
-
@current_transaction_url ||= transaction_url # the transaction_url is only returned with the first request
|
39
|
-
@query_count += 1
|
40
|
-
end
|
41
|
-
|
42
|
-
def url
|
43
|
-
return @commit_url if @commit_url && batch_full? || expires_soon?
|
44
|
-
|
45
|
-
current_transaction_url || new_transaction_url
|
46
|
-
end
|
47
|
-
|
48
|
-
def new_transaction_url
|
49
|
-
Delfos.neo4j.uri_for("/db/data/transaction")
|
50
|
-
end
|
51
|
-
|
52
|
-
def flush_if_required!
|
53
|
-
check_for_expiry!
|
54
|
-
|
55
|
-
if batch_full? || expires_soon? || large_query?
|
56
|
-
flush!
|
57
|
-
return true
|
58
|
-
end
|
59
|
-
|
60
|
-
false
|
61
|
-
end
|
62
|
-
|
63
|
-
def check_for_expiry!
|
64
|
-
return if @expires.nil? || (@clock.now <= @expires)
|
65
|
-
|
66
|
-
raise QueryExecution::ExpiredTransaction.new(@commit_url, "")
|
67
|
-
end
|
68
|
-
|
69
|
-
def batch_full?
|
70
|
-
query_count >= size
|
71
|
-
end
|
72
|
-
|
73
|
-
def expires_soon?
|
74
|
-
@expires && (@clock.now + 10 > @expires)
|
75
|
-
end
|
76
|
-
|
77
|
-
def large_query?
|
78
|
-
@total_query_length >= Delfos.max_query_size
|
79
|
-
end
|
80
|
-
|
81
|
-
def reset!
|
82
|
-
@query_count = 0
|
83
|
-
@total_query_length = 0
|
84
|
-
@current_transaction_url = nil
|
85
|
-
@commit_url = nil
|
86
|
-
@expires = nil
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
@@ -1,101 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative "execution"
|
4
|
-
|
5
|
-
module Delfos
|
6
|
-
module Neo4j
|
7
|
-
module Batch
|
8
|
-
class Retryable
|
9
|
-
class << self
|
10
|
-
MUTEX = Mutex.new
|
11
|
-
|
12
|
-
def reset!
|
13
|
-
MUTEX.synchronize do
|
14
|
-
Thread.current[:_delfos__retryable_batch] = nil
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
def ensure_instance(size: nil)
|
19
|
-
MUTEX.synchronize do
|
20
|
-
Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
|
21
|
-
end
|
22
|
-
end
|
23
|
-
|
24
|
-
def execute!(query, params: {}, size: nil)
|
25
|
-
ensure_instance(size: size).execute!(query, params: params)
|
26
|
-
end
|
27
|
-
|
28
|
-
def flush!
|
29
|
-
ensure_instance&.flush!
|
30
|
-
reset!
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
attr_reader :size, :queries, :execution
|
35
|
-
attr_accessor :retry_count
|
36
|
-
|
37
|
-
def initialize(size:)
|
38
|
-
@size = size
|
39
|
-
reset!
|
40
|
-
end
|
41
|
-
|
42
|
-
def execute!(query, params: {}, retrying: false)
|
43
|
-
queries.push([query, params]) unless retrying
|
44
|
-
|
45
|
-
with_retry(retrying) do
|
46
|
-
flushed = execution.execute!(query, params: params)
|
47
|
-
|
48
|
-
reset! if flushed
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
def flush!
|
53
|
-
execution.flush!
|
54
|
-
reset!
|
55
|
-
end
|
56
|
-
|
57
|
-
private
|
58
|
-
|
59
|
-
def with_retry(already_retrying)
|
60
|
-
yield
|
61
|
-
rescue QueryExecution::ExpiredTransaction
|
62
|
-
check_retry_limit! if already_retrying
|
63
|
-
|
64
|
-
retry_batch!
|
65
|
-
end
|
66
|
-
|
67
|
-
def check_retry_limit!
|
68
|
-
self.retry_count += 1
|
69
|
-
|
70
|
-
return if self.retry_count <= 5
|
71
|
-
|
72
|
-
self.retry_count = 0
|
73
|
-
Delfos.logger.error "Transaction expired - 5 retries failed aborting"
|
74
|
-
raise
|
75
|
-
end
|
76
|
-
|
77
|
-
def retry_batch!
|
78
|
-
Delfos.logger.error do
|
79
|
-
"Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
|
80
|
-
end
|
81
|
-
|
82
|
-
new_execution
|
83
|
-
|
84
|
-
queries.each { |q, p| execute!(q, params: p, retrying: true) }
|
85
|
-
|
86
|
-
Delfos.logger.error { "Batch retry successful" }
|
87
|
-
end
|
88
|
-
|
89
|
-
def reset!
|
90
|
-
@queries = []
|
91
|
-
new_execution
|
92
|
-
@retry_count = 0
|
93
|
-
end
|
94
|
-
|
95
|
-
def new_execution
|
96
|
-
@execution = Execution.new(size: size)
|
97
|
-
end
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
data/lib/delfos/setup.rb
DELETED
@@ -1,120 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Delfos
|
4
|
-
module Setup
|
5
|
-
extend self
|
6
|
-
attr_accessor :neo4j
|
7
|
-
|
8
|
-
def perform!(call_site_logger: nil,
|
9
|
-
application_directories: nil,
|
10
|
-
offline_query_saving: nil)
|
11
|
-
self.application_directories = application_directories if application_directories
|
12
|
-
self.call_site_logger = call_site_logger if call_site_logger
|
13
|
-
|
14
|
-
assign_offline_attributes(offline_query_saving)
|
15
|
-
|
16
|
-
require "delfos/method_trace"
|
17
|
-
::Delfos::MethodTrace.trace!
|
18
|
-
end
|
19
|
-
|
20
|
-
def application_directories=(dirs)
|
21
|
-
dirs ||= %w[app lib]
|
22
|
-
require "pathname"
|
23
|
-
Delfos.application_directories = Array(dirs).map { |f| Pathname.new(f.to_s).expand_path }
|
24
|
-
end
|
25
|
-
|
26
|
-
def call_site_logger=(call_site_logger)
|
27
|
-
Delfos.call_site_logger = call_site_logger || default_call_site_logger
|
28
|
-
end
|
29
|
-
|
30
|
-
def default_call_site_logger
|
31
|
-
if Delfos.offline_query_saving
|
32
|
-
require "delfos/neo4j/offline/call_site_logger"
|
33
|
-
Delfos:: Neo4j::Offline::CallSiteLogger.new
|
34
|
-
else
|
35
|
-
Delfos.setup_neo4j!
|
36
|
-
|
37
|
-
require "delfos/neo4j/live/call_site_logger"
|
38
|
-
Delfos:: Neo4j::Live::CallSiteLogger.new
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
def disable!
|
43
|
-
disable_tracepoint!
|
44
|
-
|
45
|
-
reset_call_stack!
|
46
|
-
reset_batch!
|
47
|
-
|
48
|
-
reset_top_level_variables!
|
49
|
-
reset_app_directories!
|
50
|
-
end
|
51
|
-
|
52
|
-
def disable_tracepoint!
|
53
|
-
require "delfos/method_trace"
|
54
|
-
::Delfos::MethodTrace.disable!
|
55
|
-
end
|
56
|
-
|
57
|
-
def reset_call_stack!
|
58
|
-
ignoring_undefined("Delfos::CallStack") do |k|
|
59
|
-
k.pop_until_top!
|
60
|
-
k.reset!
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def reset_batch!
|
65
|
-
Delfos.batch_size = nil
|
66
|
-
|
67
|
-
ignoring_undefined "Delfos::Neo4j::Batch::Retryable" do |b|
|
68
|
-
begin
|
69
|
-
with_rescue { b.flush! }
|
70
|
-
ensure
|
71
|
-
b.instance = nil
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
def reset_app_directories!
|
77
|
-
ignoring_undefined("Delfos::FileSystem::AppDirectories", &:reset!)
|
78
|
-
end
|
79
|
-
|
80
|
-
def reset_top_level_variables!
|
81
|
-
Delfos.offline_query_saving = nil
|
82
|
-
Delfos.offline_query_filename = nil
|
83
|
-
Delfos.neo4j = nil
|
84
|
-
Delfos.logger = nil
|
85
|
-
Delfos.application_directories = nil
|
86
|
-
Delfos.call_site_logger = nil
|
87
|
-
Delfos.max_query_size = nil
|
88
|
-
Delfos.neo4j = nil
|
89
|
-
end
|
90
|
-
|
91
|
-
# This method allows resetting in between every spec. So we avoid load
|
92
|
-
# order issues in cases where we have not run particular specs that require
|
93
|
-
# and define these constants
|
94
|
-
def ignoring_undefined(k)
|
95
|
-
o = Object.const_get(k)
|
96
|
-
yield(o)
|
97
|
-
rescue NameError => e
|
98
|
-
raise unless e.message[k]
|
99
|
-
end
|
100
|
-
|
101
|
-
def with_rescue
|
102
|
-
yield
|
103
|
-
rescue Delfos::Neo4j::QueryExecution::ExpiredTransaction
|
104
|
-
puts # no-op
|
105
|
-
end
|
106
|
-
|
107
|
-
def assign_offline_attributes(offline_query_saving)
|
108
|
-
return unless offline_query_saving
|
109
|
-
Delfos.offline_query_saving = offline_query_saving
|
110
|
-
|
111
|
-
Delfos.offline_query_filename = filename_from(offline_query_saving)
|
112
|
-
end
|
113
|
-
|
114
|
-
def filename_from(offline_query_saving)
|
115
|
-
return offline_query_saving if offline_query_saving.is_a?(String)
|
116
|
-
|
117
|
-
"delfos_query_parameters.json"
|
118
|
-
end
|
119
|
-
end
|
120
|
-
end
|