delfos 0.0.2.pre.rc2 → 0.0.2.pre.rc3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/exe/delfos_import +38 -0
  3. data/exe/delfos_import_offline_queries +9 -0
  4. data/exe/delfos_update_distance +11 -0
  5. data/lib/delfos.rb +29 -73
  6. data/lib/delfos/config.rb +90 -0
  7. data/lib/delfos/config/inclusion.rb +90 -0
  8. data/lib/delfos/file_system.rb +7 -2
  9. data/lib/delfos/file_system/app_directories.rb +16 -18
  10. data/lib/delfos/file_system/app_files.rb +31 -0
  11. data/lib/delfos/file_system/file_cache.rb +21 -0
  12. data/lib/delfos/file_system/path_determination.rb +10 -7
  13. data/lib/delfos/method_trace.rb +18 -6
  14. data/lib/delfos/method_trace/call_handler.rb +6 -2
  15. data/lib/delfos/method_trace/code_location.rb +2 -12
  16. data/lib/delfos/method_trace/code_location/container_method_factory.rb +50 -25
  17. data/lib/delfos/method_trace/code_location/eval_in_caller.rb +2 -4
  18. data/lib/delfos/method_trace/code_location/filename_helpers.rb +1 -1
  19. data/lib/delfos/method_trace/code_location/method.rb +13 -9
  20. data/lib/delfos/neo4j.rb +12 -3
  21. data/lib/delfos/neo4j/distance/update.rb +1 -1
  22. data/lib/delfos/neo4j/live/call_site_logger.rb +27 -0
  23. data/lib/delfos/neo4j/offline.rb +13 -0
  24. data/lib/delfos/neo4j/offline/call_site_logger.rb +6 -1
  25. data/lib/delfos/neo4j/offline/importer.rb +8 -7
  26. data/lib/delfos/neo4j/query_execution/batch/execution.rb +93 -0
  27. data/lib/delfos/neo4j/query_execution/batch/retryable.rb +103 -0
  28. data/lib/delfos/neo4j/query_execution/errors.rb +4 -3
  29. data/lib/delfos/neo4j/query_execution/http.rb +1 -1
  30. data/lib/delfos/neo4j/query_execution/sync.rb +1 -1
  31. data/lib/delfos/neo4j/query_execution/transactional.rb +1 -1
  32. data/lib/delfos/neo4j/schema.rb +1 -1
  33. metadata +38 -16
  34. data/lib/delfos/neo4j/batch/execution.rb +0 -91
  35. data/lib/delfos/neo4j/batch/retryable.rb +0 -101
  36. data/lib/delfos/setup.rb +0 -120
@@ -1,91 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "delfos/neo4j/query_execution/transactional"
4
-
5
- module Delfos
6
- module Neo4j
7
- module Batch
8
- class Execution
9
- attr_reader :size, :current_transaction_url, :commit_url, :expires, :query_count
10
-
11
- def initialize(size:, clock: Time)
12
- @size = size
13
- @clock = clock
14
- reset!
15
- end
16
-
17
- def execute!(query, params: {})
18
- check_for_expiry!
19
-
20
- perform_query(query, params)
21
- flush_if_required!
22
- end
23
-
24
- def flush!
25
- return unless query_count.positive?
26
- return unless @commit_url
27
- QueryExecution::Transactional.commit!(@commit_url)
28
-
29
- reset!
30
- end
31
-
32
- private
33
-
34
- def perform_query(query, params)
35
- transactional_query = QueryExecution::Transactional.new(query, params, url)
36
- @total_query_length += transactional_query.query_length
37
- transaction_url, @commit_url, @expires = transactional_query.perform
38
- @current_transaction_url ||= transaction_url # the transaction_url is only returned with the first request
39
- @query_count += 1
40
- end
41
-
42
- def url
43
- return @commit_url if @commit_url && batch_full? || expires_soon?
44
-
45
- current_transaction_url || new_transaction_url
46
- end
47
-
48
- def new_transaction_url
49
- Delfos.neo4j.uri_for("/db/data/transaction")
50
- end
51
-
52
- def flush_if_required!
53
- check_for_expiry!
54
-
55
- if batch_full? || expires_soon? || large_query?
56
- flush!
57
- return true
58
- end
59
-
60
- false
61
- end
62
-
63
- def check_for_expiry!
64
- return if @expires.nil? || (@clock.now <= @expires)
65
-
66
- raise QueryExecution::ExpiredTransaction.new(@commit_url, "")
67
- end
68
-
69
- def batch_full?
70
- query_count >= size
71
- end
72
-
73
- def expires_soon?
74
- @expires && (@clock.now + 10 > @expires)
75
- end
76
-
77
- def large_query?
78
- @total_query_length >= Delfos.max_query_size
79
- end
80
-
81
- def reset!
82
- @query_count = 0
83
- @total_query_length = 0
84
- @current_transaction_url = nil
85
- @commit_url = nil
86
- @expires = nil
87
- end
88
- end
89
- end
90
- end
91
- end
@@ -1,101 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative "execution"
4
-
5
- module Delfos
6
- module Neo4j
7
- module Batch
8
- class Retryable
9
- class << self
10
- MUTEX = Mutex.new
11
-
12
- def reset!
13
- MUTEX.synchronize do
14
- Thread.current[:_delfos__retryable_batch] = nil
15
- end
16
- end
17
-
18
- def ensure_instance(size: nil)
19
- MUTEX.synchronize do
20
- Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
21
- end
22
- end
23
-
24
- def execute!(query, params: {}, size: nil)
25
- ensure_instance(size: size).execute!(query, params: params)
26
- end
27
-
28
- def flush!
29
- ensure_instance&.flush!
30
- reset!
31
- end
32
- end
33
-
34
- attr_reader :size, :queries, :execution
35
- attr_accessor :retry_count
36
-
37
- def initialize(size:)
38
- @size = size
39
- reset!
40
- end
41
-
42
- def execute!(query, params: {}, retrying: false)
43
- queries.push([query, params]) unless retrying
44
-
45
- with_retry(retrying) do
46
- flushed = execution.execute!(query, params: params)
47
-
48
- reset! if flushed
49
- end
50
- end
51
-
52
- def flush!
53
- execution.flush!
54
- reset!
55
- end
56
-
57
- private
58
-
59
- def with_retry(already_retrying)
60
- yield
61
- rescue QueryExecution::ExpiredTransaction
62
- check_retry_limit! if already_retrying
63
-
64
- retry_batch!
65
- end
66
-
67
- def check_retry_limit!
68
- self.retry_count += 1
69
-
70
- return if self.retry_count <= 5
71
-
72
- self.retry_count = 0
73
- Delfos.logger.error "Transaction expired - 5 retries failed aborting"
74
- raise
75
- end
76
-
77
- def retry_batch!
78
- Delfos.logger.error do
79
- "Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
80
- end
81
-
82
- new_execution
83
-
84
- queries.each { |q, p| execute!(q, params: p, retrying: true) }
85
-
86
- Delfos.logger.error { "Batch retry successful" }
87
- end
88
-
89
- def reset!
90
- @queries = []
91
- new_execution
92
- @retry_count = 0
93
- end
94
-
95
- def new_execution
96
- @execution = Execution.new(size: size)
97
- end
98
- end
99
- end
100
- end
101
- end
@@ -1,120 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Delfos
4
- module Setup
5
- extend self
6
- attr_accessor :neo4j
7
-
8
- def perform!(call_site_logger: nil,
9
- application_directories: nil,
10
- offline_query_saving: nil)
11
- self.application_directories = application_directories if application_directories
12
- self.call_site_logger = call_site_logger if call_site_logger
13
-
14
- assign_offline_attributes(offline_query_saving)
15
-
16
- require "delfos/method_trace"
17
- ::Delfos::MethodTrace.trace!
18
- end
19
-
20
- def application_directories=(dirs)
21
- dirs ||= %w[app lib]
22
- require "pathname"
23
- Delfos.application_directories = Array(dirs).map { |f| Pathname.new(f.to_s).expand_path }
24
- end
25
-
26
- def call_site_logger=(call_site_logger)
27
- Delfos.call_site_logger = call_site_logger || default_call_site_logger
28
- end
29
-
30
- def default_call_site_logger
31
- if Delfos.offline_query_saving
32
- require "delfos/neo4j/offline/call_site_logger"
33
- Delfos:: Neo4j::Offline::CallSiteLogger.new
34
- else
35
- Delfos.setup_neo4j!
36
-
37
- require "delfos/neo4j/live/call_site_logger"
38
- Delfos:: Neo4j::Live::CallSiteLogger.new
39
- end
40
- end
41
-
42
- def disable!
43
- disable_tracepoint!
44
-
45
- reset_call_stack!
46
- reset_batch!
47
-
48
- reset_top_level_variables!
49
- reset_app_directories!
50
- end
51
-
52
- def disable_tracepoint!
53
- require "delfos/method_trace"
54
- ::Delfos::MethodTrace.disable!
55
- end
56
-
57
- def reset_call_stack!
58
- ignoring_undefined("Delfos::CallStack") do |k|
59
- k.pop_until_top!
60
- k.reset!
61
- end
62
- end
63
-
64
- def reset_batch!
65
- Delfos.batch_size = nil
66
-
67
- ignoring_undefined "Delfos::Neo4j::Batch::Retryable" do |b|
68
- begin
69
- with_rescue { b.flush! }
70
- ensure
71
- b.instance = nil
72
- end
73
- end
74
- end
75
-
76
- def reset_app_directories!
77
- ignoring_undefined("Delfos::FileSystem::AppDirectories", &:reset!)
78
- end
79
-
80
- def reset_top_level_variables!
81
- Delfos.offline_query_saving = nil
82
- Delfos.offline_query_filename = nil
83
- Delfos.neo4j = nil
84
- Delfos.logger = nil
85
- Delfos.application_directories = nil
86
- Delfos.call_site_logger = nil
87
- Delfos.max_query_size = nil
88
- Delfos.neo4j = nil
89
- end
90
-
91
- # This method allows resetting in between every spec. So we avoid load
92
- # order issues in cases where we have not run particular specs that require
93
- # and define these constants
94
- def ignoring_undefined(k)
95
- o = Object.const_get(k)
96
- yield(o)
97
- rescue NameError => e
98
- raise unless e.message[k]
99
- end
100
-
101
- def with_rescue
102
- yield
103
- rescue Delfos::Neo4j::QueryExecution::ExpiredTransaction
104
- puts # no-op
105
- end
106
-
107
- def assign_offline_attributes(offline_query_saving)
108
- return unless offline_query_saving
109
- Delfos.offline_query_saving = offline_query_saving
110
-
111
- Delfos.offline_query_filename = filename_from(offline_query_saving)
112
- end
113
-
114
- def filename_from(offline_query_saving)
115
- return offline_query_saving if offline_query_saving.is_a?(String)
116
-
117
- "delfos_query_parameters.json"
118
- end
119
- end
120
- end