delfos 0.0.2.pre.rc2 → 0.0.2.pre.rc3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/exe/delfos_import +38 -0
  3. data/exe/delfos_import_offline_queries +9 -0
  4. data/exe/delfos_update_distance +11 -0
  5. data/lib/delfos.rb +29 -73
  6. data/lib/delfos/config.rb +90 -0
  7. data/lib/delfos/config/inclusion.rb +90 -0
  8. data/lib/delfos/file_system.rb +7 -2
  9. data/lib/delfos/file_system/app_directories.rb +16 -18
  10. data/lib/delfos/file_system/app_files.rb +31 -0
  11. data/lib/delfos/file_system/file_cache.rb +21 -0
  12. data/lib/delfos/file_system/path_determination.rb +10 -7
  13. data/lib/delfos/method_trace.rb +18 -6
  14. data/lib/delfos/method_trace/call_handler.rb +6 -2
  15. data/lib/delfos/method_trace/code_location.rb +2 -12
  16. data/lib/delfos/method_trace/code_location/container_method_factory.rb +50 -25
  17. data/lib/delfos/method_trace/code_location/eval_in_caller.rb +2 -4
  18. data/lib/delfos/method_trace/code_location/filename_helpers.rb +1 -1
  19. data/lib/delfos/method_trace/code_location/method.rb +13 -9
  20. data/lib/delfos/neo4j.rb +12 -3
  21. data/lib/delfos/neo4j/distance/update.rb +1 -1
  22. data/lib/delfos/neo4j/live/call_site_logger.rb +27 -0
  23. data/lib/delfos/neo4j/offline.rb +13 -0
  24. data/lib/delfos/neo4j/offline/call_site_logger.rb +6 -1
  25. data/lib/delfos/neo4j/offline/importer.rb +8 -7
  26. data/lib/delfos/neo4j/query_execution/batch/execution.rb +93 -0
  27. data/lib/delfos/neo4j/query_execution/batch/retryable.rb +103 -0
  28. data/lib/delfos/neo4j/query_execution/errors.rb +4 -3
  29. data/lib/delfos/neo4j/query_execution/http.rb +1 -1
  30. data/lib/delfos/neo4j/query_execution/sync.rb +1 -1
  31. data/lib/delfos/neo4j/query_execution/transactional.rb +1 -1
  32. data/lib/delfos/neo4j/schema.rb +1 -1
  33. metadata +38 -16
  34. data/lib/delfos/neo4j/batch/execution.rb +0 -91
  35. data/lib/delfos/neo4j/batch/retryable.rb +0 -101
  36. data/lib/delfos/setup.rb +0 -120
@@ -1,8 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "json"
4
- require "delfos/neo4j"
5
4
  require "delfos/neo4j/call_site_query"
5
+ require "delfos/neo4j/query_execution/errors"
6
6
  require "fileutils"
7
7
 
8
8
  module Delfos
@@ -17,8 +17,6 @@ module Delfos
17
17
 
18
18
  def perform
19
19
  each_line do |params, err|
20
- params = JSON.parse(params)
21
-
22
20
  execute(CallSiteQuery::BODY, params, err)
23
21
  end
24
22
 
@@ -28,8 +26,9 @@ module Delfos
28
26
  private
29
27
 
30
28
  def execute(query, params, err)
29
+ params = JSON.parse(params)
31
30
  Neo4j.execute_sync(query, params)
32
- rescue Delfos::Neo4j::QueryExecution::InvalidQuery => e
31
+ rescue Delfos::Neo4j::QueryExecution::InvalidQuery, JSON::ParserError => e
33
32
  @no_errors = false
34
33
  Delfos.logger.error e.message.to_s
35
34
  err.puts JSON.dump(params)
@@ -48,13 +47,15 @@ module Delfos
48
47
  def with_errors
49
48
  @no_errors = true
50
49
 
51
- error_filename = "#{filename}.errors"
52
-
53
50
  File.open(error_filename, "w") do |err|
54
51
  yield err
55
52
  end
56
53
 
57
- FileUtils.rm_rf error_filename if @no_errors
54
+ FileUtils.rm_rf(error_filename) if @no_errors
55
+ end
56
+
57
+ def error_filename
58
+ "#{filename}.errors"
58
59
  end
59
60
 
60
61
  def with_input
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "delfos/neo4j/query_execution/transactional"
4
+
5
+ module Delfos
6
+ module Neo4j
7
+ module QueryExecution
8
+ module Batch
9
+ class Execution
10
+ attr_reader :size, :current_transaction_url, :commit_url, :expires, :query_count
11
+
12
+ def initialize(size:, clock: Time)
13
+ @size = size
14
+ @clock = clock
15
+ reset!
16
+ end
17
+
18
+ def execute!(query, params: {})
19
+ check_for_expiry!
20
+
21
+ perform_query(query, params)
22
+ flush_if_required!
23
+ end
24
+
25
+ def flush!
26
+ return unless query_count.positive?
27
+ return unless @commit_url
28
+ QueryExecution::Transactional.commit!(@commit_url)
29
+
30
+ reset!
31
+ end
32
+
33
+ private
34
+
35
+ def perform_query(query, params)
36
+ transactional_query = QueryExecution::Transactional.new(query, params, url)
37
+ @total_query_length += transactional_query.query_length
38
+ transaction_url, @commit_url, @expires = transactional_query.perform
39
+ @current_transaction_url ||= transaction_url # the transaction_url is only returned with the first request
40
+ @query_count += 1
41
+ end
42
+
43
+ def url
44
+ return @commit_url if @commit_url && batch_full? || expires_soon?
45
+
46
+ current_transaction_url || new_transaction_url
47
+ end
48
+
49
+ def new_transaction_url
50
+ Delfos.config.neo4j.uri_for("/db/data/transaction")
51
+ end
52
+
53
+ def flush_if_required!
54
+ check_for_expiry!
55
+
56
+ if batch_full? || expires_soon? || large_query?
57
+ flush!
58
+ return true
59
+ end
60
+
61
+ false
62
+ end
63
+
64
+ def check_for_expiry!
65
+ return if @expires.nil? || (@clock.now <= @expires)
66
+
67
+ raise QueryExecution::ExpiredTransaction.new(@commit_url, "")
68
+ end
69
+
70
+ def batch_full?
71
+ query_count >= size
72
+ end
73
+
74
+ def expires_soon?
75
+ @expires && (@clock.now + 10 > @expires)
76
+ end
77
+
78
+ def large_query?
79
+ @total_query_length >= Delfos.max_query_size
80
+ end
81
+
82
+ def reset!
83
+ @query_count = 0
84
+ @total_query_length = 0
85
+ @current_transaction_url = nil
86
+ @commit_url = nil
87
+ @expires = nil
88
+ end
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "execution"
4
+
5
+ module Delfos
6
+ module Neo4j
7
+ module QueryExecution
8
+ module Batch
9
+ class Retryable
10
+ class << self
11
+ MUTEX = Mutex.new
12
+
13
+ def reset!
14
+ MUTEX.synchronize do
15
+ Thread.current[:_delfos__retryable_batch] = nil
16
+ end
17
+ end
18
+
19
+ def ensure_instance(size: nil)
20
+ MUTEX.synchronize do
21
+ Thread.current[:_delfos__retryable_batch] ||= new(size: size || 1_000)
22
+ end
23
+ end
24
+
25
+ def execute!(query, params: {}, size: nil)
26
+ ensure_instance(size: size).execute!(query, params: params)
27
+ end
28
+
29
+ def flush!
30
+ ensure_instance&.flush!
31
+ reset!
32
+ end
33
+ end
34
+
35
+ attr_reader :size, :queries, :execution
36
+ attr_accessor :retry_count
37
+
38
+ def initialize(size:)
39
+ @size = size
40
+ reset!
41
+ end
42
+
43
+ def execute!(query, params: {}, retrying: false)
44
+ queries.push([query, params]) unless retrying
45
+
46
+ with_retry(retrying) do
47
+ flushed = execution.execute!(query, params: params)
48
+
49
+ reset! if flushed
50
+ end
51
+ end
52
+
53
+ def flush!
54
+ execution.flush!
55
+ reset!
56
+ end
57
+
58
+ private
59
+
60
+ def with_retry(already_retrying)
61
+ yield
62
+ rescue QueryExecution::ExpiredTransaction
63
+ check_retry_limit! if already_retrying
64
+
65
+ retry_batch!
66
+ end
67
+
68
+ def check_retry_limit!
69
+ self.retry_count += 1
70
+
71
+ return if self.retry_count <= 5
72
+
73
+ self.retry_count = 0
74
+ Delfos.logger.error "Transaction expired - 5 retries failed aborting"
75
+ raise
76
+ end
77
+
78
+ def retry_batch!
79
+ Delfos.logger.error do
80
+ "Transaction expired - retrying batch. #{queries.count} queries retry_count: #{retry_count}"
81
+ end
82
+
83
+ new_execution
84
+
85
+ queries.each { |q, p| execute!(q, params: p, retrying: true) }
86
+
87
+ Delfos.logger.error { "Batch retry successful" }
88
+ end
89
+
90
+ def reset!
91
+ @queries = []
92
+ new_execution
93
+ @retry_count = 0
94
+ end
95
+
96
+ def new_execution
97
+ @execution = Execution.new(size: size)
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -5,6 +5,9 @@ require "net/http"
5
5
  module Delfos
6
6
  module Neo4j
7
7
  module QueryExecution
8
+ class ConnectionError < IOError
9
+ end
10
+
8
11
  HTTP_ERRORS = [
9
12
  EOFError,
10
13
  Errno::EAGAIN,
@@ -21,6 +24,7 @@ module Delfos
21
24
  Net::ReadTimeout,
22
25
  SocketError,
23
26
  Timeout::Error,
27
+ ConnectionError,
24
28
  ].freeze
25
29
 
26
30
  class InvalidQuery < IOError
@@ -32,9 +36,6 @@ module Delfos
32
36
  super [message, { query: query.to_s }, { params: params.to_s }].join("\n\n")
33
37
  end
34
38
  end
35
-
36
- class ConnectionError < IOError
37
- end
38
39
  end
39
40
  end
40
41
  end
@@ -53,7 +53,7 @@ module Delfos
53
53
 
54
54
  def add_headers(request)
55
55
  request.initialize_http_header("Accept" => "application/json; charset=UTF-8")
56
- request.basic_auth(Delfos.neo4j.username, Delfos.neo4j.password)
56
+ request.basic_auth(Delfos.config.neo4j.username, Delfos.config.neo4j.password)
57
57
  request.content_type = "application/json"
58
58
  end
59
59
  end
@@ -28,7 +28,7 @@ module Delfos
28
28
  end
29
29
 
30
30
  def uri
31
- @uri ||= Delfos.neo4j.uri_for("/db/data/transaction/commit")
31
+ @uri ||= Delfos.config.neo4j.uri_for("/db/data/transaction/commit")
32
32
  end
33
33
  end
34
34
  end
@@ -62,7 +62,7 @@ module Delfos
62
62
  end
63
63
 
64
64
  def uri
65
- @uri ||= Delfos.neo4j.uri_for("/db/data/transaction")
65
+ @uri ||= Delfos.config.neo4j.uri_for("/db/data/transaction")
66
66
  end
67
67
  end
68
68
 
@@ -52,7 +52,7 @@ module Delfos
52
52
  end
53
53
 
54
54
  def constraints_uri
55
- Delfos.neo4j.uri_for("/db/data/schema/constraint")
55
+ Delfos.config.neo4j.uri_for("/db/data/schema/constraint")
56
56
  end
57
57
  end
58
58
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: delfos
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2.pre.rc2
4
+ version: 0.0.2.pre.rc3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Mark Burns
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-06-22 00:00:00.000000000 Z
11
+ date: 2017-06-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: binding_of_caller
@@ -34,20 +34,23 @@ dependencies:
34
34
  name: simplecov
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ">="
37
+ - - "~>"
38
38
  - !ruby/object:Gem::Version
39
39
  version: '0'
40
40
  type: :development
41
41
  prerelease: false
42
42
  version_requirements: !ruby/object:Gem::Requirement
43
43
  requirements:
44
- - - ">="
44
+ - - "~>"
45
45
  - !ruby/object:Gem::Version
46
46
  version: '0'
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: rspec_junit_formatter
49
49
  requirement: !ruby/object:Gem::Requirement
50
50
  requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '0.2'
51
54
  - - ">="
52
55
  - !ruby/object:Gem::Version
53
56
  version: 0.2.3
@@ -55,6 +58,9 @@ dependencies:
55
58
  prerelease: false
56
59
  version_requirements: !ruby/object:Gem::Requirement
57
60
  requirements:
61
+ - - "~>"
62
+ - !ruby/object:Gem::Version
63
+ version: '0.2'
58
64
  - - ">="
59
65
  - !ruby/object:Gem::Version
60
66
  version: 0.2.3
@@ -62,20 +68,23 @@ dependencies:
62
68
  name: codeclimate-test-reporter
63
69
  requirement: !ruby/object:Gem::Requirement
64
70
  requirements:
65
- - - ">="
71
+ - - "~>"
66
72
  - !ruby/object:Gem::Version
67
73
  version: '0'
68
74
  type: :development
69
75
  prerelease: false
70
76
  version_requirements: !ruby/object:Gem::Requirement
71
77
  requirements:
72
- - - ">="
78
+ - - "~>"
73
79
  - !ruby/object:Gem::Version
74
80
  version: '0'
75
81
  - !ruby/object:Gem::Dependency
76
82
  name: webmock
77
83
  requirement: !ruby/object:Gem::Requirement
78
84
  requirements:
85
+ - - "~>"
86
+ - !ruby/object:Gem::Version
87
+ version: '2.3'
79
88
  - - ">="
80
89
  - !ruby/object:Gem::Version
81
90
  version: 2.3.2
@@ -83,6 +92,9 @@ dependencies:
83
92
  prerelease: false
84
93
  version_requirements: !ruby/object:Gem::Requirement
85
94
  requirements:
95
+ - - "~>"
96
+ - !ruby/object:Gem::Version
97
+ version: '2.3'
86
98
  - - ">="
87
99
  - !ruby/object:Gem::Version
88
100
  version: 2.3.2
@@ -90,14 +102,14 @@ dependencies:
90
102
  name: rake-compiler
91
103
  requirement: !ruby/object:Gem::Requirement
92
104
  requirements:
93
- - - ">="
105
+ - - "~>"
94
106
  - !ruby/object:Gem::Version
95
107
  version: '0'
96
108
  type: :development
97
109
  prerelease: false
98
110
  version_requirements: !ruby/object:Gem::Requirement
99
111
  requirements:
100
- - - ">="
112
+ - - "~>"
101
113
  - !ruby/object:Gem::Version
102
114
  version: '0'
103
115
  - !ruby/object:Gem::Dependency
@@ -146,35 +158,45 @@ dependencies:
146
158
  requirements:
147
159
  - - "~>"
148
160
  - !ruby/object:Gem::Version
149
- version: 3.5.0
161
+ version: 3.6.0
150
162
  - - ">="
151
163
  - !ruby/object:Gem::Version
152
- version: 3.5.0
164
+ version: 3.6.0
153
165
  type: :development
154
166
  prerelease: false
155
167
  version_requirements: !ruby/object:Gem::Requirement
156
168
  requirements:
157
169
  - - "~>"
158
170
  - !ruby/object:Gem::Version
159
- version: 3.5.0
171
+ version: 3.6.0
160
172
  - - ">="
161
173
  - !ruby/object:Gem::Version
162
- version: 3.5.0
174
+ version: 3.6.0
163
175
  description: Record every method call, call-site, arguments and their types in your
164
176
  application code
165
177
  email:
166
178
  - markthedeveloper@gmail..com
167
- executables: []
179
+ executables:
180
+ - delfos_import
181
+ - delfos_import_offline_queries
182
+ - delfos_update_distance
168
183
  extensions: []
169
184
  extra_rdoc_files: []
170
185
  files:
186
+ - exe/delfos_import
187
+ - exe/delfos_import_offline_queries
188
+ - exe/delfos_update_distance
171
189
  - lib/delfos.rb
172
190
  - lib/delfos/call_stack.rb
173
191
  - lib/delfos/call_stack/stack.rb
192
+ - lib/delfos/config.rb
193
+ - lib/delfos/config/inclusion.rb
174
194
  - lib/delfos/file_system.rb
175
195
  - lib/delfos/file_system/app_directories.rb
196
+ - lib/delfos/file_system/app_files.rb
176
197
  - lib/delfos/file_system/common_path.rb
177
198
  - lib/delfos/file_system/distance_calculation.rb
199
+ - lib/delfos/file_system/file_cache.rb
178
200
  - lib/delfos/file_system/path_determination.rb
179
201
  - lib/delfos/file_system/relation.rb
180
202
  - lib/delfos/method_trace.rb
@@ -186,21 +208,21 @@ files:
186
208
  - lib/delfos/method_trace/code_location/filename_helpers.rb
187
209
  - lib/delfos/method_trace/code_location/method.rb
188
210
  - lib/delfos/neo4j.rb
189
- - lib/delfos/neo4j/batch/execution.rb
190
- - lib/delfos/neo4j/batch/retryable.rb
191
211
  - lib/delfos/neo4j/call_site_query.rb
192
212
  - lib/delfos/neo4j/distance/call_site_fetcher.rb
193
213
  - lib/delfos/neo4j/distance/update.rb
194
214
  - lib/delfos/neo4j/live/call_site_logger.rb
215
+ - lib/delfos/neo4j/offline.rb
195
216
  - lib/delfos/neo4j/offline/call_site_logger.rb
196
217
  - lib/delfos/neo4j/offline/importer.rb
218
+ - lib/delfos/neo4j/query_execution/batch/execution.rb
219
+ - lib/delfos/neo4j/query_execution/batch/retryable.rb
197
220
  - lib/delfos/neo4j/query_execution/errors.rb
198
221
  - lib/delfos/neo4j/query_execution/http.rb
199
222
  - lib/delfos/neo4j/query_execution/http_query.rb
200
223
  - lib/delfos/neo4j/query_execution/sync.rb
201
224
  - lib/delfos/neo4j/query_execution/transactional.rb
202
225
  - lib/delfos/neo4j/schema.rb
203
- - lib/delfos/setup.rb
204
226
  homepage: https://github.com/markburns/delfos
205
227
  licenses:
206
228
  - MIT