grumlin 0.21.0 → 0.22.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +8 -8
- data/README.md +63 -8
- data/lib/grumlin/action.rb +13 -8
- data/lib/grumlin/client.rb +16 -12
- data/lib/grumlin/config.rb +26 -0
- data/lib/grumlin/dummy_transaction.rb +25 -0
- data/lib/grumlin/features/feature_list.rb +19 -0
- data/lib/grumlin/features/neptune_features.rb +13 -0
- data/lib/grumlin/features/tinkergraph_features.rb +13 -0
- data/lib/grumlin/features.rb +16 -0
- data/lib/grumlin/repository/instance_methods.rb +42 -19
- data/lib/grumlin/request_dispatcher.rb +4 -43
- data/lib/grumlin/request_error_factory.rb +59 -0
- data/lib/grumlin/steppable.rb +8 -3
- data/lib/grumlin/steps.rb +3 -1
- data/lib/grumlin/steps_serializers/bytecode.rb +8 -6
- data/lib/grumlin/transaction.rb +39 -0
- data/lib/grumlin/traversal_start.rb +27 -0
- data/lib/grumlin/version.rb +1 -1
- data/lib/grumlin.rb +14 -11
- metadata +10 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f91cc8f8b58fc3d72726e876dabac6a8f167e81bd524907eef4df7d72802e79
|
4
|
+
data.tar.gz: 892fa89f75c999899ff0869c7c6f74c26526b9eb5a6da0b43bd4df8a1b64504a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5d50c9b36f4a69a431620404b333b71c8fd9ca2fd8a6b2689c6ece3da57198de4cfa294adaf631238ca1186581eaa3b74a9ec8dea6e6f50f69a977a78ea71a79
|
7
|
+
data.tar.gz: 3f201c437eee80beac09bb76f121542f7c611b68ce513545fb46d6c44790cf061527564e5f9c03e4f55e51c70c67e355802d0e63e64004d4db36de135f110750
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
grumlin (0.
|
4
|
+
grumlin (0.22.1)
|
5
5
|
async-pool (~> 0.3)
|
6
6
|
async-websocket (~> 0.19)
|
7
7
|
oj (~> 3.13)
|
@@ -21,14 +21,14 @@ GEM
|
|
21
21
|
console (~> 1.10)
|
22
22
|
nio4r (~> 2.3)
|
23
23
|
timers (~> 4.1)
|
24
|
-
async-http (0.
|
24
|
+
async-http (0.58.0)
|
25
25
|
async (>= 1.25)
|
26
26
|
async-io (>= 1.28)
|
27
27
|
async-pool (>= 0.2)
|
28
|
-
protocol-http (~> 0.
|
28
|
+
protocol-http (~> 0.23.1)
|
29
29
|
protocol-http1 (~> 0.14.0)
|
30
30
|
protocol-http2 (~> 0.14.0)
|
31
|
-
traces (
|
31
|
+
traces (>= 0.4.0)
|
32
32
|
async-io (1.33.0)
|
33
33
|
async
|
34
34
|
async-pool (0.3.10)
|
@@ -37,7 +37,7 @@ GEM
|
|
37
37
|
rspec (~> 3.0)
|
38
38
|
rspec-files (~> 1.0)
|
39
39
|
rspec-memory (~> 1.0)
|
40
|
-
async-websocket (0.19.
|
40
|
+
async-websocket (0.19.2)
|
41
41
|
async-http (~> 0.54)
|
42
42
|
async-io (~> 1.23)
|
43
43
|
protocol-websocket (~> 0.7.0)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
racc (~> 1.4)
|
73
73
|
nokogiri (1.13.6-x86_64-linux)
|
74
74
|
racc (~> 1.4)
|
75
|
-
oj (3.13.
|
75
|
+
oj (3.13.20)
|
76
76
|
overcommit (0.59.1)
|
77
77
|
childprocess (>= 0.6.3, < 5)
|
78
78
|
iniparse (~> 1.4)
|
@@ -81,7 +81,7 @@ GEM
|
|
81
81
|
parser (3.1.2.0)
|
82
82
|
ast (~> 2.4.1)
|
83
83
|
protocol-hpack (1.4.2)
|
84
|
-
protocol-http (0.
|
84
|
+
protocol-http (0.23.4)
|
85
85
|
protocol-http1 (0.14.4)
|
86
86
|
protocol-http (~> 0.22)
|
87
87
|
protocol-http2 (0.14.2)
|
@@ -157,7 +157,7 @@ GEM
|
|
157
157
|
thor (1.2.1)
|
158
158
|
tilt (2.0.10)
|
159
159
|
timers (4.3.3)
|
160
|
-
traces (0.
|
160
|
+
traces (0.6.0)
|
161
161
|
tzinfo (2.0.4)
|
162
162
|
concurrent-ruby (~> 1.0)
|
163
163
|
unicode-display_width (2.1.0)
|
data/README.md
CHANGED
@@ -46,9 +46,38 @@ Or install it yourself as:
|
|
46
46
|
```ruby
|
47
47
|
Grumlin.configure do |config|
|
48
48
|
config.url = "ws://localhost:8182/gremlin"
|
49
|
+
|
50
|
+
# make sure you select right provider for better compatibility
|
51
|
+
config.provider = :tinkergraph
|
49
52
|
end
|
50
53
|
```
|
51
54
|
|
55
|
+
#### Providers
|
56
|
+
|
57
|
+
Currently `Grumlin` supports 2 providers:
|
58
|
+
- tinkergraph (default)
|
59
|
+
- neptune
|
60
|
+
|
61
|
+
As different providers may have or may have not support for specific features it's recommended to
|
62
|
+
explicitly specify the provider you use.
|
63
|
+
|
64
|
+
#### Provider features
|
65
|
+
|
66
|
+
Every provider is described by a set of features. In the future `Grumlin` may decide to disable or enable
|
67
|
+
some parts of it's functionality to comply provider's supported features.
|
68
|
+
|
69
|
+
To check current providers supported features use
|
70
|
+
|
71
|
+
```ruby
|
72
|
+
Grumlin.features
|
73
|
+
```
|
74
|
+
|
75
|
+
Current differences between providers:
|
76
|
+
|
77
|
+
| Feature | TinkerGraph |AWS Neptune|
|
78
|
+
|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|-----------|
|
79
|
+
| Transactions | Transaction semantic is ignoroed, data is always writen, `tx.rollback` does nothing, an info is printed every time transactions are used with TinkerGraph |Full support
|
80
|
+
|
52
81
|
### Traversing graphs
|
53
82
|
|
54
83
|
**Warning**: Not all steps and expressions defined in the reference documentation are supported.
|
@@ -216,21 +245,29 @@ Each `return_mode` is mapped to a particular termination step:
|
|
216
245
|
- `:traversal` - do not execute the query and return the traversal as is
|
217
246
|
|
218
247
|
`Grumlin::Repository` also provides a set of generic CRUD operations:
|
219
|
-
- `add_vertex(label, id = nil, **properties)`
|
220
|
-
- `add_edge(label, id = nil, from:, to:, **properties)`
|
221
|
-
- `drop_vertex(id)`
|
222
|
-
- `drop_edge(id = nil, from: nil, to: nil, label: nil)`
|
248
|
+
- `add_vertex(label, id = nil, start: g, **properties)`
|
249
|
+
- `add_edge(label, id = nil, from:, to:, start: g, **properties)`
|
250
|
+
- `drop_vertex(id, start: g)`
|
251
|
+
- `drop_edge(id = nil, from: nil, to: nil, label: nil, start: g)`
|
252
|
+
- `drop_in_batches(traversal, batch_size: 10_000)`
|
223
253
|
|
224
254
|
and a few methods that emulate upserts:
|
225
|
-
- `upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, **params)`
|
226
|
-
- `upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, on_failure: :retry, **params)`
|
227
|
-
- `upsert_edges(edges, batch_size: 100, on_failure: :retry, **params)`
|
228
|
-
- `upsert_vertices(edges, batch_size: 100, on_failure: :retry, **params)`
|
255
|
+
- `upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params)`
|
256
|
+
- `upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params)`
|
257
|
+
- `upsert_edges(edges, batch_size: 100, on_failure: :retry, start: g, **params)`
|
258
|
+
- `upsert_vertices(edges, batch_size: 100, on_failure: :retry, start: g, **params)`
|
229
259
|
|
230
260
|
All of them support 3 different modes for error handling: `:retry`, `:ignore` and `:raise`. Retry mode is implemented
|
231
261
|
with [retryable](https://github.com/nfedyashev/retryable). **params will be merged to the default config for upserts
|
232
262
|
and passed to `Retryable.retryable`. In case if you want to modify retryable behaviour you are to do so.
|
233
263
|
|
264
|
+
If you want to use these methods inside a transaction simply pass your `gtx` as `start` parameter:
|
265
|
+
```ruby
|
266
|
+
g.tx do |gtx|
|
267
|
+
add_vertex(:vertex, start: gtx)
|
268
|
+
end
|
269
|
+
```
|
270
|
+
|
234
271
|
If you don't want to define you own repository, simply use
|
235
272
|
|
236
273
|
`Grumlin::Repository.new` returns an instance of an anonymous class extending `Grumlin::Repository`.
|
@@ -258,6 +295,24 @@ it may be useful for debugging. Note that one needs to call a termination step m
|
|
258
295
|
|
259
296
|
method will return profiling data of the results.
|
260
297
|
|
298
|
+
#### Transactions
|
299
|
+
|
300
|
+
Since 0.22.0 `Grumlin` supports transactions when working with providers that supports them:
|
301
|
+
```ruby
|
302
|
+
# Using Transaction directly
|
303
|
+
tx = g.tx
|
304
|
+
gtx = tx.begin
|
305
|
+
gtx.addV(:vertex).iterate
|
306
|
+
tx.commit # or tx.rollback
|
307
|
+
|
308
|
+
# Using with a block
|
309
|
+
g.tx do |gtx|
|
310
|
+
gtx.addV(:vertex).iterate
|
311
|
+
# raise Grumlin::Rollback to manually rollback
|
312
|
+
# any other exception will also rollback the transaction and will be reraised
|
313
|
+
end # commits automatically
|
314
|
+
```
|
315
|
+
|
261
316
|
#### IRB
|
262
317
|
|
263
318
|
An example of how to start an IRB session with support for executing gremlin queries:
|
data/lib/grumlin/action.rb
CHANGED
@@ -4,18 +4,19 @@ module Grumlin
|
|
4
4
|
class Action < Steppable
|
5
5
|
attr_reader :name, :args, :params, :next_step, :configuration_steps, :previous_step, :shortcut
|
6
6
|
|
7
|
-
|
8
|
-
|
7
|
+
# client is only used when a traversal is a part of transaction
|
8
|
+
def initialize(name, args: [], params: {}, previous_step: nil, pool: Grumlin.default_pool, session_id: nil)
|
9
|
+
super(pool: pool, session_id: session_id)
|
10
|
+
|
9
11
|
@name = name.to_sym
|
10
12
|
@args = args # TODO: add recursive validation: only json types or Action
|
11
13
|
@params = params # TODO: add recursive validation: only json types
|
12
14
|
@previous_step = previous_step
|
13
15
|
@shortcut = shortcuts[@name]
|
14
|
-
@pool = pool || Grumlin.default_pool
|
15
16
|
end
|
16
17
|
|
17
18
|
def configuration_step?
|
18
|
-
CONFIGURATION_STEPS.include?(@name)
|
19
|
+
CONFIGURATION_STEPS.include?(@name) || name.to_sym == :tx
|
19
20
|
end
|
20
21
|
|
21
22
|
def start_step?
|
@@ -73,14 +74,18 @@ module Grumlin
|
|
73
74
|
end
|
74
75
|
|
75
76
|
def toList
|
76
|
-
|
77
|
-
client.write(bytecode)
|
78
|
-
end
|
77
|
+
client_write(bytecode)
|
79
78
|
end
|
80
79
|
|
81
80
|
def iterate
|
81
|
+
client_write(bytecode(no_return: true))
|
82
|
+
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
def client_write(payload)
|
82
87
|
@pool.acquire do |client|
|
83
|
-
client.write(
|
88
|
+
client.write(payload, session_id: @session_id)
|
84
89
|
end
|
85
90
|
end
|
86
91
|
end
|
data/lib/grumlin/client.rb
CHANGED
@@ -24,8 +24,8 @@ module Grumlin
|
|
24
24
|
@client.close
|
25
25
|
end
|
26
26
|
|
27
|
-
def write(bytecode)
|
28
|
-
@client.write(bytecode)
|
27
|
+
def write(bytecode, session_id: nil)
|
28
|
+
@client.write(bytecode, session_id: session_id)
|
29
29
|
ensure
|
30
30
|
@count += 1
|
31
31
|
end
|
@@ -94,14 +94,14 @@ module Grumlin
|
|
94
94
|
end
|
95
95
|
|
96
96
|
# TODO: support yielding
|
97
|
-
def write(bytecode)
|
97
|
+
def write(bytecode, session_id: nil)
|
98
98
|
raise NotConnectedError unless connected?
|
99
99
|
|
100
|
-
request = to_query(bytecode)
|
101
|
-
channel = @request_dispatcher.add_request(request)
|
102
|
-
@transport.write(request)
|
100
|
+
request = to_query(bytecode, session_id: session_id)
|
103
101
|
|
102
|
+
channel = @request_dispatcher.add_request(request)
|
104
103
|
begin
|
104
|
+
@transport.write(request)
|
105
105
|
channel.dequeue.flat_map { |item| Typing.cast(item) }
|
106
106
|
rescue Async::Stop, Async::TimeoutError
|
107
107
|
close(check_requests: false)
|
@@ -124,15 +124,19 @@ module Grumlin
|
|
124
124
|
Transport.new(@url, parent: @parent, **@client_options)
|
125
125
|
end
|
126
126
|
|
127
|
-
def to_query(bytecode)
|
127
|
+
def to_query(bytecode, session_id:)
|
128
128
|
{
|
129
129
|
requestId: SecureRandom.uuid,
|
130
|
-
op:
|
131
|
-
processor:
|
130
|
+
op: :bytecode,
|
131
|
+
processor: session_id ? :session : :traversal,
|
132
132
|
args: {
|
133
|
-
gremlin: {
|
134
|
-
|
135
|
-
|
133
|
+
gremlin: {
|
134
|
+
:@type => "g:Bytecode",
|
135
|
+
:@value => bytecode.serialize
|
136
|
+
},
|
137
|
+
aliases: { g: :g },
|
138
|
+
session: session_id
|
139
|
+
}.compact
|
136
140
|
}
|
137
141
|
end
|
138
142
|
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
class Config
|
5
|
+
attr_accessor :url, :pool_size, :client_concurrency, :client_factory, :provider
|
6
|
+
|
7
|
+
SUPPORTED_PROVIDERS = %i[neptune tinkergraph].freeze
|
8
|
+
|
9
|
+
class ConfigurationError < Grumlin::Error; end
|
10
|
+
|
11
|
+
class UnknownProviderError < ConfigurationError; end
|
12
|
+
|
13
|
+
def initialize
|
14
|
+
@pool_size = 10
|
15
|
+
@client_concurrency = 5
|
16
|
+
@provider = :tinkergraph
|
17
|
+
@client_factory = ->(url, parent) { Grumlin::Client.new(url, parent: parent) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate!
|
21
|
+
return if SUPPORTED_PROVIDERS.include?(provider.to_sym)
|
22
|
+
|
23
|
+
raise UnknownProviderError, "provider '#{provider}' is unknown. Supported providers: #{SUPPORTED_PROVIDERS}"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
class DummyTransaction < Transaction
|
5
|
+
attr_reader :uuid
|
6
|
+
|
7
|
+
include Console
|
8
|
+
|
9
|
+
def initialize(traversal_start_class, pool: Grumlin.default_pool) # rubocop:disable Lint/MissingSuper, Lint/UnusedMethodArgument
|
10
|
+
@traversal_start_class = traversal_start_class
|
11
|
+
|
12
|
+
logger.info(self) do
|
13
|
+
"#{Grumlin.config.provider} does not support transactions. commit and rollback are ignored, data will be saved"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def commit
|
18
|
+
nil
|
19
|
+
end
|
20
|
+
|
21
|
+
def rollback
|
22
|
+
nil
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
module Features
|
5
|
+
class FeatureList
|
6
|
+
def user_supplied_ids?
|
7
|
+
raise(NotImplementedError) if @user_supplied_ids.nil?
|
8
|
+
|
9
|
+
@user_supplied_ids
|
10
|
+
end
|
11
|
+
|
12
|
+
def supports_transactions?
|
13
|
+
raise(NotImplementedError) if @supports_transactions.nil?
|
14
|
+
|
15
|
+
@supports_transactions
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
module Features
|
5
|
+
class << self
|
6
|
+
FEATURES = {
|
7
|
+
neptune: NeptuneFeatures.new,
|
8
|
+
tinkergraph: TinkergraphFeatures.new
|
9
|
+
}.freeze
|
10
|
+
|
11
|
+
def for(provider)
|
12
|
+
FEATURES[provider]
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -2,13 +2,13 @@
|
|
2
2
|
|
3
3
|
module Grumlin
|
4
4
|
module Repository
|
5
|
-
module InstanceMethods
|
5
|
+
module InstanceMethods # rubocop:disable Metrics/ModuleLength
|
6
6
|
include Grumlin::Expressions
|
7
7
|
|
8
8
|
extend Forwardable
|
9
9
|
|
10
10
|
UPSERT_RETRY_PARAMS = {
|
11
|
-
on: [Grumlin::AlreadyExistsError, Grumlin::
|
11
|
+
on: [Grumlin::AlreadyExistsError, Grumlin::ConcurrentModificationError],
|
12
12
|
sleep_method: ->(n) { Async::Task.current.sleep(n) },
|
13
13
|
tries: 3,
|
14
14
|
sleep: ->(n) { (n**2) + 1 + rand }
|
@@ -22,51 +22,73 @@ module Grumlin
|
|
22
22
|
self.class.shortcuts
|
23
23
|
end
|
24
24
|
|
25
|
-
def drop_vertex(id)
|
26
|
-
|
25
|
+
def drop_vertex(id, start: g)
|
26
|
+
start.V(id).drop.iterate
|
27
27
|
end
|
28
28
|
|
29
|
-
def
|
29
|
+
def drop_in_batches(traversal, batch_size: 10_000) # rubocop:disable Metrics/AbcSize
|
30
|
+
total_count = traversal.count.next
|
31
|
+
|
32
|
+
batches = (total_count / batch_size) + 1
|
33
|
+
|
34
|
+
Console.logger.info(self) do
|
35
|
+
"drop_in_batches: total_count: #{total_count}, batch_size: #{batch_size}, batches: #{batches}"
|
36
|
+
end
|
37
|
+
|
38
|
+
batches.times do |batch|
|
39
|
+
Console.logger.info(self) { "drop_in_batches: deleting batch #{batch + 1}/#{batches}..." }
|
40
|
+
traversal.limit(batch_size).drop.iterate
|
41
|
+
Console.logger.info(self) { "drop_in_batches: batch #{batch + 1}/#{batches} deleted" }
|
42
|
+
end
|
43
|
+
|
44
|
+
return if traversal.count.next.zero?
|
45
|
+
|
46
|
+
drop_in_batches(traversal, batch_size: batch_size)
|
47
|
+
|
48
|
+
Console.logger.info(self) { "drop_in_batches: finished." }
|
49
|
+
end
|
50
|
+
|
51
|
+
def drop_edge(id = nil, from: nil, to: nil, label: nil, start: g) # rubocop:disable Metrics/AbcSize
|
30
52
|
raise ArgumentError, "either id or from:, to: and label: must be passed" if [id, from, to, label].all?(&:nil?)
|
31
|
-
return
|
53
|
+
return start.E(id).drop.iterate unless id.nil?
|
32
54
|
|
33
55
|
raise ArgumentError, "from:, to: and label: must be passed" if [from, to, label].any?(&:nil?)
|
34
56
|
|
35
|
-
|
57
|
+
start.V(from).outE(label).where(__.inV.hasId(to)).limit(1).drop.iterate
|
36
58
|
end
|
37
59
|
|
38
|
-
def add_vertex(label, id = nil, **properties)
|
60
|
+
def add_vertex(label, id = nil, start: g, **properties)
|
39
61
|
id ||= properties[T.id]
|
40
62
|
properties = except(properties, T.id)
|
41
63
|
|
42
|
-
t =
|
64
|
+
t = start.addV(label)
|
43
65
|
t = t.props(T.id => id) unless id.nil?
|
44
66
|
t.props(**properties).next
|
45
67
|
end
|
46
68
|
|
47
|
-
def add_edge(label, id = nil, from:, to:, **properties)
|
69
|
+
def add_edge(label, id = nil, from:, to:, start: g, **properties)
|
48
70
|
id ||= properties[T.id]
|
49
71
|
properties = except(properties, T.label)
|
50
72
|
properties[T.id] = id
|
51
73
|
|
52
|
-
|
74
|
+
start.addE(label).from(__.V(from)).to(__.V(to)).props(**properties).next
|
53
75
|
end
|
54
76
|
|
55
|
-
def upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, **params)
|
77
|
+
def upsert_vertex(label, id, create_properties: {}, update_properties: {}, on_failure: :retry, start: g, **params) # rubocop:disable Metrics/ParameterLists
|
56
78
|
with_upsert_error_handling(on_failure, params) do
|
57
79
|
create_properties, update_properties = cleanup_properties(create_properties, update_properties)
|
58
80
|
|
59
|
-
|
81
|
+
start.upsertV(label, id, create_properties, update_properties).id.next
|
60
82
|
end
|
61
83
|
end
|
62
84
|
|
63
85
|
# vertices:
|
64
86
|
# [["label", "id", {create: :properties}, {update: properties}]]
|
65
87
|
# params can override Retryable config from UPSERT_RETRY_PARAMS
|
66
|
-
def upsert_vertices(vertices, batch_size: 100, on_failure: :retry, **params)
|
88
|
+
def upsert_vertices(vertices, batch_size: 100, on_failure: :retry, start: g, **params)
|
67
89
|
vertices.each_slice(batch_size) do |slice|
|
68
90
|
with_upsert_error_handling(on_failure, params) do
|
69
|
-
slice.reduce(
|
91
|
+
slice.reduce(start) do |t, (label, id, create_properties, update_properties)|
|
70
92
|
create_properties, update_properties = cleanup_properties(create_properties, update_properties)
|
71
93
|
|
72
94
|
t.upsertV(label, id, create_properties, update_properties)
|
@@ -77,20 +99,21 @@ module Grumlin
|
|
77
99
|
|
78
100
|
# Only from and to are used to find the existing edge, if one wants to assign an id to a created edge,
|
79
101
|
# it must be passed as T.id in create_properties.
|
80
|
-
def upsert_edge(label, from:, to:, create_properties: {}, update_properties: {},
|
102
|
+
def upsert_edge(label, from:, to:, create_properties: {}, update_properties: {}, # rubocop:disable Metrics/ParameterLists
|
103
|
+
on_failure: :retry, start: g, **params)
|
81
104
|
with_upsert_error_handling(on_failure, params) do
|
82
105
|
create_properties, update_properties = cleanup_properties(create_properties, update_properties, T.label)
|
83
|
-
|
106
|
+
start.upsertE(label, from, to, create_properties, update_properties).id.next
|
84
107
|
end
|
85
108
|
end
|
86
109
|
|
87
110
|
# edges:
|
88
111
|
# [["label", "from", "to", {create: :properties}, {update: properties}]]
|
89
112
|
# params can override Retryable config from UPSERT_RETRY_PARAMS
|
90
|
-
def upsert_edges(edges, batch_size: 100, on_failure: :retry, **params)
|
113
|
+
def upsert_edges(edges, batch_size: 100, on_failure: :retry, start: g, **params)
|
91
114
|
edges.each_slice(batch_size) do |slice|
|
92
115
|
with_upsert_error_handling(on_failure, params) do
|
93
|
-
slice.reduce(
|
116
|
+
slice.reduce(start) do |t, (label, from, to, create_properties, update_properties)|
|
94
117
|
create_properties, update_properties = cleanup_properties(create_properties, update_properties, T.label)
|
95
118
|
|
96
119
|
t.upsertE(label, from, to, create_properties, update_properties)
|
@@ -10,23 +10,6 @@ module Grumlin
|
|
10
10
|
206 => :partial_content
|
11
11
|
}.freeze
|
12
12
|
|
13
|
-
ERRORS = {
|
14
|
-
499 => InvalidRequestArgumentsError,
|
15
|
-
500 => ServerError,
|
16
|
-
597 => ScriptEvaluationError,
|
17
|
-
599 => ServerSerializationError,
|
18
|
-
598 => ServerTimeoutError,
|
19
|
-
|
20
|
-
401 => ClientSideError,
|
21
|
-
407 => ClientSideError,
|
22
|
-
498 => ClientSideError
|
23
|
-
}.freeze
|
24
|
-
|
25
|
-
VERTEX_ALREADY_EXISTS = "Vertex with id already exists:"
|
26
|
-
EDGE_ALREADY_EXISTS = "Edge with id already exists:"
|
27
|
-
CONCURRENT_VERTEX_INSERT_FAILED = "Failed to complete Insert operation for a Vertex due to conflicting concurrent"
|
28
|
-
CONCURRENT_EDGE_INSERT_FAILED = "Failed to complete Insert operation for an Edge due to conflicting concurrent"
|
29
|
-
|
30
13
|
class DispatcherError < Grumlin::Error; end
|
31
14
|
|
32
15
|
class RequestAlreadyAddedError < DispatcherError; end
|
@@ -47,14 +30,16 @@ module Grumlin
|
|
47
30
|
|
48
31
|
# builds a response object, when it's ready sends it to the client via a channel
|
49
32
|
# TODO: sometimes response does not include requestID, no idea how to handle it so far.
|
50
|
-
def add_response(response) # rubocop:disable Metrics/AbcSize
|
33
|
+
def add_response(response) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
51
34
|
request_id = response[:requestId]
|
52
35
|
raise UnknownRequestError unless ongoing_request?(request_id)
|
53
36
|
|
54
37
|
begin
|
55
38
|
request = @requests[request_id]
|
56
39
|
|
57
|
-
|
40
|
+
RequestErrorFactory.build(request, response).tap do |err|
|
41
|
+
raise err unless err.nil?
|
42
|
+
end
|
58
43
|
|
59
44
|
case SUCCESS[response.dig(:status, :code)]
|
60
45
|
when :success
|
@@ -91,29 +76,5 @@ module Grumlin
|
|
91
76
|
request = @requests.delete(request_id)
|
92
77
|
request[:channel].close
|
93
78
|
end
|
94
|
-
|
95
|
-
def check_errors!(status, query)
|
96
|
-
if (error = ERRORS[status[:code]])
|
97
|
-
raise (
|
98
|
-
already_exists_error(status) ||
|
99
|
-
concurrent_insert_error(status) ||
|
100
|
-
error
|
101
|
-
).new(status, query)
|
102
|
-
end
|
103
|
-
|
104
|
-
return unless SUCCESS[status[:code]].nil?
|
105
|
-
|
106
|
-
raise(UnknownResponseStatus, status)
|
107
|
-
end
|
108
|
-
|
109
|
-
def already_exists_error(status)
|
110
|
-
return VertexAlreadyExistsError if status[:message]&.include?(VERTEX_ALREADY_EXISTS)
|
111
|
-
return EdgeAlreadyExistsError if status[:message]&.include?(EDGE_ALREADY_EXISTS)
|
112
|
-
end
|
113
|
-
|
114
|
-
def concurrent_insert_error(status)
|
115
|
-
return ConcurrentVertexInsertFailedError if status[:message]&.include?(CONCURRENT_VERTEX_INSERT_FAILED)
|
116
|
-
return ConcurrentEdgeInsertFailedError if status[:message]&.include?(CONCURRENT_EDGE_INSERT_FAILED)
|
117
|
-
end
|
118
79
|
end
|
119
80
|
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
class RequestErrorFactory
|
5
|
+
ERRORS = {
|
6
|
+
499 => InvalidRequestArgumentsError,
|
7
|
+
500 => ServerError,
|
8
|
+
597 => ScriptEvaluationError,
|
9
|
+
599 => ServerSerializationError,
|
10
|
+
598 => ServerTimeoutError,
|
11
|
+
|
12
|
+
401 => ClientSideError,
|
13
|
+
407 => ClientSideError,
|
14
|
+
498 => ClientSideError
|
15
|
+
}.freeze
|
16
|
+
|
17
|
+
# Neptune presumably returns message as a JSON string of format
|
18
|
+
# {"detailedMessage":"",
|
19
|
+
# "requestId":"UUID",
|
20
|
+
# "code":"ConcurrentModificationException"}
|
21
|
+
# Currencly we simply search for substings to identify the exact error
|
22
|
+
# TODO: parse json and use `code` instead
|
23
|
+
VERTEX_ALREADY_EXISTS = "Vertex with id already exists:"
|
24
|
+
EDGE_ALREADY_EXISTS = "Edge with id already exists:"
|
25
|
+
CONCURRENT_VERTEX_INSERT_FAILED = "Failed to complete Insert operation for a Vertex due to conflicting concurrent"
|
26
|
+
CONCURRENT_EDGE_INSERT_FAILED = "Failed to complete Insert operation for an Edge due to conflicting concurrent"
|
27
|
+
CONCURRENCT_MODIFICATION_FAILED = "Failed to complete operation due to conflicting concurrent"
|
28
|
+
|
29
|
+
class << self
|
30
|
+
def build(request, response)
|
31
|
+
status = response[:status]
|
32
|
+
query = request[:request]
|
33
|
+
|
34
|
+
if (error = ERRORS[status[:code]])
|
35
|
+
return (
|
36
|
+
already_exists_error(status) ||
|
37
|
+
concurrent_modification_error(status) ||
|
38
|
+
error
|
39
|
+
).new(status, query)
|
40
|
+
end
|
41
|
+
|
42
|
+
return unless RequestDispatcher::SUCCESS[status[:code]].nil?
|
43
|
+
|
44
|
+
UnknownResponseStatus.new(status)
|
45
|
+
end
|
46
|
+
|
47
|
+
def already_exists_error(status)
|
48
|
+
return VertexAlreadyExistsError if status[:message]&.include?(VERTEX_ALREADY_EXISTS)
|
49
|
+
return EdgeAlreadyExistsError if status[:message]&.include?(EDGE_ALREADY_EXISTS)
|
50
|
+
end
|
51
|
+
|
52
|
+
def concurrent_modification_error(status)
|
53
|
+
return ConcurrentVertexInsertFailedError if status[:message]&.include?(CONCURRENT_VERTEX_INSERT_FAILED)
|
54
|
+
return ConcurrentEdgeInsertFailedError if status[:message]&.include?(CONCURRENT_EDGE_INSERT_FAILED)
|
55
|
+
return ConcurrentModificationError if status[:message]&.include?(CONCURRENCT_MODIFICATION_FAILED)
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
data/lib/grumlin/steppable.rb
CHANGED
@@ -4,13 +4,18 @@ module Grumlin
|
|
4
4
|
class Steppable
|
5
5
|
extend Forwardable
|
6
6
|
|
7
|
+
attr_reader :session_id
|
8
|
+
|
7
9
|
START_STEPS = Grumlin.definitions.dig(:steps, :start).map(&:to_sym).freeze
|
8
10
|
REGULAR_STEPS = Grumlin.definitions.dig(:steps, :regular).map(&:to_sym).freeze
|
9
11
|
CONFIGURATION_STEPS = Grumlin.definitions.dig(:steps, :configuration).map(&:to_sym).freeze
|
10
12
|
|
11
13
|
ALL_STEPS = START_STEPS + CONFIGURATION_STEPS + REGULAR_STEPS
|
12
14
|
|
13
|
-
def initialize
|
15
|
+
def initialize(pool: Grumlin.default_pool, session_id: nil)
|
16
|
+
@pool = pool
|
17
|
+
@session_id = session_id
|
18
|
+
|
14
19
|
return if respond_to?(:shortcuts)
|
15
20
|
|
16
21
|
raise "steppable must not be initialized directly, use Grumlin::Shortcuts::Storage#g or #__ instead"
|
@@ -18,12 +23,12 @@ module Grumlin
|
|
18
23
|
|
19
24
|
ALL_STEPS.each do |step|
|
20
25
|
define_method step do |*args, **params|
|
21
|
-
shortcuts.action_class.new(step, args: args, params: params, previous_step: self)
|
26
|
+
shortcuts.action_class.new(step, args: args, params: params, previous_step: self, session_id: @session_id)
|
22
27
|
end
|
23
28
|
end
|
24
29
|
|
25
30
|
def step(name, *args, **params)
|
26
|
-
shortcuts.action_class.new(name, args: args, params: params, previous_step: self)
|
31
|
+
shortcuts.action_class.new(name, args: args, params: params, previous_step: self, session_id: @session_id)
|
27
32
|
end
|
28
33
|
|
29
34
|
def_delegator :shortcuts, :__
|
data/lib/grumlin/steps.rb
CHANGED
@@ -32,7 +32,9 @@ module Grumlin
|
|
32
32
|
end
|
33
33
|
|
34
34
|
def add(name, args: [], params: {})
|
35
|
-
|
35
|
+
if CONFIGURATION_STEPS.include?(name) || name.to_sym == :tx
|
36
|
+
return add_configuration_step(name, args: args, params: params)
|
37
|
+
end
|
36
38
|
|
37
39
|
StepData.new(name, args: cast_arguments(args), params: params).tap do |step|
|
38
40
|
@steps << step
|
@@ -10,17 +10,19 @@ module Grumlin
|
|
10
10
|
|
11
11
|
def serialize
|
12
12
|
steps = ShortcutsApplyer.call(@steps)
|
13
|
-
no_return = @params
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
}.tap do |v|
|
18
|
-
v.merge!(source: steps.configuration_steps.map { |s| serialize_step(s) }) if steps.configuration_steps.any?
|
13
|
+
no_return = @params.fetch(:no_return, false)
|
14
|
+
{}.tap do |result|
|
15
|
+
result[:step] = serialize_steps(steps.steps + (no_return ? [NONE_STEP] : [])) if steps.steps.any?
|
16
|
+
result[:source] = serialize_steps(steps.configuration_steps) if steps.configuration_steps.any?
|
19
17
|
end
|
20
18
|
end
|
21
19
|
|
22
20
|
private
|
23
21
|
|
22
|
+
def serialize_steps(steps)
|
23
|
+
steps.map { |s| serialize_step(s) }
|
24
|
+
end
|
25
|
+
|
24
26
|
def serialize_step(step)
|
25
27
|
[step.name].tap do |result|
|
26
28
|
step.args.each do |arg|
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Grumlin
|
4
|
+
class Transaction
|
5
|
+
attr_reader :session_id
|
6
|
+
|
7
|
+
include Console
|
8
|
+
|
9
|
+
COMMIT = Grumlin::Repository.new.g.step(:tx, :commit).bytecode
|
10
|
+
ROLLBACK = Grumlin::Repository.new.g.step(:tx, :rollback).bytecode
|
11
|
+
|
12
|
+
def initialize(traversal_start_class, pool: Grumlin.default_pool)
|
13
|
+
@traversal_start_class = traversal_start_class
|
14
|
+
@pool = pool
|
15
|
+
|
16
|
+
@session_id = SecureRandom.uuid
|
17
|
+
end
|
18
|
+
|
19
|
+
def begin
|
20
|
+
@traversal_start_class.new(session_id: @session_id)
|
21
|
+
end
|
22
|
+
|
23
|
+
def commit
|
24
|
+
finalize(COMMIT)
|
25
|
+
end
|
26
|
+
|
27
|
+
def rollback
|
28
|
+
finalize(ROLLBACK)
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def finalize(action)
|
34
|
+
@pool.acquire do |client|
|
35
|
+
client.write(action, session_id: @session_id)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -4,6 +4,27 @@ module Grumlin
|
|
4
4
|
class TraversalStart < Steppable
|
5
5
|
include WithExtension
|
6
6
|
|
7
|
+
class TraversalError < Grumlin::Error; end
|
8
|
+
class AlreadyBoundToTransactionError < TraversalError; end
|
9
|
+
|
10
|
+
def tx
|
11
|
+
raise AlreadyBoundToTransactionError if @session_id
|
12
|
+
|
13
|
+
transaction = tx_class.new(self.class, pool: @pool)
|
14
|
+
return transaction unless block_given?
|
15
|
+
|
16
|
+
begin
|
17
|
+
yield transaction.begin
|
18
|
+
rescue Grumlin::Rollback
|
19
|
+
transaction.rollback
|
20
|
+
rescue StandardError
|
21
|
+
transaction.rollback
|
22
|
+
raise
|
23
|
+
else
|
24
|
+
transaction.commit
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
7
28
|
def to_s(*)
|
8
29
|
self.class.to_s
|
9
30
|
end
|
@@ -11,5 +32,11 @@ module Grumlin
|
|
11
32
|
def inspect
|
12
33
|
self.class.inspect
|
13
34
|
end
|
35
|
+
|
36
|
+
private
|
37
|
+
|
38
|
+
def tx_class
|
39
|
+
Grumlin.features.supports_transactions? ? Transaction : DummyTransaction
|
40
|
+
end
|
14
41
|
end
|
15
42
|
end
|
data/lib/grumlin/version.rb
CHANGED
data/lib/grumlin.rb
CHANGED
@@ -33,6 +33,9 @@ loader.do_not_eager_load(test_helpers)
|
|
33
33
|
module Grumlin
|
34
34
|
class Error < StandardError; end
|
35
35
|
|
36
|
+
class TransactionError < Error; end
|
37
|
+
class Rollback < TransactionError; end
|
38
|
+
|
36
39
|
class UnknownError < Error; end
|
37
40
|
|
38
41
|
class ConnectionError < Error; end
|
@@ -98,7 +101,8 @@ module Grumlin
|
|
98
101
|
class VertexAlreadyExistsError < AlreadyExistsError; end
|
99
102
|
class EdgeAlreadyExistsError < AlreadyExistsError; end
|
100
103
|
|
101
|
-
class
|
104
|
+
class ConcurrentModificationError < ServerError; end
|
105
|
+
class ConcurrentInsertFailedError < ConcurrentModificationError; end
|
102
106
|
|
103
107
|
class ConcurrentVertexInsertFailedError < ConcurrentInsertFailedError; end
|
104
108
|
class ConcurrentEdgeInsertFailedError < ConcurrentInsertFailedError; end
|
@@ -127,27 +131,26 @@ module Grumlin
|
|
127
131
|
|
128
132
|
class WrongQueryResult < RepositoryError; end
|
129
133
|
|
130
|
-
class Config
|
131
|
-
attr_accessor :url, :pool_size, :client_concurrency, :client_factory
|
132
|
-
|
133
|
-
def initialize
|
134
|
-
@pool_size = 10
|
135
|
-
@client_concurrency = 5
|
136
|
-
@client_factory = ->(url, parent) { Grumlin::Client.new(url, parent: parent) }
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
134
|
@pool_mutex = Mutex.new
|
141
135
|
|
142
136
|
class << self
|
143
137
|
def configure
|
144
138
|
yield config
|
139
|
+
|
140
|
+
config.validate!
|
145
141
|
end
|
146
142
|
|
147
143
|
def config
|
148
144
|
@config ||= Config.new
|
149
145
|
end
|
150
146
|
|
147
|
+
# returns a subset of features for currently configured backend.
|
148
|
+
# The features lists are hardcoded as there is no way to get them
|
149
|
+
# from the remote server.
|
150
|
+
def features
|
151
|
+
Features.for(config.provider) # no memoization as provider may be changed
|
152
|
+
end
|
153
|
+
|
151
154
|
def default_pool
|
152
155
|
if Thread.current.thread_variable_get(:grumlin_default_pool)
|
153
156
|
return Thread.current.thread_variable_get(:grumlin_default_pool)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: grumlin
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.22.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Gleb Sinyavskiy
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-08-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: async-pool
|
@@ -116,6 +116,8 @@ files:
|
|
116
116
|
- lib/grumlin/action.rb
|
117
117
|
- lib/grumlin/benchmark/repository.rb
|
118
118
|
- lib/grumlin/client.rb
|
119
|
+
- lib/grumlin/config.rb
|
120
|
+
- lib/grumlin/dummy_transaction.rb
|
119
121
|
- lib/grumlin/edge.rb
|
120
122
|
- lib/grumlin/expressions/cardinality.rb
|
121
123
|
- lib/grumlin/expressions/column.rb
|
@@ -128,12 +130,17 @@ files:
|
|
128
130
|
- lib/grumlin/expressions/t.rb
|
129
131
|
- lib/grumlin/expressions/text_p.rb
|
130
132
|
- lib/grumlin/expressions/with_options.rb
|
133
|
+
- lib/grumlin/features.rb
|
134
|
+
- lib/grumlin/features/feature_list.rb
|
135
|
+
- lib/grumlin/features/neptune_features.rb
|
136
|
+
- lib/grumlin/features/tinkergraph_features.rb
|
131
137
|
- lib/grumlin/path.rb
|
132
138
|
- lib/grumlin/property.rb
|
133
139
|
- lib/grumlin/repository.rb
|
134
140
|
- lib/grumlin/repository/error_handling_strategy.rb
|
135
141
|
- lib/grumlin/repository/instance_methods.rb
|
136
142
|
- lib/grumlin/request_dispatcher.rb
|
143
|
+
- lib/grumlin/request_error_factory.rb
|
137
144
|
- lib/grumlin/shortcut.rb
|
138
145
|
- lib/grumlin/shortcuts.rb
|
139
146
|
- lib/grumlin/shortcuts/properties.rb
|
@@ -151,6 +158,7 @@ files:
|
|
151
158
|
- lib/grumlin/test/rspec.rb
|
152
159
|
- lib/grumlin/test/rspec/db_cleaner_context.rb
|
153
160
|
- lib/grumlin/test/rspec/gremlin_context.rb
|
161
|
+
- lib/grumlin/transaction.rb
|
154
162
|
- lib/grumlin/transport.rb
|
155
163
|
- lib/grumlin/traversal_start.rb
|
156
164
|
- lib/grumlin/traversal_strategies/options_strategy.rb
|