aerospike 2.22.0 → 2.24.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +307 -262
  3. data/lib/aerospike/atomic/atomic.rb +1 -1
  4. data/lib/aerospike/cdt/context.rb +7 -7
  5. data/lib/aerospike/cdt/list_return_type.rb +4 -0
  6. data/lib/aerospike/cdt/map_operation.rb +6 -6
  7. data/lib/aerospike/cdt/map_return_type.rb +4 -0
  8. data/lib/aerospike/client.rb +59 -84
  9. data/lib/aerospike/command/admin_command.rb +1 -1
  10. data/lib/aerospike/command/batch_index_node.rb +1 -1
  11. data/lib/aerospike/command/batch_item.rb +1 -1
  12. data/lib/aerospike/command/command.rb +65 -25
  13. data/lib/aerospike/command/field_type.rb +25 -25
  14. data/lib/aerospike/command/login_command.rb +4 -4
  15. data/lib/aerospike/command/multi_command.rb +8 -2
  16. data/lib/aerospike/command/read_command.rb +2 -2
  17. data/lib/aerospike/connection/authenticate.rb +3 -3
  18. data/lib/aerospike/exp/exp.rb +1262 -0
  19. data/lib/aerospike/features.rb +9 -9
  20. data/lib/aerospike/host/parse.rb +2 -2
  21. data/lib/aerospike/key.rb +10 -1
  22. data/lib/aerospike/node/refresh/info.rb +1 -1
  23. data/lib/aerospike/node/verify/name.rb +1 -1
  24. data/lib/aerospike/node/verify/partition_generation.rb +1 -1
  25. data/lib/aerospike/node/verify/peers_generation.rb +1 -1
  26. data/lib/aerospike/node/verify/rebalance_generation.rb +1 -1
  27. data/lib/aerospike/policy/policy.rb +4 -1
  28. data/lib/aerospike/policy/query_policy.rb +35 -2
  29. data/lib/aerospike/policy/scan_policy.rb +19 -2
  30. data/lib/aerospike/privilege.rb +1 -1
  31. data/lib/aerospike/query/node_partitions.rb +39 -0
  32. data/lib/aerospike/query/partition_filter.rb +66 -0
  33. data/lib/aerospike/query/partition_status.rb +36 -0
  34. data/lib/aerospike/query/partition_tracker.rb +347 -0
  35. data/lib/aerospike/query/query_command.rb +1 -1
  36. data/lib/aerospike/query/query_executor.rb +73 -0
  37. data/lib/aerospike/query/query_partition_command.rb +266 -0
  38. data/lib/aerospike/query/scan_command.rb +3 -3
  39. data/lib/aerospike/query/scan_executor.rb +69 -0
  40. data/lib/aerospike/query/scan_partition_command.rb +49 -0
  41. data/lib/aerospike/query/statement.rb +8 -1
  42. data/lib/aerospike/query/stream_command.rb +15 -1
  43. data/lib/aerospike/result_code.rb +79 -4
  44. data/lib/aerospike/role.rb +2 -2
  45. data/lib/aerospike/task/execute_task.rb +2 -2
  46. data/lib/aerospike/task/index_task.rb +1 -1
  47. data/lib/aerospike/user_role.rb +1 -1
  48. data/lib/aerospike/utils/buffer.rb +32 -7
  49. data/lib/aerospike/utils/pool.rb +1 -1
  50. data/lib/aerospike/value/value.rb +6 -6
  51. data/lib/aerospike/version.rb +1 -1
  52. data/lib/aerospike.rb +8 -0
  53. metadata +14 -5
@@ -23,7 +23,7 @@ module Aerospike
23
23
 
24
24
  class ScanCommand < StreamCommand #:nodoc:
25
25
 
26
- def initialize(node, policy, namespace, set_name, bin_names, recordset, partitions)
26
+ def initialize(node, policy, namespace, set_name, bin_names, recordset, node_partitions)
27
27
  super(node)
28
28
 
29
29
  @policy = policy
@@ -31,11 +31,11 @@ module Aerospike
31
31
  @set_name = set_name
32
32
  @bin_names = bin_names
33
33
  @recordset = recordset
34
- @partitions = partitions
34
+ @node_partitions = node_partitions
35
35
  end
36
36
 
37
37
  def write_buffer
38
- set_scan(@policy, @namespace, @set_name, @bin_names, @partitions)
38
+ set_scan(@policy, @namespace, @set_name, @bin_names, @node_partitions)
39
39
  end
40
40
 
41
41
  end # class
@@ -0,0 +1,69 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2014-2020 Aerospike, Inc.
4
+ #
5
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
6
+ # license agreements.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
+ # use this file except in compliance with the License. You may obtain a copy of
10
+ # the License at http:#www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
+ # License for the specific language governing permissions and limitations under
16
+ # the License.
17
+
18
+ module Aerospike
19
+ class ScanExecutor # :nodoc:
20
+
21
+ def self.scan_partitions(policy, cluster, tracker, namespace, set_name, recordset, bin_names = nil)
22
+ interval = policy.sleep_between_retries
23
+
24
+ should_retry = false
25
+
26
+ loop do
27
+ list = tracker.assign_partitions_to_nodes(cluster, namespace)
28
+
29
+ if policy.concurrent_nodes
30
+ threads = []
31
+ # Use a thread per node
32
+ list.each do |node_partition|
33
+
34
+ threads << Thread.new do
35
+ Thread.current.abort_on_exception = true
36
+ command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
37
+ begin
38
+ command.execute
39
+ rescue => e
40
+ should_retry ||= command.should_retry(e)
41
+ Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
42
+ end
43
+ end
44
+ end
45
+ threads.each(&:join)
46
+ else
47
+ # Use a single thread for all nodes for all node
48
+ list.each do |node_partition|
49
+ command = ScanPartitionCommand.new(policy, tracker, node_partition, namespace, set_name, bin_names, recordset)
50
+ begin
51
+ command.execute
52
+ rescue => e
53
+ should_retry ||= command.should_retry(e)
54
+ Aerospike.logger.error(e.backtrace.join("\n")) unless e == SCAN_TERMINATED_EXCEPTION
55
+ end
56
+ end
57
+ end
58
+
59
+ if tracker.complete?(@cluster, policy) || !should_retry
60
+ recordset.thread_finished
61
+ return
62
+ end
63
+ sleep(interval) if policy.sleep_between_retries > 0
64
+ end
65
+ end
66
+
67
+ end
68
+
69
+ end
@@ -0,0 +1,49 @@
1
+ # encoding: utf-8
2
+ # Copyright 2014-2020 Aerospike, Inc.
3
+ #
4
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
5
+ # license agreements.
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
8
+ # use this file except in compliance with the License. You may obtain a copy of
9
+ # the License at http:#www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing, software
12
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14
+ # License for the specific language governing permissions and limitations under
15
+ # the License.
16
+
17
+ require 'aerospike/query/stream_command'
18
+ require 'aerospike/query/recordset'
19
+
20
+ module Aerospike
21
+
22
+ private
23
+
24
+ class ScanPartitionCommand < StreamCommand #:nodoc:
25
+
26
+ def initialize(policy, tracker, node_partitions, namespace, set_name, bin_names, recordset)
27
+ super(node_partitions.node)
28
+
29
+ @policy = policy
30
+ @namespace = namespace
31
+ @set_name = set_name
32
+ @bin_names = bin_names
33
+ @recordset = recordset
34
+ @node_partitions = node_partitions
35
+ @tracker = tracker
36
+ end
37
+
38
+ def write_buffer
39
+ set_scan(@policy, @namespace, @set_name, @bin_names, @node_partitions)
40
+ end
41
+
42
+ def should_retry(e)
43
+ # !! converts nil to false
44
+ !!@tracker&.should_retry(@node_partitions, e)
45
+ end
46
+
47
+ end # class
48
+
49
+ end # module
@@ -49,7 +49,7 @@ module Aerospike
49
49
  # This method is redundant because PredExp can now be set in the base Policy for
50
50
  # any transaction (including queries).
51
51
  #
52
- # NOTE : Policy.predexp takes precedence to this value. This value will be
52
+ # NOTE : Policy.predexp takes precedence to this value. This value will be
53
53
  # deprecated in the future.
54
54
  @predexp = nil
55
55
 
@@ -81,6 +81,13 @@ module Aerospike
81
81
  end
82
82
  end
83
83
 
84
+ def reset_task_id
85
+ @task_id = rand(RAND_MAX)
86
+ while @task_id == 0
87
+ @task_id = rand(RAND_MAX)
88
+ end
89
+ end
90
+
84
91
  private
85
92
 
86
93
  RAND_MAX = 2**63
@@ -39,6 +39,8 @@ module Aerospike
39
39
  case result_code
40
40
  when Aerospike::ResultCode::OK
41
41
  # noop
42
+ when Aerospike::ResultCode::PARTITION_UNAVAILABLE
43
+ # noop
42
44
  when Aerospike::ResultCode::KEY_NOT_FOUND_ERROR
43
45
  # consume the rest of the input buffer from the socket
44
46
  read_bytes(receive_size - @data_offset) if @data_offset < receive_size
@@ -59,7 +61,16 @@ module Aerospike
59
61
  key = parse_key(field_count)
60
62
 
61
63
  # If cmd is the end marker of the response, do not proceed further
62
- return true if (info3 & INFO3_PARTITION_DONE) != 0
64
+ if (info3 & INFO3_PARTITION_DONE) != 0
65
+ # When an error code is received, mark partition as unavailable
66
+ # for the current round. Unavailable partitions will be retried
67
+ # in the next round. Generation is overloaded as partitionId.
68
+ if result_code != 0
69
+ @tracker&.partition_unavailable(@node_partitions, generation)
70
+ end
71
+
72
+ next
73
+ end
63
74
 
64
75
  if result_code == 0
65
76
  if @recordset.active?
@@ -68,6 +79,9 @@ module Aerospike
68
79
  expn = @recordset.is_scan? ? SCAN_TERMINATED_EXCEPTION : QUERY_TERMINATED_EXCEPTION
69
80
  raise expn
70
81
  end
82
+
83
+ # UDF results do not return a key
84
+ @tracker&.set_last(@node_partitions, key, key.bval) if key
71
85
  end
72
86
  end # while
73
87
 
@@ -20,6 +20,42 @@ module Aerospike
20
20
 
21
21
  attr_reader :code
22
22
 
23
+ # One or more keys failed in a batch.
24
+ BATCH_FAILED = -20
25
+
26
+ # No response was received from the server.
27
+ NO_RESPONSE = -19
28
+
29
+ # A network error. Checked the wrapped error for detail.
30
+ NETWORK_ERROR = -18
31
+
32
+ # A common, none-aerospike error. Checked the wrapped error for detail.
33
+ COMMON_ERROR = -17
34
+
35
+ # Max retries limit reached.
36
+ MAX_RETRIES_EXCEEDED = -16
37
+
38
+ # Max errors limit reached.
39
+ MAX_ERROR_RATE = -15
40
+
41
+ # Requested Rack for node/namespace was not defined in the cluster.
42
+ RACK_NOT_DEFINED = -13
43
+
44
+ # Cluster has an invalid partition map, usually due to bad configuration.
45
+ INVALID_CLUSTER_PARTITION_MAP = -12
46
+
47
+ # Server is not accepting requests.
48
+ SERVER_NOT_AVAILABLE = -11
49
+
50
+ # Cluster Name does not match the ClientPolicy.ClusterName value.
51
+ CLUSTER_NAME_MISMATCH_ERROR = -10
52
+
53
+ # Recordset has already been closed or cancelled
54
+ RECORDSET_CLOSED = -9
55
+
56
+ # There were no connections available to the node in the pool, and the pool was limited
57
+ NO_AVAILABLE_CONNECTIONS_TO_NODE = -8
58
+
23
59
  # Value type not supported by Aerospike server
24
60
  TYPE_NOT_SUPPORTED = -7
25
61
 
@@ -76,8 +112,8 @@ module Aerospike
76
112
  # XDS product is not available.
77
113
  NO_XDS = 10
78
114
 
79
- # Server is not accepting requests.
80
- SERVER_NOT_AVAILABLE = 11
115
+ # Partition is unavailable.
116
+ PARTITION_UNAVAILABLE = 11
81
117
 
82
118
  # Operation is not supported with configured bin type (single-bin or
83
119
  # multi-bin).
@@ -265,6 +301,45 @@ module Aerospike
265
301
 
266
302
  def self.message(code)
267
303
  case code
304
+ when BATCH_FAILED
305
+ "one or more keys failed in a batch"
306
+
307
+ when NO_RESPONSE
308
+ "no response was received from the server"
309
+
310
+ when NETWORK_ERROR
311
+ "network error. Checked the wrapped error for detail"
312
+
313
+ when COMMON_ERROR
314
+ "common, none-aerospike error. Checked the wrapped error for detail"
315
+
316
+ when MAX_RETRIES_EXCEEDED
317
+ "Max retries exceeded"
318
+
319
+ when MAX_ERROR_RATE
320
+ "Max errors limit reached for node"
321
+
322
+ when RACK_NOT_DEFINED
323
+ "Requested Rack for node/namespace was not defined in the cluster."
324
+
325
+ when INVALID_CLUSTER_PARTITION_MAP
326
+ "Cluster has an invalid partition map, usually due to bad configuration."
327
+
328
+ when SERVER_NOT_AVAILABLE
329
+ "Server is not accepting requests."
330
+
331
+ when CLUSTER_NAME_MISMATCH_ERROR
332
+ "Cluster Name does not match the ClientPolicy.ClusterName value"
333
+
334
+ when RECORDSET_CLOSED
335
+ "Recordset has already been closed or cancelled."
336
+
337
+ when NO_AVAILABLE_CONNECTIONS_TO_NODE
338
+ "No available connections to the node. Connection Pool was empty, and limited to certain number of connections."
339
+
340
+ when TYPE_NOT_SUPPORTED
341
+ "Type cannot be converted to Value Type."
342
+
268
343
  when COMMAND_REJECTED
269
344
  "Command rejected"
270
345
 
@@ -316,8 +391,8 @@ module Aerospike
316
391
  when NO_XDS
317
392
  "XDS not available"
318
393
 
319
- when SERVER_NOT_AVAILABLE
320
- "Server not available"
394
+ when PARTITION_UNAVAILABLE
395
+ "Partition not available"
321
396
 
322
397
  when BIN_TYPE_ERROR
323
398
  "Bin type error"
@@ -49,7 +49,7 @@ module Aerospike
49
49
  def to_s
50
50
  "Role [name=#{@name}, privileges=#{@privileges}, allowlist=#{@allowlist}, readQuota=#{@read_quota}, writeQuota=#{@write_quota}]";
51
51
  end
52
-
52
+
53
53
  end # class
54
54
 
55
- end # module
55
+ end # module
@@ -70,7 +70,7 @@ module Aerospike
70
70
 
71
71
  case status
72
72
  when 'ABORTED'
73
- raise raise Aerospike::Exceptions::QueryTerminated
73
+ raise Aerospike::Exceptions::QueryTerminated
74
74
  when 'IN PROGRESS'
75
75
  return false
76
76
  when 'DONE'
@@ -82,4 +82,4 @@ module Aerospike
82
82
  end
83
83
 
84
84
  end
85
- end
85
+ end
@@ -25,7 +25,7 @@ module Aerospike
25
25
 
26
26
  class IndexTask < Task
27
27
 
28
- MATCHER = /.*load_pct=(?<load_pct>\d+(\.\d+)?).*/
28
+ MATCHER = /.*load_pct=(?<load_pct>\d+(\.\d+)?).*/.freeze
29
29
 
30
30
  def initialize(cluster, namespace, index_name, done=false)
31
31
  super(cluster, done)
@@ -52,4 +52,4 @@ module Aerospike
52
52
 
53
53
  end
54
54
 
55
- end
55
+ end
@@ -38,6 +38,7 @@ module Aerospike
38
38
  UINT32 = 'N'
39
39
  INT64 = 'q>'
40
40
  UINT64 = 'Q>'
41
+ UINT64LE = 'Q'
41
42
  DOUBLE = 'G'
42
43
 
43
44
  DEFAULT_BUFFER_SIZE = 16 * 1024
@@ -124,6 +125,11 @@ module Aerospike
124
125
  8
125
126
  end
126
127
 
128
+ def write_uint64_little_endian(i, offset)
129
+ @buf[offset, 8] = [i].pack(UINT64LE)
130
+ 8
131
+ end
132
+
127
133
  def write_double(f, offset)
128
134
  @buf[offset, 8] = [f].pack(DOUBLE)
129
135
  8
@@ -162,6 +168,11 @@ module Aerospike
162
168
  vals.unpack(INT64)[0]
163
169
  end
164
170
 
171
+ def read_uint64_little_endian(offset)
172
+ vals = @buf[offset..offset+7]
173
+ vals.unpack(UINT64LE)[0]
174
+ end
175
+
165
176
  def read_uint64(offset)
166
177
  vals = @buf[offset..offset+7]
167
178
  vals.unpack(UINT64)[0]
@@ -197,14 +208,28 @@ module Aerospike
197
208
  end
198
209
  end
199
210
 
200
- def dump(from=nil, to=nil)
201
- from ||= 0
202
- to ||= @slice_end - 1
203
-
204
- @buf.bytes[from...to].each do |c|
205
- print c.ord.to_s(16)
206
- putc ' '
211
+ def dump(start=0, finish=nil)
212
+ finish ||= @slice_end - 1
213
+ width = 16
214
+
215
+ ascii = '|'
216
+ counter = 0
217
+
218
+ print '%06x ' % start
219
+ @buf.bytes[start...finish].each do |c|
220
+ if counter >= start
221
+ print '%02x ' % c
222
+ ascii << (c.between?(32, 126) ? c : ?.)
223
+ if ascii.length >= width
224
+ ascii << '|'
225
+ puts ascii
226
+ ascii = '|'
227
+ print '%06x ' % (counter + 1)
228
+ end
229
+ end
230
+ counter += 1
207
231
  end
232
+ puts
208
233
  end
209
234
 
210
235
  end # buffer
@@ -53,7 +53,7 @@ module Aerospike
53
53
  end
54
54
 
55
55
  def empty?
56
- @pool.length == 0
56
+ @pool.empty?
57
57
  end
58
58
 
59
59
  def length
@@ -26,19 +26,19 @@ module Aerospike
26
26
  def self.of(value, allow_64bits = false)
27
27
  case value
28
28
  when Integer
29
- if !allow_64bits
30
- if value.bit_length < 64
29
+ if allow_64bits
30
+ # used in bitwise operations
31
+ if value.bit_length <= 64
31
32
  res = IntegerValue.new(value)
32
33
  else
33
- # big nums > 2**63 are not supported
34
+ # nums with more than 64 bits are not supported
34
35
  raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::TYPE_NOT_SUPPORTED, "Value type #{value.class} not supported with more than 64 bits.")
35
36
  end
36
37
  else
37
- # used in bitwise operations
38
- if value.bit_length <= 64
38
+ if value.bit_length < 64
39
39
  res = IntegerValue.new(value)
40
40
  else
41
- # nums with more than 64 bits are not supported
41
+ # big nums > 2**63 are not supported
42
42
  raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::TYPE_NOT_SUPPORTED, "Value type #{value.class} not supported with more than 64 bits.")
43
43
  end
44
44
  end
@@ -1,4 +1,4 @@
1
1
  # encoding: utf-8
2
2
  module Aerospike
3
- VERSION = "2.22.0"
3
+ VERSION = "2.24.0"
4
4
  end
data/lib/aerospike.rb CHANGED
@@ -160,6 +160,14 @@ require 'aerospike/query/query_command'
160
160
  require 'aerospike/query/scan_command'
161
161
  require 'aerospike/query/statement'
162
162
  require 'aerospike/query/pred_exp'
163
+ require 'aerospike/query/partition_tracker'
164
+ require 'aerospike/query/partition_status'
165
+ require 'aerospike/query/partition_filter'
166
+ require 'aerospike/query/node_partitions'
167
+ require 'aerospike/query/scan_executor'
168
+ require 'aerospike/query/scan_partition_command'
169
+ require 'aerospike/query/query_executor'
170
+ require 'aerospike/query/query_partition_command'
163
171
 
164
172
  require 'aerospike/query/pred_exp/and_or'
165
173
  require 'aerospike/query/pred_exp/geo_json_value'
metadata CHANGED
@@ -1,15 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: aerospike
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.22.0
4
+ version: 2.24.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Khosrow Afroozeh
8
8
  - Jan Hecking
9
- autorequire:
9
+ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2022-07-14 00:00:00.000000000 Z
12
+ date: 2022-11-15 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: msgpack
@@ -108,6 +108,7 @@ files:
108
108
  - lib/aerospike/command/write_command.rb
109
109
  - lib/aerospike/connection/authenticate.rb
110
110
  - lib/aerospike/connection/create.rb
111
+ - lib/aerospike/exp/exp.rb
111
112
  - lib/aerospike/features.rb
112
113
  - lib/aerospike/geo_json.rb
113
114
  - lib/aerospike/host.rb
@@ -155,6 +156,10 @@ files:
155
156
  - lib/aerospike/policy/write_policy.rb
156
157
  - lib/aerospike/privilege.rb
157
158
  - lib/aerospike/query/filter.rb
159
+ - lib/aerospike/query/node_partitions.rb
160
+ - lib/aerospike/query/partition_filter.rb
161
+ - lib/aerospike/query/partition_status.rb
162
+ - lib/aerospike/query/partition_tracker.rb
158
163
  - lib/aerospike/query/pred_exp.rb
159
164
  - lib/aerospike/query/pred_exp/and_or.rb
160
165
  - lib/aerospike/query/pred_exp/geo_json_value.rb
@@ -164,8 +169,12 @@ files:
164
169
  - lib/aerospike/query/pred_exp/regex_flags.rb
165
170
  - lib/aerospike/query/pred_exp/string_value.rb
166
171
  - lib/aerospike/query/query_command.rb
172
+ - lib/aerospike/query/query_executor.rb
173
+ - lib/aerospike/query/query_partition_command.rb
167
174
  - lib/aerospike/query/recordset.rb
168
175
  - lib/aerospike/query/scan_command.rb
176
+ - lib/aerospike/query/scan_executor.rb
177
+ - lib/aerospike/query/scan_partition_command.rb
169
178
  - lib/aerospike/query/statement.rb
170
179
  - lib/aerospike/query/stream_command.rb
171
180
  - lib/aerospike/record.rb
@@ -212,8 +221,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
212
221
  - !ruby/object:Gem::Version
213
222
  version: '0'
214
223
  requirements: []
215
- rubygems_version: 3.2.15
216
- signing_key:
224
+ rubygems_version: 3.3.5
225
+ signing_key:
217
226
  specification_version: 4
218
227
  summary: An Aerospike driver for Ruby.
219
228
  test_files: []