aerospike 2.13.0 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +44 -1
  3. data/lib/aerospike.rb +14 -0
  4. data/lib/aerospike/cdt/bit_operation.rb +376 -0
  5. data/lib/aerospike/cdt/bit_overflow_action.rb +46 -0
  6. data/lib/aerospike/cdt/bit_policy.rb +36 -0
  7. data/lib/aerospike/cdt/bit_resize_flags.rb +44 -0
  8. data/lib/aerospike/cdt/bit_write_flags.rb +51 -0
  9. data/lib/aerospike/cdt/context.rb +113 -0
  10. data/lib/aerospike/cdt/hll_operation.rb +200 -0
  11. data/lib/aerospike/cdt/hll_policy.rb +34 -0
  12. data/lib/aerospike/cdt/hll_write_flags.rb +53 -0
  13. data/lib/aerospike/cdt/list_operation.rb +155 -96
  14. data/lib/aerospike/cdt/list_order.rb +7 -0
  15. data/lib/aerospike/cdt/list_sort_flags.rb +10 -2
  16. data/lib/aerospike/cdt/map_operation.rb +179 -92
  17. data/lib/aerospike/cdt/map_order.rb +3 -3
  18. data/lib/aerospike/client.rb +24 -7
  19. data/lib/aerospike/cluster.rb +47 -0
  20. data/lib/aerospike/cluster/rack_parser.rb +117 -0
  21. data/lib/aerospike/command/batch_direct_command.rb +1 -0
  22. data/lib/aerospike/command/batch_index_command.rb +1 -0
  23. data/lib/aerospike/command/command.rb +76 -7
  24. data/lib/aerospike/command/multi_command.rb +44 -1
  25. data/lib/aerospike/command/read_command.rb +34 -2
  26. data/lib/aerospike/command/touch_command.rb +34 -1
  27. data/lib/aerospike/features.rb +5 -0
  28. data/lib/aerospike/node.rb +18 -1
  29. data/lib/aerospike/node/rebalance.rb +50 -0
  30. data/lib/aerospike/node/refresh/info.rb +4 -1
  31. data/lib/aerospike/node/refresh/racks.rb +47 -0
  32. data/lib/aerospike/node/refresh/reset.rb +1 -0
  33. data/lib/aerospike/node/verify/rebalance_generation.rb +43 -0
  34. data/lib/aerospike/operation.rb +7 -2
  35. data/lib/aerospike/policy/client_policy.rb +15 -0
  36. data/lib/aerospike/policy/policy.rb +11 -3
  37. data/lib/aerospike/policy/replica.rb +7 -0
  38. data/lib/aerospike/result_code.rb +102 -0
  39. data/lib/aerospike/socket/base.rb +3 -2
  40. data/lib/aerospike/utils/buffer.rb +13 -3
  41. data/lib/aerospike/utils/unpacker.rb +2 -2
  42. data/lib/aerospike/value/particle_type.rb +1 -1
  43. data/lib/aerospike/value/value.rb +107 -5
  44. data/lib/aerospike/version.rb +1 -1
  45. metadata +15 -2
@@ -20,15 +20,15 @@ module Aerospike
20
20
 
21
21
  ##
22
22
  # Map is not ordered. This is the default.
23
- UNORDERED = 0
23
+ UNORDERED = {attr: 0, flag: 0x40}
24
24
 
25
25
  ##
26
26
  # Order map by key.
27
- KEY_ORDERED = 1
27
+ KEY_ORDERED = {attr: 1, flag: 0x80}
28
28
 
29
29
  ##
30
30
  # Order map by key, then value.
31
- KEY_VALUE_ORDERED = 3
31
+ KEY_VALUE_ORDERED = {attr: 3, flag: 0xc0}
32
32
 
33
33
  ##
34
34
  # Default order
@@ -44,6 +44,8 @@ module Aerospike
44
44
  attr_accessor :default_read_policy
45
45
  attr_accessor :default_scan_policy
46
46
  attr_accessor :default_write_policy
47
+ attr_accessor :default_operate_policy
48
+ attr_accessor :cluster
47
49
 
48
50
  def initialize(hosts = nil, policy: ClientPolicy.new, connect: true)
49
51
 
@@ -224,8 +226,19 @@ module Aerospike
224
226
  def truncate(namespace, set_name = nil, before_last_update = nil, options = {})
225
227
  policy = create_policy(options, Policy, default_info_policy)
226
228
 
227
- str_cmd = "truncate:namespace=#{namespace}"
228
- str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
229
+ node = @cluster.random_node
230
+ conn = node.get_connection(policy.timeout)
231
+
232
+ if set_name && !set_name.to_s.strip.empty?
233
+ str_cmd = "truncate:namespace=#{namespace}"
234
+ str_cmd << ";set=#{set_name}" unless set_name.to_s.strip.empty?
235
+ else
236
+ if node.supports_feature?(Aerospike::Features::TRUNCATE_NAMESPACE)
237
+ str_cmd = "truncate-namespace:namespace=#{namespace}"
238
+ else
239
+ str_cmd = "truncate:namespace=#{namespace}"
240
+ end
241
+ end
229
242
 
230
243
  if before_last_update
231
244
  lut_nanos = (before_last_update.to_f * 1_000_000_000.0).round
@@ -235,8 +248,7 @@ module Aerospike
235
248
  str_cmd << ";lut=now"
236
249
  end
237
250
 
238
- # Send index command to one node. That node will distribute the command to other nodes.
239
- response = send_info_command(policy, str_cmd).upcase
251
+ response = send_info_command(policy, str_cmd, node).upcase
240
252
  return if response == 'OK'
241
253
  raise Aerospike::Exceptions::Aerospike.new(Aerospike::ResultCode::SERVER_ERROR, "Truncate failed: #{response}")
242
254
  end
@@ -372,7 +384,7 @@ module Aerospike
372
384
  # read the result, all in one database call. Operations are executed in
373
385
  # the order they are specified.
374
386
  def operate(key, operations, options = nil)
375
- policy = create_policy(options, WritePolicy, default_write_policy)
387
+ policy = create_policy(options, OperatePolicy, default_operate_policy)
376
388
 
377
389
  command = OperateCommand.new(@cluster, policy, key, operations)
378
390
  execute_command(command)
@@ -816,11 +828,16 @@ module Aerospike
816
828
  self.default_query_policy = create_policy(policies[:query], QueryPolicy)
817
829
  self.default_scan_policy = create_policy(policies[:scan], ScanPolicy)
818
830
  self.default_write_policy = create_policy(policies[:write], WritePolicy)
831
+ self.default_operate_policy = create_policy(policies[:operate], OperatePolicy)
819
832
  end
820
833
 
821
- def send_info_command(policy, command)
834
+ def send_info_command(policy, command, node = nil)
822
835
  Aerospike.logger.debug { "Sending info command: #{command}" }
823
- _, response = @cluster.request_info(policy, command).first
836
+ if node
837
+ _, response = @cluster.request_node_info(node, policy, command).first
838
+ else
839
+ _, response = @cluster.request_info(policy, command).first
840
+ end
824
841
  response.to_s
825
842
  end
826
843
 
@@ -28,6 +28,7 @@ module Aerospike
28
28
  attr_reader :features, :tls_options
29
29
  attr_reader :cluster_id, :aliases
30
30
  attr_reader :cluster_name
31
+ attr_accessor :rack_aware, :rack_id
31
32
 
32
33
  def initialize(policy, hosts)
33
34
  @cluster_seeds = hosts
@@ -37,6 +38,8 @@ module Aerospike
37
38
  @tend_interval = policy.tend_interval
38
39
  @cluster_name = policy.cluster_name
39
40
  @tls_options = policy.tls
41
+ @rack_aware = policy.rack_aware
42
+ @rack_id = policy.rack_id
40
43
 
41
44
  @replica_index = Atomic.new(0)
42
45
 
@@ -111,6 +114,8 @@ module Aerospike
111
114
  return master_node(partition)
112
115
  when Aerospike::Replica::MASTER_PROLES
113
116
  return master_proles_node(partition)
117
+ when Aerospike::Replica::PREFER_RACK
118
+ return rack_node(partition, seq)
114
119
  when Aerospike::Replica::RANDOM
115
120
  return random_node
116
121
  else
@@ -125,6 +130,8 @@ module Aerospike
125
130
  return master_node(partition)
126
131
  when Aerospike::Replica::MASTER_PROLES
127
132
  return master_proles_node(partition)
133
+ when Aerospike::Replica::PREFER_RACK
134
+ return rack_node(partition, seq)
128
135
  when Aerospike::Replica::SEQUENCE
129
136
  return sequence_node(partition, seq)
130
137
  when Aerospike::Replica::RANDOM
@@ -149,6 +156,38 @@ module Aerospike
149
156
  node
150
157
  end
151
158
 
159
+ # Returns a node on the cluster
160
+ def rack_node(partition, seq)
161
+ partition_map = partitions
162
+ replica_array = partition_map[partition.namespace]
163
+ raise Aerospike::Exceptions::InvalidNamespace("namespace not found in the partition map") if !replica_array
164
+
165
+ replica_array = replica_array.get
166
+
167
+ is_retry = seq.value > -1
168
+
169
+ node = nil
170
+ fallback = nil
171
+ for i in 1..replica_array.length
172
+ idx = (seq.update{|v| v.succ} % replica_array.size).abs
173
+ node = (replica_array[idx].get)[partition.partition_id]
174
+
175
+ next if !node
176
+
177
+ fallback = node
178
+
179
+ # If fallback exists, do not retry on node where command failed,
180
+ # even if fallback is not on the same rack.
181
+ return fallback if is_retry && fallback && i == replica_array.length
182
+
183
+ return node if node && node.active? && node.has_rack(partition.namespace, @rack_id)
184
+ end
185
+
186
+ return fallback if fallback
187
+
188
+ raise Aerospike::Exceptions::InvalidNode
189
+ end
190
+
152
191
  # Returns a node on the cluster for read operations
153
192
  def master_proles_node(partition)
154
193
  partition_map = partitions
@@ -256,6 +295,13 @@ module Aerospike
256
295
  end
257
296
  end
258
297
 
298
+ def request_node_info(node, policy, *commands)
299
+ conn = node.get_connection(policy.timeout)
300
+ Info.request(conn, *commands).tap do
301
+ node.put_connection(conn)
302
+ end
303
+ end
304
+
259
305
  def supports_feature?(feature)
260
306
  @features.get.include?(feature.to_s)
261
307
  end
@@ -352,6 +398,7 @@ module Aerospike
352
398
 
353
399
  nodes.each do |node|
354
400
  node.refresh_partitions(peers) if node.partition_generation.changed?
401
+ node.refresh_racks if node.rebalance_generation.changed?
355
402
  end
356
403
 
357
404
  if peers.generation_changed? || !peers.use_peers?
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2014-2020 Aerospike, Inc.
4
+ #
5
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
6
+ # license agreements.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
+ # use this file except in compliance with the License. You may obtain a copy of
10
+ # the License at http:#www.apache.org/licenses/LICENSE-2.0
11
+ #
12
+ # Unless required by applicable law or agreed to in writing, software
13
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15
+ # License for the specific language governing permissions and limitations under
16
+ # the License.
17
+
18
+ require 'base64'
19
+
20
+ module Aerospike
21
+
22
+ class RackParser #:nodoc:
23
+
24
+ attr_accessor :rebalance_generation, :racks
25
+
26
+ REBALANCE_GENERATION = "rebalance-generation"
27
+ RACK_IDS = "rack-ids"
28
+
29
+ def initialize(node, conn)
30
+ @node = node
31
+ @conn = conn
32
+ @racks = nil
33
+ end
34
+
35
+ def update_racks
36
+ # Use low-level info methods and parse byte array directly for maximum performance.
37
+ # Receive format: rack-ids\t
38
+ # <ns1>:<rack-id>...;
39
+ # <ns2>:<rack-id>...; ...
40
+ info_map = Info.request(@conn, REBALANCE_GENERATION, RACK_IDS)
41
+
42
+ @rebalance_generation = info_map[REBALANCE_GENERATION].to_i
43
+
44
+ info = info_map[RACK_IDS]
45
+ if !info || info.length == 0
46
+ raise Aerospike::Exceptions::Connection.new("#{RACK_IDS} response for node #{@node.name} is empty")
47
+ end
48
+
49
+ @buffer = info
50
+ @length = info.length
51
+ @offset = 0
52
+
53
+ while @offset < @length && @buffer[@offset] != '\n'
54
+ namespace = parse_name
55
+ rack_id = parse_rack_id
56
+
57
+ @racks = {} if !@racks
58
+ @racks[namespace] = rack_id
59
+ end
60
+
61
+ @racks
62
+ end
63
+
64
+ private
65
+
66
+ def parse_name
67
+ beginning = @offset
68
+ while @offset < @length
69
+ break if @buffer[@offset] == ':'
70
+ @offset+=1
71
+ end
72
+
73
+ # Parse namespace.
74
+ namespace = @buffer[beginning...@offset].strip
75
+
76
+ if namespace.length <= 0 || namespace.length >= 32
77
+ response = get_truncated_response
78
+ raise Aerospike::Exceptions::Parse.new(
79
+ "Invalid rack namespace #{namespace}. Response=#{response}"
80
+ )
81
+ end
82
+
83
+ @offset+=1
84
+ namespace
85
+ end
86
+
87
+ def parse_rack_id
88
+ beginning = @offset
89
+ while @offset < @length
90
+ break if @buffer[@offset] == ';'
91
+ @offset+=1
92
+ end
93
+
94
+ # Parse rack_id
95
+ rack_id = @buffer[beginning...@offset].strip.to_i
96
+
97
+ if rack_id < 0
98
+ response = get_truncated_response
99
+ raise Aerospike::Exceptions::Parse.new(
100
+ "Invalid rack_id #{rack_id}. Response=#{response}"
101
+ )
102
+ end
103
+
104
+ @offset+=1
105
+ rack_id
106
+ end
107
+
108
+ def get_truncated_response
109
+ max = @length
110
+ @length = max if @length > 200
111
+ @buffer[0...max]
112
+ end
113
+
114
+
115
+ end # class
116
+
117
+ end # module
@@ -75,6 +75,7 @@ module Aerospike
75
75
  end
76
76
 
77
77
  end_cmd
78
+ mark_compressed(@policy)
78
79
  end
79
80
 
80
81
  # Parse all results in the batch. Add records to shared list.
@@ -98,6 +98,7 @@ module Aerospike
98
98
  end
99
99
  end
100
100
  end_cmd
101
+ mark_compressed(@policy)
101
102
  end
102
103
 
103
104
  # Parse all results in the batch. Add records to shared list.
@@ -16,6 +16,7 @@
16
16
  # the License.
17
17
 
18
18
  require 'time'
19
+ require 'zlib'
19
20
 
20
21
  require 'msgpack'
21
22
  require 'aerospike/result_code'
@@ -42,6 +43,9 @@ module Aerospike
42
43
  # Involve all replicas in read operation.
43
44
  INFO1_CONSISTENCY_ALL = Integer(1 << 6)
44
45
 
46
+ # Tell server to compress it's response.
47
+ INFO1_COMPRESS_RESPONSE = (1 << 7)
48
+
45
49
  # Create or update record
46
50
  INFO2_WRITE = Integer(1 << 0)
47
51
  # Fling a record into the belly of Moloch.
@@ -55,6 +59,9 @@ module Aerospike
55
59
  # Create only. Fail if record already exists.
56
60
  INFO2_CREATE_ONLY = Integer(1 << 5)
57
61
 
62
+ # Return a result for every operation.
63
+ INFO2_RESPOND_ALL_OPS = Integer(1 << 7)
64
+
58
65
  # This is the last of a multi-part message.
59
66
  INFO3_LAST = Integer(1 << 0)
60
67
  # Commit to master only before declaring success.
@@ -72,8 +79,10 @@ module Aerospike
72
79
  OPERATION_HEADER_SIZE = 8
73
80
  MSG_REMAINING_HEADER_SIZE = 22
74
81
  DIGEST_SIZE = 20
82
+ COMPRESS_THRESHOLD = 128
75
83
  CL_MSG_VERSION = 2
76
84
  AS_MSG_TYPE = 3
85
+ AS_MSG_TYPE_COMPRESSED = 4
77
86
 
78
87
  class Command #:nodoc:
79
88
 
@@ -83,6 +92,8 @@ module Aerospike
83
92
 
84
93
  @node = node
85
94
 
95
+ @compress = false
96
+
86
97
  # will add before use
87
98
  @sequence = Atomic.new(-1)
88
99
 
@@ -118,6 +129,7 @@ module Aerospike
118
129
  end
119
130
 
120
131
  end_cmd
132
+ mark_compressed(policy)
121
133
  end
122
134
 
123
135
  # Writes the command for delete operations
@@ -245,10 +257,12 @@ module Aerospike
245
257
  read_attr = 0
246
258
  write_attr = 0
247
259
  read_header = false
260
+ record_bin_multiplicity = policy.record_bin_multiplicity == RecordBinMultiplicity::ARRAY
248
261
 
249
262
  operations.each do |operation|
250
263
  case operation.op_type
251
- when Aerospike::Operation::READ
264
+ when Aerospike::Operation::READ, Aerospike::Operation::CDT_READ,
265
+ Aerospike::Operation::HLL_READ, Aerospike::Operation::BIT_READ
252
266
  read_attr |= INFO1_READ
253
267
 
254
268
  # Read all bins if no bin is specified.
@@ -262,17 +276,22 @@ module Aerospike
262
276
  read_attr |= INFO1_READ
263
277
  read_header = true
264
278
 
265
- when Aerospike::Operation::CDT_READ
266
- read_attr |= INFO1_READ
267
-
268
279
  else
269
280
  write_attr = INFO2_WRITE
270
281
  end
271
282
 
283
+ if [Aerospike::Operation::HLL_MODIFY, Aerospike::Operation::HLL_READ,
284
+ Aerospike::Operation::BIT_MODIFY, Aerospike::Operation::BIT_READ].include?(operation.op_type)
285
+ record_bin_multiplicity = true
286
+ end
287
+
272
288
  estimate_operation_size_for_operation(operation)
273
289
  end
274
290
  size_buffer
275
291
 
292
+
293
+ write_attr |= INFO2_RESPOND_ALL_OPS if write_attr != 0 && record_bin_multiplicity
294
+
276
295
  if write_attr != 0
277
296
  write_header_with_policy(policy, read_attr, write_attr, field_count, operations.length)
278
297
  else
@@ -288,6 +307,7 @@ module Aerospike
288
307
  write_operation_for_bin(nil, Aerospike::Operation::READ) if read_header
289
308
 
290
309
  end_cmd
310
+ mark_compressed(policy)
291
311
  end
292
312
 
293
313
  def set_udf(policy, key, package_name, function_name, args)
@@ -310,6 +330,7 @@ module Aerospike
310
330
  write_field_bytes(arg_bytes, Aerospike::FieldType::UDF_ARGLIST)
311
331
 
312
332
  end_cmd
333
+ mark_compressed(policy)
313
334
  end
314
335
 
315
336
  def set_scan(policy, namespace, set_name, bin_names)
@@ -425,10 +446,14 @@ module Aerospike
425
446
  @node = get_node
426
447
  @conn = @node.get_connection(@policy.timeout)
427
448
  rescue => e
428
- # Socket connection error has occurred. Decrease health and retry.
429
- @node.decrease_health
449
+ if @node
450
+ # Socket connection error has occurred. Decrease health and retry.
451
+ @node.decrease_health
430
452
 
431
- Aerospike.logger.error("Node #{@node.to_s}: #{e}")
453
+ Aerospike.logger.error("Node #{@node.to_s}: #{e}")
454
+ else
455
+ Aerospike.logger.error("No node available for transaction: #{e}")
456
+ end
432
457
  next
433
458
  end
434
459
 
@@ -579,6 +604,7 @@ module Aerospike
579
604
  # Generic header write.
580
605
  def write_header(policy, read_attr, write_attr, field_count, operation_count)
581
606
  read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
607
+ read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
582
608
 
583
609
  # Write all header data except total size which must be written last.
584
610
  @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -628,6 +654,7 @@ module Aerospike
628
654
  info_attr |= INFO3_COMMIT_MASTER if policy.commit_level == Aerospike::CommitLevel::COMMIT_MASTER
629
655
  read_attr |= INFO1_CONSISTENCY_ALL if policy.consistency_level == Aerospike::ConsistencyLevel::CONSISTENCY_ALL
630
656
  write_attr |= INFO2_DURABLE_DELETE if policy.durable_delete
657
+ read_attr |= INFO1_COMPRESS_RESPONSE if policy.use_compression
631
658
 
632
659
  # Write all header data except total size which must be written last.
633
660
  @data_buffer.write_byte(MSG_REMAINING_HEADER_SIZE, 8) # Message heade.length.
@@ -810,6 +837,48 @@ module Aerospike
810
837
  @data_buffer.write_int64(size, 0)
811
838
  end
812
839
 
840
+ def use_compression?
841
+ @compress
842
+ end
843
+
844
+ def compress_buffer
845
+ if @data_offset > COMPRESS_THRESHOLD
846
+ compressed = Zlib::deflate(@data_buffer.buf, Zlib::DEFAULT_COMPRESSION)
847
+
848
+ # write original size as header
849
+ proto_s = "%08d" % 0
850
+ proto_s[0, 8] = [@data_offset].pack('q>')
851
+ compressed.prepend(proto_s)
852
+
853
+ # write proto
854
+ proto = (compressed.size+8) | Integer(CL_MSG_VERSION << 56) | Integer(AS_MSG_TYPE << 48)
855
+ proto_s = "%08d" % 0
856
+ proto_s[0, 8] = [proto].pack('q>')
857
+ compressed.prepend(proto_s)
858
+
859
+ @data_buffer = Buffer.new(-1, compressed)
860
+ @data_offset = @data_buffer.size
861
+ end
862
+ end
863
+
864
+ # isCompressed returns the length of the compressed buffer.
865
+ # If the buffer is not compressed, the result will be -1
866
+ def compressed_size
867
+ # A number of these are commented out because we just don't care enough to read
868
+ # that section of the header. If we do care, uncomment and check!
869
+ proto = @data_buffer.read_int64(0)
870
+ size = proto & 0xFFFFFFFFFFFF
871
+ msg_type = (proto >> 48) & 0xFF
872
+
873
+ return nil if msg_type != AS_MSG_TYPE_COMPRESSED
874
+
875
+ size
876
+ end
877
+
878
+ def mark_compressed(policy)
879
+ @compress = policy.use_compression
880
+ end
881
+
813
882
  end # class
814
883
 
815
884
  end # module