aerospike 2.13.0 → 2.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +44 -1
  3. data/lib/aerospike.rb +14 -0
  4. data/lib/aerospike/cdt/bit_operation.rb +376 -0
  5. data/lib/aerospike/cdt/bit_overflow_action.rb +46 -0
  6. data/lib/aerospike/cdt/bit_policy.rb +36 -0
  7. data/lib/aerospike/cdt/bit_resize_flags.rb +44 -0
  8. data/lib/aerospike/cdt/bit_write_flags.rb +51 -0
  9. data/lib/aerospike/cdt/context.rb +113 -0
  10. data/lib/aerospike/cdt/hll_operation.rb +200 -0
  11. data/lib/aerospike/cdt/hll_policy.rb +34 -0
  12. data/lib/aerospike/cdt/hll_write_flags.rb +53 -0
  13. data/lib/aerospike/cdt/list_operation.rb +155 -96
  14. data/lib/aerospike/cdt/list_order.rb +7 -0
  15. data/lib/aerospike/cdt/list_sort_flags.rb +10 -2
  16. data/lib/aerospike/cdt/map_operation.rb +179 -92
  17. data/lib/aerospike/cdt/map_order.rb +3 -3
  18. data/lib/aerospike/client.rb +24 -7
  19. data/lib/aerospike/cluster.rb +47 -0
  20. data/lib/aerospike/cluster/rack_parser.rb +117 -0
  21. data/lib/aerospike/command/batch_direct_command.rb +1 -0
  22. data/lib/aerospike/command/batch_index_command.rb +1 -0
  23. data/lib/aerospike/command/command.rb +76 -7
  24. data/lib/aerospike/command/multi_command.rb +44 -1
  25. data/lib/aerospike/command/read_command.rb +34 -2
  26. data/lib/aerospike/command/touch_command.rb +34 -1
  27. data/lib/aerospike/features.rb +5 -0
  28. data/lib/aerospike/node.rb +18 -1
  29. data/lib/aerospike/node/rebalance.rb +50 -0
  30. data/lib/aerospike/node/refresh/info.rb +4 -1
  31. data/lib/aerospike/node/refresh/racks.rb +47 -0
  32. data/lib/aerospike/node/refresh/reset.rb +1 -0
  33. data/lib/aerospike/node/verify/rebalance_generation.rb +43 -0
  34. data/lib/aerospike/operation.rb +7 -2
  35. data/lib/aerospike/policy/client_policy.rb +15 -0
  36. data/lib/aerospike/policy/policy.rb +11 -3
  37. data/lib/aerospike/policy/replica.rb +7 -0
  38. data/lib/aerospike/result_code.rb +102 -0
  39. data/lib/aerospike/socket/base.rb +3 -2
  40. data/lib/aerospike/utils/buffer.rb +13 -3
  41. data/lib/aerospike/utils/unpacker.rb +2 -2
  42. data/lib/aerospike/value/particle_type.rb +1 -1
  43. data/lib/aerospike/value/value.rb +107 -5
  44. data/lib/aerospike/version.rb +1 -1
  45. metadata +15 -2
@@ -29,6 +29,9 @@ module Aerospike
29
29
  @valid = true
30
30
  @mutex = Mutex.new
31
31
 
32
+ @compressed_data_buffer = nil
33
+ @compressed_data_offset = nil
34
+
32
35
  self
33
36
  end
34
37
 
@@ -42,12 +45,41 @@ module Aerospike
42
45
  status = true
43
46
 
44
47
  while status
48
+ @data_offset = 0
49
+ @compressed_data_buffer = nil
50
+
45
51
  # Read header.
46
52
  read_bytes(8)
47
53
 
48
54
  size = @data_buffer.read_int64(0)
49
55
  receive_size = size & 0xFFFFFFFFFFFF
50
56
 
57
+ # inflate if compressed
58
+ compressed_sz = compressed_size
59
+ if compressed_sz
60
+ begin
61
+ # read compressed msg header
62
+ @conn.read(@data_buffer, 8)
63
+
64
+ # read compressed message
65
+ @conn.read(@data_buffer, compressed_sz - 8)
66
+
67
+ # inflate the results
68
+ # TODO: reuse the current buffer
69
+ uncompressed = Zlib::inflate(@data_buffer.buf)
70
+ receive_size = uncompressed.size - 8
71
+
72
+ @compressed_data_buffer = Buffer.new(-1, uncompressed)
73
+ @compressed_data_offset = 0
74
+
75
+ # waste the first 8 header bytes
76
+ @compressed_data_buffer.eat!(8)
77
+ rescue => e
78
+ Aerospike.logger.error("parse result error: #{e}")
79
+ raise e
80
+ end
81
+ end
82
+
51
83
  if receive_size > 0
52
84
  status = parse_group(receive_size)
53
85
  else
@@ -157,7 +189,13 @@ module Aerospike
157
189
  @data_buffer = Buffer.new(length)
158
190
  end
159
191
 
160
- @conn.read(@data_buffer, length)
192
+ if compressed?
193
+ @data_buffer.write_binary(@compressed_data_buffer.buf[@compressed_data_offset...@compressed_data_offset+length], 0)
194
+ @compressed_data_offset += length
195
+ else
196
+ @conn.read(@data_buffer, length)
197
+ end
198
+
161
199
  @data_offset += length
162
200
  end
163
201
 
@@ -176,6 +214,11 @@ module Aerospike
176
214
  res
177
215
  end
178
216
 
217
+ def compressed?
218
+ @compressed_data_buffer != nil
219
+ end
220
+
221
+
179
222
  end # class
180
223
 
181
224
  end # module
@@ -15,6 +15,8 @@
15
15
  # License for the specific language governing permissions and limitations under
16
16
  # the License.
17
17
 
18
+ require 'zlib'
19
+
18
20
  require 'aerospike/record'
19
21
 
20
22
  require 'aerospike/command/single_command'
@@ -50,12 +52,40 @@ module Aerospike
50
52
  def parse_result
51
53
  # Read header.
52
54
  begin
53
- @conn.read(@data_buffer, MSG_TOTAL_HEADER_SIZE)
55
+ @conn.read(@data_buffer, 8)
54
56
  rescue => e
55
57
  Aerospike.logger.error("parse result error: #{e}")
56
58
  raise e
57
59
  end
58
60
 
61
+ # inflate if compressed
62
+ compressed_sz = compressed_size
63
+ if compressed_sz
64
+ begin
65
+ # waste 8 size bytes
66
+ @conn.read(@data_buffer, 8)
67
+
68
+ # read compressed message
69
+ @conn.read(@data_buffer, compressed_sz - 8)
70
+
71
+ # inflate the results
72
+ # TODO: reuse the current buffer
73
+ uncompressed = Zlib::inflate(@data_buffer.buf)
74
+
75
+ @data_buffer = Buffer.new(-1, uncompressed)
76
+ rescue => e
77
+ Aerospike.logger.error("parse result error: #{e}")
78
+ raise e
79
+ end
80
+ else
81
+ begin
82
+ bytes_read = @conn.read(@data_buffer, MSG_TOTAL_HEADER_SIZE - 8, 8)
83
+ rescue => e
84
+ Aerospike.logger.error("parse result error: #{e}")
85
+ raise e
86
+ end
87
+ end
88
+
59
89
  # A number of these are commented out because we just don't care enough to read
60
90
  # that section of the header. If we do care, uncomment and check!
61
91
  sz = @data_buffer.read_int64(0)
@@ -68,7 +98,9 @@ module Aerospike
68
98
  receive_size = (sz & 0xFFFFFFFFFFFF) - header_length
69
99
 
70
100
  # Read remaining message bytes.
71
- if receive_size > 0
101
+ if compressed_sz
102
+ @data_buffer.eat!(MSG_TOTAL_HEADER_SIZE)
103
+ elsif receive_size > 0
72
104
  size_buffer_sz(receive_size)
73
105
 
74
106
  begin
@@ -40,7 +40,40 @@ module Aerospike
40
40
 
41
41
  def parse_result
42
42
  # Read header.
43
- @conn.read(@data_buffer, MSG_TOTAL_HEADER_SIZE)
43
+ begin
44
+ @conn.read(@data_buffer, 8)
45
+ rescue => e
46
+ Aerospike.logger.error("parse result error: #{e}")
47
+ raise e
48
+ end
49
+
50
+ # inflate if compressed
51
+ compressed_sz = compressed_size
52
+ if compressed_sz
53
+ begin
54
+ #waste 8 size bytes
55
+ @conn.read(@data_buffer, 8)
56
+
57
+ # read compressed message
58
+ @conn.read(@data_buffer, sz - 8)
59
+
60
+ # inflate the results
61
+ # TODO: reuse the current buffer
62
+ uncompressed = Zlib::inflate(@data_buffer.buf)
63
+
64
+ @data_buffer = Buffer.new(-1, uncompressed)
65
+ rescue => e
66
+ Aerospike.logger.error("parse result error: #{e}")
67
+ raise e
68
+ end
69
+ else
70
+ begin
71
+ bytes_read = @conn.read(@data_buffer, MSG_TOTAL_HEADER_SIZE - 8, 8)
72
+ rescue => e
73
+ Aerospike.logger.error("parse result error: #{e}")
74
+ raise e
75
+ end
76
+ end
44
77
 
45
78
  result_code = @data_buffer.read(13).ord & 0xFF
46
79
 
@@ -40,5 +40,10 @@ module Aerospike
40
40
  # Server supports the new "peers" protocol for automatic node discovery
41
41
  PEERS = :peers
42
42
 
43
+ # Server supports the "truncate-namespace" command
44
+ TRUNCATE_NAMESPACE = :"truncate-namespace"
45
+
46
+ # Server supports the "blob-bits" command
47
+ BLOB_BITS = :"blob-bits"
43
48
  end
44
49
  end
@@ -22,7 +22,7 @@ require 'aerospike/atomic/atomic'
22
22
  module Aerospike
23
23
  class Node
24
24
 
25
- attr_reader :reference_count, :responded, :name, :features, :cluster_name, :partition_generation, :peers_generation, :failures, :cluster, :peers_count, :host
25
+ attr_reader :reference_count, :responded, :name, :features, :cluster_name, :partition_generation, :rebalance_generation, :peers_generation, :failures, :cluster, :peers_count, :host
26
26
 
27
27
  PARTITIONS = 4096
28
28
  FULL_HEALTH = 100
@@ -46,16 +46,29 @@ module Aerospike
46
46
  @peers_count = Atomic.new(0)
47
47
  @peers_generation = ::Aerospike::Node::Generation.new
48
48
  @partition_generation = ::Aerospike::Node::Generation.new
49
+ @rebalance_generation = ::Aerospike::Node::Rebalance.new
49
50
  @reference_count = Atomic.new(0)
50
51
  @responded = Atomic.new(false)
51
52
  @active = Atomic.new(true)
52
53
  @failures = Atomic.new(0)
53
54
 
54
55
  @replica_index = Atomic.new(0)
56
+ @racks = Atomic.new(nil)
55
57
 
56
58
  @connections = ::Aerospike::ConnectionPool.new(cluster, host)
57
59
  end
58
60
 
61
+ def update_racks(parser)
62
+ new_racks = parser.update_racks
63
+ @racks.value = new_racks if new_racks
64
+ end
65
+
66
+ def has_rack(ns, rack_id)
67
+ racks = @racks.value
68
+ return false if !racks
69
+ racks[ns] == rack_id
70
+ end
71
+
59
72
  # Get a connection to the node. If no cached connection is not available,
60
73
  # a new connection will be created
61
74
  def get_connection(timeout)
@@ -199,6 +212,10 @@ module Aerospike
199
212
  Node::Refresh::Partitions.(self, peers)
200
213
  end
201
214
 
215
+ def refresh_racks()
216
+ Node::Refresh::Racks.(self)
217
+ end
218
+
202
219
  def refresh_peers(peers)
203
220
  Node::Refresh::Peers.(self, peers)
204
221
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2018 Aerospike, Inc.
4
+ #
5
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
6
+ # license agreements.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
+ # use this file except in compliance with the License. You may obtain a copy of
10
+ # the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
+ # License for the specific language governing permissions and limitations under
18
+ # the License.
19
+
20
+ module Aerospike
21
+ class Node
22
+ # generic class for representing changes in eg. peer and partition generation
23
+ class Rebalance
24
+ attr_reader :generation
25
+
26
+ def initialize(generation = -1)
27
+ @generation = ::Aerospike::Atomic.new(generation)
28
+ @changed = ::Aerospike::Atomic.new(false)
29
+ end
30
+
31
+ def changed?
32
+ @changed.value == true
33
+ end
34
+
35
+ def eql?(generation)
36
+ @generation.value == generation
37
+ end
38
+
39
+ def reset_changed!
40
+ @changed.value = false
41
+ end
42
+
43
+ def update(new_generation)
44
+ return if @generation.value == new_generation
45
+ @generation.value = new_generation
46
+ @changed.value = true
47
+ end
48
+ end
49
+ end
50
+ end
@@ -24,14 +24,17 @@ module Aerospike
24
24
  CMDS_BASE = %w[node partition-generation cluster-name].freeze
25
25
  CMDS_PEERS = (CMDS_BASE + ['peers-generation']).freeze
26
26
  CMDS_SERVICES = (CMDS_BASE + ['services']).freeze
27
+ CMDS_REBALANCE = (CMDS_PEERS + ['rebalance-generation']).freeze
27
28
 
28
29
  class << self
29
30
  def call(node, peers)
30
31
  conn = node.tend_connection
31
32
  if peers.use_peers?
32
- info_map = ::Aerospike::Info.request(conn, *CMDS_PEERS)
33
+ cmds = node.cluster.rack_aware ? CMDS_REBALANCE : CMDS_PEERS
34
+ info_map = ::Aerospike::Info.request(conn, *cmds)
33
35
  Verify::PeersGeneration.(node, info_map, peers)
34
36
  Verify::PartitionGeneration.(node, info_map)
37
+ Verify::RebalanceGeneration.(node, info_map) if node.cluster.rack_aware
35
38
  Verify::Name.(node, info_map)
36
39
  Verify::ClusterName.(node, info_map)
37
40
  else
@@ -0,0 +1,47 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2018-2020 Aerospike, Inc.
4
+ #
5
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
6
+ # license agreements.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
+ # use this file except in compliance with the License. You may obtain a copy of
10
+ # the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
+ # License for the specific language governing permissions and limitations under
18
+ # the License.
19
+
20
+ module Aerospike
21
+ class Node
22
+ module Refresh
23
+ module Racks
24
+ class << self
25
+ def call(node)
26
+ return unless should_refresh?(node)
27
+
28
+ Aerospike.logger.info("Updating racks for node #{node.name}")
29
+ conn = node.tend_connection
30
+ parser = RackParser.new(node, conn)
31
+ node.update_racks(parser)
32
+ rescue ::Aerospike::Exceptions::Aerospike => e
33
+ conn.close
34
+ Refresh::Failed.(node, e)
35
+ end
36
+
37
+ # Do not refresh racks when node connection has already failed
38
+ # during this cluster tend iteration.
39
+ def should_refresh?(node)
40
+ return false if node.failed? || !node.active?
41
+ true
42
+ end
43
+ end
44
+ end
45
+ end
46
+ end
47
+ end
@@ -28,6 +28,7 @@ module Aerospike
28
28
  node.reset_reference_count!
29
29
  node.reset_responded!
30
30
  node.partition_generation.reset_changed!
31
+ node.rebalance_generation.reset_changed!
31
32
  end
32
33
  end
33
34
  end
@@ -0,0 +1,43 @@
1
+ # frozen_string_literal: true
2
+
3
+ # Copyright 2018 Aerospike, Inc.
4
+ #
5
+ # Portions may be licensed to Aerospike, Inc. under one or more contributor
6
+ # license agreements.
7
+ #
8
+ # Licensed under the Apache License, Version 2.0 (the "License"); you may not
9
+ # use this file except in compliance with the License. You may obtain a copy of
10
+ # the License at
11
+ #
12
+ # http://www.apache.org/licenses/LICENSE-2.0
13
+ #
14
+ # Unless required by applicable law or agreed to in writing, software
15
+ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16
+ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17
+ # License for the specific language governing permissions and limitations under
18
+ # the License.
19
+
20
+ module Aerospike
21
+ class Node
22
+ module Verify
23
+ # Fetch and set rebalance generation. If racks needs to be refreshed
24
+ # this will be indicated in node.rebalance_changed
25
+ module RebalanceGeneration
26
+ class << self
27
+ def call(node, info_map)
28
+ gen_string = info_map.fetch('rebalance-generation', nil)
29
+
30
+ raise Aerospike::Exceptions::Parse.new('rebalance-generation is empty') if gen_string.to_s.empty?
31
+
32
+ generation = gen_string.to_i
33
+
34
+ node.rebalance_generation.update(generation)
35
+
36
+ return unless node.rebalance_generation.changed?
37
+ Aerospike.logger.info("Node #{node.name} rebalance generation #{generation} changed")
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
@@ -20,7 +20,7 @@ module Aerospike
20
20
 
21
21
  class Operation
22
22
 
23
- attr_reader :op_type, :bin_name, :bin_value
23
+ attr_reader :op_type, :bin_name, :bin_value, :ctx
24
24
 
25
25
  READ = 1
26
26
  READ_HEADER = 1
@@ -31,12 +31,17 @@ module Aerospike
31
31
  APPEND = 9
32
32
  PREPEND = 10
33
33
  TOUCH = 11
34
+ BIT_READ = 12
35
+ BIT_MODIFY = 13
34
36
  DELETE = 14
37
+ HLL_READ = 15
38
+ HLL_MODIFY = 16
35
39
 
36
- def initialize(op_type, bin_name=nil, bin_value=NullValue.new)
40
+ def initialize(op_type, bin_name=nil, bin_value=NullValue.new, ctx = nil)
37
41
  @op_type = op_type
38
42
  @bin_name = bin_name
39
43
  @bin_value = Value.of(bin_value)
44
+ @ctx = ctx
40
45
  self
41
46
  end
42
47
 
@@ -27,6 +27,7 @@ module Aerospike
27
27
  attr_accessor :cluster_name
28
28
  attr_accessor :tls
29
29
  attr_accessor :policies
30
+ attr_accessor :rack_aware, :rack_id
30
31
 
31
32
  def initialize(opt={})
32
33
  # Initial host connection timeout in seconds. The timeout when opening a connection
@@ -56,6 +57,20 @@ module Aerospike
56
57
 
57
58
  # Default Policies
58
59
  @policies = opt.fetch(:policies) { Hash.new }
60
+
61
+ # Track server rack data. This field is useful when directing read commands to the server node
62
+ # that contains the key and exists on the same rack as the client. This serves to lower cloud
63
+ # provider costs when nodes are distributed across different racks/data centers.
64
+ #
65
+ # ClientPolicy#rack_id, Replica#PREFER_RACK and server rack
66
+ # configuration must also be set to enable this functionality.
67
+ @rack_aware = opt[:rack_aware] || false
68
+
69
+ # Rack where this client instance resides.
70
+ #
71
+ # ClientPolicy#rack_aware, Replica#PREFER_RACK and server rack
72
+ # configuration must also be set to enable this functionality.
73
+ @rack_id = opt[:rack_id] || 0
59
74
  end
60
75
 
61
76
  def requires_authentication