redis_cluster 0.2.9 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d5f4f4dd87bea457c22bcb596945c12e2cffad6b
4
- data.tar.gz: 9cce133f24ba35be564f9a241513a8ae28a49254
3
+ metadata.gz: f4e3f46f9197fb5e41afea6d2b4e0f41af7defbc
4
+ data.tar.gz: 21e8e6613f409e4d2ea6942cac0c5a44dc855bff
5
5
  SHA512:
6
- metadata.gz: 5d88d716725acf7b52ca0c99690b86d4203a3a725f5589a4af3f5020317b73139626105a5858336c559131c566d109a542b6a2f4dd1cfae05936deac3bb57ee4
7
- data.tar.gz: 55428c277ba8d8a7accc20a4d5cdf572b7df74df550b73409570fe60fd5131f90cf1551133968f8cd4c387bcedc738f82330b13ed077f8582579198d6f1a9015
6
+ metadata.gz: d01fd2610f8e69cec1344105227b0ae007aa60fd066d4f5bc9d77320dbd6eb0b8af6bd524bc4fb9771a3a125cf43b686c4838ac3ec85d414dd7aa9d41a1fdab3
7
+ data.tar.gz: e8006dabb95bfc9ff141fdf0b8120b308b649e65f3183fe6400ff77c4b655f00d91db5ceb5e4131ebfa48d343afe9dc1ca2f2382db0fb77a84acf379626ef3cf
data/Gemfile CHANGED
@@ -2,3 +2,11 @@ source 'https://rubygems.org'
2
2
 
3
3
  # Specify your gem's dependencies in redis_cluster.gemspec
4
4
  gemspec
5
+
6
+ group :development do
7
+ platforms :mri do
8
+ if RUBY_VERSION >= "2.0.0"
9
+ gem "pry-byebug"
10
+ end
11
+ end
12
+ end
data/README.md CHANGED
@@ -28,13 +28,15 @@ Or install it yourself as:
28
28
  First you need to configure redis cluster with some nodes! Please see: [https://redis.io/topics/cluster-tutorial](https://redis.io/topics/cluster-tutorial)
29
29
 
30
30
  ```ruby
31
- # don't need all, gem can auto detect all nodes, and process failover if some master nodes down
31
+ # Don't need all, gem can auto detect all nodes, and process failover if some master nodes down
32
32
  hosts = [{host: '127.0.0.1', port: 7000}, {host: '127.0.0.1', port: 7001}]
33
33
  rs = RedisCluster.new hosts
34
34
  rs.set "test", 1
35
35
  rs.get "test"
36
36
  ```
37
37
 
38
+ At development environment with single redis node, you can set hosts a hash value: {host: 'xx', port: 6379}.
39
+
38
40
  If masterauth & requirepass configed, you can initialize below:
39
41
  ```ruby
40
42
  RedisCluster.new hosts, password: 'password'
@@ -74,20 +76,16 @@ rs.eval "return 'hello redis!'", [:foo]
74
76
 
75
77
  ## Benchmark test
76
78
 
77
- A simple benchmark at my macbook, start 4 master nodes (and 4 cold slave nodes), running with one ruby process.
78
- This only testing redis_cluster can work, not for redis Performance. When I fork 8 ruby process same time and run get command,redis can run 80,000 - 110,000 times per second at my macbook.
79
-
80
-
81
79
  ```ruby
82
80
  Benchmark.bm do |x|
83
81
  x.report do
84
82
  1.upto(100_000).each do |i|
85
- redis.get "test#{i}"
83
+ redis.set "test#{i}", i
86
84
  end
87
85
  end
88
86
  x.report do
89
87
  1.upto(100_000).each do |i|
90
- redis.set "test#{i}", i
88
+ redis.get "test#{i}"
91
89
  end
92
90
  end
93
91
  x.report do
@@ -4,40 +4,77 @@ module RedisCluster
4
4
 
5
5
  class Client
6
6
 
7
- def initialize(startup_hosts, global_configs = {})
7
+ def initialize(startup_hosts, configs = {})
8
8
  @startup_hosts = startup_hosts
9
- @pool = Pool.new(global_configs)
9
+
10
+ # Extract configuration options relevant to Redis Cluster.
11
+
12
+ # force_cluster defaults to true to match the client's behavior before
13
+ # the option existed
14
+ @force_cluster = configs.delete(:force_cluster) { |_key| true }
15
+
16
+ # The number of times to retry a failed execute. Redis errors, `MOVE`, or
17
+ # `ASK` are all considered failures that will count towards this tally. A
18
+ # count of at least 2 is probably sensible because if a node disappears
19
+ # the first try will be a Redis error, the second try retry will probably
20
+ # be a `MOVE` (whereupon the node pool is reloaded), and it will take
21
+ # until the third try to succeed.
22
+ @retry_count = configs.delete(:retry_count) { |_key| 3 }
23
+
24
+ # Any leftover configuration goes through to the pool and onto individual
25
+ # Redis clients.
26
+ @pool = Pool.new(configs)
10
27
  @mutex = Mutex.new
28
+
11
29
  reload_pool_nodes(true)
12
30
  end
13
31
 
14
32
  def execute(method, args, &block)
15
- ttl = Configuration::REQUEST_TTL
16
33
  asking = false
34
+ last_error = nil
35
+ retries_left = @retry_count
17
36
  try_random_node = false
18
37
 
19
- while ttl > 0
20
- ttl -= 1
38
+ # We use `>= 0` instead of `> 0` because we decrement this counter on the
39
+ # first run.
40
+ while retries_left >= 0
41
+ retries_left -= 1
42
+
21
43
  begin
22
44
  return @pool.execute(method, args, {asking: asking, random_node: try_random_node}, &block)
23
- rescue Errno::ECONNREFUSED, Redis::TimeoutError, Redis::CannotConnectError, Errno::EACCES
45
+
46
+ rescue Errno::ECONNREFUSED, Redis::TimeoutError, Redis::CannotConnectError, Errno::EACCES => e
47
+ last_error = e
48
+
49
+ # Getting an error while executing may be an indication that we've
50
+ # lost the node that we were talking to and in that case it makes
51
+ # sense to try a different node and maybe reload our node pool (if
52
+ # the new node issues a `MOVE`).
24
53
  try_random_node = true
25
- sleep 0.1 if ttl < Configuration::REQUEST_TTL/2
54
+
26
55
  rescue => e
56
+ last_error = e
57
+
27
58
  err_code = e.to_s.split.first
28
59
  raise e unless %w(MOVED ASK).include?(err_code)
29
60
 
30
61
  if err_code == 'ASK'
31
62
  asking = true
32
63
  else
33
- reload_pool_nodes
34
- sleep 0.1 if ttl < Configuration::REQUEST_TTL/2
64
+ # `MOVED` indicates a permanent redirect which means that our slot
65
+ # mappings are stale: reload them.
66
+ reload_pool_nodes(false)
35
67
  end
36
68
  end
37
69
  end
70
+
71
+ # If we ran out of retries (the maximum number may have been set to 0),
72
+ # surface any error that was thrown back to the caller. We'd otherwise
73
+ # suppress the error, which would return something quite unexpected.
74
+ raise last_error
38
75
  end
39
76
 
40
- Configuration::SUPPORT_SINGLE_NODE_METHODS.each do |method_name|
77
+ Configuration.method_names.each do |method_name|
41
78
  define_method method_name do |*args, &block|
42
79
  execute(method_name, args, &block)
43
80
  end
@@ -49,33 +86,86 @@ module RedisCluster
49
86
 
50
87
  private
51
88
 
52
- def reload_pool_nodes(raise_error = false)
53
- return @pool.add_node!(@startup_hosts, [(0..Configuration::HASH_SLOTS)]) unless @startup_hosts.is_a? Array
89
+ # Adds only a single node to the client pool and sets it result for the
90
+ # entire space of slots. This is useful when running either a standalone
91
+ # Redis or a single-node Redis Cluster.
92
+ def create_single_node_pool
93
+ host = @startup_hosts
94
+ if host.is_a?(Array)
95
+ if host.length > 1
96
+ raise ArgumentError, "Can only create single node pool for single host"
97
+ end
54
98
 
55
- @mutex.synchronize do
56
- @startup_hosts.each do |options|
57
- begin
58
- redis = Node.redis(@pool.global_configs.merge(options))
59
- slots_mapping = redis.cluster("slots").group_by{|x| x[2]}
60
- @pool.delete_except!(slots_mapping.keys)
61
- slots_mapping.each do |host, infos|
62
- slots_ranges = infos.map {|x| x[0]..x[1] }
63
- @pool.add_node!({host: host[0], port: host[1]}, slots_ranges)
99
+ # Flatten the configured host so that we can easily add it to the
100
+ # client pool.
101
+ host = host.first
102
+ end
103
+
104
+ @pool.add_node!(host, [(0..Configuration::HASH_SLOTS)])
105
+ end
106
+
107
+ def create_multi_node_pool(raise_error)
108
+ unless @startup_hosts.is_a?(Array)
109
+ raise ArgumentError, "Can only create multi-node pool for multiple hosts"
110
+ end
111
+
112
+ @startup_hosts.each do |options|
113
+ begin
114
+ redis = Node.redis(@pool.global_configs.merge(options))
115
+ slots_mapping = redis.cluster("slots").group_by{|x| x[2]}
116
+ @pool.delete_except!(slots_mapping.keys)
117
+ slots_mapping.each do |host, infos|
118
+ slots_ranges = infos.map {|x| x[0]..x[1] }
119
+ @pool.add_node!({host: host[0], port: host[1]}, slots_ranges)
120
+ end
121
+ rescue Redis::CommandError => e
122
+ if e.message =~ /cluster\ support\ disabled$/
123
+ if !@force_cluster
124
+ # We're running outside of cluster-mode -- just create a
125
+ # single-node pool and move on. The exception is if we've been
126
+ # asked for force Redis Cluster, in which case we assume this is
127
+ # a configuration problem and maybe raise an error.
128
+ create_single_node_pool
129
+ return
130
+ elsif raise_error
131
+ raise e
64
132
  end
65
- rescue Redis::CommandError => e
66
- raise e if raise_error && e.message =~ /cluster\ support\ disabled$/
67
- raise e if e.message =~ /NOAUTH\ Authentication\ required/
68
- next
69
- rescue
70
- next
71
133
  end
72
- break
134
+
135
+ raise e if e.message =~ /NOAUTH\ Authentication\ required/
136
+
137
+ # TODO: log error for visibility
138
+ next
139
+ rescue
140
+ # TODO: log error for visibility
141
+ next
142
+ end
143
+
144
+ # We only need to see a `CLUSTER SLOTS` result from a single host, so
145
+ # break after one success.
146
+ break
147
+ end
148
+ end
149
+
150
+ # Reloads the client node pool by requesting new information with `CLUSTER
151
+ # SLOTS` or just adding a node directly if running on standalone. Clients
152
+ # are "upserted" so that we don't necessarily drop clients that are still
153
+ # relevant.
154
+ def reload_pool_nodes(raise_error)
155
+ @mutex.synchronize do
156
+ if @startup_hosts.is_a?(Array)
157
+ create_multi_node_pool(raise_error)
158
+ refresh_startup_nodes
159
+ else
160
+ create_single_node_pool
73
161
  end
74
- fresh_startup_nodes
75
162
  end
76
163
  end
77
164
 
78
- def fresh_startup_nodes
165
+ # Refreshes the contents of @startup_hosts based on the hosts currently in
166
+ # the client pool. This is useful because we may have been told about new
167
+ # hosts after running `CLUSTER SLOTS`.
168
+ def refresh_startup_nodes
79
169
  @pool.nodes.each {|node| @startup_hosts.push(node.host_hash) }
80
170
  @startup_hosts.uniq!
81
171
  end
@@ -2,7 +2,6 @@ module RedisCluster
2
2
 
3
3
  class Configuration
4
4
  HASH_SLOTS = 16384
5
- REQUEST_TTL = 16
6
5
  DEFAULT_TIMEOUT = 1
7
6
 
8
7
  SUPPORT_SINGLE_NODE_METHODS = %w(
@@ -18,6 +17,10 @@ module RedisCluster
18
17
  ).freeze
19
18
 
20
19
  SUPPORT_MULTI_NODE_METHODS = %w(keys script multi pipelined).freeze
20
+
21
+ def self.method_names
22
+ SUPPORT_SINGLE_NODE_METHODS + SUPPORT_MULTI_NODE_METHODS
23
+ end
21
24
  end
22
25
 
23
26
  end
@@ -1,7 +1,26 @@
1
1
  module RedisCluster
2
- NotSupportError = Class.new StandardError
2
+ class CommandNotSupportedError < StandardError
3
+ def initialize(command)
4
+ super("Command #{command} is not supported for Redis Cluster")
5
+ end
6
+ end
3
7
 
4
- KeysNotAtSameSlotError = Class.new StandardError
8
+ class KeysNotAtSameSlotError < StandardError
9
+ def initialize(keys)
10
+ super("Keys must map to the same Redis Cluster slot when using " \
11
+ "EVAL/EVALSHA. Consider using Redis Cluster 'hash tags' (see " \
12
+ "documentation). Keys: #{keys}")
13
+ end
14
+ end
5
15
 
6
- KeyNotAppointError = Class.new StandardError
16
+ class KeysNotSpecifiedError < StandardError
17
+ def initialize(command)
18
+ super("Keys must be specified for command #{command}")
19
+ end
20
+ end
21
+
22
+ # These error classes were renamed. These aliases are here for backwards
23
+ # compatibility.
24
+ KeyNotAppointError = KeysNotSpecifiedError
25
+ NotSupportError = CommandNotSupportedError
7
26
  end
@@ -28,7 +28,7 @@ module RedisCluster
28
28
  return send(method, args, &block) if Configuration::SUPPORT_MULTI_NODE_METHODS.include?(method.to_s)
29
29
 
30
30
  key = key_by_command(method, args)
31
- raise NotSupportError if key.nil?
31
+ raise CommandNotSupportedError.new(method.upcase) if key.nil?
32
32
 
33
33
  node = other_options[:random_node] ? random_node : node_by(key)
34
34
  node.asking if other_options[:asking]
@@ -70,8 +70,14 @@ module RedisCluster
70
70
  when 'info', 'exec', 'slaveof', 'config', 'shutdown'
71
71
  nil
72
72
  when 'eval', 'evalsha'
73
- raise KeyNotAppointError if args[1].nil? || args[1].empty?
74
- raise KeysNotAtSameSlotError unless Slot.at_one?(args[1])
73
+ if args[1].nil? || args[1].empty?
74
+ raise KeysNotSpecifiedError.new(method.upcase)
75
+ end
76
+
77
+ unless Slot.at_one?(args[1])
78
+ raise KeysNotAtSameSlotError.new(args[1])
79
+ end
80
+
75
81
  return args[1][0]
76
82
  else
77
83
  return args.first
@@ -1,3 +1,3 @@
1
1
  module RedisCluster
2
- VERSION = "0.2.9"
2
+ VERSION = "0.3.0"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: redis_cluster
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.9
4
+ version: 0.3.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - wangzc
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-04-04 00:00:00.000000000 Z
11
+ date: 2018-02-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
142
142
  version: '0'
143
143
  requirements: []
144
144
  rubyforge_project:
145
- rubygems_version: 2.6.10
145
+ rubygems_version: 2.6.13
146
146
  signing_key:
147
147
  specification_version: 4
148
148
  summary: redis cluster client