redis2-namespaced 3.0.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +11 -0
  3. data/.order +170 -0
  4. data/.travis/Gemfile +11 -0
  5. data/.travis.yml +55 -0
  6. data/.yardopts +3 -0
  7. data/CHANGELOG.md +285 -0
  8. data/LICENSE +20 -0
  9. data/README.md +251 -0
  10. data/Rakefile +403 -0
  11. data/benchmarking/logging.rb +71 -0
  12. data/benchmarking/pipeline.rb +51 -0
  13. data/benchmarking/speed.rb +21 -0
  14. data/benchmarking/suite.rb +24 -0
  15. data/benchmarking/worker.rb +71 -0
  16. data/examples/basic.rb +15 -0
  17. data/examples/dist_redis.rb +43 -0
  18. data/examples/incr-decr.rb +17 -0
  19. data/examples/list.rb +26 -0
  20. data/examples/pubsub.rb +37 -0
  21. data/examples/sets.rb +36 -0
  22. data/examples/unicorn/config.ru +3 -0
  23. data/examples/unicorn/unicorn.rb +20 -0
  24. data/lib/redis2/client.rb +419 -0
  25. data/lib/redis2/connection/command_helper.rb +44 -0
  26. data/lib/redis2/connection/hiredis.rb +63 -0
  27. data/lib/redis2/connection/registry.rb +12 -0
  28. data/lib/redis2/connection/ruby.rb +322 -0
  29. data/lib/redis2/connection/synchrony.rb +124 -0
  30. data/lib/redis2/connection.rb +9 -0
  31. data/lib/redis2/distributed.rb +853 -0
  32. data/lib/redis2/errors.rb +40 -0
  33. data/lib/redis2/hash_ring.rb +131 -0
  34. data/lib/redis2/pipeline.rb +141 -0
  35. data/lib/redis2/subscribe.rb +83 -0
  36. data/lib/redis2/version.rb +3 -0
  37. data/lib/redis2.rb +2533 -0
  38. data/redis.gemspec +43 -0
  39. data/test/bitpos_test.rb +69 -0
  40. data/test/blocking_commands_test.rb +42 -0
  41. data/test/command_map_test.rb +30 -0
  42. data/test/commands_on_hashes_test.rb +21 -0
  43. data/test/commands_on_lists_test.rb +20 -0
  44. data/test/commands_on_sets_test.rb +77 -0
  45. data/test/commands_on_sorted_sets_test.rb +109 -0
  46. data/test/commands_on_strings_test.rb +101 -0
  47. data/test/commands_on_value_types_test.rb +131 -0
  48. data/test/connection_handling_test.rb +189 -0
  49. data/test/db/.gitkeep +0 -0
  50. data/test/distributed_blocking_commands_test.rb +46 -0
  51. data/test/distributed_commands_on_hashes_test.rb +10 -0
  52. data/test/distributed_commands_on_lists_test.rb +22 -0
  53. data/test/distributed_commands_on_sets_test.rb +83 -0
  54. data/test/distributed_commands_on_sorted_sets_test.rb +18 -0
  55. data/test/distributed_commands_on_strings_test.rb +59 -0
  56. data/test/distributed_commands_on_value_types_test.rb +95 -0
  57. data/test/distributed_commands_requiring_clustering_test.rb +164 -0
  58. data/test/distributed_connection_handling_test.rb +23 -0
  59. data/test/distributed_internals_test.rb +70 -0
  60. data/test/distributed_key_tags_test.rb +52 -0
  61. data/test/distributed_persistence_control_commands_test.rb +26 -0
  62. data/test/distributed_publish_subscribe_test.rb +92 -0
  63. data/test/distributed_remote_server_control_commands_test.rb +66 -0
  64. data/test/distributed_scripting_test.rb +102 -0
  65. data/test/distributed_sorting_test.rb +20 -0
  66. data/test/distributed_test.rb +58 -0
  67. data/test/distributed_transactions_test.rb +32 -0
  68. data/test/encoding_test.rb +18 -0
  69. data/test/error_replies_test.rb +59 -0
  70. data/test/helper.rb +218 -0
  71. data/test/helper_test.rb +24 -0
  72. data/test/internals_test.rb +410 -0
  73. data/test/lint/blocking_commands.rb +150 -0
  74. data/test/lint/hashes.rb +162 -0
  75. data/test/lint/lists.rb +143 -0
  76. data/test/lint/sets.rb +125 -0
  77. data/test/lint/sorted_sets.rb +238 -0
  78. data/test/lint/strings.rb +260 -0
  79. data/test/lint/value_types.rb +122 -0
  80. data/test/persistence_control_commands_test.rb +26 -0
  81. data/test/pipelining_commands_test.rb +242 -0
  82. data/test/publish_subscribe_test.rb +210 -0
  83. data/test/remote_server_control_commands_test.rb +117 -0
  84. data/test/scanning_test.rb +413 -0
  85. data/test/scripting_test.rb +78 -0
  86. data/test/sorting_test.rb +59 -0
  87. data/test/support/connection/hiredis.rb +1 -0
  88. data/test/support/connection/ruby.rb +1 -0
  89. data/test/support/connection/synchrony.rb +17 -0
  90. data/test/support/redis_mock.rb +115 -0
  91. data/test/support/wire/synchrony.rb +24 -0
  92. data/test/support/wire/thread.rb +5 -0
  93. data/test/synchrony_driver.rb +88 -0
  94. data/test/test.conf +9 -0
  95. data/test/thread_safety_test.rb +32 -0
  96. data/test/transactions_test.rb +264 -0
  97. data/test/unknown_commands_test.rb +14 -0
  98. data/test/url_param_test.rb +132 -0
  99. metadata +226 -0
@@ -0,0 +1,40 @@
1
+ class Redis2
2
+ # Base error for all redis-rb errors.
3
+ class BaseError < RuntimeError
4
+ end
5
+
6
+ # Raised by the connection when a protocol error occurs.
7
+ class ProtocolError < BaseError
8
+ def initialize(reply_type)
9
+ super(<<-EOS.gsub(/(?:^|\n)\s*/, " "))
10
+ Got '#{reply_type}' as initial reply byte.
11
+ If you're in a forking environment, such as Unicorn, you need to
12
+ connect to Redis2 after forking.
13
+ EOS
14
+ end
15
+ end
16
+
17
+ # Raised by the client when command execution returns an error reply.
18
+ class CommandError < BaseError
19
+ end
20
+
21
+ # Base error for connection related errors.
22
+ class BaseConnectionError < BaseError
23
+ end
24
+
25
+ # Raised when connection to a Redis2 server cannot be made.
26
+ class CannotConnectError < BaseConnectionError
27
+ end
28
+
29
+ # Raised when connection to a Redis2 server is lost.
30
+ class ConnectionError < BaseConnectionError
31
+ end
32
+
33
+ # Raised when performing I/O times out.
34
+ class TimeoutError < BaseConnectionError
35
+ end
36
+
37
+ # Raised when the connection was inherited by a child process.
38
+ class InheritedError < BaseConnectionError
39
+ end
40
+ end
@@ -0,0 +1,131 @@
1
+ require 'zlib'
2
+
3
+ class Redis2
4
+ class HashRing
5
+
6
+ POINTS_PER_SERVER = 160 # this is the default in libmemcached
7
+
8
+ attr_reader :ring, :sorted_keys, :replicas, :nodes
9
+
10
+ # nodes is a list of objects that have a proper to_s representation.
11
+ # replicas indicates how many virtual points should be used pr. node,
12
+ # replicas are required to improve the distribution.
13
+ def initialize(nodes=[], replicas=POINTS_PER_SERVER)
14
+ @replicas = replicas
15
+ @ring = {}
16
+ @nodes = []
17
+ @sorted_keys = []
18
+ nodes.each do |node|
19
+ add_node(node)
20
+ end
21
+ end
22
+
23
+ # Adds a `node` to the hash ring (including a number of replicas).
24
+ def add_node(node)
25
+ @nodes << node
26
+ @replicas.times do |i|
27
+ key = Zlib.crc32("#{node.id}:#{i}")
28
+ @ring[key] = node
29
+ @sorted_keys << key
30
+ end
31
+ @sorted_keys.sort!
32
+ end
33
+
34
+ def remove_node(node)
35
+ @nodes.reject!{|n| n.id == node.id}
36
+ @replicas.times do |i|
37
+ key = Zlib.crc32("#{node.id}:#{i}")
38
+ @ring.delete(key)
39
+ @sorted_keys.reject! {|k| k == key}
40
+ end
41
+ end
42
+
43
+ # get the node in the hash ring for this key
44
+ def get_node(key)
45
+ get_node_pos(key)[0]
46
+ end
47
+
48
+ def get_node_pos(key)
49
+ return [nil,nil] if @ring.size == 0
50
+ crc = Zlib.crc32(key)
51
+ idx = HashRing.binary_search(@sorted_keys, crc)
52
+ return [@ring[@sorted_keys[idx]], idx]
53
+ end
54
+
55
+ def iter_nodes(key)
56
+ return [nil,nil] if @ring.size == 0
57
+ _, pos = get_node_pos(key)
58
+ @ring.size.times do |n|
59
+ yield @ring[@sorted_keys[(pos+n) % @ring.size]]
60
+ end
61
+ end
62
+
63
+ class << self
64
+
65
+ # gem install RubyInline to use this code
66
+ # Native extension to perform the binary search within the hashring.
67
+ # There's a pure ruby version below so this is purely optional
68
+ # for performance. In testing 20k gets and sets, the native
69
+ # binary search shaved about 12% off the runtime (9sec -> 8sec).
70
+ begin
71
+ require 'inline'
72
+ inline do |builder|
73
+ builder.c <<-EOM
74
+ int binary_search(VALUE ary, unsigned int r) {
75
+ int upper = RARRAY_LEN(ary) - 1;
76
+ int lower = 0;
77
+ int idx = 0;
78
+
79
+ while (lower <= upper) {
80
+ idx = (lower + upper) / 2;
81
+
82
+ VALUE continuumValue = RARRAY_PTR(ary)[idx];
83
+ unsigned int l = NUM2UINT(continuumValue);
84
+ if (l == r) {
85
+ return idx;
86
+ }
87
+ else if (l > r) {
88
+ upper = idx - 1;
89
+ }
90
+ else {
91
+ lower = idx + 1;
92
+ }
93
+ }
94
+ if (upper < 0) {
95
+ upper = RARRAY_LEN(ary) - 1;
96
+ }
97
+ return upper;
98
+ }
99
+ EOM
100
+ end
101
+ rescue Exception
102
+ # Find the closest index in HashRing with value <= the given value
103
+ def binary_search(ary, value, &block)
104
+ upper = ary.size - 1
105
+ lower = 0
106
+ idx = 0
107
+
108
+ while(lower <= upper) do
109
+ idx = (lower + upper) / 2
110
+ comp = ary[idx] <=> value
111
+
112
+ if comp == 0
113
+ return idx
114
+ elsif comp > 0
115
+ upper = idx - 1
116
+ else
117
+ lower = idx + 1
118
+ end
119
+ end
120
+
121
+ if upper < 0
122
+ upper = ary.size - 1
123
+ end
124
+ return upper
125
+ end
126
+
127
+ end
128
+ end
129
+
130
+ end
131
+ end
@@ -0,0 +1,141 @@
1
+ class Redis2
2
+ unless defined?(::BasicObject)
3
+ class BasicObject
4
+ instance_methods.each { |meth| undef_method(meth) unless meth =~ /\A(__|instance_eval)/ }
5
+ end
6
+ end
7
+
8
+ class Pipeline
9
+ attr_accessor :db
10
+
11
+ attr :futures
12
+
13
+ def initialize
14
+ @with_reconnect = true
15
+ @shutdown = false
16
+ @futures = []
17
+ end
18
+
19
+ def with_reconnect?
20
+ @with_reconnect
21
+ end
22
+
23
+ def without_reconnect?
24
+ !@with_reconnect
25
+ end
26
+
27
+ def shutdown?
28
+ @shutdown
29
+ end
30
+
31
+ def call(command, &block)
32
+ # A pipeline that contains a shutdown should not raise ECONNRESET when
33
+ # the connection is gone.
34
+ @shutdown = true if command.first == :shutdown
35
+ future = Future.new(command, block)
36
+ @futures << future
37
+ future
38
+ end
39
+
40
+ def call_pipeline(pipeline)
41
+ @shutdown = true if pipeline.shutdown?
42
+ @futures.concat(pipeline.futures)
43
+ @db = pipeline.db
44
+ nil
45
+ end
46
+
47
+ def commands
48
+ @futures.map { |f| f._command }
49
+ end
50
+
51
+ def with_reconnect(val=true)
52
+ @with_reconnect = false unless val
53
+ yield
54
+ end
55
+
56
+ def without_reconnect(&blk)
57
+ with_reconnect(false, &blk)
58
+ end
59
+
60
+ def finish(replies, &blk)
61
+ if blk
62
+ futures.each_with_index.map do |future, i|
63
+ future._set(blk.call(replies[i]))
64
+ end
65
+ else
66
+ futures.each_with_index.map do |future, i|
67
+ future._set(replies[i])
68
+ end
69
+ end
70
+ end
71
+
72
+ class Multi < self
73
+ def finish(replies)
74
+ exec = replies.last
75
+
76
+ return if exec.nil? # The transaction failed because of WATCH.
77
+
78
+ # EXEC command failed.
79
+ raise exec if exec.is_a?(CommandError)
80
+
81
+ if exec.size < futures.size
82
+ # Some command wasn't recognized by Redis2.
83
+ raise replies.detect { |r| r.is_a?(CommandError) }
84
+ end
85
+
86
+ super(exec) do |reply|
87
+ # Because an EXEC returns nested replies, hiredis won't be able to
88
+ # convert an error reply to a CommandError instance itself. This is
89
+ # specific to MULTI/EXEC, so we solve this here.
90
+ reply.is_a?(::RuntimeError) ? CommandError.new(reply.message) : reply
91
+ end
92
+ end
93
+
94
+ def commands
95
+ [[:multi]] + super + [[:exec]]
96
+ end
97
+ end
98
+ end
99
+
100
+ class FutureNotReady < RuntimeError
101
+ def initialize
102
+ super("Value will be available once the pipeline executes.")
103
+ end
104
+ end
105
+
106
+ class Future < BasicObject
107
+ FutureNotReady = ::Redis2::FutureNotReady.new
108
+
109
+ def initialize(command, transformation)
110
+ @command = command
111
+ @transformation = transformation
112
+ @object = FutureNotReady
113
+ end
114
+
115
+ def inspect
116
+ "<Redis2::Future #{@command.inspect}>"
117
+ end
118
+
119
+ def _set(object)
120
+ @object = @transformation ? @transformation.call(object) : object
121
+ value
122
+ end
123
+
124
+ def _command
125
+ @command
126
+ end
127
+
128
+ def value
129
+ ::Kernel.raise(@object) if @object.kind_of?(::RuntimeError)
130
+ @object
131
+ end
132
+
133
+ def is_a?(other)
134
+ self.class.ancestors.include?(other)
135
+ end
136
+
137
+ def class
138
+ Future
139
+ end
140
+ end
141
+ end
@@ -0,0 +1,83 @@
1
+ class Redis2
2
+ class SubscribedClient
3
+ def initialize(client)
4
+ @client = client
5
+ end
6
+
7
+ def call(command)
8
+ @client.process([command])
9
+ end
10
+
11
+ def subscribe(*channels, &block)
12
+ subscription("subscribe", "unsubscribe", channels, block)
13
+ end
14
+
15
+ def psubscribe(*channels, &block)
16
+ subscription("psubscribe", "punsubscribe", channels, block)
17
+ end
18
+
19
+ def unsubscribe(*channels)
20
+ call([:unsubscribe, *channels])
21
+ end
22
+
23
+ def punsubscribe(*channels)
24
+ call([:punsubscribe, *channels])
25
+ end
26
+
27
+ protected
28
+
29
+ def subscription(start, stop, channels, block)
30
+ sub = Subscription.new(&block)
31
+
32
+ unsubscribed = false
33
+
34
+ begin
35
+ @client.call_loop([start, *channels]) do |line|
36
+ type, *rest = line
37
+ sub.callbacks[type].call(*rest)
38
+ unsubscribed = type == stop && rest.last == 0
39
+ break if unsubscribed
40
+ end
41
+ ensure
42
+ # No need to unsubscribe here. The real client closes the connection
43
+ # whenever an exception is raised (see #ensure_connected).
44
+ end
45
+ end
46
+ end
47
+
48
+ class Subscription
49
+ attr :callbacks
50
+
51
+ def initialize
52
+ @callbacks = Hash.new do |hash, key|
53
+ hash[key] = lambda { |*_| }
54
+ end
55
+
56
+ yield(self)
57
+ end
58
+
59
+ def subscribe(&block)
60
+ @callbacks["subscribe"] = block
61
+ end
62
+
63
+ def unsubscribe(&block)
64
+ @callbacks["unsubscribe"] = block
65
+ end
66
+
67
+ def message(&block)
68
+ @callbacks["message"] = block
69
+ end
70
+
71
+ def psubscribe(&block)
72
+ @callbacks["psubscribe"] = block
73
+ end
74
+
75
+ def punsubscribe(&block)
76
+ @callbacks["punsubscribe"] = block
77
+ end
78
+
79
+ def pmessage(&block)
80
+ @callbacks["pmessage"] = block
81
+ end
82
+ end
83
+ end
@@ -0,0 +1,3 @@
1
+ class Redis2
2
+ VERSION = "3.0.7"
3
+ end