redis_migrator 0.0.1 → 0.1.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/Gemfile +3 -0
  4. data/README.md +13 -1
  5. data/Rakefile +6 -0
  6. data/lib/redis_migrator/redis_helper.rb +5 -36
  7. data/lib/redis_migrator/redis_native_migrator.rb +27 -0
  8. data/lib/redis_migrator/redis_pipe_migrator.rb +66 -0
  9. data/lib/redis_migrator/redis_populator.rb +1 -1
  10. data/lib/redis_migrator.rb +19 -20
  11. data/migrator_benchmark.rb +1 -2
  12. data/redis_migrator.gemspec +8 -4
  13. data/spec/different_redis_type_migrator.rb +67 -0
  14. data/spec/pretested_migrator.rb +47 -0
  15. data/spec/redis_migrator_spec.rb +41 -0
  16. data/spec/redis_native_migrator_spec.rb +44 -0
  17. data/spec/redis_pipe_migrator_spec.rb +51 -0
  18. data/spec/shared_hosts_context.rb +10 -0
  19. data/spec/spec_helper.rb +9 -7
  20. metadata +85 -49
  21. data/spec/migrator_spec.rb +0 -63
  22. data/spec/mock_redis/lib/mock_redis/assertions.rb +0 -13
  23. data/spec/mock_redis/lib/mock_redis/database.rb +0 -432
  24. data/spec/mock_redis/lib/mock_redis/distributed.rb +0 -6
  25. data/spec/mock_redis/lib/mock_redis/exceptions.rb +0 -3
  26. data/spec/mock_redis/lib/mock_redis/expire_wrapper.rb +0 -25
  27. data/spec/mock_redis/lib/mock_redis/hash_methods.rb +0 -118
  28. data/spec/mock_redis/lib/mock_redis/list_methods.rb +0 -187
  29. data/spec/mock_redis/lib/mock_redis/multi_db_wrapper.rb +0 -86
  30. data/spec/mock_redis/lib/mock_redis/set_methods.rb +0 -126
  31. data/spec/mock_redis/lib/mock_redis/string_methods.rb +0 -203
  32. data/spec/mock_redis/lib/mock_redis/transaction_wrapper.rb +0 -80
  33. data/spec/mock_redis/lib/mock_redis/undef_redis_methods.rb +0 -11
  34. data/spec/mock_redis/lib/mock_redis/utility_methods.rb +0 -25
  35. data/spec/mock_redis/lib/mock_redis/version.rb +0 -3
  36. data/spec/mock_redis/lib/mock_redis/zset.rb +0 -110
  37. data/spec/mock_redis/lib/mock_redis/zset_methods.rb +0 -210
  38. data/spec/mock_redis/lib/mock_redis.rb +0 -119
  39. data/spec/redis_helper_spec.rb +0 -58
@@ -1,203 +0,0 @@
1
- require 'mock_redis/assertions'
2
-
3
- class MockRedis
4
- module StringMethods
5
- include Assertions
6
-
7
- def append(key, value)
8
- assert_stringy(key)
9
- data[key] ||= ""
10
- data[key] << value
11
- data[key].length
12
- end
13
-
14
- def decr(key)
15
- decrby(key, 1)
16
- end
17
-
18
- def decrby(key, n)
19
- incrby(key, -n)
20
- end
21
-
22
- def get(key)
23
- assert_stringy(key)
24
- data[key]
25
- end
26
-
27
- def [](key)
28
- get(key)
29
- end
30
-
31
- def getbit(key, offset)
32
- assert_stringy(key)
33
-
34
- offset_of_byte = offset / 8
35
- offset_within_byte = offset % 8
36
-
37
- # String#getbyte would be lovely, but it's not in 1.8.7.
38
- byte = (data[key] || "").each_byte.drop(offset_of_byte).first
39
-
40
- if byte
41
- (byte & (2**7 >> offset_within_byte)) > 0 ? 1 : 0
42
- else
43
- 0
44
- end
45
- end
46
-
47
- def getrange(key, start, stop)
48
- assert_stringy(key)
49
- (data[key] || "")[start..stop]
50
- end
51
-
52
- def getset(key, value)
53
- retval = get(key)
54
- set(key, value)
55
- retval
56
- end
57
-
58
- def incr(key)
59
- incrby(key, 1)
60
- end
61
-
62
- def incrby(key, n)
63
- assert_stringy(key)
64
- unless can_incr?(data[key])
65
- raise RuntimeError, "ERR value is not an integer or out of range"
66
- end
67
-
68
- unless looks_like_integer?(n.to_s)
69
- raise RuntimeError, "ERR value is not an integer or out of range"
70
- end
71
-
72
- new_value = data[key].to_i + n.to_i
73
- data[key] = new_value.to_s
74
- # for some reason, redis-rb doesn't return this as a string.
75
- new_value
76
- end
77
-
78
- def mget(*keys)
79
- assert_has_args(keys, 'mget')
80
-
81
- keys.map do |key|
82
- get(key) if stringy?(key)
83
- end
84
- end
85
-
86
- def mset(*kvpairs)
87
- assert_has_args(kvpairs, 'mset')
88
- if kvpairs.length.odd?
89
- raise RuntimeError, "ERR wrong number of arguments for MSET"
90
- end
91
-
92
- kvpairs.each_slice(2) do |(k,v)|
93
- set(k,v)
94
- end
95
-
96
- "OK"
97
- end
98
-
99
- def msetnx(*kvpairs)
100
- assert_has_args(kvpairs, 'msetnx')
101
-
102
- if kvpairs.each_slice(2).any? {|(k,v)| exists(k)}
103
- 0
104
- else
105
- mset(*kvpairs)
106
- 1
107
- end
108
- end
109
-
110
- def set(key, value)
111
- data[key] = value.to_s
112
- 'OK'
113
- end
114
-
115
- def []=(key, value)
116
- set(key, value)
117
- end
118
-
119
- def setbit(key, offset, value)
120
- assert_stringy(key, "ERR bit is not an integer or out of range")
121
- retval = getbit(key, offset)
122
-
123
- str = data[key] || ""
124
-
125
- offset_of_byte = offset / 8
126
- offset_within_byte = offset % 8
127
-
128
- if offset_of_byte >= str.bytesize
129
- str = zero_pad(str, offset_of_byte+1)
130
- end
131
-
132
- char_index = byte_index = offset_within_char = 0
133
- str.each_char do |c|
134
- if byte_index < offset_of_byte
135
- char_index += 1
136
- byte_index += c.bytesize
137
- else
138
- offset_within_char = byte_index - offset_of_byte
139
- break
140
- end
141
- end
142
-
143
- char = str[char_index]
144
- char = char.chr if char.respond_to?(:chr) # ruby 1.8 vs 1.9
145
- char_as_number = char.each_byte.reduce(0) do |a, byte|
146
- (a << 8) + byte
147
- end
148
- char_as_number |=
149
- (2**((char.bytesize * 8)-1) >>
150
- (offset_within_char * 8 + offset_within_byte))
151
- str[char_index] = char_as_number.chr
152
-
153
- data[key] = str
154
- retval
155
- end
156
-
157
- def setex(key, seconds, value)
158
- set(key, value)
159
- expire(key, seconds)
160
- 'OK'
161
- end
162
-
163
- def setnx(key, value)
164
- if exists(key)
165
- false
166
- else
167
- set(key, value)
168
- true
169
- end
170
- end
171
-
172
- def setrange(key, offset, value)
173
- assert_stringy(key)
174
- value = value.to_s
175
- old_value = (data[key] || "")
176
-
177
- prefix = zero_pad(old_value[0...offset], offset)
178
- data[key] = prefix + value + (old_value[(offset + value.length)..-1] || "")
179
- data[key].length
180
- end
181
-
182
- def strlen(key)
183
- assert_stringy(key)
184
- (data[key] || "").bytesize
185
- end
186
-
187
-
188
-
189
-
190
- private
191
- def stringy?(key)
192
- data[key].nil? || data[key].kind_of?(String)
193
- end
194
-
195
- def assert_stringy(key,
196
- message="ERR Operation against a key holding the wrong kind of value")
197
- unless stringy?(key)
198
- raise RuntimeError, message
199
- end
200
- end
201
-
202
- end
203
- end
@@ -1,80 +0,0 @@
1
- require 'mock_redis/undef_redis_methods'
2
-
3
- class MockRedis
4
- class TransactionWrapper
5
- include UndefRedisMethods
6
-
7
- def respond_to?(method, include_private=false)
8
- super || @db.respond_to?(method)
9
- end
10
-
11
- def initialize(db)
12
- @db = db
13
- @queued_commands = []
14
- @in_multi = false
15
- end
16
-
17
- def method_missing(method, *args)
18
- if @in_multi
19
- @queued_commands << [method, *args]
20
- 'QUEUED'
21
- else
22
- @db.expire_keys
23
- @db.send(method, *args)
24
- end
25
- end
26
-
27
- def initialize_copy(source)
28
- super
29
- @db = @db.clone
30
- @queued_commands = @queued_commands.clone
31
- end
32
-
33
- def discard
34
- unless @in_multi
35
- raise RuntimeError, "ERR DISCARD without MULTI"
36
- end
37
- @in_multi = false
38
- @queued_commands = []
39
- 'OK'
40
- end
41
-
42
- def exec
43
- unless @in_multi
44
- raise RuntimeError, "ERR EXEC without MULTI"
45
- end
46
- @in_multi = false
47
- responses = @queued_commands.map do |cmd|
48
- begin
49
- send(*cmd)
50
- rescue => e
51
- e
52
- end
53
- end
54
- @queued_commands = []
55
- responses
56
- end
57
-
58
- def multi
59
- if @in_multi
60
- raise RuntimeError, "ERR MULTI calls can not be nested"
61
- end
62
- @in_multi = true
63
- if block_given?
64
- yield(self)
65
- self.exec
66
- else
67
- 'OK'
68
- end
69
- end
70
-
71
- def unwatch
72
- 'OK'
73
- end
74
-
75
- def watch(_)
76
- 'OK'
77
- end
78
-
79
- end
80
- end
@@ -1,11 +0,0 @@
1
- class MockRedis
2
- module UndefRedisMethods
3
- def self.included(klass)
4
- if klass.instance_methods.map(&:to_s).include?('type')
5
- klass.send(:undef_method, 'type')
6
- end
7
- klass.send(:undef_method, 'exec')
8
- klass.send(:undef_method, 'select')
9
- end
10
- end
11
- end
@@ -1,25 +0,0 @@
1
- class MockRedis
2
- module UtilityMethods
3
- private
4
-
5
- def with_thing_at(key, assertion, empty_thing_generator)
6
- begin
7
- send(assertion, key)
8
- data[key] ||= empty_thing_generator.call
9
- data_key_ref = data[key]
10
- ret = yield data[key]
11
- data[key] = data_key_ref if data[key].nil?
12
- ret
13
- ensure
14
- clean_up_empties_at(key)
15
- end
16
- end
17
-
18
- def clean_up_empties_at(key)
19
- if data[key] && data[key].empty?
20
- del(key)
21
- end
22
- end
23
-
24
- end
25
- end
@@ -1,3 +0,0 @@
1
- class MockRedis
2
- VERSION = '0.4.1'
3
- end
@@ -1,110 +0,0 @@
1
- require 'forwardable'
2
- require 'set'
3
-
4
- class MockRedis
5
- class Zset
6
- include Enumerable
7
- extend Forwardable
8
-
9
- attr_reader :members, :scores
10
-
11
- def_delegators :members, :empty?, :include?, :size
12
-
13
- def initialize
14
- @members = Set.new
15
- @scores = Hash.new
16
- end
17
-
18
- def initialize_copy(source)
19
- super
20
- @members = @members.clone
21
- @scores = @scores.clone
22
- end
23
-
24
- def add(score, member)
25
- members.add(member)
26
- if score.to_f.to_i == score.to_f
27
- scores[member] = score.to_f.to_i
28
- else
29
- scores[member] = score.to_f
30
- end
31
- self
32
- end
33
-
34
- def delete?(member)
35
- scores.delete(member)
36
- members.delete?(member) and self
37
- end
38
-
39
- def each
40
- members.each {|m| yield score(m), m}
41
- end
42
-
43
- def in_range(min, max)
44
- in_from_the_left = case min
45
- when "-inf"
46
- lambda {|_| true }
47
- when "+inf"
48
- lambda {|_| false }
49
- when /\((.*)$/
50
- val = $1.to_f
51
- lambda {|x| x.to_f > val }
52
- else
53
- lambda {|x| x.to_f >= min.to_f }
54
- end
55
-
56
- in_from_the_right = case max
57
- when "-inf"
58
- lambda {|_| false }
59
- when "+inf"
60
- lambda {|_| true }
61
- when /\((.*)$/
62
- val = $1.to_f
63
- lambda {|x| x.to_f < val }
64
- else
65
- lambda {|x| x.to_f <= max.to_f }
66
- end
67
-
68
- sorted.find_all do |(score, member)|
69
- in_from_the_left[score] && in_from_the_right[score]
70
- end
71
- end
72
-
73
- def intersection(other)
74
- if !block_given?
75
- intersection(other, &:+)
76
- else
77
- self.members.intersection(other.members).reduce(self.class.new) do |acc, m|
78
- new_score = yield(self.score(m), other.score(m))
79
- acc.add(new_score, m)
80
- end
81
- end
82
- end
83
-
84
- def score(member)
85
- scores[member]
86
- end
87
-
88
- def sorted
89
- members.map do |m|
90
- [score(m), m]
91
- end.sort_by(&:first)
92
- end
93
-
94
- def sorted_members
95
- sorted.map(&:last)
96
- end
97
-
98
- def union(other)
99
- if !block_given?
100
- union(other, &:+)
101
- else
102
- self.members.union(other.members).reduce(self.class.new) do |acc, m|
103
- new_score = yield(self.score(m), other.score(m))
104
- acc.add(new_score, m)
105
- end
106
- end
107
- end
108
-
109
- end
110
- end
@@ -1,210 +0,0 @@
1
- require 'mock_redis/assertions'
2
- require 'mock_redis/utility_methods'
3
- require 'mock_redis/zset'
4
-
5
- class MockRedis
6
- module ZsetMethods
7
- include Assertions
8
- include UtilityMethods
9
-
10
- def zadd(key, score, member)
11
- assert_scorey(score)
12
-
13
- retval = !zscore(key, member)
14
- with_zset_at(key) {|z| z.add(score, member.to_s)}
15
- retval
16
- end
17
-
18
- def zcard(key)
19
- with_zset_at(key, &:size)
20
- end
21
-
22
- def zcount(key, min, max)
23
- assert_scorey(min, 'min or max')
24
- assert_scorey(max, 'min or max')
25
-
26
- with_zset_at(key) do |z|
27
- z.count do |score, _|
28
- score >= min && score <= max
29
- end
30
- end
31
- end
32
-
33
- def zincrby(key, increment, member)
34
- assert_scorey(increment)
35
- member = member.to_s
36
- with_zset_at(key) do |z|
37
- old_score = z.include?(member) ? z.score(member) : 0
38
- new_score = old_score + increment
39
- z.add(new_score, member)
40
- new_score.to_s
41
- end
42
- end
43
-
44
- def zinterstore(destination, keys, options={})
45
- assert_has_args(keys, 'zinterstore')
46
-
47
- data[destination] = combine_weighted_zsets(keys, options, :intersection)
48
- zcard(destination)
49
- end
50
-
51
- def zrange(key, start, stop, options={})
52
- with_zset_at(key) do |z|
53
- to_response(z.sorted[start..stop] || [], options)
54
- end
55
- end
56
-
57
- def zrangebyscore(key, min, max, options={})
58
- with_zset_at(key) do |zset|
59
- all_results = zset.in_range(min, max)
60
- to_response(apply_limit(all_results, options[:limit]), options)
61
- end
62
- end
63
-
64
- def zrank(key, member)
65
- with_zset_at(key) {|z| z.sorted_members.index(member.to_s) }
66
- end
67
-
68
- def zrem(key, member)
69
- with_zset_at(key) {|z| !!z.delete?(member.to_s)}
70
- end
71
-
72
- def zrevrange(key, start, stop, options={})
73
- with_zset_at(key) do |z|
74
- to_response(z.sorted.reverse[start..stop], options)
75
- end
76
- end
77
-
78
- def zremrangebyrank(key, start, stop)
79
- zrange(key, start, stop).
80
- each {|member| zrem(key, member)}.
81
- size
82
- end
83
-
84
- def zremrangebyscore(key, min, max)
85
- zrangebyscore(key, min, max).
86
- each {|member| zrem(key, member)}.
87
- size
88
- end
89
-
90
- def zrevrangebyscore(key, max, min, options={})
91
- with_zset_at(key) do |zset|
92
- to_response(
93
- apply_limit(
94
- zset.in_range(min, max).reverse,
95
- options[:limit]),
96
- options)
97
- end
98
- end
99
-
100
- def zrevrank(key, member)
101
- with_zset_at(key) {|z| z.sorted_members.reverse.index(member.to_s) }
102
- end
103
-
104
- def zscore(key, member)
105
- with_zset_at(key) do |z|
106
- score = z.score(member.to_s)
107
- score.to_s if score
108
- end
109
- end
110
-
111
- def zunionstore(destination, keys, options={})
112
- assert_has_args(keys, 'zunionstore')
113
-
114
- data[destination] = combine_weighted_zsets(keys, options, :union)
115
- zcard(destination)
116
- end
117
-
118
- private
119
- def apply_limit(collection, limit)
120
- if limit
121
- if limit.is_a?(Array) && limit.length == 2
122
- offset, count = limit
123
- collection.drop(offset).take(count)
124
- else
125
- raise RuntimeError, "ERR syntax error"
126
- end
127
- else
128
- collection
129
- end
130
- end
131
-
132
- def to_response(score_member_pairs, options)
133
- score_member_pairs.map do |(score,member)|
134
- if options[:with_scores] || options[:withscores]
135
- [member, score.to_s]
136
- else
137
- member
138
- end
139
- end.flatten
140
- end
141
-
142
- def combine_weighted_zsets(keys, options, how)
143
- weights = options.fetch(:weights, keys.map { 1 })
144
- if weights.length != keys.length
145
- raise RuntimeError, "ERR syntax error"
146
- end
147
-
148
- aggregator = case options.fetch(:aggregate, :sum).to_s.downcase.to_sym
149
- when :sum
150
- proc {|a,b| [a,b].compact.reduce(&:+)}
151
- when :min
152
- proc {|a,b| [a,b].compact.min}
153
- when :max
154
- proc {|a,b| [a,b].compact.max}
155
- else
156
- raise RuntimeError, "ERR syntax error"
157
- end
158
-
159
- with_zsets_at(*keys) do |*zsets|
160
- zsets.zip(weights).map do |(zset, weight)|
161
- zset.reduce(Zset.new) do |acc, (score, member)|
162
- acc.add(score * weight, member)
163
- end
164
- end.reduce do |za, zb|
165
- za.send(how, zb, &aggregator)
166
- end
167
- end
168
-
169
- end
170
-
171
- def with_zset_at(key, &blk)
172
- with_thing_at(key, :assert_zsety, proc {Zset.new}, &blk)
173
- end
174
-
175
- def with_zsets_at(*keys, &blk)
176
- if keys.length == 1
177
- with_zset_at(keys.first, &blk)
178
- else
179
- with_zset_at(keys.first) do |set|
180
- with_zsets_at(*(keys[1..-1])) do |*sets|
181
- blk.call(*([set] + sets))
182
- end
183
- end
184
- end
185
- end
186
-
187
- def zsety?(key)
188
- data[key].nil? || data[key].kind_of?(Zset)
189
- end
190
-
191
- def assert_zsety(key)
192
- unless zsety?(key)
193
- raise RuntimeError,
194
- "ERR Operation against a key holding the wrong kind of value"
195
- end
196
- end
197
-
198
- def looks_like_float?(x)
199
- # ugh, exceptions for flow control.
200
- !!Float(x) rescue false
201
- end
202
-
203
- def assert_scorey(value, what='value')
204
- unless looks_like_float?(value)
205
- raise RuntimeError, "ERR #{what} is not a double"
206
- end
207
- end
208
-
209
- end
210
- end