redis_migrator 0.0.1 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/Gemfile +3 -0
  4. data/README.md +13 -1
  5. data/Rakefile +6 -0
  6. data/lib/redis_migrator/redis_helper.rb +5 -36
  7. data/lib/redis_migrator/redis_native_migrator.rb +27 -0
  8. data/lib/redis_migrator/redis_pipe_migrator.rb +66 -0
  9. data/lib/redis_migrator/redis_populator.rb +1 -1
  10. data/lib/redis_migrator.rb +19 -20
  11. data/migrator_benchmark.rb +1 -2
  12. data/redis_migrator.gemspec +8 -4
  13. data/spec/different_redis_type_migrator.rb +67 -0
  14. data/spec/pretested_migrator.rb +47 -0
  15. data/spec/redis_migrator_spec.rb +41 -0
  16. data/spec/redis_native_migrator_spec.rb +44 -0
  17. data/spec/redis_pipe_migrator_spec.rb +51 -0
  18. data/spec/shared_hosts_context.rb +10 -0
  19. data/spec/spec_helper.rb +9 -7
  20. metadata +85 -49
  21. data/spec/migrator_spec.rb +0 -63
  22. data/spec/mock_redis/lib/mock_redis/assertions.rb +0 -13
  23. data/spec/mock_redis/lib/mock_redis/database.rb +0 -432
  24. data/spec/mock_redis/lib/mock_redis/distributed.rb +0 -6
  25. data/spec/mock_redis/lib/mock_redis/exceptions.rb +0 -3
  26. data/spec/mock_redis/lib/mock_redis/expire_wrapper.rb +0 -25
  27. data/spec/mock_redis/lib/mock_redis/hash_methods.rb +0 -118
  28. data/spec/mock_redis/lib/mock_redis/list_methods.rb +0 -187
  29. data/spec/mock_redis/lib/mock_redis/multi_db_wrapper.rb +0 -86
  30. data/spec/mock_redis/lib/mock_redis/set_methods.rb +0 -126
  31. data/spec/mock_redis/lib/mock_redis/string_methods.rb +0 -203
  32. data/spec/mock_redis/lib/mock_redis/transaction_wrapper.rb +0 -80
  33. data/spec/mock_redis/lib/mock_redis/undef_redis_methods.rb +0 -11
  34. data/spec/mock_redis/lib/mock_redis/utility_methods.rb +0 -25
  35. data/spec/mock_redis/lib/mock_redis/version.rb +0 -3
  36. data/spec/mock_redis/lib/mock_redis/zset.rb +0 -110
  37. data/spec/mock_redis/lib/mock_redis/zset_methods.rb +0 -210
  38. data/spec/mock_redis/lib/mock_redis.rb +0 -119
  39. data/spec/redis_helper_spec.rb +0 -58
@@ -1,203 +0,0 @@
1
- require 'mock_redis/assertions'
2
-
3
- class MockRedis
4
- module StringMethods
5
- include Assertions
6
-
7
- def append(key, value)
8
- assert_stringy(key)
9
- data[key] ||= ""
10
- data[key] << value
11
- data[key].length
12
- end
13
-
14
- def decr(key)
15
- decrby(key, 1)
16
- end
17
-
18
- def decrby(key, n)
19
- incrby(key, -n)
20
- end
21
-
22
- def get(key)
23
- assert_stringy(key)
24
- data[key]
25
- end
26
-
27
- def [](key)
28
- get(key)
29
- end
30
-
31
- def getbit(key, offset)
32
- assert_stringy(key)
33
-
34
- offset_of_byte = offset / 8
35
- offset_within_byte = offset % 8
36
-
37
- # String#getbyte would be lovely, but it's not in 1.8.7.
38
- byte = (data[key] || "").each_byte.drop(offset_of_byte).first
39
-
40
- if byte
41
- (byte & (2**7 >> offset_within_byte)) > 0 ? 1 : 0
42
- else
43
- 0
44
- end
45
- end
46
-
47
- def getrange(key, start, stop)
48
- assert_stringy(key)
49
- (data[key] || "")[start..stop]
50
- end
51
-
52
- def getset(key, value)
53
- retval = get(key)
54
- set(key, value)
55
- retval
56
- end
57
-
58
- def incr(key)
59
- incrby(key, 1)
60
- end
61
-
62
- def incrby(key, n)
63
- assert_stringy(key)
64
- unless can_incr?(data[key])
65
- raise RuntimeError, "ERR value is not an integer or out of range"
66
- end
67
-
68
- unless looks_like_integer?(n.to_s)
69
- raise RuntimeError, "ERR value is not an integer or out of range"
70
- end
71
-
72
- new_value = data[key].to_i + n.to_i
73
- data[key] = new_value.to_s
74
- # for some reason, redis-rb doesn't return this as a string.
75
- new_value
76
- end
77
-
78
- def mget(*keys)
79
- assert_has_args(keys, 'mget')
80
-
81
- keys.map do |key|
82
- get(key) if stringy?(key)
83
- end
84
- end
85
-
86
- def mset(*kvpairs)
87
- assert_has_args(kvpairs, 'mset')
88
- if kvpairs.length.odd?
89
- raise RuntimeError, "ERR wrong number of arguments for MSET"
90
- end
91
-
92
- kvpairs.each_slice(2) do |(k,v)|
93
- set(k,v)
94
- end
95
-
96
- "OK"
97
- end
98
-
99
- def msetnx(*kvpairs)
100
- assert_has_args(kvpairs, 'msetnx')
101
-
102
- if kvpairs.each_slice(2).any? {|(k,v)| exists(k)}
103
- 0
104
- else
105
- mset(*kvpairs)
106
- 1
107
- end
108
- end
109
-
110
- def set(key, value)
111
- data[key] = value.to_s
112
- 'OK'
113
- end
114
-
115
- def []=(key, value)
116
- set(key, value)
117
- end
118
-
119
- def setbit(key, offset, value)
120
- assert_stringy(key, "ERR bit is not an integer or out of range")
121
- retval = getbit(key, offset)
122
-
123
- str = data[key] || ""
124
-
125
- offset_of_byte = offset / 8
126
- offset_within_byte = offset % 8
127
-
128
- if offset_of_byte >= str.bytesize
129
- str = zero_pad(str, offset_of_byte+1)
130
- end
131
-
132
- char_index = byte_index = offset_within_char = 0
133
- str.each_char do |c|
134
- if byte_index < offset_of_byte
135
- char_index += 1
136
- byte_index += c.bytesize
137
- else
138
- offset_within_char = byte_index - offset_of_byte
139
- break
140
- end
141
- end
142
-
143
- char = str[char_index]
144
- char = char.chr if char.respond_to?(:chr) # ruby 1.8 vs 1.9
145
- char_as_number = char.each_byte.reduce(0) do |a, byte|
146
- (a << 8) + byte
147
- end
148
- char_as_number |=
149
- (2**((char.bytesize * 8)-1) >>
150
- (offset_within_char * 8 + offset_within_byte))
151
- str[char_index] = char_as_number.chr
152
-
153
- data[key] = str
154
- retval
155
- end
156
-
157
- def setex(key, seconds, value)
158
- set(key, value)
159
- expire(key, seconds)
160
- 'OK'
161
- end
162
-
163
- def setnx(key, value)
164
- if exists(key)
165
- false
166
- else
167
- set(key, value)
168
- true
169
- end
170
- end
171
-
172
- def setrange(key, offset, value)
173
- assert_stringy(key)
174
- value = value.to_s
175
- old_value = (data[key] || "")
176
-
177
- prefix = zero_pad(old_value[0...offset], offset)
178
- data[key] = prefix + value + (old_value[(offset + value.length)..-1] || "")
179
- data[key].length
180
- end
181
-
182
- def strlen(key)
183
- assert_stringy(key)
184
- (data[key] || "").bytesize
185
- end
186
-
187
-
188
-
189
-
190
- private
191
- def stringy?(key)
192
- data[key].nil? || data[key].kind_of?(String)
193
- end
194
-
195
- def assert_stringy(key,
196
- message="ERR Operation against a key holding the wrong kind of value")
197
- unless stringy?(key)
198
- raise RuntimeError, message
199
- end
200
- end
201
-
202
- end
203
- end
@@ -1,80 +0,0 @@
1
- require 'mock_redis/undef_redis_methods'
2
-
3
- class MockRedis
4
- class TransactionWrapper
5
- include UndefRedisMethods
6
-
7
- def respond_to?(method, include_private=false)
8
- super || @db.respond_to?(method)
9
- end
10
-
11
- def initialize(db)
12
- @db = db
13
- @queued_commands = []
14
- @in_multi = false
15
- end
16
-
17
- def method_missing(method, *args)
18
- if @in_multi
19
- @queued_commands << [method, *args]
20
- 'QUEUED'
21
- else
22
- @db.expire_keys
23
- @db.send(method, *args)
24
- end
25
- end
26
-
27
- def initialize_copy(source)
28
- super
29
- @db = @db.clone
30
- @queued_commands = @queued_commands.clone
31
- end
32
-
33
- def discard
34
- unless @in_multi
35
- raise RuntimeError, "ERR DISCARD without MULTI"
36
- end
37
- @in_multi = false
38
- @queued_commands = []
39
- 'OK'
40
- end
41
-
42
- def exec
43
- unless @in_multi
44
- raise RuntimeError, "ERR EXEC without MULTI"
45
- end
46
- @in_multi = false
47
- responses = @queued_commands.map do |cmd|
48
- begin
49
- send(*cmd)
50
- rescue => e
51
- e
52
- end
53
- end
54
- @queued_commands = []
55
- responses
56
- end
57
-
58
- def multi
59
- if @in_multi
60
- raise RuntimeError, "ERR MULTI calls can not be nested"
61
- end
62
- @in_multi = true
63
- if block_given?
64
- yield(self)
65
- self.exec
66
- else
67
- 'OK'
68
- end
69
- end
70
-
71
- def unwatch
72
- 'OK'
73
- end
74
-
75
- def watch(_)
76
- 'OK'
77
- end
78
-
79
- end
80
- end
@@ -1,11 +0,0 @@
1
- class MockRedis
2
- module UndefRedisMethods
3
- def self.included(klass)
4
- if klass.instance_methods.map(&:to_s).include?('type')
5
- klass.send(:undef_method, 'type')
6
- end
7
- klass.send(:undef_method, 'exec')
8
- klass.send(:undef_method, 'select')
9
- end
10
- end
11
- end
@@ -1,25 +0,0 @@
1
- class MockRedis
2
- module UtilityMethods
3
- private
4
-
5
- def with_thing_at(key, assertion, empty_thing_generator)
6
- begin
7
- send(assertion, key)
8
- data[key] ||= empty_thing_generator.call
9
- data_key_ref = data[key]
10
- ret = yield data[key]
11
- data[key] = data_key_ref if data[key].nil?
12
- ret
13
- ensure
14
- clean_up_empties_at(key)
15
- end
16
- end
17
-
18
- def clean_up_empties_at(key)
19
- if data[key] && data[key].empty?
20
- del(key)
21
- end
22
- end
23
-
24
- end
25
- end
@@ -1,3 +0,0 @@
1
- class MockRedis
2
- VERSION = '0.4.1'
3
- end
@@ -1,110 +0,0 @@
1
- require 'forwardable'
2
- require 'set'
3
-
4
- class MockRedis
5
- class Zset
6
- include Enumerable
7
- extend Forwardable
8
-
9
- attr_reader :members, :scores
10
-
11
- def_delegators :members, :empty?, :include?, :size
12
-
13
- def initialize
14
- @members = Set.new
15
- @scores = Hash.new
16
- end
17
-
18
- def initialize_copy(source)
19
- super
20
- @members = @members.clone
21
- @scores = @scores.clone
22
- end
23
-
24
- def add(score, member)
25
- members.add(member)
26
- if score.to_f.to_i == score.to_f
27
- scores[member] = score.to_f.to_i
28
- else
29
- scores[member] = score.to_f
30
- end
31
- self
32
- end
33
-
34
- def delete?(member)
35
- scores.delete(member)
36
- members.delete?(member) and self
37
- end
38
-
39
- def each
40
- members.each {|m| yield score(m), m}
41
- end
42
-
43
- def in_range(min, max)
44
- in_from_the_left = case min
45
- when "-inf"
46
- lambda {|_| true }
47
- when "+inf"
48
- lambda {|_| false }
49
- when /\((.*)$/
50
- val = $1.to_f
51
- lambda {|x| x.to_f > val }
52
- else
53
- lambda {|x| x.to_f >= min.to_f }
54
- end
55
-
56
- in_from_the_right = case max
57
- when "-inf"
58
- lambda {|_| false }
59
- when "+inf"
60
- lambda {|_| true }
61
- when /\((.*)$/
62
- val = $1.to_f
63
- lambda {|x| x.to_f < val }
64
- else
65
- lambda {|x| x.to_f <= max.to_f }
66
- end
67
-
68
- sorted.find_all do |(score, member)|
69
- in_from_the_left[score] && in_from_the_right[score]
70
- end
71
- end
72
-
73
- def intersection(other)
74
- if !block_given?
75
- intersection(other, &:+)
76
- else
77
- self.members.intersection(other.members).reduce(self.class.new) do |acc, m|
78
- new_score = yield(self.score(m), other.score(m))
79
- acc.add(new_score, m)
80
- end
81
- end
82
- end
83
-
84
- def score(member)
85
- scores[member]
86
- end
87
-
88
- def sorted
89
- members.map do |m|
90
- [score(m), m]
91
- end.sort_by(&:first)
92
- end
93
-
94
- def sorted_members
95
- sorted.map(&:last)
96
- end
97
-
98
- def union(other)
99
- if !block_given?
100
- union(other, &:+)
101
- else
102
- self.members.union(other.members).reduce(self.class.new) do |acc, m|
103
- new_score = yield(self.score(m), other.score(m))
104
- acc.add(new_score, m)
105
- end
106
- end
107
- end
108
-
109
- end
110
- end
@@ -1,210 +0,0 @@
1
- require 'mock_redis/assertions'
2
- require 'mock_redis/utility_methods'
3
- require 'mock_redis/zset'
4
-
5
- class MockRedis
6
- module ZsetMethods
7
- include Assertions
8
- include UtilityMethods
9
-
10
- def zadd(key, score, member)
11
- assert_scorey(score)
12
-
13
- retval = !zscore(key, member)
14
- with_zset_at(key) {|z| z.add(score, member.to_s)}
15
- retval
16
- end
17
-
18
- def zcard(key)
19
- with_zset_at(key, &:size)
20
- end
21
-
22
- def zcount(key, min, max)
23
- assert_scorey(min, 'min or max')
24
- assert_scorey(max, 'min or max')
25
-
26
- with_zset_at(key) do |z|
27
- z.count do |score, _|
28
- score >= min && score <= max
29
- end
30
- end
31
- end
32
-
33
- def zincrby(key, increment, member)
34
- assert_scorey(increment)
35
- member = member.to_s
36
- with_zset_at(key) do |z|
37
- old_score = z.include?(member) ? z.score(member) : 0
38
- new_score = old_score + increment
39
- z.add(new_score, member)
40
- new_score.to_s
41
- end
42
- end
43
-
44
- def zinterstore(destination, keys, options={})
45
- assert_has_args(keys, 'zinterstore')
46
-
47
- data[destination] = combine_weighted_zsets(keys, options, :intersection)
48
- zcard(destination)
49
- end
50
-
51
- def zrange(key, start, stop, options={})
52
- with_zset_at(key) do |z|
53
- to_response(z.sorted[start..stop] || [], options)
54
- end
55
- end
56
-
57
- def zrangebyscore(key, min, max, options={})
58
- with_zset_at(key) do |zset|
59
- all_results = zset.in_range(min, max)
60
- to_response(apply_limit(all_results, options[:limit]), options)
61
- end
62
- end
63
-
64
- def zrank(key, member)
65
- with_zset_at(key) {|z| z.sorted_members.index(member.to_s) }
66
- end
67
-
68
- def zrem(key, member)
69
- with_zset_at(key) {|z| !!z.delete?(member.to_s)}
70
- end
71
-
72
- def zrevrange(key, start, stop, options={})
73
- with_zset_at(key) do |z|
74
- to_response(z.sorted.reverse[start..stop], options)
75
- end
76
- end
77
-
78
- def zremrangebyrank(key, start, stop)
79
- zrange(key, start, stop).
80
- each {|member| zrem(key, member)}.
81
- size
82
- end
83
-
84
- def zremrangebyscore(key, min, max)
85
- zrangebyscore(key, min, max).
86
- each {|member| zrem(key, member)}.
87
- size
88
- end
89
-
90
- def zrevrangebyscore(key, max, min, options={})
91
- with_zset_at(key) do |zset|
92
- to_response(
93
- apply_limit(
94
- zset.in_range(min, max).reverse,
95
- options[:limit]),
96
- options)
97
- end
98
- end
99
-
100
- def zrevrank(key, member)
101
- with_zset_at(key) {|z| z.sorted_members.reverse.index(member.to_s) }
102
- end
103
-
104
- def zscore(key, member)
105
- with_zset_at(key) do |z|
106
- score = z.score(member.to_s)
107
- score.to_s if score
108
- end
109
- end
110
-
111
- def zunionstore(destination, keys, options={})
112
- assert_has_args(keys, 'zunionstore')
113
-
114
- data[destination] = combine_weighted_zsets(keys, options, :union)
115
- zcard(destination)
116
- end
117
-
118
- private
119
- def apply_limit(collection, limit)
120
- if limit
121
- if limit.is_a?(Array) && limit.length == 2
122
- offset, count = limit
123
- collection.drop(offset).take(count)
124
- else
125
- raise RuntimeError, "ERR syntax error"
126
- end
127
- else
128
- collection
129
- end
130
- end
131
-
132
- def to_response(score_member_pairs, options)
133
- score_member_pairs.map do |(score,member)|
134
- if options[:with_scores] || options[:withscores]
135
- [member, score.to_s]
136
- else
137
- member
138
- end
139
- end.flatten
140
- end
141
-
142
- def combine_weighted_zsets(keys, options, how)
143
- weights = options.fetch(:weights, keys.map { 1 })
144
- if weights.length != keys.length
145
- raise RuntimeError, "ERR syntax error"
146
- end
147
-
148
- aggregator = case options.fetch(:aggregate, :sum).to_s.downcase.to_sym
149
- when :sum
150
- proc {|a,b| [a,b].compact.reduce(&:+)}
151
- when :min
152
- proc {|a,b| [a,b].compact.min}
153
- when :max
154
- proc {|a,b| [a,b].compact.max}
155
- else
156
- raise RuntimeError, "ERR syntax error"
157
- end
158
-
159
- with_zsets_at(*keys) do |*zsets|
160
- zsets.zip(weights).map do |(zset, weight)|
161
- zset.reduce(Zset.new) do |acc, (score, member)|
162
- acc.add(score * weight, member)
163
- end
164
- end.reduce do |za, zb|
165
- za.send(how, zb, &aggregator)
166
- end
167
- end
168
-
169
- end
170
-
171
- def with_zset_at(key, &blk)
172
- with_thing_at(key, :assert_zsety, proc {Zset.new}, &blk)
173
- end
174
-
175
- def with_zsets_at(*keys, &blk)
176
- if keys.length == 1
177
- with_zset_at(keys.first, &blk)
178
- else
179
- with_zset_at(keys.first) do |set|
180
- with_zsets_at(*(keys[1..-1])) do |*sets|
181
- blk.call(*([set] + sets))
182
- end
183
- end
184
- end
185
- end
186
-
187
- def zsety?(key)
188
- data[key].nil? || data[key].kind_of?(Zset)
189
- end
190
-
191
- def assert_zsety(key)
192
- unless zsety?(key)
193
- raise RuntimeError,
194
- "ERR Operation against a key holding the wrong kind of value"
195
- end
196
- end
197
-
198
- def looks_like_float?(x)
199
- # ugh, exceptions for flow control.
200
- !!Float(x) rescue false
201
- end
202
-
203
- def assert_scorey(value, what='value')
204
- unless looks_like_float?(value)
205
- raise RuntimeError, "ERR #{what} is not a double"
206
- end
207
- end
208
-
209
- end
210
- end