dynport_tools 0.2.12 → 0.2.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/VERSION +1 -1
- data/dynport_tools.gemspec +1 -1
- data/lib/dynport_tools/redis_q.rb +16 -13
- data/spec/dynport_tools/redis_q_spec.rb +42 -2
- data/spec/spec_helper.rb +6 -0
- metadata +3 -3
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.2.
|
1
|
+
0.2.13
|
data/dynport_tools.gemspec
CHANGED
@@ -20,8 +20,10 @@ class DynportTools::RedisQ
|
|
20
20
|
|
21
21
|
def push_many(array, options = {})
|
22
22
|
redis.multi do
|
23
|
-
|
24
|
-
|
23
|
+
array.each do | (id, popularity) |
|
24
|
+
(id.is_a?(Hash) ? id : { id => popularity }).each do |id2, popularity2|
|
25
|
+
push(id2, popularity2, options.merge(:no_multi => true))
|
26
|
+
end
|
25
27
|
end
|
26
28
|
end
|
27
29
|
end
|
@@ -38,27 +40,28 @@ class DynportTools::RedisQ
|
|
38
40
|
redis.zcard(redis_key)
|
39
41
|
end
|
40
42
|
|
41
|
-
def pop
|
42
|
-
redis.multi do
|
43
|
-
redis.zrevrange(redis_key, 0,
|
44
|
-
redis.zremrangebyrank(redis_key, -
|
45
|
-
end.first
|
43
|
+
def pop(number = 1)
|
44
|
+
Hash[*redis.multi do
|
45
|
+
redis.zrevrange(redis_key, 0, number - 1, :with_scores => true)
|
46
|
+
redis.zremrangebyrank(redis_key, 0 - number, -1)
|
47
|
+
end.first]
|
46
48
|
end
|
47
49
|
|
48
50
|
def failed_tries
|
49
51
|
@failed_tries ||= Hash.new(0)
|
50
52
|
end
|
51
53
|
|
52
|
-
def each
|
54
|
+
def each(options = {})
|
53
55
|
entries_with_errors = []
|
54
56
|
stats = { :errors => {}, :ok => [] }
|
55
|
-
|
57
|
+
batch_size = options[:batch_size].to_i if options[:batch_size].to_i > 0
|
58
|
+
while (result_hash = pop(batch_size || 1)).any?
|
56
59
|
begin
|
57
|
-
yield(
|
58
|
-
stats[:ok] <<
|
60
|
+
yield(batch_size ? result_hash.to_a.sort_by { |a| a.last }.reverse.map { |a| a.first } : result_hash.keys.first)
|
61
|
+
stats[:ok] << result_hash.keys.join(",")
|
59
62
|
rescue => err
|
60
|
-
stats[:errors][
|
61
|
-
entries_with_errors <<
|
63
|
+
stats[:errors][result_hash.keys.join(",")] = ([err.message] + err.backtrace[0,5]).join("\n")
|
64
|
+
entries_with_errors << result_hash if mark_failed(result_hash.keys.join(",")) < retry_count
|
62
65
|
end
|
63
66
|
end
|
64
67
|
push_many(entries_with_errors, :failed => true) if entries_with_errors.any?
|
@@ -37,6 +37,23 @@ describe DynportTools::RedisQ do
|
|
37
37
|
queue.redis_key.should == "some/queue"
|
38
38
|
end
|
39
39
|
|
40
|
+
describe "#push_many" do
|
41
|
+
it "pushes nested arrays" do
|
42
|
+
queue.push_many([[1, 2], [3, 4]])
|
43
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == ["3", "4", "1", "2"]
|
44
|
+
end
|
45
|
+
|
46
|
+
it "also pushes hashes" do
|
47
|
+
queue.push_many({ 2 => 4, 6 => 8})
|
48
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == ["6", "8", "2", "4"]
|
49
|
+
end
|
50
|
+
|
51
|
+
it "adds arrays of hashes" do
|
52
|
+
queue.push_many([{ 2 => 4 }, { 6 => 8}])
|
53
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == ["6", "8", "2", "4"]
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
40
57
|
describe "#push" do
|
41
58
|
it "pushes the records with negative timestamps" do
|
42
59
|
Timecop.freeze(Time.at(112233))
|
@@ -98,7 +115,18 @@ describe DynportTools::RedisQ do
|
|
98
115
|
end
|
99
116
|
|
100
117
|
it "returns the highest member and rank" do
|
101
|
-
queue.pop.should ==
|
118
|
+
queue.pop.should == { "101" => "100" }
|
119
|
+
end
|
120
|
+
|
121
|
+
it "can also return 2 elements" do
|
122
|
+
queue.pop(2).should == { "101" => "100", "98" => "10" }
|
123
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == ["99", "1"]
|
124
|
+
end
|
125
|
+
|
126
|
+
it "can also return 3 elements" do
|
127
|
+
queue.push("1", "0")
|
128
|
+
queue.pop(3).should == { "101" => "100", "98" => "10", "99" => "1" }
|
129
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == ["1", "0"]
|
102
130
|
end
|
103
131
|
|
104
132
|
it "removes the member from the set" do
|
@@ -110,7 +138,7 @@ describe DynportTools::RedisQ do
|
|
110
138
|
queue.pop
|
111
139
|
queue.pop
|
112
140
|
queue.pop
|
113
|
-
queue.pop.should ==
|
141
|
+
queue.pop.should == {}
|
114
142
|
end
|
115
143
|
end
|
116
144
|
|
@@ -127,6 +155,18 @@ describe DynportTools::RedisQ do
|
|
127
155
|
redis.zrevrange(key, 0, -1, :with_scores => true).should == []
|
128
156
|
end
|
129
157
|
|
158
|
+
it "is able to yield batches" do
|
159
|
+
queue.push(99, 1)
|
160
|
+
queue.push(100, 9)
|
161
|
+
queue.push(101, 0)
|
162
|
+
all_ids = []
|
163
|
+
queue.each(:batch_size => 2) do |ids|
|
164
|
+
all_ids << ids
|
165
|
+
end
|
166
|
+
all_ids.should == [["100", "99"], ["101"]]
|
167
|
+
redis.zrevrange(key, 0, -1, :with_scores => true).should == []
|
168
|
+
end
|
169
|
+
|
130
170
|
it "readds all ids to the queue which raised errors" do
|
131
171
|
queue.push(99, 1)
|
132
172
|
queue.push(100, 9)
|
data/spec/spec_helper.rb
CHANGED
@@ -3,6 +3,12 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
|
|
3
3
|
require 'rspec'
|
4
4
|
require 'dynport_tools'
|
5
5
|
require "timecop"
|
6
|
+
require "ruby-debug"
|
7
|
+
|
8
|
+
if defined?(Debugger) && Debugger.respond_to?(:settings)
|
9
|
+
Debugger.settings[:autolist] = 1
|
10
|
+
Debugger.settings[:autoeval] = true
|
11
|
+
end
|
6
12
|
|
7
13
|
# Requires supporting files with custom matchers and macros, etc,
|
8
14
|
# in ./support/ and its subdirectories.
|
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynport_tools
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 13
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 13
|
10
|
+
version: 0.2.13
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Tobias Schwab
|