dynport_tools 0.2.12 → 0.2.13

data/VERSION

@@ -1 +1 @@
-0.2.12
+0.2.13

data/dynport_tools.gemspec

@@ -5,7 +5,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{dynport_tools}
-  s.version = "0.2.12"
+  s.version = "0.2.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Tobias Schwab"]

data/lib/dynport_tools/redis_q.rb

@@ -20,8 +20,10 @@ class DynportTools::RedisQ
   
   def push_many(array, options = {})
     redis.multi do
-     array.each do | (id, popularity) |
-        push(id, popularity, options.merge(:no_multi => true))
+      array.each do | (id, popularity) |
+        (id.is_a?(Hash) ? id : { id => popularity }).each do |id2, popularity2|
+          push(id2, popularity2, options.merge(:no_multi => true))
+        end
       end
     end
   end
@@ -38,27 +40,28 @@ class DynportTools::RedisQ
     redis.zcard(redis_key)
   end
   
-  def pop
-    redis.multi do
-      redis.zrevrange(redis_key, 0, 0, :with_scores => true)
-      redis.zremrangebyrank(redis_key, -1, -1)
-    end.first
+  def pop(number = 1)
+    Hash[*redis.multi do
+      redis.zrevrange(redis_key, 0, number - 1, :with_scores => true)
+      redis.zremrangebyrank(redis_key, 0 - number, -1)
+    end.first]
   end
   
   def failed_tries
     @failed_tries ||= Hash.new(0)
   end
   
-  def each
+  def each(options = {})
     entries_with_errors = []
     stats = { :errors => {}, :ok => [] }
-    while (result = pop).any?
+    batch_size = options[:batch_size].to_i if options[:batch_size].to_i > 0
+    while (result_hash = pop(batch_size || 1)).any?
       begin
-        yield(result.first)
-        stats[:ok] << result.first
+        yield(batch_size ? result_hash.to_a.sort_by { |a| a.last }.reverse.map { |a| a.first } : result_hash.keys.first)
+        stats[:ok] << result_hash.keys.join(",")
       rescue => err
-        stats[:errors][result.first] = ([err.message] + err.backtrace[0,5]).join("\n")
-        entries_with_errors << result if mark_failed(result.first) < retry_count
+        stats[:errors][result_hash.keys.join(",")] = ([err.message] + err.backtrace[0,5]).join("\n")
+        entries_with_errors << result_hash if mark_failed(result_hash.keys.join(",")) < retry_count
       end
     end
     push_many(entries_with_errors, :failed => true) if entries_with_errors.any?

data/spec/dynport_tools/redis_q_spec.rb

@@ -37,6 +37,23 @@ describe DynportTools::RedisQ do
     queue.redis_key.should == "some/queue"
   end
   
+  describe "#push_many" do
+    it "pushes nested arrays" do
+      queue.push_many([[1, 2], [3, 4]])
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == ["3", "4", "1", "2"]
+    end
+    
+    it "also pushes hashes" do
+      queue.push_many({ 2 => 4, 6 => 8})
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == ["6", "8", "2", "4"]
+    end
+    
+    it "adds arrays of hashes" do
+      queue.push_many([{ 2 => 4 }, { 6 => 8}])
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == ["6", "8", "2", "4"]
+    end
+  end
+  
   describe "#push" do
     it "pushes the records with negative timestamps" do
       Timecop.freeze(Time.at(112233))
@@ -98,7 +115,18 @@ describe DynportTools::RedisQ do
     end
     
     it "returns the highest member and rank" do
-      queue.pop.should == ["101", "100"]
+      queue.pop.should == { "101" => "100" }
+    end
+    
+    it "can also return 2 elements" do
+      queue.pop(2).should == { "101" => "100", "98" => "10" }
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == ["99", "1"]
+    end
+    
+    it "can also return 3 elements" do
+      queue.push("1", "0")
+      queue.pop(3).should == { "101" => "100", "98" => "10", "99" => "1" }
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == ["1", "0"]
     end
     
     it "removes the member from the set" do
@@ -110,7 +138,7 @@ describe DynportTools::RedisQ do
       queue.pop
       queue.pop
       queue.pop
-      queue.pop.should == []
+      queue.pop.should == {}
     end
   end
   
@@ -127,6 +155,18 @@ describe DynportTools::RedisQ do
       redis.zrevrange(key, 0, -1, :with_scores => true).should == []
     end
     
+    it "is able to yield batches" do
+      queue.push(99, 1)
+      queue.push(100, 9)
+      queue.push(101, 0)
+      all_ids = []
+      queue.each(:batch_size => 2) do |ids|
+        all_ids << ids
+      end
+      all_ids.should == [["100", "99"], ["101"]]
+      redis.zrevrange(key, 0, -1, :with_scores => true).should == []
+    end
+    
     it "readds all ids to the queue which raised errors" do
       queue.push(99, 1)
       queue.push(100, 9)

data/spec/spec_helper.rb

@@ -3,6 +3,12 @@ $LOAD_PATH.unshift(File.dirname(__FILE__))
 require 'rspec'
 require 'dynport_tools'
 require "timecop"
+require "ruby-debug"
+
+if defined?(Debugger) && Debugger.respond_to?(:settings)
+  Debugger.settings[:autolist] = 1
+  Debugger.settings[:autoeval] = true
+end
 
 # Requires supporting files with custom matchers and macros, etc,
 # in ./support/ and its subdirectories.

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: dynport_tools
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: 
   segments: 
   - 0
   - 2
-  - 12
-  version: 0.2.12
+  - 13
+  version: 0.2.13
 platform: ruby
 authors: 
 - Tobias Schwab