redis_queue 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 478d7768be5fbb3a39f9b578d1a0398f221e4dfa
-  data.tar.gz: d818a1f802cb83ff38f1a639bd8011dc0dedab7d
+  metadata.gz: f49c187b5bb4deb1ca5849534ff7ee9760bc2fcd
+  data.tar.gz: 10468e1f322dc6a4e68fc5f385ed04c79d28829c
 SHA512:
-  metadata.gz: f1be4795444eb4700248dded81c56c880cba7cb1490149acbf73913df7890331ad469274a192583f0f5255c5227ba0228862ed2d1015d32b79f680e020051706
-  data.tar.gz: 6a6c522e9b3542eff0dd875ff83567294795f69f6645a4e35a5fb5bb1d122c01fadfd3e1beb0e3040b0646e4b8232ba33c4b704f7b16d9969c700ec0d239c1b8
+  metadata.gz: ce68bfda0e8abdbd3a5b189f47d0496004808e1be7f320349127fc6ff749a0cb66d25d75f0537b8158a6dfaf3e7a1aa8157747f26bc4542afbf0a4bc3c8aa5a2
+  data.tar.gz: 9ee5a1184dacaa73a10906aa03587a6a37eb50599c5e6a56a1d9ecea72fafe7d15c3c2b2a445b40924fb3c06c6c6eb07ad94d95f5cac9c5e12b67c142ea6113a

data/lib/redis_queue.rb

@@ -18,7 +18,7 @@ class RedisQueue
         message = redis.call('lpop', ARGV[1])
       end
       if message then
-        redis.call('sadd', ARGV[1]..'_in_use', message)
+        redis.call('hset', ARGV[1]..'_in_use', message, ARGV[2])
       end
       return message
     "'',
@@ -34,25 +34,30 @@ class RedisQueue
     "'',
     repush: ''"
       #{PUSH_CODE}
-      redis.call('srem', ARGV[1]..'_in_use', ARGV[2])
+      redis.call('hdel', ARGV[1]..'_in_use', ARGV[2])
     "'',
     fail: ''"
-      redis.call('sadd', ARGV[1]..'_failed', ARGV[2])
-      redis.call('srem', ARGV[1]..'_in_use', ARGV[2])
+      redis.call('hset', ARGV[1]..'_failed', ARGV[2], ARGV[3])
+      redis.call('hdel', ARGV[1]..'_in_use', ARGV[2])
     "'',
     done: ''"
-      redis.call('sadd', ARGV[1]..'_done', ARGV[2])
-      redis.call('srem', ARGV[1]..'_in_use', ARGV[2])
+      redis.call('hset', ARGV[1]..'_done', ARGV[2], ARGV[3])
+      redis.call('hdel', ARGV[1]..'_in_use', ARGV[2])
     "'',
     unpop: ''"
       redis.call('lpush', ARGV[1], ARGV[2])
-      redis.call('srem', ARGV[1]..'_in_use', ARGV[2])
+      redis.call('hdel', ARGV[1]..'_in_use', ARGV[2])
     "'',
     init_from: ''"
-      local vals = redis.call('smembers', ARGV[2])
+      local vals = redis.call('hkeys', ARGV[2])
       for i = 1, table.getn(vals) do
-        redis.call('lpush', ARGV[1], vals[i])
-      end"''
+        local timestamp = redis.call('hget', ARGV[2], vals[i])
+        if timestamp < ARGV[3] then
+          redis.call('lpush', ARGV[1], vals[i])
+          redis.call('hdel', ARGV[2], vals[i])
+        end
+      end
+      "''
   }.freeze
 
   def initialize(args = {})
@@ -66,7 +71,7 @@ class RedisQueue
   def pop(block: true)
     return nonblpop unless block
     message = blpop
-    @redis.run { |redis| redis.sadd "#{@id}_in_use", message } if message
+    @redis.run { |redis| redis.hset "#{@id}_in_use", message, now } if message
     message
   end
 
@@ -75,11 +80,11 @@ class RedisQueue
   end
 
   def fail(message)
-    script :fail, @id, message
+    script :fail, @id, message, now
   end
 
   def done(message)
-    script :done, @id, message
+    script :done, @id, message, now
   end
 
   def unpop(message)
@@ -91,7 +96,7 @@ class RedisQueue
   end
 
   def forget(message)
-    @redis.run { |redis| redis.srem "#{@id}_in_use", message }
+    @redis.run { |redis| redis.hdel "#{@id}_in_use", message }
   end
 
   def remove(message)
@@ -105,18 +110,12 @@ class RedisQueue
     message
   end
 
-  def reset
-    init_from "#{@id}_in_use"
-    @redis.run { |redis| redis.del "#{@id}_in_use" }
+  def reset(older_than: nil)
+    init_from "#{@id}_in_use", older_than
   end
 
   def restart
     init_from "#{@id}_done"
-    @redis.run { |redis| redis.del "#{@id}_done" }
-  end
-
-  def init_from(set)
-    script(:init_from, @id, set)
   end
 
   def size
@@ -124,15 +123,15 @@ class RedisQueue
   end
 
   def done_size
-    @redis.run { |redis| redis.scard "#{@id}_done" }.to_i
+    @redis.run { |redis| redis.hlen "#{@id}_done" }.to_i
   end
 
   def failed_size
-    @redis.run { |redis| redis.scard "#{@id}_failed" }.to_i
+    @redis.run { |redis| redis.hlen "#{@id}_failed" }.to_i
   end
 
   def in_use_size
-    @redis.run { |redis| redis.scard "#{@id}_in_use" }.to_i
+    @redis.run { |redis| redis.hlen "#{@id}_in_use" }.to_i
   end
 
   def list
@@ -140,15 +139,15 @@ class RedisQueue
   end
 
   def done_list
-    @redis.run { |redis| redis.smembers "#{@id}_done" }
+    Hash[@redis.run { |redis| redis.hgetall "#{@id}_done" }]
   end
 
   def failed_list
-    @redis.run { |redis| redis.smembers "#{@id}_failed" }
+    Hash[@redis.run { |redis| redis.hgetall "#{@id}_failed" }]
   end
 
   def in_use_list
-    @redis.run { |redis| redis.smembers "#{@id}_in_use" }
+    Hash[@redis.run { |redis| redis.hgetall "#{@id}_in_use" }]
   end
 
   def print_stats
@@ -177,20 +176,24 @@ class RedisQueue
   private
 
   def blpop
-    begin
+    loop do
       message = @redis_blocking.run { |redis| redis.blpop(@id) }.last
-    end while message == ''
-    message
+      return message unless message == ''
+    end
   end
 
   def nonblpop
-    script :nonblpop, @id
+    script :nonblpop, @id, now
   end
 
   def nonbltouch
     script :touch, @id
   end
 
+  def init_from(key, older_than = nil)
+    script(:init_from, @id, key, older_than || now + 100_000)
+  end
+
   def load_scripts
     @scripts = {}
     @redis.run do |redis|
@@ -203,6 +206,10 @@ class RedisQueue
   def script(name, *args)
     @redis.run { |redis| redis.evalsha @scripts[name], argv: args }
   end
+
+  def now
+    (Time.now.to_f * 1000).to_i
+  end
 end
 
 require_relative 'redis_connection'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: redis_queue
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Jose Ignacio Fernandez