refilling_queue 0.0.4 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eb46dfea534dbaac56b171c985c0182bb9b766a7
+  data.tar.gz: 432cec3c6c0d9eb76218be864a86a012b4bab0d1
+SHA512:
+  metadata.gz: 2e309d6f2d14c1234b0e8bfdbc6bcdd8d67a6e8a42451d63d3f155ee5004a7d1864a00d41ac753149c908098c5caec729e194afc0b7bc03e62f937f7608c6703
+  data.tar.gz: 7050b24da1fd1e3b06098d63c747c20f6218f930fb3f0aff111bd9fd15b8b6e410c2bd76c3db56b54ad563ee6e89d18a3dd33ff477db570fa3fb212d49a33868

data/lib/refilling_queue/version.rb

@@ -1,3 +1,3 @@
 class RefillingQueue
-  VERSION = '0.0.4'
+  VERSION = '0.1.0'
 end

data/lib/refilling_queue.rb

@@ -6,11 +6,13 @@ class RefillingQueue
 
   DEFAULT_OPTIONS = {
     :lock_timeout => 60,
-    :refresh_every => nil
+    :refresh_every => nil,
+    :paginate => false
   }
 
   def initialize(client, name, options={}, &block)
     @client, @name, @block = client, name, block
+    @page_name = name + "/page"
     @options = DEFAULT_OPTIONS.merge(options)
     raise "Invalid keys" if (options.keys - DEFAULT_OPTIONS.keys).any?
   end
@@ -22,21 +24,48 @@ class RefillingQueue
     _pop
   end
 
+  def clear
+    lock do
+      mark_as_empty
+      _refill
+    end
+  end
+
   private
 
+  def paginate?
+    @options[:paginate]
+  end
+
   def _pop
     @client.lpop @name
   end
 
   def refill
     lock do
-      results = @block.call
-      return if results.empty?
+      _refill
+    end
+  end
+
+  def _refill
+    results = if paginate?
+      page = (@client.get(@page_name) || 0).to_i
+      @block.call(page + 1)
+    else
+      @block.call
+    end
+    if results.empty?
+      mark_as_empty
+      return
+    end
 
-      @client.pipelined do
-        @client.del @name
-        results.each{ |r| @client.rpush @name, r } # TODO https://github.com/redis/redis-rb/issues/253
-        @client.expire @name, @options[:refresh_every] if @options[:refresh_every]
+    @client.pipelined do
+      @client.del @name
+      @client.rpush @name, results
+      @client.expire @name, @options[:refresh_every] if @options[:refresh_every]
+      if paginate?
+        @client.incr @page_name
+        @client.expire @page_name, @options[:refresh_every] if @options[:refresh_every]
       end
     end
   end
@@ -45,6 +74,10 @@ class RefillingQueue
     @client.llen(@name) == 0
   end
 
+  def mark_as_empty
+    @client.del @page_name if paginate?
+  end
+
   def lock
     lock = "#{@name}_lock"
 

metadata

@@ -1,67 +1,102 @@
 --- !ruby/object:Gem::Specification
 name: refilling_queue
 version: !ruby/object:Gem::Version
-  version: 0.0.4
-  prerelease: 
+  version: 0.1.0
 platform: ruby
 authors:
 - Michael Grosser
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-20 00:00:00.000000000 Z
-dependencies: []
+date: 2014-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: bump
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
 description: 
 email: michael@grosser.it
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .travis.yml
-- Appraisals
-- Gemfile
-- Gemfile.lock
-- Rakefile
-- Readme.md
-- gemfiles/redis2.gemfile
-- gemfiles/redis2.gemfile.lock
-- gemfiles/redis3.gemfile
-- gemfiles/redis3.gemfile.lock
 - lib/refilling_queue.rb
 - lib/refilling_queue/version.rb
-- refilling_queue.gemspec
-- spec/refilling_queue_spec.rb
-- spec/spec_helper.rb
 homepage: http://github.com/grosser/refilling_queue
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 949949052394646117
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: 949949052394646117
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A queue that refreshes itself when it gets empty or stale, so you can keep
   popping
 test_files: []

data/.travis.yml

@@ -1,4 +0,0 @@
-rvm:
-  - ree
-  - 1.9.2
-  - 1.9.3

data/Appraisals

@@ -1,5 +0,0 @@
-["2", "3"].each do |version|
-  appraise "redis#{version}" do
-    gem "redis", "~>#{version}.0"
-  end
-end

data/Gemfile

@@ -1,8 +0,0 @@
-source :rubygems
-gemspec
-
-gem 'appraisal'
-gem 'bump'
-gem 'rake'
-gem 'rspec', '~>2'
-gem 'redis'

data/Gemfile.lock

@@ -1,34 +0,0 @@
-PATH
-  remote: .
-  specs:
-    refilling_queue (0.0.4)
-
-GEM
-  remote: http://rubygems.org/
-  specs:
-    appraisal (0.5.1)
-      bundler
-      rake
-    bump (0.3.5)
-    diff-lcs (1.1.3)
-    rake (0.9.2)
-    redis (3.0.1)
-    rspec (2.6.0)
-      rspec-core (~> 2.6.0)
-      rspec-expectations (~> 2.6.0)
-      rspec-mocks (~> 2.6.0)
-    rspec-core (2.6.4)
-    rspec-expectations (2.6.0)
-      diff-lcs (~> 1.1.2)
-    rspec-mocks (2.6.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  appraisal
-  bump
-  rake
-  redis
-  refilling_queue!
-  rspec (~> 2)

data/Rakefile

@@ -1,11 +0,0 @@
-require 'bundler/gem_tasks'
-require 'appraisal'
-require 'bump/tasks'
-
-task :spec do
-  sh "rspec spec/"
-end
-
-task :default do
-  sh "bundle exec rake appraisal:install && bundle exec rake appraisal spec"
-end

data/Readme.md

@@ -1,34 +0,0 @@
-A queue that refreshes itself when it gets empty or stale, so you can keep popping
-
-Install
-=======
-
-    gem install refilling_queue
-
-Usage
-=====
-
-    queue = RefillingQueue.new resque_client, "my_queue", :refresh_every => 30.seconds do
-      expensive_operation.map(&:id)
-    end
-
-    begin
-      queue.pop
-    rescue RefillingQueue::Locked
-      # queue was empty, refilling failed because other process is already trying it
-    end
-
-    queue.pop -> return id
-    ... # queue empty ?
-    queue.pop -> run block -> store new ids -> return id
-    ... # 30 seconds elapsed (global expires_at stored in reque_client) ?
-    queue.pop -> run block -> store new ids -> return id
-    ...
-    queue.pop -> run block -> empty result -> return nil
-
-Author
-======
-[Michael Grosser](http://grosser.it)<br/>
-michael@grosser.it<br/>
-License: MIT<br/>
-[![Build Status](https://secure.travis-ci.org/grosser/refilling_queue.png)](http://travis-ci.org/grosser/refilling_queue)

data/gemfiles/redis2.gemfile

@@ -1,11 +0,0 @@
-# This file was generated by Appraisal
-
-source :rubygems
-
-gem "appraisal"
-gem "bump"
-gem "rake"
-gem "rspec", "~>2"
-gem "redis", "~>2.0"
-
-gemspec :path=>"../"

data/gemfiles/redis2.gemfile.lock

@@ -1,34 +0,0 @@
-PATH
-  remote: /Users/mgrosser/code/tools/refilling_queue
-  specs:
-    refilling_queue (0.0.3)
-
-GEM
-  remote: http://rubygems.org/
-  specs:
-    appraisal (0.5.1)
-      bundler
-      rake
-    bump (0.3.5)
-    diff-lcs (1.1.3)
-    rake (10.0.2)
-    redis (2.2.2)
-    rspec (2.11.0)
-      rspec-core (~> 2.11.0)
-      rspec-expectations (~> 2.11.0)
-      rspec-mocks (~> 2.11.0)
-    rspec-core (2.11.1)
-    rspec-expectations (2.11.3)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  appraisal
-  bump
-  rake
-  redis (~> 2.0)
-  refilling_queue!
-  rspec (~> 2)

data/gemfiles/redis3.gemfile

@@ -1,11 +0,0 @@
-# This file was generated by Appraisal
-
-source :rubygems
-
-gem "appraisal"
-gem "bump"
-gem "rake"
-gem "rspec", "~>2"
-gem "redis", "~>3.0"
-
-gemspec :path=>"../"

data/gemfiles/redis3.gemfile.lock

@@ -1,34 +0,0 @@
-PATH
-  remote: /Users/mgrosser/code/tools/refilling_queue
-  specs:
-    refilling_queue (0.0.3)
-
-GEM
-  remote: http://rubygems.org/
-  specs:
-    appraisal (0.5.1)
-      bundler
-      rake
-    bump (0.3.5)
-    diff-lcs (1.1.3)
-    rake (10.0.2)
-    redis (3.0.2)
-    rspec (2.11.0)
-      rspec-core (~> 2.11.0)
-      rspec-expectations (~> 2.11.0)
-      rspec-mocks (~> 2.11.0)
-    rspec-core (2.11.1)
-    rspec-expectations (2.11.3)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.3)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  appraisal
-  bump
-  rake
-  redis (~> 3.0)
-  refilling_queue!
-  rspec (~> 2)

data/refilling_queue.gemspec

@@ -1,12 +0,0 @@
-$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
-name = "refilling_queue"
-require "#{name}/version"
-
-Gem::Specification.new name, RefillingQueue::VERSION do |s|
-  s.summary = "A queue that refreshes itself when it gets empty or stale, so you can keep popping"
-  s.authors = ["Michael Grosser"]
-  s.email = "michael@grosser.it"
-  s.homepage = "http://github.com/grosser/#{name}"
-  s.files = `git ls-files`.split("\n")
-  s.license = 'MIT'
-end

data/spec/refilling_queue_spec.rb

@@ -1,112 +0,0 @@
-require 'spec_helper'
-
-describe RefillingQueue do
-  def kill_all_threads
-    Thread.list.each {|thread| thread.exit unless thread == Thread.current }
-  end
-
-  let(:client){ Redis.new }
-
-  before do
-    kill_all_threads
-    client.flushdb
-  end
-
-  after :all do
-    `rm -f dump.rdb`
-  end
-
-  it "has a VERSION" do
-    RefillingQueue::VERSION.should =~ /^[\.\da-z]+$/
-  end
-
-  context "#initialize" do
-    it "does not fill itself when started full" do
-      x = 0
-      RefillingQueue.new(client, "x"){ x = 1 }
-      x.should == 0
-    end
-  end
-
-  context "#pop" do
-    it "removes an element" do
-      queue = RefillingQueue.new(client, "x"){ [1,2,3,4] }
-      queue.pop.should == "1"
-      queue.pop.should == "2"
-      queue.pop.should == "3"
-    end
-
-    it "only tries to refill once" do
-      calls = []
-      RefillingQueue.new(client, "x"){ calls << 1; [] }.pop.should == nil
-      calls.should == [1]
-    end
-
-    it "refills itself if queue gets empty" do
-      content = [1]
-      queue = RefillingQueue.new(client, "x"){ content }
-      queue.pop.should == "1"
-      content.replace [4,5]
-      queue.pop.should == "4"
-      queue.pop.should == "5"
-      queue.pop.should == "4"
-    end
-
-    it "refills itself when it expires" do
-      content = [1,2,3]
-      queue = RefillingQueue.new(client, "x", :refresh_every => 1){ content }
-
-      queue.pop.should == "1"
-      content.replace [4,5]
-      queue.pop.should == "2"
-      sleep 2
-
-      queue.pop.should == "4"
-    end
-  end
-
-  context "with multiple actors" do
-    it "only refills once" do
-      called = []
-
-      # lock-blocker
-      Thread.new do
-        RefillingQueue.new(client, "x"){ called << 1; sleep 0.3; called << 2; [] }.pop
-      end
-      sleep 0.1
-
-      # blocked
-      locked = false
-      queue = RefillingQueue.new(client, "x"){ called << 3; [] }
-      begin
-        queue.pop
-        fail
-      rescue RefillingQueue::Locked
-        locked = true
-      end
-      sleep 0.3
-
-      called.should == [1, 2]
-      locked.should == true
-    end
-
-    it "can refill after refill is complete" do
-      called = []
-      RefillingQueue.new(client, "x"){ called << 1; [1] }.pop
-      RefillingQueue.new(client, "x"){ called << 2; [1] }.pop
-      called.should == [1,2]
-    end
-
-    it "can refill if lock expired" do
-      called = []
-      RefillingQueue.new(client, "x", :lock_timeout => 1){ called << 1; [1] }.pop
-      Thread.new do
-        # lock-blocker
-        RefillingQueue.new(client, "x", :lock_timeout => 1){ sleep 3; called << 2; [1] }.pop
-      end
-      sleep 2
-      RefillingQueue.new(client, "x", :lock_timeout => 1){ called << 3; [1] }.pop
-      called.should == [1,3]
-    end
-  end
-end

data/spec/spec_helper.rb

@@ -1,3 +0,0 @@
-$LOAD_PATH.unshift 'lib'
-require 'refilling_queue'
-require 'redis'