refilling_queue 0.0.1

data/.travis.yml

@@ -0,0 +1,4 @@
+rvm:
+  - ree
+  - 1.9.2
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,6 @@
+source :rubygems
+gemspec
+
+gem 'rake'
+gem 'rspec', '~>2'
+gem 'fakeredis'

data/Gemfile.lock

@@ -0,0 +1,30 @@
+PATH
+  remote: .
+  specs:
+    refilling_queue (0.0.1)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    diff-lcs (1.1.3)
+    fakeredis (0.4.0)
+      redis (~> 3.0.0)
+    rake (0.9.2)
+    redis (3.0.1)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fakeredis
+  rake
+  refilling_queue!
+  rspec (~> 2)

data/Rakefile

@@ -0,0 +1,22 @@
+require 'bundler/gem_tasks'
+
+task :default do
+  sh "rspec spec/"
+end
+
+# extracted from https://github.com/grosser/project_template
+rule /^version:bump:.*/ do |t|
+  sh "git status | grep 'nothing to commit'" # ensure we are not dirty
+  index = ['major', 'minor','patch'].index(t.name.split(':').last)
+  file = 'lib/refilling_queue/version.rb'
+
+  version_file = File.read(file)
+  old_version, *version_parts = version_file.match(/(\d+)\.(\d+)\.(\d+)/).to_a
+  version_parts[index] = version_parts[index].to_i + 1
+  version_parts[2] = 0 if index < 2 # remove patch for minor
+  version_parts[1] = 0 if index < 1 # remove minor for major
+  new_version = version_parts * '.'
+  File.open(file,'w'){|f| f.write(version_file.sub(old_version, new_version)) }
+
+  sh "bundle && git add #{file} Gemfile.lock && git commit -m 'bump version to #{new_version}'"
+end

data/Readme.md

@@ -0,0 +1,32 @@
+A queue that refreshes itself when it gets empty or stale, so you can keep popping
+
+Install
+=======
+
+    gem install refilling_queue
+
+Usage
+=====
+
+    queue = RefillingQueue.new resque_client, "my_queue", :refresh_every => 30.seconds do
+      expensive_operation.map(&:id)
+    end
+
+    begin
+      queue.pop
+    rescue
+      RefillingQueue::EmptyRefill # queue was empty, refilled but is still empty
+    end
+
+    queue.pop -> return id
+    ... # queue empty ?
+    queue.pop -> run block -> store new ids -> return id
+    ... # 30 seconds elapsed (global expires_at stored in reque_client) ?
+    queue.pop -> run block -> store new ids -> return id
+
+Author
+======
+[Michael Grosser](http://grosser.it)<br/>
+michael@grosser.it<br/>
+License: MIT<br/>
+[![Build Status](https://secure.travis-ci.org/grosser/refilling_queue.png)](http://travis-ci.org/grosser/refilling_queue)

data/lib/refilling_queue/version.rb

@@ -0,0 +1,3 @@
+class RefillingQueue
+  VERSION = '0.0.1'
+end

data/lib/refilling_queue.rb

@@ -0,0 +1,57 @@
+require 'refilling_queue/version'
+
+class RefillingQueue
+  class EmptyRefill < RuntimeError
+  end
+
+  DEFAULT_OPTIONS = {
+    :lock_timeout => 60,
+    :refresh_every => nil
+  }
+
+  def initialize(client, name, options={}, &block)
+    @client, @name, @block = client, name, block
+    @options = DEFAULT_OPTIONS.merge(options)
+    raise "Invalid keys" if (options.keys - DEFAULT_OPTIONS.keys).any?
+  end
+
+  def pop
+    item = @client.lpop @name
+    return item unless item.nil?
+
+    refill
+
+    item = @client.lpop @name
+    return item unless item.nil?
+
+    raise RefillingQueue::EmptyRefill
+  end
+
+  def refill
+    lock do
+      results = @block.call
+      @client.pipelined do
+        @client.del @name
+        results.each{ |r| @client.rpush @name, r } # TODO https://github.com/redis/redis-rb/issues/253
+        @client.expire @name, @options[:refresh_every] if @options[:refresh_every]
+      end
+    end
+  end
+
+  def empty?
+    @client.llen(@name) == 0
+  end
+
+  private
+
+  def lock
+    lock = "#{@name}_lock"
+    return unless @client.setnx lock, "1"
+    @client.expire lock, @options[:lock_timeout]
+    begin
+      yield
+    ensure
+      @client.del lock, "1"
+    end
+  end
+end

data/refilling_queue.gemspec

@@ -0,0 +1,12 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+name = "refilling_queue"
+require "#{name}/version"
+
+Gem::Specification.new name, RefillingQueue::VERSION do |s|
+  s.summary = "A queue that refreshes itself when it gets empty or stale, so you can keep popping"
+  s.authors = ["Michael Grosser"]
+  s.email = "michael@grosser.it"
+  s.homepage = "http://github.com/grosser/#{name}"
+  s.files = `git ls-files`.split("\n")
+  s.license = 'MIT'
+end

data/spec/refilling_queue_spec.rb

@@ -0,0 +1,99 @@
+require 'spec_helper'
+require 'fakeredis'
+
+describe RefillingQueue do
+  def kill_all_threads
+    Thread.list.each {|thread| thread.exit unless thread == Thread.current }
+  end
+
+  let(:client){ FakeRedis::Redis.new }
+
+  before do
+    kill_all_threads
+  end
+
+  it "has a VERSION" do
+    RefillingQueue::VERSION.should =~ /^[\.\da-z]+$/
+  end
+
+  context "#initialize" do
+    it "does not fill itself when started full" do
+      x = 0
+      RefillingQueue.new(client, "x"){ x = 1 }
+      x.should == 0
+    end
+  end
+
+  context "#pop" do
+    it "removes an element" do
+      queue = RefillingQueue.new(client, "x"){ [1,2,3,4] }
+      queue.pop.should == 1
+      queue.pop.should == 2
+      queue.pop.should == 3
+    end
+
+    it "only tries to refill once / raises on empty refill" do
+      calls = []
+      queue = RefillingQueue.new(client, "x"){ calls << 1; [] }
+      expect{
+        queue.pop
+      }.to raise_error(RefillingQueue::EmptyRefill)
+      calls.should == [1]
+    end
+
+    it "refills itself if queue gets empty" do
+      content = [1]
+      queue = RefillingQueue.new(client, "x"){ content }
+      queue.pop.should == 1
+      content.replace [4,5]
+      queue.pop.should == 4
+      queue.pop.should == 5
+      queue.pop.should == 4
+    end
+
+    it "refills itself when it expires" do
+      content = [1,2,3]
+      queue = RefillingQueue.new(client, "x", :refresh_every => 0.1){ content }
+
+      queue.pop.should == 1
+      content.replace [4,5]
+      queue.pop.should == 2
+      sleep 0.5
+
+      queue.pop.should == 4
+    end
+  end
+
+  context "with multiple actors" do
+    it "only refills once" do
+      called = []
+      Thread.new do
+        RefillingQueue.new(client, "x"){ sleep 0.2; called << 1; [] }.pop
+      end
+      Thread.new do
+        RefillingQueue.new(client, "x"){ sleep 0.2; called << 1; [] }.pop
+      end
+      sleep 0.3
+      called.should == [1]
+    end
+
+    it "can refill after refill is complete" do
+      called = []
+      RefillingQueue.new(client, "x"){ called << 1; [1] }.pop
+      RefillingQueue.new(client, "x"){ called << 2; [1] }.pop
+      called.should == [1,2]
+    end
+
+    it "can refill if lock expired" do
+      called = []
+      RefillingQueue.new(client, "x", :lock_timeout => 1){ called << 1; [1] }.pop
+      Thread.new do
+        # lock-blocker
+        RefillingQueue.new(client, "x", :lock_timeout => 1){ sleep 2; called << 2; [1] }.pop
+      end
+      sleep 1.5
+      RefillingQueue.new(client, "x", :lock_timeout => 1){ called << 3; [1] }.pop
+      called.should == [1,3]
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require "fakeredis"
+
+$LOAD_PATH.unshift 'lib'
+require 'refilling_queue'

metadata

@@ -0,0 +1,62 @@
+--- !ruby/object:Gem::Specification
+name: refilling_queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Michael Grosser
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-12 00:00:00.000000000 Z
+dependencies: []
+description: 
+email: michael@grosser.it
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .travis.yml
+- Gemfile
+- Gemfile.lock
+- Rakefile
+- Readme.md
+- lib/refilling_queue.rb
+- lib/refilling_queue/version.rb
+- refilling_queue.gemspec
+- spec/refilling_queue_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/grosser/refilling_queue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2693512841144003384
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: -2693512841144003384
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: A queue that refreshes itself when it gets empty or stale, so you can keep
+  popping
+test_files: []