qmore 0.6.3 → 0.7.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 34dc454ee618a38dedc1f352dfaa2165cbd8f399
+  data.tar.gz: 81c85cdf9bd56911966eb74b6ee6e7fcc0530241
+SHA512:
+  metadata.gz: e96673443f8265ed6efb052532c15675fa3d52ed443623d1ce4b673cee527afc4d60495b6434ddd04fe8e98ac1a0d79915a125b8fc23d40e67aedda33b2621a4
+  data.tar.gz: fdb5218c001f5995fa0640e07e7ac89b2bd82dd36ad4f78af29e952d0e848a1300302341cf65bb576cf9f2e913d699c78257ac2e7c3e5d9b1c6a8cccd6b156f3

data/CHANGELOG

@@ -1,3 +1,19 @@
+0.7.0 (06/23/2014)
+------------------
+
+Refactoring of qmore reserver to break it into its component pieces
+making it more flexible.
+
+Added strategies for the reservers:
+Sources - where we pull the Qless::Queue from (Enumeration that returns Qless:Queue)
+Filters - filters out queues from Sources based on some criteria (acts as a Source)
+Ordering- Reorders the results of an iteration through an enumeration.
+
+Added a Delegating Reserver which delegates down to a collect of reservers to
+find jobs.
+
+Added Background Source which updates its available queues asynchronously
+
 0.6.3 (06/13/2014)
 ------------------
 

data/README.md

@@ -8,11 +8,11 @@ Authored against Qless 0.9.3, so it at least works with that - try running the t
 Usage
 -----
 
-To use the rake tasks built into qless, just adding qless to your Gemfile should cause it to get required before the task executes.  If you aren't using a gemfile, then you'll need to require qmore directly so that it sets up ENV['JOB_RESERVER'] to use Qmore::JobReserver.
+To use the rake tasks built into qless, just adding qless to your Gemfile should cause it to get required before the task executes.  If you aren't using a gemfile, then you'll need to require qmore directly so that it sets up ENV['JOB_RESERVER'] to use Qmore::Reservers::Default.
 
 Alternatively, if you have some other way of launching workers (e.g. qless-pool), you can assign the reserver explicitly in the setup rake task or some other initializer:
 
-    Qless::Pool.pool_factory.reserver_class = Qmore::JobReserver
+    Qless::Pool.pool_factory.reserver_class = Qmore::Reservers::Default
     Qmore.client == Qless::Pool.pool_factory.client
 
     # Enabling Monitoring
@@ -129,3 +129,4 @@ Contributors
 
 Matt Conway ( https://github.com/wr0ngway )
 Bert Goethals ( https://github.com/Bertg )
+James Lawrence ( https://github.com/jambli )

data/lib/qmore.rb

@@ -4,10 +4,9 @@ require 'gem_logger'
 require 'qmore/configuration'
 require 'qmore/persistence'
 require 'qmore/attributes'
-require 'qmore/job_reserver'
+require 'qmore/reservers'
 
 module Qmore
-
   def self.client=(client)
     @client = client
   end
@@ -43,7 +42,7 @@ end
 
 module Qless
   module JobReservers
-    QmoreReserver = Qmore::JobReserver
+    QmoreReserver = Qmore::Reservers::Default
   end
 end
 ENV['JOB_RESERVER'] ||= 'QmoreReserver'

data/lib/qmore/reservers.rb

@@ -0,0 +1,7 @@
+module Qmore
+  module Reservers
+    require 'qmore/reservers/strategies'
+    require 'qmore/reservers/default'
+    require 'qmore/reservers/delegating'
+  end
+end

data/lib/qmore/reservers/default.rb

@@ -0,0 +1,21 @@
+module Qmore::Reservers
+  # @param [Enumerable] queues - qless queues to check for work
+  class Default < Struct.new(:queues)
+    def description
+      @description ||= queues.collect(&:name).uniq.join(', ') + " (qmore)"
+    end
+
+    def prep_for_work!
+      # nothing here on purpose
+    end
+
+    def reserve
+      queues.each do |queue|
+        if (job = queue.pop)
+          return job
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/qmore/reservers/delegating.rb

@@ -0,0 +1,30 @@
+module Qmore::Reservers
+  # @param [Enumerable] reservers - a set of reservers
+  # to check for work.
+  class Delegating < Struct.new(:reservers)
+    def description
+      "Delegating Reserver"
+    end
+
+    def prep_for_work!
+      # nothing here on purpose
+    end
+
+    def queues
+      Enumerator.new do |yielder|
+        reservers.each do |reserver|
+          reserver.queues.each {|queue| yielder << queue}
+        end
+      end
+    end
+
+    def reserve
+      reservers.each do |reserver|
+        if (job = reserver.reserve)
+          return job
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/qmore/reservers/strategies.rb

@@ -0,0 +1,7 @@
+module Qmore::Reservers
+  module Strategies
+    require 'qmore/reservers/strategies/filtering'
+    require 'qmore/reservers/strategies/ordering'
+    require 'qmore/reservers/strategies/sources'
+  end
+end

data/lib/qmore/reservers/strategies/filtering.rb

@@ -0,0 +1,28 @@
+module Qmore::Reservers::Strategies
+  module Filtering
+    extend Qmore::Attributes
+    # @param [Enumerable] queues - a source of queues
+    # @param [Array] regexes - a list of regexes to match against.
+    # Return an enumerator of the filtered queues in
+    # in prioritized order.
+    def self.default(queues, regexes)
+      Enumerator.new do |yielder|
+        # Map queues to their names
+        mapped_queues = queues.reduce({}) do |hash,queue|
+          hash[queue.name] = queue
+          hash
+        end
+
+        # Filter the queue names against the regexes provided.
+        matches = Filtering.expand_queues(regexes, mapped_queues.keys)
+
+        # Prioritize the queues.
+        prioritized_names = Filtering.prioritize_queues(Qmore.configuration.priority_buckets, matches)
+
+        prioritized_names.each do |name|
+          yielder << mapped_queues[name]
+        end
+      end
+    end
+  end
+end

data/lib/qmore/reservers/strategies/ordering.rb

@@ -0,0 +1,24 @@
+module Qmore::Reservers::Strategies::Ordering
+  # Shuffles the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  def self.shuffled(enumerable)
+    Enumerator.new do |yielder|
+      enumerable.to_a.shuffle.each do |e|
+        yielder << e
+      end
+    end
+  end
+
+  # Samples a subset of the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  # @param [Integer] sample_size   - number of items to take per iteration
+  def self.sampled(enumerable, sample_size = 5)
+    Enumerator.new do |yielder|
+      enumerable.to_a.sample(sample_size).each do |e|
+        yielder << e
+      end
+    end
+  end
+end

data/lib/qmore/reservers/strategies/sources.rb

@@ -0,0 +1,57 @@
+# This module provides the different kinds of queue sources used by qmore
+module Qmore::Reservers::Strategies::Sources
+  # Direct source uses a client to generate the queues we should
+  # pull work from. Ignores any queues that do not have tasks available.
+  def self.direct(client)
+    Enumerator.new do |yielder|
+      queues = client.queues.counts.reject do |queue|
+        total = %w(waiting recurring depends stalled scheduled).inject(0) { |sum, state| sum += queue[state].to_i }
+        total == 0
+      end
+
+      queues.each do |queue|
+        yielder << client.queues[queue['name']]
+      end
+    end
+  end
+
+  # Background Queue source runs in a background thread
+  # to periodically update the queues available.
+  class Background
+    include Enumerable
+    attr_reader :delegate, :delay
+    # @param [Enumerator] delegate queue source to load the queues from.
+    # @param [Integer] delay - how long between updates
+    def initialize(delegate, delay)
+      @delegate = delegate
+      @delay = delay
+    end
+
+    # Spawns a thread to periodically update the
+    # queues.
+    # @return [Thread] returns the spawned thread.
+    def start
+      @stop   = false
+      @queues = delegate.to_a
+      Thread.new do
+        begin
+          loop do
+            sleep delay
+            break if @stop
+            @queues = delegate.to_a
+          end
+        rescue => e
+          retry
+        end
+      end
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def each(&block)
+      @queues.each { |q| block.call(q) }
+    end
+  end
+end

data/lib/qmore/version.rb

@@ -1,3 +1,3 @@
 module Qmore
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 end

data/spec/attributes_spec.rb

@@ -164,9 +164,9 @@ describe "Attributes" do
       priority_buckets = [{'pattern' => 'other*', 'fairly' => true},
                           {'pattern' => 'default', 'fairly' => false}]
       queues = prioritize_queues(priority_buckets, @real_queues)
-      queues[0..4].sort.should == others.sort
-      queues[5..-1].should == ["high_x", "foo", "high_y", "superhigh_z"]
-      queues.should_not == others.sort + ["high_x", "foo", "high_y", "superhigh_z"]
+      (queues[0..4].sort).should eq(others.sort)
+      queues[5..-1].should eq(["high_x", "foo", "high_y", "superhigh_z"])
+      expect(queues).should_not eq(others.sort + ["high_x", "foo", "high_y", "superhigh_z"])
     end
   end
 end

data/spec/reservers/default_spec.rb

@@ -0,0 +1,21 @@
+require "spec_helper"
+
+describe "Reservers::Default" do
+  before(:each) do
+    Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
+  end
+
+  it "can reserve from multiple queues" do
+    high_queue = Qmore.client.queues['high']
+    critical_queue = Qmore.client.queues['critical']
+
+    high_queue.put(SomeJob, {})
+    critical_queue.put(SomeJob, {})
+
+    reserver = Qmore::Reservers::Default.new([critical_queue, high_queue])
+
+    expect(reserver.reserve.queue.name).to eq('critical')
+    expect(reserver.reserve.queue.name).to eq('high')
+  end
+end

data/spec/reservers/delegating_spec.rb

@@ -0,0 +1,79 @@
+require "spec_helper"
+
+describe 'Reservers::Delegating' do
+  before(:each) do
+    Redis.connect(:port => 6380).flushall
+    Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
+  end
+
+  it 'should implement #queues' do
+    qless1 = Qless::Client.new(:redis => Redis.connect(:port => 6379))
+    qless2 = Qless::Client.new(:redis => Redis.connect(:port => 6380))
+    queue_a = qless1.queues["a"]
+    queue_b = qless2.queues["b"]
+
+    job1 = queue_a.put(SomeJob, {})
+    job2 = queue_b.put(SomeJob, {})
+
+    expect(queue_a.length).to eq(1)
+    expect(queue_b.length).to eq(1)
+
+    reservers = []
+    reservers << Qmore::Reservers::Default.new([queue_a])
+    reservers << Qmore::Reservers::Default.new([queue_b])
+
+    reserver = Qmore::Reservers::Delegating.new(reservers)
+
+    queues = reserver.queues.to_a
+    expect(queues).to include(queue_a)
+    expect(queues).to include(queue_b)
+  end
+
+  it 'can delegate to multiple reservers' do
+    qless1 = Qless::Client.new(:redis => Redis.connect(:port => 6379))
+    qless2 = Qless::Client.new(:redis => Redis.connect(:port => 6380))
+    queue_a = qless1.queues["a"]
+    queue_b = qless2.queues["b"]
+
+    job1 = queue_a.put(SomeJob, {})
+    job2 = queue_b.put(SomeJob, {})
+
+    expect(queue_a.length).to eq(1)
+    expect(queue_b.length).to eq(1)
+
+    reserver1 = Qmore::Reservers::Default.new([queue_a])
+    reserver2 = Qmore::Reservers::Default.new([queue_b])
+
+    reserver = Qmore::Reservers::Delegating.new([reserver1, reserver2])
+    expect(reserver.reserve.queue.name).to eq('a')
+    expect(reserver.reserve.queue.name).to eq('b')
+  end
+
+  context 'with ordering' do
+    it 'should work with ordering strategy' do
+      qless1 = Qless::Client.new(:redis => Redis.connect(:port => 6379))
+      qless2 = Qless::Client.new(:redis => Redis.connect(:port => 6380))
+      queue_a = qless1.queues["a"]
+      queue_b = qless2.queues["b"]
+
+      job1 = queue_a.put(SomeJob, {})
+      job2 = queue_b.put(SomeJob, {})
+
+      expect(queue_a.length).to eq(1)
+      expect(queue_b.length).to eq(1)
+
+      reservers = []
+      reservers << Qmore::Reservers::Default.new([queue_a])
+      reservers << Qmore::Reservers::Default.new([queue_b])
+
+      reservers = Qmore::Reservers::Strategies::Ordering.shuffled(reservers)
+      reserver = Qmore::Reservers::Delegating.new(reservers)
+
+      expected = ['a', 'b']
+      expected.delete(reserver.reserve.queue.name)
+      expected.delete(reserver.reserve.queue.name)
+      expect(expected).to be_empty
+    end
+  end
+end

data/spec/reservers/strategies/filtering_spec.rb

@@ -0,0 +1,62 @@
+require "spec_helper"
+
+describe "Reservers::Strategies::Filtering" do
+  before(:each) do
+    Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
+  end
+
+  context "default qless filtering behavior" do
+    it "can filter multiple queues" do
+      high_queue = Qmore.client.queues['high']
+      critical_queue = Qmore.client.queues['critical']
+
+      high_queue.put(SomeJob, {})
+      critical_queue.put(SomeJob, {})
+
+      queues = [high_queue, critical_queue]
+      filter = Qmore::Reservers::Strategies::Filtering.default(queues, ["*"])
+
+      queues = filter.collect(&:name)
+      expect(queues).to include('critical')
+      expect(queues).to include('high')
+    end
+
+    it "should only return matching queues" do
+      high_queue = Qmore.client.queues['high']
+      critical_queue = Qmore.client.queues['critical']
+
+      high_queue.put(SomeJob, {})
+      critical_queue.put(SomeJob, {})
+
+      queues = [high_queue, critical_queue]
+      filter = Qmore::Reservers::Strategies::Filtering.default(queues, ['critical'])
+
+      queues = filter.collect(&:name)
+      expect(queues).to include('critical')
+      expect(queues).to_not include('high')
+    end
+
+    it "handles priorities" do
+      Qmore.configuration.priority_buckets = [{'pattern' => 'foo*', 'fairly' => false},
+                            {'pattern' => 'default', 'fairly' => false},
+                            {'pattern' => 'bar', 'fairly' => true}]
+
+      queues = []
+      ['other', 'blah', 'foobie', 'bar', 'foo'].each do |q|
+        queue = Qmore.client.queues[q]
+        queue.put(SomeJob, {})
+        expect(queue.length).to be(1)
+        queues << queue
+      end
+
+      filter = Qmore::Reservers::Strategies::Filtering.default(queues, ['*', '!blah'])
+
+      expect(filter.next.name).to eq('foo')
+      expect(filter.next.name).to eq('foobie')
+      expect(filter.next.name).to eq('other')
+      expect(filter.next.name).to eq('bar')
+      expect { filter.next }.to raise_error(StopIteration)
+    end
+  end
+end

data/spec/reservers/strategies/ordering_spec.rb

@@ -0,0 +1,42 @@
+require "spec_helper"
+
+describe "Reservers::Strategies::Ordering" do
+  before(:each) do
+    Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
+  end
+
+  context 'shuffled order' do
+    it 'should return all items in a shuffled order' do
+      input = [1,2,3,4,5,6,7,8]
+      ordering = Qmore::Reservers::Strategies::Ordering.shuffled(input)
+
+      round1 = ordering.to_a
+      round2 = ordering.to_a
+
+      expect(round1).not_to eq(input)
+      expect(round2).not_to eq(input)
+      expect(round1).not_to eq(round2)
+
+      input.each do |element|
+        expect(round1).to include(element)
+        expect(round2).to include(element)
+      end
+    end
+  end
+
+  context 'sampled subset' do
+    it 'should return the specified number of elements' do
+      input = [1,2,3,4,5,6,7,8]
+      ordering = Qmore::Reservers::Strategies::Ordering.sampled(input, 4)
+
+      round1 = ordering.to_a
+      round2 = ordering.to_a
+
+      expect(round1.length).to be(4)
+      expect(round2.length).to be(4)
+
+      expect(round1).not_to eq(round2)
+    end
+  end
+end

data/spec/reservers/strategies/sources_spec.rb

@@ -0,0 +1,128 @@
+require "spec_helper"
+
+describe "Reservers::Strategies::Sources" do
+  before(:each) do
+    Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
+  end
+
+  context 'direct source' do
+    it "does not return queues that have no work available" do
+      no_work_queue = Qmore.client.queues['no-work']
+      has_work_queue = Qmore.client.queues['has-work']
+
+      no_work_queue.put(SomeJob, {})
+      has_work_queue.put(SomeJob, {})
+
+      # drain the no work queue
+      no_work_queue.pop
+
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+
+      queues = source.collect(&:name)
+      expect(queues).to include("has-work")
+      expect(queues).not_to include("no-work")
+    end
+
+    it "should not ignore queues that have work in scheduled state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.put(SomeJob, {}, {:delay => 1600})
+
+      %w(waiting recurring depends stalled).each do |state|
+        expect(work_queue.counts[state]).to be(0)
+      end
+      expect(work_queue.counts["scheduled"]).to be(1)
+
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+
+      queues = source.collect(&:name)
+      expect(queues).to include("work")
+    end
+
+    it "should not ignore queues that have work in the depends state" do
+      work_queue = Qmore.client.queues['work']
+      jid = work_queue.put(SomeJob, {})
+      work_queue.put(SomeJob, {}, {:depends => [jid]})
+
+      work_queue.pop
+
+      %w(waiting recurring stalled scheduled).each do |state|
+        expect(work_queue.counts[state]).to be(0)
+      end
+      expect(work_queue.counts["depends"]).to be(1)
+
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+
+      queues = source.collect(&:name)
+      expect(queues).to include("work")
+    end
+
+    it "should not ignore queues that have work in the recurring state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.recur(SomeJob, {}, 1000)
+
+      %w(waiting depends stalled scheduled).each do |state|
+        expect(work_queue.counts[state]).to be(0)
+      end
+      expect(work_queue.counts["recurring"]).to be(1)
+
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+
+      queues = source.collect(&:name)
+      expect(queues).to include("work")
+    end
+
+    it "should not ignore queues that have work in the waiting state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.put(SomeJob, {})
+
+      %w(recurring depends stalled scheduled).each do |state|
+        expect(work_queue.counts[state]).to be(0)
+      end
+      expect(work_queue.counts["waiting"]).to be(1)
+
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+
+      queues = source.collect(&:name)
+      expect(queues).to include("work")
+    end
+  end
+
+  context 'background source' do
+    it 'should return the results from the delegate' do
+      work_queue = Qmore.client.queues['work']
+      work_queue.put(SomeJob, {})
+      source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+      source = Qmore::Reservers::Strategies::Sources::Background.new(source, 0.1)
+      thread = source.start # Start the update
+      source.stop
+      thread.join
+
+      queues = source.collect(&:name)
+      expect(queues).to include("work")
+    end
+
+    context 'start' do
+      it 'should update from the source' do
+        work_queue = Qmore.client.queues['work']
+        work_queue.put(SomeJob, {})
+        source = Qmore::Reservers::Strategies::Sources.direct(Qmore.client)
+        source = Qmore::Reservers::Strategies::Sources::Background.new(source, 0.1)
+        thread = source.start # Start the update
+
+        # Add another queue to the source
+        queue = Qmore.client.queues['work-queue']
+        queue.put(SomeJob, {})
+
+        # Sleep long enough for multiple updates to occur
+        sleep 0.3
+        source.stop
+        thread.join
+
+        queues = source.collect(&:name)
+        expect(queues).to include("work")
+        expect(queues).to include("work-queue")
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -1,6 +1,14 @@
 require 'rspec'
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+  config.mock_with :rspec do |c|
+    c.syntax = [:should, :expect]
+  end
+end
+
 require 'coveralls'
-require 'pry'
 Coveralls.wear!
 
 require 'qmore'

metadata

@@ -1,144 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: qmore
 version: !ruby/object:Gem::Version
-  version: 0.6.3
-  prerelease: 
+  version: 0.7.0
 platform: ruby
 authors:
 - Matt Conway
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-13 00:00:00.000000000 Z
+date: 2014-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: qless
-  type: :runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-    none: false
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: multi_json
-  type: :runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
-    none: false
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.7'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: gem_logger
-  type: :runtime
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: rake
-  type: :development
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: rspec
-  type: :development
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: rack-test
-  type: :development
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: pry
-  type: :development
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 - !ruby/object:Gem::Dependency
-  prerelease: false
   name: orderedhash
-  type: :development
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
+  type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-    none: false
 description: Qmore allows one to specify the queues a worker processes by the use
   of wildcards, negations, or dynamic look up from redis.  It also allows one to specify
   the relative priority between queues (rather than within a single queue).  It plugs
@@ -149,8 +132,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG
 - Gemfile
 - LICENSE
@@ -161,8 +144,14 @@ files:
 - lib/qmore.rb
 - lib/qmore/attributes.rb
 - lib/qmore/configuration.rb
-- lib/qmore/job_reserver.rb
 - lib/qmore/persistence.rb
+- lib/qmore/reservers.rb
+- lib/qmore/reservers/default.rb
+- lib/qmore/reservers/delegating.rb
+- lib/qmore/reservers/strategies.rb
+- lib/qmore/reservers/strategies/filtering.rb
+- lib/qmore/reservers/strategies/ordering.rb
+- lib/qmore/reservers/strategies/sources.rb
 - lib/qmore/server.rb
 - lib/qmore/server/views/dynamicqueues.erb
 - lib/qmore/server/views/priorities.erb
@@ -170,40 +159,49 @@ files:
 - qmore.gemspec
 - spec/attributes_spec.rb
 - spec/configuration_spec.rb
-- spec/job_reserver_spec.rb
 - spec/persistance_spec.rb
 - spec/redis/qless01-test.conf
 - spec/redis/qless02-test.conf
+- spec/reservers/default_spec.rb
+- spec/reservers/delegating_spec.rb
+- spec/reservers/strategies/filtering_spec.rb
+- spec/reservers/strategies/ordering_spec.rb
+- spec/reservers/strategies/sources_spec.rb
 - spec/server_spec.rb
 - spec/spec_helper.rb
 homepage: ''
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -593121205799441535
-  none: false
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -593121205799441535
-  none: false
 requirements: []
 rubyforge_project: qmore
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A qless plugin that gives more control over how queues are processed
-test_files: []
+test_files:
+- spec/attributes_spec.rb
+- spec/configuration_spec.rb
+- spec/persistance_spec.rb
+- spec/redis/qless01-test.conf
+- spec/redis/qless02-test.conf
+- spec/reservers/default_spec.rb
+- spec/reservers/delegating_spec.rb
+- spec/reservers/strategies/filtering_spec.rb
+- spec/reservers/strategies/ordering_spec.rb
+- spec/reservers/strategies/sources_spec.rb
+- spec/server_spec.rb
+- spec/spec_helper.rb

data/lib/qmore/job_reserver.rb

@@ -1,63 +0,0 @@
-module Qmore
-  class JobReserver
-    include Qmore::Attributes
-    # define queues for Qless worker to invoke.
-    attr_reader :queues
-    attr_reader :clients
-
-    def initialize(queues)
-      @queues = queues
-      # Pull the regex off of the Qless::Queue#name, we want to keep the same interface
-      # that Qless reservers use.
-      @regexes = queues.collect(&:name).uniq
-      @clients = {}
-      queues.each do |q|
-        @clients[q.client] ||= []
-        @clients[q.client] << q.name
-      end
-    end
-
-    def description
-      @description ||= @regexes.join(', ') + " (qmore)"
-    end
-
-    def prep_for_work!
-      # nothing here on purpose
-    end
-
-    def reserve
-      self.clients.keys.shuffle.each do |client|
-        self.extract_queues(client, self.clients[client]).each do |q|
-          job = q.pop
-          return job if job
-        end
-      end
-
-      nil
-    end
-
-    # @param [Qless::Client] client - client to pull queues from.
-    # @param [Array] regexes - array of regular expressions to match
-    # queues against.
-    def extract_queues(client, regexes)
-      # Cache the queues so we don't make multiple calls.
-      # and remove any queues that don't have work.
-      actual_queues = client.queues.counts.reject do |queue|
-        total = %w(waiting recurring depends stalled scheduled).inject(0) { |sum, state| sum += queue[state].to_i }
-        total == 0
-      end
-
-      # Grab all the actual queue names from the client.
-      queue_names = actual_queues.collect {|h| h['name'] }
-
-      # Match the queue names against the regexes provided.
-      matched_names = expand_queues(regexes, queue_names)
-
-      # Prioritize the queues.
-      prioritized_names = prioritize_queues(Qmore.configuration.priority_buckets, matched_names)
-
-      # collect the matched queues names in prioritized order.
-      prioritized_names.collect {|name| client.queues[name] }
-    end
-  end
-end

data/spec/job_reserver_spec.rb

@@ -1,195 +0,0 @@
-require "spec_helper"
-
-describe "JobReserver" do
-  include Qmore::Attributes
-
-  before(:each) do
-    Qmore.client.redis.flushall
-    Qmore.configuration = Qmore::Configuration.new
-  end
-
-  context "multiple qless server environment" do
-    it "can reserve jobs from regex queue names on multiple clients" do
-      qless1 = Qless::Client.new(:redis => Redis.connect(:port => 6379))
-      qless2 = Qless::Client.new(:redis => Redis.connect(:port => 6380))
-      queue_a = qless1.queues["a"]
-      queue_b = qless2.queues["b"]
-      queue_a.put(SomeJob, {})
-      queue_b.put(SomeJob, {})
-
-      queue_a.length.should == 1
-      queue_b.length.should == 1
-
-      reserver = Qmore::JobReserver.new([qless1.queues["*"], qless2.queues["*"]])
-      worker = Qless::Worker.new(reserver, :run_as_single_process => true)
-      worker.work(0)
-
-      queue_a.length.should == 0
-      queue_b.length.should == 0
-    end
-
-    it "shuffles the order of the clients" do
-      queue = Qmore.client.queues['queue']
-
-      reserver = Qmore::JobReserver.new([queue])
-      k = reserver.clients.keys
-      k.should_receive(:shuffle).once.and_return(reserver.clients.keys)
-      reserver.clients.should_receive(:keys).and_return(k)
-      reserver.reserve
-    end
-  end
-
-  context "basic qless behavior still works" do
-    it "ignores queues that have no work available" do
-      no_work_queue = Qmore.client.queues['no-work']
-      has_work_queue = Qmore.client.queues['has-work']
-
-      no_work_queue.put(SomeJob, {})
-      has_work_queue.put(SomeJob, {})
-
-      # drain the no work queue
-      no_work_queue.pop
-
-      reserver = Qmore::JobReserver.new([no_work_queue, has_work_queue])
-
-      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
-      queues.should include("has-work")
-      queues.should_not include("no-work")
-    end
-
-    it "should not ignore queues that have work in scheduled state" do
-      work_queue = Qmore.client.queues['work']
-      work_queue.put(SomeJob, {}, {:delay => 1600})
-
-      %w(waiting recurring depends stalled).each do |state|
-        work_queue.counts[state].should equal(0)
-      end
-      work_queue.counts["scheduled"].should equal(1)
-
-      reserver = Qmore::JobReserver.new([work_queue])
-      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
-      queues.should include("work")
-    end
-
-    it "should not ignore queues that have work in the depends state" do
-      work_queue = Qmore.client.queues['work']
-      jid = work_queue.put(SomeJob, {})
-      work_queue.put(SomeJob, {}, {:depends => [jid]})
-
-      work_queue.pop
-
-      %w(waiting recurring stalled scheduled).each do |state|
-        work_queue.counts[state].should equal(0)
-      end
-      work_queue.counts["depends"].should equal(1)
-
-      reserver = Qmore::JobReserver.new([work_queue])
-      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
-      queues.should include("work")
-    end
-
-    it "should not ignore queues that have work in the recurring state" do
-      work_queue = Qmore.client.queues['work']
-      work_queue.recur(SomeJob, {}, 1000)
-
-      %w(waiting depends stalled scheduled).each do |state|
-        work_queue.counts[state].should equal(0)
-      end
-      work_queue.counts["recurring"].should equal(1)
-
-      reserver = Qmore::JobReserver.new([work_queue])
-      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
-      queues.should include("work")
-    end
-
-    it "should not ignore queues that have work in the waiting state" do
-      work_queue = Qmore.client.queues['work']
-      work_queue.put(SomeJob, {})
-
-      %w(recurring depends stalled scheduled).each do |state|
-        work_queue.counts[state].should equal(0)
-      end
-      work_queue.counts["waiting"].should equal(1)
-
-      reserver = Qmore::JobReserver.new([work_queue])
-      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
-      queues.should include("work")
-    end
-
-    it "can reserve from multiple queues" do
-      high_queue = Qmore.client.queues['high']
-      critical_queue = Qmore.client.queues['critical']
-
-      high_queue.put(SomeJob, {})
-      critical_queue.put(SomeJob, {})
-
-      reserver = Qmore::JobReserver.new([critical_queue, high_queue])
-
-      reserver.reserve.queue.name.should == 'critical'
-      reserver.reserve.queue.name.should == 'high'
-    end
-
-    it "can work on multiple queues" do
-      high_queue = Qmore.client.queues['high']
-      critical_queue = Qmore.client.queues['critical']
-      high_queue.put(SomeJob, {})
-      critical_queue.put(SomeJob, {})
-
-      high_queue.length.should == 1
-      critical_queue.length.should == 1
-
-      reserver = Qmore::JobReserver.new([critical_queue, high_queue])
-
-      worker = Qless::Worker.new(reserver,
-                                 :run_as_single_process => true)
-      worker.work(0)
-
-      high_queue.length.should == 0
-      critical_queue.length.should == 0
-    end
-
-    it "can work on all queues" do
-      queues = []
-      ['high', 'critical', 'blahblah'].each do |q|
-        queue = Qmore.client.queues[q]
-        queue.put(SomeJob, {})
-        queue.length.should == 1
-        queues << queue
-      end
-
-      reserver = Qmore::JobReserver.new([Qmore.client.queues['*']])
-      worker = Qless::Worker.new(reserver,
-                                 :run_as_single_process => true)
-      worker.work(0)
-
-      queues.each do |q|
-        q.length.should == 0
-      end
-    end
-
-    it "handles priorities" do
-      Qmore.configuration.priority_buckets = [{'pattern' => 'foo*', 'fairly' => false},
-                            {'pattern' => 'default', 'fairly' => false},
-                            {'pattern' => 'bar', 'fairly' => true}]
-
-
-      queues = []
-      ['other', 'blah', 'foobie', 'bar', 'foo'].each do |q|
-        queue = Qmore.client.queues[q]
-        queue.put(SomeJob, {})
-        queue.length.should == 1
-        queues << queue
-      end
-
-      reserver = Qmore::JobReserver.new([Qmore.client.queues['*'], Qmore.client.queues['!blah']])
-
-      reserver.reserve.queue.name.should == 'foo'
-      reserver.reserve.queue.name.should == 'foobie'
-      reserver.reserve.queue.name.should == 'other'
-      reserver.reserve.queue.name.should == 'bar'
-      reserver.reserve.should be_nil
-    end
-
-  end
-
-end