qmore 0.5.0

data/lib/qmore/server/views/dynamicqueues.erb

@@ -0,0 +1,64 @@
+
+<div class="page-header">
+  <h1>Dynamic Queues</h1>
+</div>
+<p class="intro">
+  The list below shows the dynamic queues currently defined.  When you start a worker with a dynamic queue key (@key_name), that key is looked up from the list below to determine the actual queues the worker should pull from.  Wildcards (*) and negation (leading !) can be used to select the queues the worker should process. There is always a fallback key - @default, which workers will use if the key for that worker is empty.  If both the key and the fallback are empty, the worker defaults to processing '*'
+</p>
+
+<form action="/dynamicqueues" method="POST" style="float:none; margin-top:10px">
+
+  <table class='queues'>
+    <tr>
+      <th>Name</th>
+      <th>Value</th>
+      <th>Expanded</th>
+      <th></th>
+    </tr>
+    <% @queues.each_with_index do |data, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-name" name="queues[][name]" value="<%= data['name'] %>" /></td>
+        <td><input type="text" id="input-<%= i %>-value" name="queues[][value]" value="<%= data['value'] %>" /></td>
+        <td class="expanded"><%= data['expanded'] %></td>
+        <td>
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+  
+  jQuery(function($) {
+
+    $("input").live("keypress", markDirty);
+    
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.queues");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("td.expanded").html("")
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+ 
+  });
+</script>

data/lib/qmore/server/views/priorities.erb

@@ -0,0 +1,78 @@
+<div class="page-header">
+  <h1>Queue Priority</h1>
+</div>
+<p class="intro">
+  The list below orders queue name patterns by the priority you wish them to be executed in.  The "Fairly" option allows you to indicate you want the queues within that pattern space be selected in a fair (random) manner, i.e. like resque-fairly.  The 'default' pattern must always exist, and matches against all queues that aren't in any of the other patterns.
+</p>
+
+<form action="/queuepriority" method="POST" style="float:none; margin-top:10px">
+
+  <table class="priorities">
+    <tr>
+      <th>Pattern</th>
+      <th>Fairly</th>
+      <th></th>
+    </tr>
+    <% @priorities.each_with_index do |priority, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-pattern" name="priorities[][pattern]" value="<%= priority["pattern"] %>" /></td>
+        <td><input type="checkbox" id="input-<%= i %>-fairly" name="priorities[][fairly]" value="true" <%= "checked" if priority["fairly"] %> /></td>
+        <td>
+          <a href="#up" class="up">Up</a> |
+          <a href="#down" class="down">Down</a> |
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+    
+  jQuery(function($) {
+    $("input").live("keypress", markDirty);
+    $("input[type=checkbox]").live("click", markDirty);
+    
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.priorities");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("input[type=checkbox]").attr("checked", false);
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+    $("a.up").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.prev(".line").before($row);
+      markDirty();
+    });
+
+    $("a.down").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.next(".line").after($row);
+      markDirty();
+    });
+
+  });
+</script>

data/lib/qmore/version.rb

@@ -0,0 +1,3 @@
+module Qmore
+  VERSION = "0.5.0"
+end

data/qmore.gemspec

@@ -0,0 +1,33 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'qmore/version'
+
+
+Gem::Specification.new do |s|
+  s.name        = "qmore"
+  s.version     = Qmore::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Matt Conway"]
+  s.email       = ["matt@conwaysplace.com"]
+  s.homepage    = ""
+  s.summary     = %q{A qless plugin that gives more control over how queues are processed}
+  s.description = %q{Qmore allows one to specify the queues a worker processes by the use of wildcards, negations, or dynamic look up from redis.  It also allows one to specify the relative priority between queues (rather than within a single queue).  It plugs into the Qless webapp to make it easy to manage the queues.}
+
+  s.rubyforge_project = "qmore"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency("qless", '~> 0.9.3')
+  s.add_dependency("multi_json", '~> 1.7.7')
+
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec', '~> 2.5')
+  s.add_development_dependency('rack-test', '~> 0.5.4')
+  # Needed for correct ordering when passing hash params to rack-test
+  s.add_development_dependency('orderedhash')
+
+end
+

data/spec/attributes_spec.rb

@@ -0,0 +1,235 @@
+require "spec_helper"
+
+describe "Attributes" do
+  include Qmore::Attributes
+
+  before(:each) do
+    Qmore.client.redis.flushall
+    @real_queues = ["high_x", "foo", "high_y", "superhigh_z"]
+  end
+
+  context "dynamic attributes" do
+
+    it "should always have a fallback pattern" do
+      get_dynamic_queues.should == {'default' => ['*']}
+    end
+
+    it "should allow setting single patterns" do
+      get_dynamic_queue('foo').should == ['*']
+      set_dynamic_queue('foo', ['bar'])
+      get_dynamic_queue('foo').should == ['bar']
+    end
+
+    it "should allow setting multiple patterns" do
+      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
+      get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
+    end
+
+    it "should remove mapping when setting empty value" do
+      get_dynamic_queues
+      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
+      get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
+
+      set_dynamic_queues({'foo' => [], 'baz' => ['boo']})
+      get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
+      set_dynamic_queues({'baz' => nil})
+      get_dynamic_queues.should == {'default' => ['*']}
+
+      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
+      set_dynamic_queue('foo', [])
+      get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
+      set_dynamic_queue('baz', nil)
+      get_dynamic_queues.should == {'default' => ['*']}
+    end
+
+  end
+
+  context "priority attributes" do
+
+    it "can lookup a default priority" do
+      get_priority_buckets.should == [{'pattern' => 'default'}]
+    end
+
+    it "can set priorities" do
+      set_priority_buckets [{'pattern' => 'foo', 'fairly' => 'false'}]
+      get_priority_buckets.should == [{'pattern' => 'foo', 'fairly' => 'false'},
+                                      {'pattern' => 'default'}]
+    end
+
+    it "can set priorities including default" do
+      set_priority_buckets [{'pattern' => 'foo', 'fairly' => false},
+                            {'pattern' => 'default', 'fairly' => false},
+                            {'pattern' => 'bar', 'fairly' => true}]
+      get_priority_buckets.should == [{'pattern' => 'foo', 'fairly' => false},
+                                      {'pattern' => 'default', 'fairly' => false},
+                                      {'pattern' => 'bar', 'fairly' => true}]
+    end
+
+  end
+
+  context "basic queue patterns" do
+
+    it "can specify simple queues" do
+      expand_queues(["foo"], @real_queues).should == ["foo"]
+      expand_queues(["foo", "bar"], @real_queues).should == ["bar", "foo"]
+    end
+
+    it "can specify simple wildcard" do
+      worker = Qless::Worker.new("*")
+      expand_queues(["*"], @real_queues).should == ["foo", "high_x", "high_y", "superhigh_z"]
+    end
+
+    it "can include queues with pattern" do
+      expand_queues(["high*"], @real_queues).should == ["high_x", "high_y"]
+      expand_queues(["*high_z"], @real_queues).should == ["superhigh_z"]
+      expand_queues(["*high*"], @real_queues).should == ["high_x", "high_y", "superhigh_z"]
+    end
+
+    it "can blacklist queues" do
+      expand_queues(["*", "!foo"], @real_queues).should == ["high_x", "high_y", "superhigh_z"]
+    end
+
+    it "can blacklist queues with pattern" do
+      expand_queues(["*", "!*high*"], @real_queues).should == ["foo"]
+    end
+
+  end
+
+  context "redis backed queues" do
+
+    it "can dynamically lookup queues" do
+      set_dynamic_queue("mykey", ["foo", "bar"])
+      expand_queues(["@mykey"], @real_queues).should == ["bar", "foo"]
+    end
+
+    it "can blacklist dynamic queues" do
+      set_dynamic_queue("mykey", ["foo"])
+      expand_queues(["*", "!@mykey"], @real_queues).should == ["high_x", "high_y", "superhigh_z"]
+    end
+
+    it "can blacklist dynamic queues with negation" do
+      set_dynamic_queue("mykey", ["!foo", "high_x"])
+      expand_queues(["!@mykey"], @real_queues).should == ["foo"]
+    end
+
+    it "will not bloat the given real_queues" do
+      orig = @real_queues.dup
+      expand_queues(["@mykey"], @real_queues)
+      @real_queues.should == orig
+    end
+
+    it "uses hostname as default key in dynamic queues" do
+      host = `hostname`.chomp
+      set_dynamic_queue(host, ["foo", "bar"])
+      expand_queues(["@"], @real_queues).should == ["bar", "foo"]
+    end
+
+    it "can use wildcards in dynamic queues" do
+      set_dynamic_queue("mykey", ["*high*", "!high_y"])
+      expand_queues(["@mykey"], @real_queues).should == ["high_x", "superhigh_z"]
+    end
+
+    it "falls back to default queues when missing" do
+      set_dynamic_queue("default", ["foo", "bar"])
+      expand_queues(["@mykey"], @real_queues).should == ["bar", "foo"]
+    end
+
+    it "falls back to all queues when missing and no default" do
+      expand_queues(["@mykey"], @real_queues).should == ["foo", "high_x", "high_y", "superhigh_z"]
+    end
+
+    it "falls back to all queues when missing and no default and keep up to date" do
+      expand_queues(["@mykey"], @real_queues).should == ["foo", "high_x", "high_y", "superhigh_z"]
+      @real_queues << "bar"
+      expand_queues(["@mykey"], @real_queues).should == ["bar", "foo", "high_x", "high_y", "superhigh_z"]
+    end
+
+  end
+
+  context "queue priorities" do
+
+    it "should pick up all queues with default priority" do
+      priority_buckets = [{'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "foo", "high_y", "superhigh_z"]
+    end
+
+    it "should pick up all queues fairly" do
+      # do a bunch to reduce likelyhood of random match causing test failure
+      @real_queues = 50.times.collect { |i| "auto_#{i}" }
+      priority_buckets = [{'pattern' => 'default', 'fairly' => true}]
+      prioritize_queues(priority_buckets, @real_queues).should_not == @real_queues.sort
+      prioritize_queues(priority_buckets, @real_queues).sort.should == @real_queues.sort
+    end
+
+    it "should prioritize simple pattern" do
+      priority_buckets = [{'pattern' => 'superhigh_z', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["superhigh_z", "high_x", "foo", "high_y"]
+    end
+
+    it "should prioritize multiple simple patterns" do
+      priority_buckets = [{'pattern' => 'superhigh_z', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false},
+                          {'pattern' => 'foo', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["superhigh_z", "high_x", "high_y", "foo"]
+    end
+
+    it "should prioritize simple wildcard pattern" do
+      priority_buckets = [{'pattern' => 'high*', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "high_y", "foo", "superhigh_z"]
+    end
+
+    it "should prioritize simple wildcard pattern with correct matching" do
+      priority_buckets = [{'pattern' => '*high*', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "high_y", "superhigh_z", "foo"]
+    end
+
+    it "should prioritize negation patterns" do
+      @real_queues.delete("high_x")
+      @real_queues << "high_x"
+      priority_buckets = [{'pattern' => 'high*,!high_x', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_y", "foo", "superhigh_z", "high_x"]
+    end
+
+    it "should not be affected by standalone negation patterns" do
+      priority_buckets = [{'pattern' => '!high_x', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "foo", "high_y", "superhigh_z"]
+    end
+
+    it "should allow multiple inclusive patterns" do
+      priority_buckets = [{'pattern' => 'high_x, superhigh*', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "superhigh_z", "foo", "high_y"]
+    end
+
+    it "should prioritize fully inclusive wildcard pattern" do
+      priority_buckets = [{'pattern' => '*high*', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "high_y", "superhigh_z", "foo"]
+    end
+
+    it "should handle empty default match" do
+      priority_buckets = [{'pattern' => '*', 'fairly' => false},
+                          {'pattern' => 'default', 'fairly' => false}]
+      prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "foo", "high_y", "superhigh_z"]
+    end
+
+    it "should pickup wildcard queues fairly" do
+      others = 5.times.collect { |i| "other#{i}" }
+      @real_queues = @real_queues + others
+
+      priority_buckets = [{'pattern' => 'other*', 'fairly' => true},
+                          {'pattern' => 'default', 'fairly' => false}]
+      queues = prioritize_queues(priority_buckets, @real_queues)
+      queues[0..4].sort.should == others.sort
+      queues[5..-1].should == ["high_x", "foo", "high_y", "superhigh_z"]
+      queues.should_not == others.sort + ["high_x", "foo", "high_y", "superhigh_z"]
+    end
+
+  end
+
+end

data/spec/job_reserver_spec.rb

@@ -0,0 +1,87 @@
+require "spec_helper"
+
+describe "JobReserver" do
+  include Qmore::Attributes
+  
+  before(:each) do
+    Qmore.client.redis.flushall
+  end
+  
+  context "basic qless behavior still works" do
+
+    it "can reserve from multiple queues" do
+      high_queue = Qmore.client.queues['high'] 
+      critical_queue = Qmore.client.queues['critical'] 
+
+      high_queue.put(SomeJob, [])
+      critical_queue.put(SomeJob, [])
+      
+      reserver = Qmore::JobReserver.new([critical_queue, high_queue])
+      reserver.reserve.queue.name.should == 'critical'
+      reserver.reserve.queue.name.should == 'high'
+    end
+
+    it "can work on multiple queues" do
+      high_queue = Qmore.client.queues['high'] 
+      critical_queue = Qmore.client.queues['critical'] 
+      high_queue.put(SomeJob, [])
+      critical_queue.put(SomeJob, [])
+
+      high_queue.length.should == 1
+      critical_queue.length.should == 1
+      
+      reserver = Qmore::JobReserver.new([critical_queue, high_queue])
+      
+      worker = Qless::Worker.new(reserver,
+                                 :run_as_single_process => true)
+      worker.work(0)
+      
+      high_queue.length.should == 0
+      critical_queue.length.should == 0
+    end
+
+    it "can work on all queues" do
+      queues = []
+      ['high', 'critical', 'blahblah'].each do |q|
+        queue = Qmore.client.queues[q] 
+        queue.put(SomeJob, [])
+        queue.length.should == 1
+        queues << queue
+      end
+
+      reserver = Qmore::JobReserver.new([Qmore.client.queues['*']])
+      worker = Qless::Worker.new(reserver,
+                                 :run_as_single_process => true)
+      worker.work(0)
+      
+      queues.each do |q|
+        q.length.should == 0
+      end
+    end
+
+    it "handles priorities" do
+      set_priority_buckets [{'pattern' => 'foo*', 'fairly' => false},
+                            {'pattern' => 'default', 'fairly' => false},
+                            {'pattern' => 'bar', 'fairly' => true}]
+      
+      
+      queues = []
+      ['other', 'blah', 'foobie', 'bar', 'foo'].each do |q|
+        queue = Qmore.client.queues[q] 
+        queue.put(SomeJob, [])
+        queue.length.should == 1
+        queues << queue
+      end
+
+      reserver = Qmore::JobReserver.new([Qmore.client.queues['*'], Qmore.client.queues['!blah']])
+      
+      reserver.reserve.queue.name.should == 'foo'
+      reserver.reserve.queue.name.should == 'foobie'
+      reserver.reserve.queue.name.should == 'other'
+      reserver.reserve.queue.name.should == 'bar'
+      reserver.reserve.should be_nil
+    end
+
+  end
+
+end