qmore 0.6.2 → 0.6.3

data/lib/qmore/server.rb

@@ -8,7 +8,7 @@ module Qmore
     def self.registered(app)
 
       app.helpers do
-        
+
         def qmore_view(filename, options = {}, locals = {})
           options = {:layout => true, :locals => { :title => filename.to_s.capitalize }}.merge(options)
           dir = File.expand_path("../server/views/", __FILE__)
@@ -24,7 +24,7 @@ module Qmore
           queue_tab_index = original_tabs.index {|t| t[:name] == 'Queues' }
           original_tabs.insert(queue_tab_index + 1, *qmore_tabs)
         end
-        
+
       end
 
       #
@@ -34,7 +34,11 @@ module Qmore
       app.get "/dynamicqueues" do
         @queues = []
         real_queues = Qmore.client.queues.counts.collect {|q| q['name'] }
-        dqueues = Attr.get_dynamic_queues
+
+        # For the UI we always want the latest persisted data
+        configuration = Qmore.persistence.load
+
+        dqueues = configuration.dynamic_queues
         dqueues.each do |k, v|
           expanded = Attr.expand_queues(["@#{k}"], real_queues)
           expanded = expanded.collect { |q| q.split(":").last }
@@ -69,7 +73,14 @@ module Qmore
           values = queue['value'].to_s.split(',').collect { |q| q.gsub(/\s/, '') }
           queues[key] = values
         end
-        Attr.set_dynamic_queues(queues)
+
+        # For the UI we always want the latest persisted data
+        configuration = Qmore.persistence.load
+        configuration.dynamic_queues = queues
+
+        Qmore.persistence.write(configuration)
+        Qmore.configuration.dynamic_queues = configuration.dynamic_queues
+
         redirect to("/dynamicqueues")
       end
 
@@ -78,16 +89,25 @@ module Qmore
       #
 
       app.get "/queuepriority" do
-        @priorities = Attr.get_priority_buckets
+        # For the UI we always want the latest persisted data
+        configuration = Qmore.persistence.load
+
+        @priorities = configuration.priority_buckets
         qmore_view :priorities
       end
 
       app.post "/queuepriority" do
         priorities = params['priorities']
-        Attr.set_priority_buckets priorities
+
+        # For the UI we always want the latest persisted data
+        configuration = Qmore.persistence.load
+        configuration.priority_buckets = priorities
+
+        Qmore.persistence.write(configuration)
+        Qmore.configuration.priority_buckets = configuration.priority_buckets
+
         redirect to("/queuepriority")
       end
-
     end
   end
 end

data/lib/qmore/version.rb

@@ -1,3 +1,3 @@
 module Qmore
-  VERSION = "0.6.2"
+  VERSION = "0.6.3"
 end

data/qmore.gemspec

@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
 
   s.add_dependency("qless", '~> 0.9')
   s.add_dependency("multi_json", '~> 1.7')
+  s.add_dependency("gem_logger")
 
   s.add_development_dependency('rake')
   s.add_development_dependency('rspec')

data/spec/attributes_spec.rb

@@ -5,70 +5,12 @@ describe "Attributes" do
 
   before(:each) do
     Qmore.client.redis.flushall
-    @real_queues = ["high_x", "foo", "high_y", "superhigh_z"]
-  end
-
-  context "dynamic attributes" do
-
-    it "should always have a fallback pattern" do
-      get_dynamic_queues.should == {'default' => ['*']}
-    end
-
-    it "should allow setting single patterns" do
-      get_dynamic_queue('foo').should == ['*']
-      set_dynamic_queue('foo', ['bar'])
-      get_dynamic_queue('foo').should == ['bar']
-    end
-
-    it "should allow setting multiple patterns" do
-      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
-      get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
-    end
-
-    it "should remove mapping when setting empty value" do
-      get_dynamic_queues
-      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
-      get_dynamic_queues.should == {'foo' => ['bar'], 'baz' => ['boo'], 'default' => ['*']}
-
-      set_dynamic_queues({'foo' => [], 'baz' => ['boo']})
-      get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
-      set_dynamic_queues({'baz' => nil})
-      get_dynamic_queues.should == {'default' => ['*']}
-
-      set_dynamic_queues({'foo' => ['bar'], 'baz' => ['boo']})
-      set_dynamic_queue('foo', [])
-      get_dynamic_queues.should == {'baz' => ['boo'], 'default' => ['*']}
-      set_dynamic_queue('baz', nil)
-      get_dynamic_queues.should == {'default' => ['*']}
-    end
-
-  end
-
-  context "priority attributes" do
-
-    it "can lookup a default priority" do
-      get_priority_buckets.should == [{'pattern' => 'default'}]
-    end
-
-    it "can set priorities" do
-      set_priority_buckets [{'pattern' => 'foo', 'fairly' => 'false'}]
-      get_priority_buckets.should == [{'pattern' => 'foo', 'fairly' => 'false'},
-                                      {'pattern' => 'default'}]
-    end
-
-    it "can set priorities including default" do
-      set_priority_buckets [{'pattern' => 'foo', 'fairly' => false},
-                            {'pattern' => 'default', 'fairly' => false},
-                            {'pattern' => 'bar', 'fairly' => true}]
-      get_priority_buckets.should == [{'pattern' => 'foo', 'fairly' => false},
-                                      {'pattern' => 'default', 'fairly' => false},
-                                      {'pattern' => 'bar', 'fairly' => true}]
-    end
+    Qmore.configuration = Qmore::Configuration.new
 
+    @real_queues = ["high_x", "foo", "high_y", "superhigh_z"]
   end
 
   context "basic queue patterns" do
-
     it "can specify simple queues" do
       expand_queues(["foo"], @real_queues).should == ["foo"]
       expand_queues(["foo", "bar"], @real_queues).should == ["bar", "foo"]
@@ -92,23 +34,21 @@ describe "Attributes" do
     it "can blacklist queues with pattern" do
       expand_queues(["*", "!*high*"], @real_queues).should == ["foo"]
     end
-
   end
 
-  context "redis backed queues" do
-
+  context "expanding queues" do
     it "can dynamically lookup queues" do
-      set_dynamic_queue("mykey", ["foo", "bar"])
+      Qmore.configuration.dynamic_queues = {"mykey" => ["foo", "bar"]}
       expand_queues(["@mykey"], @real_queues).should == ["bar", "foo"]
     end
 
     it "can blacklist dynamic queues" do
-      set_dynamic_queue("mykey", ["foo"])
+      Qmore.configuration.dynamic_queues["mykey"] = ["foo"]
       expand_queues(["*", "!@mykey"], @real_queues).should == ["high_x", "high_y", "superhigh_z"]
     end
 
     it "can blacklist dynamic queues with negation" do
-      set_dynamic_queue("mykey", ["!foo", "high_x"])
+      Qmore.configuration.dynamic_queues["mykey"] = ["!foo", "high_x"]
       expand_queues(["!@mykey"], @real_queues).should == ["foo"]
     end
 
@@ -119,18 +59,19 @@ describe "Attributes" do
     end
 
     it "uses hostname as default key in dynamic queues" do
-      host = `hostname`.chomp
-      set_dynamic_queue(host, ["foo", "bar"])
+      host = Socket.gethostname
+      Qmore.configuration.dynamic_queues[host] = ["foo", "bar"]
+
       expand_queues(["@"], @real_queues).should == ["bar", "foo"]
     end
 
     it "can use wildcards in dynamic queues" do
-      set_dynamic_queue("mykey", ["*high*", "!high_y"])
+      Qmore.configuration.dynamic_queues["mykey"] = ["*high*", "!high_y"]
       expand_queues(["@mykey"], @real_queues).should == ["high_x", "superhigh_z"]
     end
 
     it "falls back to default queues when missing" do
-      set_dynamic_queue("default", ["foo", "bar"])
+      Qmore.configuration.dynamic_queues["default"] = ["foo", "bar"]
       expand_queues(["@mykey"], @real_queues).should == ["bar", "foo"]
     end
 
@@ -143,11 +84,9 @@ describe "Attributes" do
       @real_queues << "bar"
       expand_queues(["@mykey"], @real_queues).should == ["bar", "foo", "high_x", "high_y", "superhigh_z"]
     end
-
   end
 
   context "queue priorities" do
-
     it "should pick up all queues with default priority" do
       priority_buckets = [{'pattern' => 'default', 'fairly' => false}]
       prioritize_queues(priority_buckets, @real_queues).should == ["high_x", "foo", "high_y", "superhigh_z"]
@@ -229,7 +168,5 @@ describe "Attributes" do
       queues[5..-1].should == ["high_x", "foo", "high_y", "superhigh_z"]
       queues.should_not == others.sort + ["high_x", "foo", "high_y", "superhigh_z"]
     end
-
   end
-
 end

data/spec/configuration_spec.rb

@@ -0,0 +1,112 @@
+require "spec_helper"
+
+describe "Qmore::LegacyConfiguration" do
+  it "should always have a fallback pattern" do
+    configuration = Qmore::LegacyConfiguration.new(Qmore.persistence)
+
+    configuration.dynamic_queues['default'].should == ['*']
+  end
+
+  it "should load the latest dynamic queues from persistence" do
+    configuration = Qmore::LegacyConfiguration.new(Qmore.persistence)
+    expected_configuration = Qmore::Configuration.new
+
+    expected_configuration.dynamic_queues["foo"] = ["bar"]
+    expected_configuration.dynamic_queues["baz"] = ["biz"]
+    Qmore.persistence.write(expected_configuration)
+    configuration.dynamic_queues.should == expected_configuration.dynamic_queues
+
+    expected_configuration.dynamic_queues["baz"] = []
+    Qmore.persistence.write(expected_configuration)
+    # Ensuring baz was removed
+    configuration.dynamic_queues.should == {"foo" => ["bar"], "default" => ["*"]}
+    configuration.dynamic_queues.should == expected_configuration.dynamic_queues
+  end
+
+  it "should load the latest priority buckets from persistence" do
+    configuration = Qmore::LegacyConfiguration.new(Qmore.persistence)
+    expected_configuration = Qmore::Configuration.new
+    expected_configuration.priority_buckets = [{'pattern' => 'foo', 'fairly' => 'false'}]
+
+    Qmore.persistence.write(expected_configuration)
+
+    configuration.priority_buckets.should == expected_configuration.priority_buckets
+  end
+end
+
+describe "Qmore::Configuration" do
+  before(:each) do
+    Qmore.client.redis.flushall
+  end
+
+  context "dynamic queues" do
+    it "should always have a fallback pattern" do
+      configuration = Qmore::Configuration.new
+
+      configuration.dynamic_queues['default'].should == ['*']
+    end
+
+    it "should default any unspecified keys to default pattern" do
+      configuration = Qmore::Configuration.new
+      configuration.dynamic_queues['foo'].should == ['*']
+
+      configuration.dynamic_queues[Qmore::Configuration::DYNAMIC_FALLBACK_KEY] = ["foo", "bar"]
+      configuration.dynamic_queues['foo'].should == ["foo", "bar"]
+    end
+
+    it "should allow setting single patterns" do
+      configuration = Qmore::Configuration.new
+
+      configuration.dynamic_queues['foo'].should == ['*']
+      configuration.dynamic_queues['foo'] = ['bar']
+      configuration.dynamic_queues['foo'].should == ['bar']
+    end
+
+    it "should allow changing the pattern of the fallback key" do
+      configuration = Qmore::Configuration.new
+      configuration.dynamic_queues[Qmore::Configuration::DYNAMIC_FALLBACK_KEY].should == ['*']
+      configuration.dynamic_queues[Qmore::Configuration::DYNAMIC_FALLBACK_KEY] = ['foo', 'bar']
+      configuration.dynamic_queues[Qmore::Configuration::DYNAMIC_FALLBACK_KEY].should == ['foo', 'bar']
+    end
+
+    it "should ignore mappings when setting empty value" do
+      configuration = Qmore::Configuration.new
+      configuration.dynamic_queues = {'foo' => ['bar'], 'baz' => ['boo']}
+      configuration.dynamic_queues.should == {'default' => ['*'], 'foo' => ['bar'], 'baz' => ['boo']}
+
+      configuration.dynamic_queues = {'foo' => [], 'baz' => ['boo']}
+      configuration.dynamic_queues.should == {'default' => ['*'], 'baz' => ['boo']}
+      configuration.dynamic_queues = {'baz' => nil}
+      configuration.dynamic_queues.should == {'default' => ['*']}
+
+      configuration.dynamic_queues = {'foo' => ['bar'], 'baz' => ['boo']}
+      configuration.dynamic_queues['foo'] = []
+      configuration.dynamic_queues.should == {'default' => ["*"], 'baz' => ['boo']}
+      configuration.dynamic_queues['baz'] = nil
+      configuration.dynamic_queues.should == {'default' => ['*']}
+    end
+  end
+
+  context "priority attributes" do
+    it "should have a default priority" do
+      configuration = Qmore::Configuration.new
+      configuration.priority_buckets.should == [{'pattern' => 'default'}]
+    end
+
+    it "can set priorities" do
+      expected_priority_buckets = [{'pattern' => 'foo', 'fairly' => 'false'},{'pattern' => 'default'}]
+      configuration = Qmore::Configuration.new
+      configuration.priority_buckets = [{'pattern' => 'foo', 'fairly' => 'false'}]
+      configuration.priority_buckets.should == expected_priority_buckets
+    end
+
+    it "can set priorities including default" do
+      expected_priority_buckets = [{'pattern' => 'foo', 'fairly' => false},
+                                   {'pattern' => 'default', 'fairly' => false},
+                                   {'pattern' => 'bar', 'fairly' => true}]
+      configuration = Qmore::Configuration.new
+      configuration.priority_buckets = expected_priority_buckets
+      configuration.priority_buckets.should == expected_priority_buckets
+    end
+  end
+end

data/spec/job_reserver_spec.rb

@@ -5,6 +5,7 @@ describe "JobReserver" do
 
   before(:each) do
     Qmore.client.redis.flushall
+    Qmore.configuration = Qmore::Configuration.new
   end
 
   context "multiple qless server environment" do
@@ -13,8 +14,8 @@ describe "JobReserver" do
       qless2 = Qless::Client.new(:redis => Redis.connect(:port => 6380))
       queue_a = qless1.queues["a"]
       queue_b = qless2.queues["b"]
-      queue_a.put(SomeJob, [])
-      queue_b.put(SomeJob, [])
+      queue_a.put(SomeJob, {})
+      queue_b.put(SomeJob, {})
 
       queue_a.length.should == 1
       queue_b.length.should == 1
@@ -39,12 +40,88 @@ describe "JobReserver" do
   end
 
   context "basic qless behavior still works" do
+    it "ignores queues that have no work available" do
+      no_work_queue = Qmore.client.queues['no-work']
+      has_work_queue = Qmore.client.queues['has-work']
+
+      no_work_queue.put(SomeJob, {})
+      has_work_queue.put(SomeJob, {})
+
+      # drain the no work queue
+      no_work_queue.pop
+
+      reserver = Qmore::JobReserver.new([no_work_queue, has_work_queue])
+
+      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
+      queues.should include("has-work")
+      queues.should_not include("no-work")
+    end
+
+    it "should not ignore queues that have work in scheduled state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.put(SomeJob, {}, {:delay => 1600})
+
+      %w(waiting recurring depends stalled).each do |state|
+        work_queue.counts[state].should equal(0)
+      end
+      work_queue.counts["scheduled"].should equal(1)
+
+      reserver = Qmore::JobReserver.new([work_queue])
+      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
+      queues.should include("work")
+    end
+
+    it "should not ignore queues that have work in the depends state" do
+      work_queue = Qmore.client.queues['work']
+      jid = work_queue.put(SomeJob, {})
+      work_queue.put(SomeJob, {}, {:depends => [jid]})
+
+      work_queue.pop
+
+      %w(waiting recurring stalled scheduled).each do |state|
+        work_queue.counts[state].should equal(0)
+      end
+      work_queue.counts["depends"].should equal(1)
+
+      reserver = Qmore::JobReserver.new([work_queue])
+      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
+      queues.should include("work")
+    end
+
+    it "should not ignore queues that have work in the recurring state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.recur(SomeJob, {}, 1000)
+
+      %w(waiting depends stalled scheduled).each do |state|
+        work_queue.counts[state].should equal(0)
+      end
+      work_queue.counts["recurring"].should equal(1)
+
+      reserver = Qmore::JobReserver.new([work_queue])
+      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
+      queues.should include("work")
+    end
+
+    it "should not ignore queues that have work in the waiting state" do
+      work_queue = Qmore.client.queues['work']
+      work_queue.put(SomeJob, {})
+
+      %w(recurring depends stalled scheduled).each do |state|
+        work_queue.counts[state].should equal(0)
+      end
+      work_queue.counts["waiting"].should equal(1)
+
+      reserver = Qmore::JobReserver.new([work_queue])
+      queues = reserver.extract_queues(Qmore.client, ["*"]).collect(&:name)
+      queues.should include("work")
+    end
+
     it "can reserve from multiple queues" do
       high_queue = Qmore.client.queues['high']
       critical_queue = Qmore.client.queues['critical']
 
-      high_queue.put(SomeJob, [])
-      critical_queue.put(SomeJob, [])
+      high_queue.put(SomeJob, {})
+      critical_queue.put(SomeJob, {})
 
       reserver = Qmore::JobReserver.new([critical_queue, high_queue])
 
@@ -55,8 +132,8 @@ describe "JobReserver" do
     it "can work on multiple queues" do
       high_queue = Qmore.client.queues['high']
       critical_queue = Qmore.client.queues['critical']
-      high_queue.put(SomeJob, [])
-      critical_queue.put(SomeJob, [])
+      high_queue.put(SomeJob, {})
+      critical_queue.put(SomeJob, {})
 
       high_queue.length.should == 1
       critical_queue.length.should == 1
@@ -75,7 +152,7 @@ describe "JobReserver" do
       queues = []
       ['high', 'critical', 'blahblah'].each do |q|
         queue = Qmore.client.queues[q]
-        queue.put(SomeJob, [])
+        queue.put(SomeJob, {})
         queue.length.should == 1
         queues << queue
       end
@@ -91,7 +168,7 @@ describe "JobReserver" do
     end
 
     it "handles priorities" do
-      set_priority_buckets [{'pattern' => 'foo*', 'fairly' => false},
+      Qmore.configuration.priority_buckets = [{'pattern' => 'foo*', 'fairly' => false},
                             {'pattern' => 'default', 'fairly' => false},
                             {'pattern' => 'bar', 'fairly' => true}]
 
@@ -99,7 +176,7 @@ describe "JobReserver" do
       queues = []
       ['other', 'blah', 'foobie', 'bar', 'foo'].each do |q|
         queue = Qmore.client.queues[q]
-        queue.put(SomeJob, [])
+        queue.put(SomeJob, {})
         queue.length.should == 1
         queues << queue
       end