qmore 0.6.2 → 0.6.3

data/CHANGELOG

@@ -1,3 +1,10 @@
+0.6.3 (06/13/2014)
+------------------
+
+Prevents updating configuration every time we want to retrieve queues.
+(see monitoring in README)
+Optimizations to job reserving - doesn't look at empty queues.
+
 0.6.2 (03/10/2014)
 ------------------
 

data/README.md

@@ -15,11 +15,23 @@ Alternatively, if you have some other way of launching workers (e.g. qless-pool)
     Qless::Pool.pool_factory.reserver_class = Qmore::JobReserver
     Qmore.client == Qless::Pool.pool_factory.client
 
+    # Enabling Monitoring
+    # Redis persistence defaults to using the same connection
+    # used for reserving jobs, however it is not required that they
+    # be the same redis connection. I.e. you can store configuration
+    # on a completely separate instance of redis.
+    Qmore.persistence = Qless::Persistence::Redis.new(Qmore.client.redis)
+    # Configure the monitor thread with the persistence type, and the interval at which to update
+    # Monitor defaults to using Qmore.persistence and 2 minutes
+    Qmore.monitor = Qless::persistence::Monitor.new(Qmore.persistence, 120)
+    # Start up monitor thread
+    Qmore.monitor.start
+
 To enable the web UI, use a config.ru similar to the following depending on your environment:
 
     require 'qless/server'
     require 'qmore-server'
-    
+
     Qless::Server.client = Qless::Client.new(:host => "some-host", :port => 7000)
     Qmore.client = Qless::Server.client
     run Qless::Server.new(Qmore.client)
@@ -94,18 +106,18 @@ And I run my worker with QUEUES=\*
 
 If I set my patterns like:
 
-high\_\* (fairly unchecked)  
-default (fairly unchecked)  
-low\_\* (fairly unchecked)  
+high\_\* (fairly unchecked)
+default (fairly unchecked)
+low\_\* (fairly unchecked)
 
 Then, the worker will scan the queues for work in this order:
 high_bar, high_baz, high_foo, myqueue, otherqueue, somequeue, low_bar, low_baz, low_foo
 
 If I set my patterns like:
 
-high\_\* (fairly checked)  
-default (fairly checked)  
-low\_\* (fairly checked)  
+high\_\* (fairly checked)
+default (fairly checked)
+low\_\* (fairly checked)
 
 Then, the worker will scan the queues for work in this order:
 

data/lib/qmore.rb

@@ -1,16 +1,43 @@
 require 'qless'
 require 'qless/worker'
+require 'gem_logger'
+require 'qmore/configuration'
+require 'qmore/persistence'
 require 'qmore/attributes'
 require 'qmore/job_reserver'
 
 module Qmore
-  
+
   def self.client=(client)
     @client = client
   end
-  
+
   def self.client
-    @client ||= Qless::Client.new 
+    @client ||= Qless::Client.new
+  end
+
+  def self.configuration
+    @configuration ||= Qmore::LegacyConfiguration.new(Qmore.persistence)
+  end
+
+  def self.configuration=(configuration)
+    @configuration = configuration
+  end
+
+  def self.persistence
+    @persistence ||= Qmore::Persistence::Redis.new(self.client.redis)
+  end
+
+  def self.persistence=(manager)
+    @persistence = manager
+  end
+
+  def self.monitor
+    @monitor ||= Qmore::Persistence::Monitor.new(self.persistence, 120)
+  end
+
+  def self.monitor=(monitor)
+    @monitor = monitor
   end
 end
 

data/lib/qmore/attributes.rb

@@ -1,82 +1,9 @@
 require 'multi_json'
 
 module Qmore
-  DYNAMIC_QUEUE_KEY = "qmore:dynamic"
-  PRIORITY_KEY = "qmore:priority"
-  DYNAMIC_FALLBACK_KEY = "default"
-
   module Attributes
     extend self
 
-    def redis
-      Qmore.client.redis
-    end
-    
-    def decode(data)
-      MultiJson.load(data) if data
-    end
-    
-    def encode(data)
-      MultiJson.dump(data)
-    end
-    
-    def get_dynamic_queue(key, fallback=['*'])
-      data = redis.hget(DYNAMIC_QUEUE_KEY, key)
-      queue_names = decode(data)
-
-      if queue_names.nil? || queue_names.size == 0
-        data = redis.hget(DYNAMIC_QUEUE_KEY, DYNAMIC_FALLBACK_KEY)
-        queue_names = decode(data)
-      end
-      
-      if queue_names.nil? || queue_names.size == 0
-        queue_names = fallback
-      end
-
-      return queue_names
-    end
-
-    def set_dynamic_queue(key, values)
-      if values.nil? or values.size == 0
-        redis.hdel(DYNAMIC_QUEUE_KEY, key)
-      else
-        redis.hset(DYNAMIC_QUEUE_KEY, key, encode(values))
-      end
-    end
-    
-    def set_dynamic_queues(dynamic_queues)
-      redis.multi do
-        redis.del(DYNAMIC_QUEUE_KEY)
-        dynamic_queues.each do |k, v|
-          set_dynamic_queue(k, v)
-        end
-      end
-    end
-
-    def get_dynamic_queues
-      result = {}
-      queues = redis.hgetall(DYNAMIC_QUEUE_KEY)
-      queues.each {|k, v| result[k] = decode(v) }
-      result[DYNAMIC_FALLBACK_KEY] ||= ['*']
-      return result
-    end
-    
-    def get_priority_buckets
-      priorities = Array(redis.lrange(PRIORITY_KEY, 0, -1))
-      priorities = priorities.collect {|p| decode(p) }
-      priorities << {'pattern' => 'default'} unless priorities.find {|b| b['pattern'] == 'default' }
-      return priorities
-    end
-
-    def set_priority_buckets(data)
-      redis.multi do
-        redis.del(PRIORITY_KEY)
-        Array(data).each do |v|
-           redis.rpush(PRIORITY_KEY, encode(v))
-        end
-      end
-    end
-
     # Returns a list of queues to use when searching for a job.
     #
     # A splat ("*") means you want every queue (in alpha order) - this
@@ -93,7 +20,7 @@ module Qmore
     def expand_queues(queue_patterns, real_queues)
       queue_patterns = queue_patterns.dup
       real_queues = real_queues.dup
-      
+
       matched_queues = []
 
       while q = queue_patterns.shift
@@ -103,7 +30,7 @@ module Qmore
           key = $2.strip
           key = Socket.gethostname if key.size == 0
 
-          add_queues = get_dynamic_queue(key)
+          add_queues = Qmore.configuration.dynamic_queues[key]
           add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
 
           queue_patterns.concat(add_queues)
@@ -149,34 +76,34 @@ module Qmore
         end
 
         bucket_queues, remaining = [], []
-        
+
         patterns = bucket_pattern.split(',')
         patterns.each do |pattern|
           pattern = pattern.strip
-          
+
           if pattern =~ /^!/
             negated = true
             pattern = pattern[1..-1]
           end
-          
+
           patstr = pattern.gsub(/\*/, ".*")
           pattern = /^#{patstr}$/
-        
-        
+
+
           if negated
             bucket_queues -= bucket_queues.grep(pattern)
           else
             bucket_queues.concat(real_queues.grep(pattern))
           end
-        
+
         end
-        
+
         bucket_queues.uniq!
         bucket_queues.shuffle! if fairly
         real_queues = real_queues - bucket_queues
-        
+
         result << bucket_queues
-        
+
       end
 
       # insert the remaining queues at the position the default item was at (or last)
@@ -186,6 +113,6 @@ module Qmore
 
       return result
     end
-    
+
   end
 end

data/lib/qmore/configuration.rb

@@ -0,0 +1,74 @@
+module Qmore
+  class Configuration
+    DYNAMIC_FALLBACK_KEY = "default".freeze
+
+    attr_accessor :dynamic_queues, :priority_buckets
+
+    def initialize
+      # Initialize the dynamic queues
+      self.dynamic_queues = {}
+      self.priority_buckets = []
+    end
+
+    def dynamic_queues=(hash)
+      queues = DynamicQueueHash.new
+      queues[DYNAMIC_FALLBACK_KEY] = ['*']
+      hash.each do |key, values|
+        queues[key] = values
+      end
+      @dynamic_queues = queues
+    end
+
+    # @param [Array] priorities
+    def priority_buckets=(priorities)
+      priorities << {'pattern' => 'default'} unless priorities.find {|b| b['pattern'] == 'default' }
+      @priority_buckets = priorities
+    end
+
+    private
+
+    class DynamicQueueHash < Hash
+      # @param key [String]
+      # @param values [Array]
+      def []=(key,values)
+        # remove any keys that have been set to empty hash or nil.
+        if values.nil? || values.size == 0
+          self.delete(key)
+          return
+        end
+
+        super
+      end
+
+      def [](key)
+        super(key) || super(DYNAMIC_FALLBACK_KEY)
+      end
+    end
+  end
+
+  # Legacy style configuration which loads the configuration on each access.
+  class LegacyConfiguration
+    def initialize(persistence)
+      @configuration = Configuration.new
+      @persistence = persistence
+    end
+
+    def dynamic_queues=(hash)
+      @configuration.dynamic_queues = hash
+    end
+
+    def priority_buckets=(priorities)
+      @configuration.priority_buckets = priorities
+    end
+
+    def priority_buckets
+      @configuration.priority_buckets = @persistence.read_priority_buckets
+      @configuration.priority_buckets
+    end
+
+    def dynamic_queues
+      @configuration.dynamic_queues = @persistence.read_dynamic_queues
+      @configuration.dynamic_queues
+    end
+  end
+end

data/lib/qmore/job_reserver.rb

@@ -36,23 +36,28 @@ module Qmore
       nil
     end
 
-    protected
-
+    # @param [Qless::Client] client - client to pull queues from.
+    # @param [Array] regexes - array of regular expressions to match
+    # queues against.
     def extract_queues(client, regexes)
       # Cache the queues so we don't make multiple calls.
-      actual_queues = client.queues
+      # and remove any queues that don't have work.
+      actual_queues = client.queues.counts.reject do |queue|
+        total = %w(waiting recurring depends stalled scheduled).inject(0) { |sum, state| sum += queue[state].to_i }
+        total == 0
+      end
 
       # Grab all the actual queue names from the client.
-      queue_names = actual_queues.counts.collect {|h| h['name'] }
+      queue_names = actual_queues.collect {|h| h['name'] }
 
       # Match the queue names against the regexes provided.
       matched_names = expand_queues(regexes, queue_names)
 
       # Prioritize the queues.
-      prioritized_names = prioritize_queues(get_priority_buckets, matched_names)
+      prioritized_names = prioritize_queues(Qmore.configuration.priority_buckets, matched_names)
 
       # collect the matched queues names in prioritized order.
-      prioritized_names.collect {|name| actual_queues[name] }
+      prioritized_names.collect {|name| client.queues[name] }
     end
   end
 end

data/lib/qmore/persistence.rb

@@ -0,0 +1,105 @@
+module Qmore::Persistence
+  class Monitor
+    include GemLogger::LoggerSupport
+
+    attr_reader :updating, :interval
+    # @param [Qmore::persistence] persistence - responsible for reading the configuration
+    # from some source (redis, file, db, etc)
+    # @param [Integer] interval - the period, in seconds, to wait between updates to the configuration.
+    # defaults to 1 minute
+    def initialize(persistence, interval)
+      @persistence = persistence
+      @interval = interval
+    end
+
+    def start
+      return if @updating
+      @updating = true
+
+      # Ensure we load the configuration once from persistence before
+      # the background thread.
+      Qmore.configuration = @persistence.load
+
+      Thread.new do
+        while(@updating) do
+          sleep @interval
+          begin
+            Qmore.configuration = @persistence.load
+          rescue => e
+            logger.error "#{e.class.name} : #{e.message}"
+          end
+        end
+      end
+    end
+
+    def stop
+      @updating = false
+    end
+  end
+
+  class Redis
+    DYNAMIC_QUEUE_KEY = "qmore:dynamic".freeze
+    PRIORITY_KEY = "qmore:priority".freeze
+
+    attr_reader :redis
+
+    def initialize(redis)
+      @redis = redis
+    end
+
+    def decode(data)
+      MultiJson.load(data) if data
+    end
+
+    def encode(data)
+      MultiJson.dump(data)
+    end
+
+    # Returns a Qmore::Configuration from the underlying data storage mechanism
+    # @return [Qmore::Configuration]
+    def load
+      configuration = Qmore::Configuration.new
+      configuration.dynamic_queues = self.read_dynamic_queues
+      configuration.priority_buckets = self.read_priority_buckets
+      configuration
+    end
+
+    # Writes out the configuration to the underlying data storage mechanism.
+    # @param[Qmore::Configuration] configuration to be persisted
+    def write(configuration)
+      write_dynamic_queues(configuration.dynamic_queues)
+      write_priority_buckets(configuration.priority_buckets)
+    end
+
+    def read_dynamic_queues
+      result = {}
+      queues = redis.hgetall(DYNAMIC_QUEUE_KEY)
+      queues.each {|k, v| result[k] = decode(v) }
+      return result
+    end
+
+    def read_priority_buckets
+      priorities = Array(redis.lrange(PRIORITY_KEY, 0, -1))
+      priorities = priorities.collect {|p| decode(p) }
+      return priorities
+    end
+
+    def write_priority_buckets(data)
+      redis.multi do
+        redis.del(PRIORITY_KEY)
+        Array(data).each do |v|
+           redis.rpush(PRIORITY_KEY, encode(v))
+        end
+      end
+    end
+
+    def write_dynamic_queues(dynamic_queues)
+      redis.multi do
+        redis.del(DYNAMIC_QUEUE_KEY)
+        dynamic_queues.each do |k, v|
+          redis.hset(DYNAMIC_QUEUE_KEY, k, encode(v))
+        end
+      end
+    end
+  end
+end