reqless 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1042e561166bf47c7938f8d372d6ff1b69e6cdf999577b44e3c4a2ac3326cf1
-  data.tar.gz: 983e5f31909dca78405b16208064460f830048c9a277fdbc76f9a4b5b8ea194c
+  metadata.gz: d89345ee8de2c77b0083cd7c93eb4aff552d57a298f6c0f55953093e5c69bd98
+  data.tar.gz: c41fac4baeaebe950d1f72fc69647b4aeca870b31bdd8b595cb821f1cea4bce5
 SHA512:
-  metadata.gz: 89613e228c2b8bc014c25125eb65db6bd60816397b6e6df712e433d7901f4b21bd1b9444d73e8583d63209baeee513aca5e75f4c6a554008d9b78be0ac1bc9aa
-  data.tar.gz: fa149f59031bc460de0395bc1c5b768129f741ba4a646b6772049be60fb491e04df9584c8ba42293b88810c35180559b33292922fa855d9380d33873a9672978
+  metadata.gz: b60f2d5e8d40f41179b418a09397dd2669126c351108fffc3781907793e3b4dee25dfa970b0bed8de2e7c05466bd1d10d1b53e0b4bb5b5f891327e8b63c688f2
+  data.tar.gz: 8afc28f017ec955c72994a1eeca4f7402c294603ab8a62ef991c1955a763614e66f3cdf0567728a80eeb45b6eeecbe3511c114e277bedde338ebf88bb82a476b

data/lib/reqless/job_reservers/delegating.rb

@@ -0,0 +1,30 @@
+module Reqless::JobReservers
+  # @param [Enumerable] reservers - a set of reservers
+  # to check for work.
+  class Delegating < Struct.new(:reservers)
+    def description
+      "Delegating Reserver"
+    end
+
+    def prep_for_work!
+      # nothing here on purpose
+    end
+
+    def queues
+      Enumerator.new do |yielder|
+        reservers.each do |reserver|
+          reserver.queues.each {|queue| yielder << queue}
+        end
+      end
+    end
+
+    def reserve
+      reservers.each do |reserver|
+        if (job = reserver.reserve)
+          return job
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/filtering.rb

@@ -0,0 +1,42 @@
+require 'reqless/queue_patterns_helper'
+
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+module Reqless::JobReservers::Strategies::Filtering
+  # @param [Enumerable] queues - a source of queues
+  # @param [Array] regexes - a list of regexes to match against.
+  # Return an enumerator of the filtered queues in
+  # in prioritized order.
+  def self.default(
+    queues,
+    regexes,
+    queue_identifier_patterns,
+    queue_priority_patterns
+  )
+    Enumerator.new do |yielder|
+      # Map queues to their names
+      mapped_queues = queues.reduce({}) do |hash,queue|
+        hash[queue.name] = queue
+        hash
+      end
+
+      # Filter the queue names against the regexes provided.
+      matches = Reqless::QueuePatternsHelper.expand_queues(regexes, mapped_queues.keys, queue_identifier_patterns)
+
+      # Prioritize the queues.
+      prioritized_names = Reqless::QueuePatternsHelper.prioritize_queues(queue_priority_patterns, matches)
+
+      prioritized_names.each do |name|
+        queue = mapped_queues[name]
+        if queue
+          yielder << queue
+        end
+      end
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/ordering.rb

@@ -0,0 +1,31 @@
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+module Reqless::JobReservers::Strategies::Ordering
+  # Shuffles the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  def self.shuffled(enumerable)
+    Enumerator.new do |yielder|
+      enumerable.to_a.shuffle.each do |e|
+        yielder << e
+      end
+    end
+  end
+
+  # Samples a subset of the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  # @param [Integer] sample_size   - number of items to take per iteration
+  def self.sampled(enumerable, sample_size = 5)
+    Enumerator.new do |yielder|
+      enumerable.to_a.sample(sample_size).each do |e|
+        yielder << e
+      end
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/sources.rb

@@ -0,0 +1,63 @@
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+# This module provides the different kinds of queue sources used by qmore
+module Reqless::JobReservers::Strategies::Sources
+  # Direct source uses a client to generate the queues we should
+  # pull work from. Ignores any queues that do not have tasks available.
+  def self.direct(client)
+    Enumerator.new do |yielder|
+      queues = client.queues.counts.select do |queue|
+        %w(waiting recurring depends stalled scheduled).any? {|state| queue[state].to_i > 0 }
+      end
+
+      queues.each do |queue|
+        yielder << client.queues[queue['name']]
+      end
+    end
+  end
+
+  # Background Queue source runs in a background thread
+  # to periodically update the queues available.
+  class Background
+    include Enumerable
+    attr_reader :delegate, :delay
+    # @param [Enumerator] delegate queue source to load the queues from.
+    # @param [Integer] delay - how long between updates
+    def initialize(delegate, delay)
+      @delegate = delegate
+      @delay = delay
+    end
+
+    # Spawns a thread to periodically update the
+    # queues.
+    # @return [Thread] returns the spawned thread.
+    def start
+      @stop   = false
+      @queues = delegate.to_a
+      Thread.new do
+        begin
+          loop do
+            sleep delay
+            break if @stop
+            @queues = delegate.to_a
+          end
+        rescue => e
+          retry
+        end
+      end
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def each(&block)
+      @queues.each { |q| block.call(q) }
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies.rb

@@ -0,0 +1,7 @@
+module Reqless::JobReservers
+  module Strategies
+    require 'reqless/job_reservers/strategies/filtering'
+    require 'reqless/job_reservers/strategies/ordering'
+    require 'reqless/job_reservers/strategies/sources'
+  end
+end

data/lib/reqless/queue_patterns_helper.rb

@@ -0,0 +1,113 @@
+module Reqless
+  module QueuePatternsHelper
+    extend self
+
+    # Returns a list of queues to use when searching for a job.
+    #
+    # A splat ("*") means you want every queue (in alpha order) - this
+    # can be useful for dynamically adding new queues.
+    #
+    # The splat can also be used as a wildcard within a queue name,
+    # e.g. "*high*", and negation can be indicated with a prefix of "!"
+    #
+    # An @key can be used to dynamically look up the queue list for key from redis.
+    # If no key is supplied, it defaults to the worker's hostname, and wildcards
+    # and negations can be used inside this dynamic queue list.
+    def expand_queues(queue_patterns, real_queues, queue_identifier_patterns)
+      queue_patterns = queue_patterns.dup
+      real_queues = real_queues.dup
+
+      matched_queues = []
+
+      while q = queue_patterns.shift
+        q = q.to_s
+        negated = false
+
+        if q =~ /^(!)?@(.*)/
+          key = $2.strip
+          key = Socket.gethostname if key.size == 0
+
+          add_queues = queue_identifier_patterns.fetch(key, nil) ||
+            queue_identifier_patterns.fetch('default', ['*'])
+          add_queues = add_queues.map { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
+
+          queue_patterns.concat(add_queues)
+          next
+        end
+
+        if q =~ /^!/
+          negated = true
+          q = q[1..-1]
+        end
+
+        patstr = q.gsub(/\*/, ".*")
+        pattern = /^#{patstr}$/
+        if negated
+          matched_queues -= matched_queues.grep(pattern)
+        else
+          matches = real_queues.grep(/^#{pattern}$/)
+          matches = [q] if matches.size == 0 && q == patstr
+          matched_queues.concat(matches)
+        end
+      end
+
+      return matched_queues.uniq.sort
+    end
+
+    def prioritize_queues(priority_buckets, real_queues)
+      real_queues = real_queues.dup
+      priority_buckets = priority_buckets.dup
+
+      result = []
+      default_idx = -1
+      default_fairly = false
+
+      # Walk the priority patterns, extract each into its own bucket
+      priority_buckets.each do |bucket|
+        bucket_pattern = bucket.pattern
+        fairly = bucket.should_distribute_fairly
+
+        # note the position of the default bucket for inserting the remaining queues at that location
+        if bucket_pattern == ['default']
+          default_idx = result.size
+          default_fairly = fairly
+          next
+        end
+
+        bucket_queues, remaining = [], []
+
+        bucket_pattern.each do |pattern|
+          pattern = pattern.strip
+
+          if pattern =~ /^!/
+            negated = true
+            pattern = pattern[1..-1]
+          end
+
+          patstr = pattern.gsub(/\*/, ".*")
+          pattern = /^#{patstr}$/
+
+          if negated
+            bucket_queues -= bucket_queues.grep(pattern)
+          else
+            bucket_queues.concat(real_queues.grep(pattern))
+          end
+
+        end
+
+        bucket_queues.uniq!
+        bucket_queues.shuffle! if fairly
+        real_queues = real_queues - bucket_queues
+
+        result << bucket_queues
+      end
+
+      # insert the remaining queues at the position the default item was at (or last)
+      real_queues.shuffle! if default_fairly
+      result.insert(default_idx, real_queues)
+      result.flatten!
+
+      return result
+    end
+  end
+end

data/lib/reqless/server.rb

@@ -2,6 +2,7 @@
 
 require 'sinatra/base'
 require 'reqless'
+require 'reqless/queue_patterns_helper'
 
 module Reqless
   # The Reqless web interface
@@ -158,60 +159,6 @@ module Reqless
           formatted
         end
       end
-
-      # Returns a list of queues to use when searching for a job.
-      #
-      # A splat ("*") means you want every queue (in alpha order) - this
-      # can be useful for dynamically adding new queues.
-      #
-      # The splat can also be used as a wildcard within a queue name,
-      # e.g. "*high*", and negation can be indicated with a prefix of "!"
-      #
-      # An @key can be used to dynamically look up the queue list for key from redis.
-      # If no key is supplied, it defaults to the worker's hostname, and wildcards
-      # and negations can be used inside this dynamic queue list.   Set the queue
-      # list for a key with set_dynamic_queue(key, ["q1", "q2"]
-      #
-      def expand_queues(queue_patterns, real_queues)
-        queue_patterns = queue_patterns.dup
-        real_queues = real_queues.dup
-        dynamic_queues = client.queue_patterns.get_queue_identifier_patterns
-
-        matched_queues = []
-
-        while q = queue_patterns.shift
-          q = q.to_s
-          negated = false
-
-          if q =~ /^(!)?@(.*)/
-            key = $2.strip
-            key = Socket.gethostname if key.size == 0
-
-            add_queues = dynamic_queues[key]
-            add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
-
-            queue_patterns.concat(add_queues)
-            next
-          end
-
-          if q =~ /^!/
-            negated = true
-            q = q[1..-1]
-          end
-
-          patstr = q.gsub(/\*/, ".*")
-          pattern = /^#{patstr}$/
-          if negated
-            matched_queues -= matched_queues.grep(pattern)
-          else
-            matches = real_queues.grep(/^#{pattern}$/)
-            matches = [q] if matches.size == 0 && q == patstr
-            matched_queues.concat(matches)
-          end
-        end
-
-        return matched_queues.uniq.sort
-      end
     end
 
     get '/?' do
@@ -603,9 +550,11 @@ module Reqless
       @queues = []
       real_queues = client.queues.counts.collect {|q| q['name'] }
 
-      dqueues = client.queue_patterns.get_queue_identifier_patterns
-      dqueues.each do |k, v|
-        expanded = expand_queues(["@#{k}"], real_queues)
+      queue_identifier_patterns = client.queue_patterns.get_queue_identifier_patterns
+      queue_identifier_patterns.each do |k, v|
+        expanded = QueuePatternsHelper.expand_queues(
+          ["@#{k}"], real_queues, queue_identifier_patterns
+        )
         expanded = expanded.collect { |q| q.split(":").last }
         view_data = {
             'name' => k,

data/lib/reqless/version.rb

@@ -2,5 +2,5 @@
 
 module Reqless
   # VERSION is overwritten dynamically as part of the release process.
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reqless
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Dan Lecocq
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-28 00:00:00.000000000 Z
+date: 2024-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -303,9 +303,14 @@ files:
 - lib/reqless/config.rb
 - lib/reqless/failure_formatter.rb
 - lib/reqless/job.rb
+- lib/reqless/job_reservers/delegating.rb
 - lib/reqless/job_reservers/ordered.rb
 - lib/reqless/job_reservers/round_robin.rb
 - lib/reqless/job_reservers/shuffled_round_robin.rb
+- lib/reqless/job_reservers/strategies.rb
+- lib/reqless/job_reservers/strategies/filtering.rb
+- lib/reqless/job_reservers/strategies/ordering.rb
+- lib/reqless/job_reservers/strategies/sources.rb
 - lib/reqless/lua/reqless-lib.lua
 - lib/reqless/lua/reqless.lua
 - lib/reqless/lua_script.rb
@@ -314,6 +319,7 @@ files:
 - lib/reqless/middleware/sentry.rb
 - lib/reqless/middleware/timeout.rb
 - lib/reqless/queue.rb
+- lib/reqless/queue_patterns_helper.rb
 - lib/reqless/queue_priority_pattern.rb
 - lib/reqless/server.rb
 - lib/reqless/server/static/css/bootstrap-responsive.css