reqless 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f560c6821304b941d057e62a8894c7a16b1b14320702cf6dc0d8314f7a071100
-  data.tar.gz: 2a866a3a4115196ac8d4c4294fbdd09d892efa2e073e75e2a1091009225210d2
+  metadata.gz: d89345ee8de2c77b0083cd7c93eb4aff552d57a298f6c0f55953093e5c69bd98
+  data.tar.gz: c41fac4baeaebe950d1f72fc69647b4aeca870b31bdd8b595cb821f1cea4bce5
 SHA512:
-  metadata.gz: '000538a55f9345823635cf0c1b0bc3df1eb4145c590e85f84de99229076c1fc8ba41f589f72dbe292677ec74cae5d09a217b2376ec4e3b08adb43d26587c5f83'
-  data.tar.gz: 6ba2ec904801c0ca9e0d81f2095e28abeac834a736cc13499d0f0bf54e5a94086055b829c148f62183e65a5cfe362669f20638a2d3eac4dd54a2fd13dc5b54ef
+  metadata.gz: b60f2d5e8d40f41179b418a09397dd2669126c351108fffc3781907793e3b4dee25dfa970b0bed8de2e7c05466bd1d10d1b53e0b4bb5b5f891327e8b63c688f2
+  data.tar.gz: 8afc28f017ec955c72994a1eeca4f7402c294603ab8a62ef991c1955a763614e66f3cdf0567728a80eeb45b6eeecbe3511c114e277bedde338ebf88bb82a476b

data/lib/reqless/job_reservers/delegating.rb

@@ -0,0 +1,30 @@
+module Reqless::JobReservers
+  # @param [Enumerable] reservers - a set of reservers
+  # to check for work.
+  class Delegating < Struct.new(:reservers)
+    def description
+      "Delegating Reserver"
+    end
+
+    def prep_for_work!
+      # nothing here on purpose
+    end
+
+    def queues
+      Enumerator.new do |yielder|
+        reservers.each do |reserver|
+          reserver.queues.each {|queue| yielder << queue}
+        end
+      end
+    end
+
+    def reserve
+      reservers.each do |reserver|
+        if (job = reserver.reserve)
+          return job
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/filtering.rb

@@ -0,0 +1,42 @@
+require 'reqless/queue_patterns_helper'
+
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+module Reqless::JobReservers::Strategies::Filtering
+  # @param [Enumerable] queues - a source of queues
+  # @param [Array] regexes - a list of regexes to match against.
+  # Return an enumerator of the filtered queues in
+  # in prioritized order.
+  def self.default(
+    queues,
+    regexes,
+    queue_identifier_patterns,
+    queue_priority_patterns
+  )
+    Enumerator.new do |yielder|
+      # Map queues to their names
+      mapped_queues = queues.reduce({}) do |hash,queue|
+        hash[queue.name] = queue
+        hash
+      end
+
+      # Filter the queue names against the regexes provided.
+      matches = Reqless::QueuePatternsHelper.expand_queues(regexes, mapped_queues.keys, queue_identifier_patterns)
+
+      # Prioritize the queues.
+      prioritized_names = Reqless::QueuePatternsHelper.prioritize_queues(queue_priority_patterns, matches)
+
+      prioritized_names.each do |name|
+        queue = mapped_queues[name]
+        if queue
+          yielder << queue
+        end
+      end
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/ordering.rb

@@ -0,0 +1,31 @@
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+module Reqless::JobReservers::Strategies::Ordering
+  # Shuffles the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  def self.shuffled(enumerable)
+    Enumerator.new do |yielder|
+      enumerable.to_a.shuffle.each do |e|
+        yielder << e
+      end
+    end
+  end
+
+  # Samples a subset of the underlying enumerable
+  # for each iteration.
+  # @param [Enumerable] enumerable - underlying enumerator to iterate over
+  # @param [Integer] sample_size   - number of items to take per iteration
+  def self.sampled(enumerable, sample_size = 5)
+    Enumerator.new do |yielder|
+      enumerable.to_a.sample(sample_size).each do |e|
+        yielder << e
+      end
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies/sources.rb

@@ -0,0 +1,63 @@
+module Reqless
+  module JobReservers
+    module Strategies
+    end
+  end
+end
+
+# This module provides the different kinds of queue sources used by qmore
+module Reqless::JobReservers::Strategies::Sources
+  # Direct source uses a client to generate the queues we should
+  # pull work from. Ignores any queues that do not have tasks available.
+  def self.direct(client)
+    Enumerator.new do |yielder|
+      queues = client.queues.counts.select do |queue|
+        %w(waiting recurring depends stalled scheduled).any? {|state| queue[state].to_i > 0 }
+      end
+
+      queues.each do |queue|
+        yielder << client.queues[queue['name']]
+      end
+    end
+  end
+
+  # Background Queue source runs in a background thread
+  # to periodically update the queues available.
+  class Background
+    include Enumerable
+    attr_reader :delegate, :delay
+    # @param [Enumerator] delegate queue source to load the queues from.
+    # @param [Integer] delay - how long between updates
+    def initialize(delegate, delay)
+      @delegate = delegate
+      @delay = delay
+    end
+
+    # Spawns a thread to periodically update the
+    # queues.
+    # @return [Thread] returns the spawned thread.
+    def start
+      @stop   = false
+      @queues = delegate.to_a
+      Thread.new do
+        begin
+          loop do
+            sleep delay
+            break if @stop
+            @queues = delegate.to_a
+          end
+        rescue => e
+          retry
+        end
+      end
+    end
+
+    def stop
+      @stop = true
+    end
+
+    def each(&block)
+      @queues.each { |q| block.call(q) }
+    end
+  end
+end

data/lib/reqless/job_reservers/strategies.rb

@@ -0,0 +1,7 @@
+module Reqless::JobReservers
+  module Strategies
+    require 'reqless/job_reservers/strategies/filtering'
+    require 'reqless/job_reservers/strategies/ordering'
+    require 'reqless/job_reservers/strategies/sources'
+  end
+end

data/lib/reqless/queue_patterns_helper.rb

@@ -0,0 +1,113 @@
+module Reqless
+  module QueuePatternsHelper
+    extend self
+
+    # Returns a list of queues to use when searching for a job.
+    #
+    # A splat ("*") means you want every queue (in alpha order) - this
+    # can be useful for dynamically adding new queues.
+    #
+    # The splat can also be used as a wildcard within a queue name,
+    # e.g. "*high*", and negation can be indicated with a prefix of "!"
+    #
+    # An @key can be used to dynamically look up the queue list for key from redis.
+    # If no key is supplied, it defaults to the worker's hostname, and wildcards
+    # and negations can be used inside this dynamic queue list.
+    def expand_queues(queue_patterns, real_queues, queue_identifier_patterns)
+      queue_patterns = queue_patterns.dup
+      real_queues = real_queues.dup
+
+      matched_queues = []
+
+      while q = queue_patterns.shift
+        q = q.to_s
+        negated = false
+
+        if q =~ /^(!)?@(.*)/
+          key = $2.strip
+          key = Socket.gethostname if key.size == 0
+
+          add_queues = queue_identifier_patterns.fetch(key, nil) ||
+            queue_identifier_patterns.fetch('default', ['*'])
+          add_queues = add_queues.map { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
+
+          queue_patterns.concat(add_queues)
+          next
+        end
+
+        if q =~ /^!/
+          negated = true
+          q = q[1..-1]
+        end
+
+        patstr = q.gsub(/\*/, ".*")
+        pattern = /^#{patstr}$/
+        if negated
+          matched_queues -= matched_queues.grep(pattern)
+        else
+          matches = real_queues.grep(/^#{pattern}$/)
+          matches = [q] if matches.size == 0 && q == patstr
+          matched_queues.concat(matches)
+        end
+      end
+
+      return matched_queues.uniq.sort
+    end
+
+    def prioritize_queues(priority_buckets, real_queues)
+      real_queues = real_queues.dup
+      priority_buckets = priority_buckets.dup
+
+      result = []
+      default_idx = -1
+      default_fairly = false
+
+      # Walk the priority patterns, extract each into its own bucket
+      priority_buckets.each do |bucket|
+        bucket_pattern = bucket.pattern
+        fairly = bucket.should_distribute_fairly
+
+        # note the position of the default bucket for inserting the remaining queues at that location
+        if bucket_pattern == ['default']
+          default_idx = result.size
+          default_fairly = fairly
+          next
+        end
+
+        bucket_queues, remaining = [], []
+
+        bucket_pattern.each do |pattern|
+          pattern = pattern.strip
+
+          if pattern =~ /^!/
+            negated = true
+            pattern = pattern[1..-1]
+          end
+
+          patstr = pattern.gsub(/\*/, ".*")
+          pattern = /^#{patstr}$/
+
+          if negated
+            bucket_queues -= bucket_queues.grep(pattern)
+          else
+            bucket_queues.concat(real_queues.grep(pattern))
+          end
+
+        end
+
+        bucket_queues.uniq!
+        bucket_queues.shuffle! if fairly
+        real_queues = real_queues - bucket_queues
+
+        result << bucket_queues
+      end
+
+      # insert the remaining queues at the position the default item was at (or last)
+      real_queues.shuffle! if default_fairly
+      result.insert(default_idx, real_queues)
+      result.flatten!
+
+      return result
+    end
+  end
+end

data/lib/reqless/server/views/dynamic_queues.erb

@@ -0,0 +1,64 @@
+
+<div class="page-header">
+  <h1>Dynamic Queues</h1>
+</div>
+<p class="intro">
+  The list below shows the dynamic queues currently defined.  When you start a worker with a dynamic queue key (@key_name), that key is looked up from the list below to determine the actual queues the worker should pull from.  Wildcards (*) and negation (leading !) can be used to select the queues the worker should process. There is always a fallback key - @default, which workers will use if the key for that worker is empty.  If both the key and the fallback are empty, the worker defaults to processing '*'
+</p>
+
+<form action="<%= u '/dynamicqueues' %>" method="POST" style="float:none; margin-top:10px">
+
+  <table class='queues'>
+    <tr>
+      <th>Name</th>
+      <th>Value</th>
+      <th>Expanded</th>
+      <th></th>
+    </tr>
+    <% @queues.each_with_index do |data, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-name" name="queues[][name]" value="<%= data['name'] %>" /></td>
+        <td><input type="text" id="input-<%= i %>-value" name="queues[][value]" value="<%= data['value'] %>" /></td>
+        <td class="expanded"><%= data['expanded'] %></td>
+        <td>
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+
+  jQuery(function($) {
+
+    $("input").live("keypress", markDirty);
+
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.queues");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("td.expanded").html("")
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+
+  });
+</script>

data/lib/reqless/server/views/priorities.erb

@@ -0,0 +1,78 @@
+<div class="page-header">
+  <h1>Queue Priority</h1>
+</div>
+<p class="intro">
+  The list below orders queue name patterns by the priority you wish them to be executed in.  The "Fairly" option allows you to indicate you want the queues within that pattern space be selected in a fair (random) manner, i.e. like resque-fairly.  The 'default' pattern must always exist, and matches against all queues that aren't in any of the other patterns.
+</p>
+
+<form action="<%= u '/queuepriority' %>" method="POST" style="float:none; margin-top:10px">
+
+  <table class="priorities">
+    <tr>
+      <th>Pattern</th>
+      <th>Fairly</th>
+      <th></th>
+    </tr>
+    <% @priorities.each_with_index do |priority, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-pattern" name="priorities[][pattern]" value="<%= priority["pattern"] %>" /></td>
+        <td><input type="checkbox" id="input-<%= i %>-fairly" name="priorities[][fairly]" value="true" <%= "checked" if priority["fairly"] %> /></td>
+        <td>
+          <a href="#up" class="up">Up</a> |
+          <a href="#down" class="down">Down</a> |
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+
+  jQuery(function($) {
+    $("input").live("keypress", markDirty);
+    $("input[type=checkbox]").live("click", markDirty);
+
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.priorities");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("input[type=checkbox]").attr("checked", false);
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+    $("a.up").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.prev(".line").before($row);
+      markDirty();
+    });
+
+    $("a.down").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.next(".line").after($row);
+      markDirty();
+    });
+
+  });
+</script>

data/lib/reqless/server.rb

@@ -2,6 +2,7 @@
 
 require 'sinatra/base'
 require 'reqless'
+require 'reqless/queue_patterns_helper'
 
 module Reqless
   # The Reqless web interface
@@ -78,6 +79,8 @@ module Reqless
       def tabs
         [
           { name: 'Queues'   , path: '/queues'   },
+          {:name => 'DynamicQueues', :path => '/dynamicqueues'},
+          {:name => 'QueuePriority', :path => '/queuepriority'},
           { name: 'Throttles', path: '/throttles'},
           { name: 'Workers'  , path: '/workers'  },
           { name: 'Track'    , path: '/track'    },
@@ -543,6 +546,77 @@ module Reqless
       end
     end
 
+    get "/dynamicqueues" do
+      @queues = []
+      real_queues = client.queues.counts.collect {|q| q['name'] }
+
+      queue_identifier_patterns = client.queue_patterns.get_queue_identifier_patterns
+      queue_identifier_patterns.each do |k, v|
+        expanded = QueuePatternsHelper.expand_queues(
+          ["@#{k}"], real_queues, queue_identifier_patterns
+        )
+        expanded = expanded.collect { |q| q.split(":").last }
+        view_data = {
+            'name' => k,
+            'value' => Array(v).join(", "),
+            'expanded' => expanded.join(", ")
+        }
+        @queues << view_data
+      end
+
+      @queues.sort! do |a, b|
+        an = a['name']
+        bn = b['name']
+        if an == 'default'
+          1
+        elsif bn == 'default'
+          -1
+        else
+          an <=> bn
+        end
+      end
+
+      erb :dynamic_queues, {:layout => true, :locals => { :title => 'Dynamic Queues' }}
+    end
+
+    post "/dynamicqueues" do
+      queues = params['queues']
+      dynamic_queues = {}
+      queues.each do |queue|
+        values = queue['value'].to_s.split(',').collect { |q| q.gsub(/\s/, '') }
+        dynamic_queues[queue['name']] = values
+      end
+
+      client.queue_patterns.set_queue_identifier_patterns(dynamic_queues)
+      redirect to("/dynamicqueues")
+    end
+
+    get "/queuepriority" do
+      # For the UI we always want the latest persisted data
+      @priorities = client.queue_patterns.get_queue_priority_patterns.map do |priority_pattern|
+        {
+          'fairly' => priority_pattern.should_distribute_fairly,
+          'pattern' => priority_pattern.pattern.join(', '),
+        }
+      end
+      erb :priorities, {:layout => true, :locals => { :title => 'Queue Priorities' }}
+    end
+
+    post "/queuepriority" do
+      priorities = params['priorities']
+      priority_patterns = priorities.map do |priority_pattern|
+        fairly = priority_pattern.fetch('fairly', 'false')
+        should_distribute_fairly = fairly == true || fairly == 'true'
+        Reqless::QueuePriorityPattern.new(
+          priority_pattern['pattern'].to_s.split(',').collect { |q| q.gsub(/\s/, '') },
+          should_distribute_fairly,
+        )
+      end
+      client.queue_patterns.set_queue_priority_patterns(priority_patterns)
+
+      redirect to("/queuepriority")
+    end
+
     # start the server if ruby file executed directly
     run! if app_file == $PROGRAM_NAME
   end

data/lib/reqless/version.rb

@@ -2,5 +2,5 @@
 
 module Reqless
   # VERSION is overwritten dynamically as part of the release process.
-  VERSION = '0.0.3'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reqless
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
 platform: ruby
 authors:
 - Dan Lecocq
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-27 00:00:00.000000000 Z
+date: 2024-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -303,9 +303,14 @@ files:
 - lib/reqless/config.rb
 - lib/reqless/failure_formatter.rb
 - lib/reqless/job.rb
+- lib/reqless/job_reservers/delegating.rb
 - lib/reqless/job_reservers/ordered.rb
 - lib/reqless/job_reservers/round_robin.rb
 - lib/reqless/job_reservers/shuffled_round_robin.rb
+- lib/reqless/job_reservers/strategies.rb
+- lib/reqless/job_reservers/strategies/filtering.rb
+- lib/reqless/job_reservers/strategies/ordering.rb
+- lib/reqless/job_reservers/strategies/sources.rb
 - lib/reqless/lua/reqless-lib.lua
 - lib/reqless/lua/reqless.lua
 - lib/reqless/lua_script.rb
@@ -314,6 +319,7 @@ files:
 - lib/reqless/middleware/sentry.rb
 - lib/reqless/middleware/timeout.rb
 - lib/reqless/queue.rb
+- lib/reqless/queue_patterns_helper.rb
 - lib/reqless/queue_priority_pattern.rb
 - lib/reqless/server.rb
 - lib/reqless/server/static/css/bootstrap-responsive.css
@@ -353,11 +359,13 @@ files:
 - lib/reqless/server/views/about.erb
 - lib/reqless/server/views/completed.erb
 - lib/reqless/server/views/config.erb
+- lib/reqless/server/views/dynamic_queues.erb
 - lib/reqless/server/views/failed.erb
 - lib/reqless/server/views/failed_type.erb
 - lib/reqless/server/views/job.erb
 - lib/reqless/server/views/layout.erb
 - lib/reqless/server/views/overview.erb
+- lib/reqless/server/views/priorities.erb
 - lib/reqless/server/views/queue.erb
 - lib/reqless/server/views/queues.erb
 - lib/reqless/server/views/tag.erb