reqless 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 615bf39dfe0d7ef11275877a19e85e0e1e638929eb02d9fcedcc9af4b11e0298
-  data.tar.gz: fbdd419cb97a5ea5d6ea7b972e53c667ab2a8d91db62a839a45acd594c06d45b
+  metadata.gz: f1042e561166bf47c7938f8d372d6ff1b69e6cdf999577b44e3c4a2ac3326cf1
+  data.tar.gz: 983e5f31909dca78405b16208064460f830048c9a277fdbc76f9a4b5b8ea194c
 SHA512:
-  metadata.gz: cc94b71787a67e7b4e50a3bb7ad3fd37ec9ff1bf92ee5e6c3ed261c116b8341d414607f1a2d78d75065abbdd6836ebadba1db6789972cb60d00e9c14d2bd5a1c
-  data.tar.gz: 4500d6d7554d00c68ea2b392931a35d3d03aa804d0a2454929bb5b7d48bc9a31c804c73aa6d22444f631ecf6169e53ce267f34f0b02b6e865dd81b71159e8275
+  metadata.gz: 89613e228c2b8bc014c25125eb65db6bd60816397b6e6df712e433d7901f4b21bd1b9444d73e8583d63209baeee513aca5e75f4c6a554008d9b78be0ac1bc9aa
+  data.tar.gz: fa149f59031bc460de0395bc1c5b768129f741ba4a646b6772049be60fb491e04df9584c8ba42293b88810c35180559b33292922fa855d9380d33873a9672978

data/lib/reqless/lua/reqless-lib.lua

@@ -1,4 +1,4 @@
--- Current SHA: 8b6600adb988e7f4922f606798b6ad64c06a245d
+-- Current SHA: 54efb0679992da71b576cf16c043e9c9f985f426
 -- This is a generated file
 -- cjson can't tell an empty array from an empty object, so empty arrays end up
 -- encoded as objects. This function makes empty arrays look like empty arrays.
@@ -2527,6 +2527,7 @@ function ReqlessQueue.counts(now, name)
 end
 local ReqlessQueuePatterns = {
   default_identifiers_default_pattern = '["*"]',
+  default_priority_pattern = '{"fairly": false, "pattern": ["default"]}',
   ns = Reqless.ns .. "qp:",
 }
 ReqlessQueuePatterns.__index = ReqlessQueuePatterns
@@ -2599,6 +2600,10 @@ ReqlessQueuePatterns['getPriorityPatterns'] = function(now)
     reply = redis.call('lrange', 'qmore:priority', 0, -1)
   end
 
+  if #reply == 0 then
+    reply = {ReqlessQueuePatterns.default_priority_pattern}
+  end
+
   return reply
 end
 
@@ -2610,7 +2615,21 @@ ReqlessQueuePatterns['setPriorityPatterns'] = function(now, ...)
   redis.call('del', key)
   -- Clear out the legacy key
   redis.call('del', 'qmore:priority')
+
   if #arg > 0 then
+    -- Check for the default priority pattern and add one if none is given.
+    local found_default = false
+    for i = 1, #arg do
+      local pattern = cjson.decode(arg[i])['pattern']
+      if #pattern == 1 and pattern[1] == 'default' then
+        found_default = true
+        break
+      end
+    end
+    if not found_default then
+      table.insert(arg, ReqlessQueuePatterns.default_priority_pattern)
+    end
+
     redis.call('rpush', key, unpack(arg))
   end
 end

data/lib/reqless/lua/reqless.lua

@@ -1,4 +1,4 @@
--- Current SHA: 8b6600adb988e7f4922f606798b6ad64c06a245d
+-- Current SHA: 54efb0679992da71b576cf16c043e9c9f985f426
 -- This is a generated file
 local function cjsonArrayDegenerationWorkaround(array)
   if #array == 0 then
@@ -1898,6 +1898,7 @@ function ReqlessQueue.counts(now, name)
 end
 local ReqlessQueuePatterns = {
   default_identifiers_default_pattern = '["*"]',
+  default_priority_pattern = '{"fairly": false, "pattern": ["default"]}',
   ns = Reqless.ns .. "qp:",
 }
 ReqlessQueuePatterns.__index = ReqlessQueuePatterns
@@ -1961,6 +1962,10 @@ ReqlessQueuePatterns['getPriorityPatterns'] = function(now)
     reply = redis.call('lrange', 'qmore:priority', 0, -1)
   end
 
+  if #reply == 0 then
+    reply = {ReqlessQueuePatterns.default_priority_pattern}
+  end
+
   return reply
 end
 
@@ -1968,7 +1973,20 @@ ReqlessQueuePatterns['setPriorityPatterns'] = function(now, ...)
   local key = ReqlessQueuePatterns.ns .. 'priorities'
   redis.call('del', key)
   redis.call('del', 'qmore:priority')
+
   if #arg > 0 then
+    local found_default = false
+    for i = 1, #arg do
+      local pattern = cjson.decode(arg[i])['pattern']
+      if #pattern == 1 and pattern[1] == 'default' then
+        found_default = true
+        break
+      end
+    end
+    if not found_default then
+      table.insert(arg, ReqlessQueuePatterns.default_priority_pattern)
+    end
+
     redis.call('rpush', key, unpack(arg))
   end
 end

data/lib/reqless/server/views/dynamic_queues.erb

@@ -0,0 +1,64 @@
+
+<div class="page-header">
+  <h1>Dynamic Queues</h1>
+</div>
+<p class="intro">
+  The list below shows the dynamic queues currently defined.  When you start a worker with a dynamic queue key (@key_name), that key is looked up from the list below to determine the actual queues the worker should pull from.  Wildcards (*) and negation (leading !) can be used to select the queues the worker should process. There is always a fallback key - @default, which workers will use if the key for that worker is empty.  If both the key and the fallback are empty, the worker defaults to processing '*'
+</p>
+
+<form action="<%= u '/dynamicqueues' %>" method="POST" style="float:none; margin-top:10px">
+
+  <table class='queues'>
+    <tr>
+      <th>Name</th>
+      <th>Value</th>
+      <th>Expanded</th>
+      <th></th>
+    </tr>
+    <% @queues.each_with_index do |data, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-name" name="queues[][name]" value="<%= data['name'] %>" /></td>
+        <td><input type="text" id="input-<%= i %>-value" name="queues[][value]" value="<%= data['value'] %>" /></td>
+        <td class="expanded"><%= data['expanded'] %></td>
+        <td>
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+
+  jQuery(function($) {
+
+    $("input").live("keypress", markDirty);
+
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.queues");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("td.expanded").html("")
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+
+  });
+</script>

data/lib/reqless/server/views/priorities.erb

@@ -0,0 +1,78 @@
+<div class="page-header">
+  <h1>Queue Priority</h1>
+</div>
+<p class="intro">
+  The list below orders queue name patterns by the priority you wish them to be executed in.  The "Fairly" option allows you to indicate you want the queues within that pattern space be selected in a fair (random) manner, i.e. like resque-fairly.  The 'default' pattern must always exist, and matches against all queues that aren't in any of the other patterns.
+</p>
+
+<form action="<%= u '/queuepriority' %>" method="POST" style="float:none; margin-top:10px">
+
+  <table class="priorities">
+    <tr>
+      <th>Pattern</th>
+      <th>Fairly</th>
+      <th></th>
+    </tr>
+    <% @priorities.each_with_index do |priority, i| %>
+      <tr class="line">
+        <td><input type="text" id="input-<%= i %>-pattern" name="priorities[][pattern]" value="<%= priority["pattern"] %>" /></td>
+        <td><input type="checkbox" id="input-<%= i %>-fairly" name="priorities[][fairly]" value="true" <%= "checked" if priority["fairly"] %> /></td>
+        <td>
+          <a href="#up" class="up">Up</a> |
+          <a href="#down" class="down">Down</a> |
+          <a href="#remove" class="remove">Remove</a>
+        </td>
+      </tr>
+    <% end %>
+  </table>
+
+  <a href="#add" class="add">Add</a>
+  <input type="submit" value="Save"/>
+
+</form>
+
+<script type="text/javascript" charset="utf-8">
+  function markDirty()
+  {
+    $("input[type=submit]").css({border:"3px orange solid"});
+  }
+
+  jQuery(function($) {
+    $("input").live("keypress", markDirty);
+    $("input[type=checkbox]").live("click", markDirty);
+
+    $("a.add").live("click", function(e) {
+      e.preventDefault();
+      var $table = $("table.priorities");
+      var $newRow = $table.find("tr.line:first").clone();
+      $newRow.find("input[type=text]").attr("value", "");
+      $newRow.find("input[type=checkbox]").attr("checked", false);
+      $newRow.appendTo($table);
+      markDirty();
+    });
+
+    $("a.remove").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      $link.parents("tr").remove();
+      markDirty();
+    });
+
+    $("a.up").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.prev(".line").before($row);
+      markDirty();
+    });
+
+    $("a.down").live("click", function(e) {
+      e.preventDefault();
+      var $link = $(this);
+      var $row = $link.parents("tr");
+      $row.next(".line").after($row);
+      markDirty();
+    });
+
+  });
+</script>

data/lib/reqless/server.rb

@@ -78,6 +78,8 @@ module Reqless
       def tabs
         [
           { name: 'Queues'   , path: '/queues'   },
+          {:name => 'DynamicQueues', :path => '/dynamicqueues'},
+          {:name => 'QueuePriority', :path => '/queuepriority'},
           { name: 'Throttles', path: '/throttles'},
           { name: 'Workers'  , path: '/workers'  },
           { name: 'Track'    , path: '/track'    },
@@ -156,6 +158,60 @@ module Reqless
           formatted
         end
       end
+
+      # Returns a list of queues to use when searching for a job.
+      #
+      # A splat ("*") means you want every queue (in alpha order) - this
+      # can be useful for dynamically adding new queues.
+      #
+      # The splat can also be used as a wildcard within a queue name,
+      # e.g. "*high*", and negation can be indicated with a prefix of "!"
+      #
+      # An @key can be used to dynamically look up the queue list for key from redis.
+      # If no key is supplied, it defaults to the worker's hostname, and wildcards
+      # and negations can be used inside this dynamic queue list.   Set the queue
+      # list for a key with set_dynamic_queue(key, ["q1", "q2"]
+      #
+      def expand_queues(queue_patterns, real_queues)
+        queue_patterns = queue_patterns.dup
+        real_queues = real_queues.dup
+        dynamic_queues = client.queue_patterns.get_queue_identifier_patterns
+
+        matched_queues = []
+
+        while q = queue_patterns.shift
+          q = q.to_s
+          negated = false
+
+          if q =~ /^(!)?@(.*)/
+            key = $2.strip
+            key = Socket.gethostname if key.size == 0
+
+            add_queues = dynamic_queues[key]
+            add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
+
+            queue_patterns.concat(add_queues)
+            next
+          end
+
+          if q =~ /^!/
+            negated = true
+            q = q[1..-1]
+          end
+
+          patstr = q.gsub(/\*/, ".*")
+          pattern = /^#{patstr}$/
+          if negated
+            matched_queues -= matched_queues.grep(pattern)
+          else
+            matches = real_queues.grep(/^#{pattern}$/)
+            matches = [q] if matches.size == 0 && q == patstr
+            matched_queues.concat(matches)
+          end
+        end
+
+        return matched_queues.uniq.sort
+      end
     end
 
     get '/?' do
@@ -543,6 +599,75 @@ module Reqless
       end
     end
 
+    get "/dynamicqueues" do
+      @queues = []
+      real_queues = client.queues.counts.collect {|q| q['name'] }
+
+      dqueues = client.queue_patterns.get_queue_identifier_patterns
+      dqueues.each do |k, v|
+        expanded = expand_queues(["@#{k}"], real_queues)
+        expanded = expanded.collect { |q| q.split(":").last }
+        view_data = {
+            'name' => k,
+            'value' => Array(v).join(", "),
+            'expanded' => expanded.join(", ")
+        }
+        @queues << view_data
+      end
+
+      @queues.sort! do |a, b|
+        an = a['name']
+        bn = b['name']
+        if an == 'default'
+          1
+        elsif bn == 'default'
+          -1
+        else
+          an <=> bn
+        end
+      end
+
+      erb :dynamic_queues, {:layout => true, :locals => { :title => 'Dynamic Queues' }}
+    end
+
+    post "/dynamicqueues" do
+      queues = params['queues']
+      dynamic_queues = {}
+      queues.each do |queue|
+        values = queue['value'].to_s.split(',').collect { |q| q.gsub(/\s/, '') }
+        dynamic_queues[queue['name']] = values
+      end
+
+      client.queue_patterns.set_queue_identifier_patterns(dynamic_queues)
+      redirect to("/dynamicqueues")
+    end
+
+    get "/queuepriority" do
+      # For the UI we always want the latest persisted data
+      @priorities = client.queue_patterns.get_queue_priority_patterns.map do |priority_pattern|
+        {
+          'fairly' => priority_pattern.should_distribute_fairly,
+          'pattern' => priority_pattern.pattern.join(', '),
+        }
+      end
+      erb :priorities, {:layout => true, :locals => { :title => 'Queue Priorities' }}
+    end
+
+    post "/queuepriority" do
+      priorities = params['priorities']
+      priority_patterns = priorities.map do |priority_pattern|
+        fairly = priority_pattern.fetch('fairly', 'false')
+        should_distribute_fairly = fairly == true || fairly == 'true'
+        Reqless::QueuePriorityPattern.new(
+          priority_pattern['pattern'].to_s.split(',').collect { |q| q.gsub(/\s/, '') },
+          should_distribute_fairly,
+        )
+      end
+      client.queue_patterns.set_queue_priority_patterns(priority_patterns)
+
+      redirect to("/queuepriority")
+    end
+
     # start the server if ruby file executed directly
     run! if app_file == $PROGRAM_NAME
   end

data/lib/reqless/version.rb

@@ -1,5 +1,6 @@
 # Encoding: utf-8
 
 module Reqless
-  VERSION = '0.0.2'
+  # VERSION is overwritten dynamically as part of the release process.
+  VERSION = '0.0.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: reqless
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
 platform: ruby
 authors:
 - Dan Lecocq
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -353,11 +353,13 @@ files:
 - lib/reqless/server/views/about.erb
 - lib/reqless/server/views/completed.erb
 - lib/reqless/server/views/config.erb
+- lib/reqless/server/views/dynamic_queues.erb
 - lib/reqless/server/views/failed.erb
 - lib/reqless/server/views/failed_type.erb
 - lib/reqless/server/views/job.erb
 - lib/reqless/server/views/layout.erb
 - lib/reqless/server/views/overview.erb
+- lib/reqless/server/views/priorities.erb
 - lib/reqless/server/views/queue.erb
 - lib/reqless/server/views/queues.erb
 - lib/reqless/server/views/tag.erb