reqless 0.0.3 → 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f560c6821304b941d057e62a8894c7a16b1b14320702cf6dc0d8314f7a071100
4
- data.tar.gz: 2a866a3a4115196ac8d4c4294fbdd09d892efa2e073e75e2a1091009225210d2
3
+ metadata.gz: d89345ee8de2c77b0083cd7c93eb4aff552d57a298f6c0f55953093e5c69bd98
4
+ data.tar.gz: c41fac4baeaebe950d1f72fc69647b4aeca870b31bdd8b595cb821f1cea4bce5
5
5
  SHA512:
6
- metadata.gz: '000538a55f9345823635cf0c1b0bc3df1eb4145c590e85f84de99229076c1fc8ba41f589f72dbe292677ec74cae5d09a217b2376ec4e3b08adb43d26587c5f83'
7
- data.tar.gz: 6ba2ec904801c0ca9e0d81f2095e28abeac834a736cc13499d0f0bf54e5a94086055b829c148f62183e65a5cfe362669f20638a2d3eac4dd54a2fd13dc5b54ef
6
+ metadata.gz: b60f2d5e8d40f41179b418a09397dd2669126c351108fffc3781907793e3b4dee25dfa970b0bed8de2e7c05466bd1d10d1b53e0b4bb5b5f891327e8b63c688f2
7
+ data.tar.gz: 8afc28f017ec955c72994a1eeca4f7402c294603ab8a62ef991c1955a763614e66f3cdf0567728a80eeb45b6eeecbe3511c114e277bedde338ebf88bb82a476b
@@ -0,0 +1,30 @@
1
+ module Reqless::JobReservers
2
+ # @param [Enumerable] reservers - a set of reservers
3
+ # to check for work.
4
+ class Delegating < Struct.new(:reservers)
5
+ def description
6
+ "Delegating Reserver"
7
+ end
8
+
9
+ def prep_for_work!
10
+ # nothing here on purpose
11
+ end
12
+
13
+ def queues
14
+ Enumerator.new do |yielder|
15
+ reservers.each do |reserver|
16
+ reserver.queues.each {|queue| yielder << queue}
17
+ end
18
+ end
19
+ end
20
+
21
+ def reserve
22
+ reservers.each do |reserver|
23
+ if (job = reserver.reserve)
24
+ return job
25
+ end
26
+ end
27
+ nil
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,42 @@
1
+ require 'reqless/queue_patterns_helper'
2
+
3
+ module Reqless
4
+ module JobReservers
5
+ module Strategies
6
+ end
7
+ end
8
+ end
9
+
10
+ module Reqless::JobReservers::Strategies::Filtering
11
+ # @param [Enumerable] queues - a source of queues
12
+ # @param [Array] regexes - a list of regexes to match against.
13
+ # Return an enumerator of the filtered queues in
14
+ # in prioritized order.
15
+ def self.default(
16
+ queues,
17
+ regexes,
18
+ queue_identifier_patterns,
19
+ queue_priority_patterns
20
+ )
21
+ Enumerator.new do |yielder|
22
+ # Map queues to their names
23
+ mapped_queues = queues.reduce({}) do |hash,queue|
24
+ hash[queue.name] = queue
25
+ hash
26
+ end
27
+
28
+ # Filter the queue names against the regexes provided.
29
+ matches = Reqless::QueuePatternsHelper.expand_queues(regexes, mapped_queues.keys, queue_identifier_patterns)
30
+
31
+ # Prioritize the queues.
32
+ prioritized_names = Reqless::QueuePatternsHelper.prioritize_queues(queue_priority_patterns, matches)
33
+
34
+ prioritized_names.each do |name|
35
+ queue = mapped_queues[name]
36
+ if queue
37
+ yielder << queue
38
+ end
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,31 @@
1
+ module Reqless
2
+ module JobReservers
3
+ module Strategies
4
+ end
5
+ end
6
+ end
7
+
8
+ module Reqless::JobReservers::Strategies::Ordering
9
+ # Shuffles the underlying enumerable
10
+ # for each iteration.
11
+ # @param [Enumerable] enumerable - underlying enumerator to iterate over
12
+ def self.shuffled(enumerable)
13
+ Enumerator.new do |yielder|
14
+ enumerable.to_a.shuffle.each do |e|
15
+ yielder << e
16
+ end
17
+ end
18
+ end
19
+
20
+ # Samples a subset of the underlying enumerable
21
+ # for each iteration.
22
+ # @param [Enumerable] enumerable - underlying enumerator to iterate over
23
+ # @param [Integer] sample_size - number of items to take per iteration
24
+ def self.sampled(enumerable, sample_size = 5)
25
+ Enumerator.new do |yielder|
26
+ enumerable.to_a.sample(sample_size).each do |e|
27
+ yielder << e
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,63 @@
1
+ module Reqless
2
+ module JobReservers
3
+ module Strategies
4
+ end
5
+ end
6
+ end
7
+
8
+ # This module provides the different kinds of queue sources used by qmore
9
+ module Reqless::JobReservers::Strategies::Sources
10
+ # Direct source uses a client to generate the queues we should
11
+ # pull work from. Ignores any queues that do not have tasks available.
12
+ def self.direct(client)
13
+ Enumerator.new do |yielder|
14
+ queues = client.queues.counts.select do |queue|
15
+ %w(waiting recurring depends stalled scheduled).any? {|state| queue[state].to_i > 0 }
16
+ end
17
+
18
+ queues.each do |queue|
19
+ yielder << client.queues[queue['name']]
20
+ end
21
+ end
22
+ end
23
+
24
+ # Background Queue source runs in a background thread
25
+ # to periodically update the queues available.
26
+ class Background
27
+ include Enumerable
28
+ attr_reader :delegate, :delay
29
+ # @param [Enumerator] delegate queue source to load the queues from.
30
+ # @param [Integer] delay - how long between updates
31
+ def initialize(delegate, delay)
32
+ @delegate = delegate
33
+ @delay = delay
34
+ end
35
+
36
+ # Spawns a thread to periodically update the
37
+ # queues.
38
+ # @return [Thread] returns the spawned thread.
39
+ def start
40
+ @stop = false
41
+ @queues = delegate.to_a
42
+ Thread.new do
43
+ begin
44
+ loop do
45
+ sleep delay
46
+ break if @stop
47
+ @queues = delegate.to_a
48
+ end
49
+ rescue => e
50
+ retry
51
+ end
52
+ end
53
+ end
54
+
55
+ def stop
56
+ @stop = true
57
+ end
58
+
59
+ def each(&block)
60
+ @queues.each { |q| block.call(q) }
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,7 @@
1
+ module Reqless::JobReservers
2
+ module Strategies
3
+ require 'reqless/job_reservers/strategies/filtering'
4
+ require 'reqless/job_reservers/strategies/ordering'
5
+ require 'reqless/job_reservers/strategies/sources'
6
+ end
7
+ end
@@ -0,0 +1,113 @@
1
+ module Reqless
2
+ module QueuePatternsHelper
3
+ extend self
4
+
5
+ # Returns a list of queues to use when searching for a job.
6
+ #
7
+ # A splat ("*") means you want every queue (in alpha order) - this
8
+ # can be useful for dynamically adding new queues.
9
+ #
10
+ # The splat can also be used as a wildcard within a queue name,
11
+ # e.g. "*high*", and negation can be indicated with a prefix of "!"
12
+ #
13
+ # An @key can be used to dynamically look up the queue list for key from redis.
14
+ # If no key is supplied, it defaults to the worker's hostname, and wildcards
15
+ # and negations can be used inside this dynamic queue list.
16
+ def expand_queues(queue_patterns, real_queues, queue_identifier_patterns)
17
+ queue_patterns = queue_patterns.dup
18
+ real_queues = real_queues.dup
19
+
20
+ matched_queues = []
21
+
22
+ while q = queue_patterns.shift
23
+ q = q.to_s
24
+ negated = false
25
+
26
+ if q =~ /^(!)?@(.*)/
27
+ key = $2.strip
28
+ key = Socket.gethostname if key.size == 0
29
+
30
+ add_queues = queue_identifier_patterns.fetch(key, nil) ||
31
+ queue_identifier_patterns.fetch('default', ['*'])
32
+ add_queues = add_queues.map { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
33
+
34
+ queue_patterns.concat(add_queues)
35
+ next
36
+ end
37
+
38
+ if q =~ /^!/
39
+ negated = true
40
+ q = q[1..-1]
41
+ end
42
+
43
+ patstr = q.gsub(/\*/, ".*")
44
+ pattern = /^#{patstr}$/
45
+ if negated
46
+ matched_queues -= matched_queues.grep(pattern)
47
+ else
48
+ matches = real_queues.grep(/^#{pattern}$/)
49
+ matches = [q] if matches.size == 0 && q == patstr
50
+ matched_queues.concat(matches)
51
+ end
52
+ end
53
+
54
+ return matched_queues.uniq.sort
55
+ end
56
+
57
+ def prioritize_queues(priority_buckets, real_queues)
58
+ real_queues = real_queues.dup
59
+ priority_buckets = priority_buckets.dup
60
+
61
+ result = []
62
+ default_idx = -1
63
+ default_fairly = false
64
+
65
+ # Walk the priority patterns, extract each into its own bucket
66
+ priority_buckets.each do |bucket|
67
+ bucket_pattern = bucket.pattern
68
+ fairly = bucket.should_distribute_fairly
69
+
70
+ # note the position of the default bucket for inserting the remaining queues at that location
71
+ if bucket_pattern == ['default']
72
+ default_idx = result.size
73
+ default_fairly = fairly
74
+ next
75
+ end
76
+
77
+ bucket_queues, remaining = [], []
78
+
79
+ bucket_pattern.each do |pattern|
80
+ pattern = pattern.strip
81
+
82
+ if pattern =~ /^!/
83
+ negated = true
84
+ pattern = pattern[1..-1]
85
+ end
86
+
87
+ patstr = pattern.gsub(/\*/, ".*")
88
+ pattern = /^#{patstr}$/
89
+
90
+ if negated
91
+ bucket_queues -= bucket_queues.grep(pattern)
92
+ else
93
+ bucket_queues.concat(real_queues.grep(pattern))
94
+ end
95
+
96
+ end
97
+
98
+ bucket_queues.uniq!
99
+ bucket_queues.shuffle! if fairly
100
+ real_queues = real_queues - bucket_queues
101
+
102
+ result << bucket_queues
103
+ end
104
+
105
+ # insert the remaining queues at the position the default item was at (or last)
106
+ real_queues.shuffle! if default_fairly
107
+ result.insert(default_idx, real_queues)
108
+ result.flatten!
109
+
110
+ return result
111
+ end
112
+ end
113
+ end
@@ -0,0 +1,64 @@
1
+
2
+ <div class="page-header">
3
+ <h1>Dynamic Queues</h1>
4
+ </div>
5
+ <p class="intro">
6
+ The list below shows the dynamic queues currently defined. When you start a worker with a dynamic queue key (@key_name), that key is looked up from the list below to determine the actual queues the worker should pull from. Wildcards (*) and negation (leading !) can be used to select the queues the worker should process. There is always a fallback key - @default, which workers will use if the key for that worker is empty. If both the key and the fallback are empty, the worker defaults to processing '*'
7
+ </p>
8
+
9
+ <form action="<%= u '/dynamicqueues' %>" method="POST" style="float:none; margin-top:10px">
10
+
11
+ <table class='queues'>
12
+ <tr>
13
+ <th>Name</th>
14
+ <th>Value</th>
15
+ <th>Expanded</th>
16
+ <th></th>
17
+ </tr>
18
+ <% @queues.each_with_index do |data, i| %>
19
+ <tr class="line">
20
+ <td><input type="text" id="input-<%= i %>-name" name="queues[][name]" value="<%= data['name'] %>" /></td>
21
+ <td><input type="text" id="input-<%= i %>-value" name="queues[][value]" value="<%= data['value'] %>" /></td>
22
+ <td class="expanded"><%= data['expanded'] %></td>
23
+ <td>
24
+ <a href="#remove" class="remove">Remove</a>
25
+ </td>
26
+ </tr>
27
+ <% end %>
28
+ </table>
29
+
30
+ <a href="#add" class="add">Add</a>
31
+ <input type="submit" value="Save"/>
32
+
33
+ </form>
34
+
35
+ <script type="text/javascript" charset="utf-8">
36
+ function markDirty()
37
+ {
38
+ $("input[type=submit]").css({border:"3px orange solid"});
39
+ }
40
+
41
+ jQuery(function($) {
42
+
43
+ $("input").live("keypress", markDirty);
44
+
45
+ $("a.add").live("click", function(e) {
46
+ e.preventDefault();
47
+ var $table = $("table.queues");
48
+ var $newRow = $table.find("tr.line:first").clone();
49
+ $newRow.find("input[type=text]").attr("value", "");
50
+ $newRow.find("td.expanded").html("")
51
+ $newRow.appendTo($table);
52
+ markDirty();
53
+ });
54
+
55
+ $("a.remove").live("click", function(e) {
56
+ e.preventDefault();
57
+ var $link = $(this);
58
+ $link.parents("tr").remove();
59
+ markDirty();
60
+ });
61
+
62
+
63
+ });
64
+ </script>
@@ -0,0 +1,78 @@
1
+ <div class="page-header">
2
+ <h1>Queue Priority</h1>
3
+ </div>
4
+ <p class="intro">
5
+ The list below orders queue name patterns by the priority you wish them to be executed in. The "Fairly" option allows you to indicate you want the queues within that pattern space be selected in a fair (random) manner, i.e. like resque-fairly. The 'default' pattern must always exist, and matches against all queues that aren't in any of the other patterns.
6
+ </p>
7
+
8
+ <form action="<%= u '/queuepriority' %>" method="POST" style="float:none; margin-top:10px">
9
+
10
+ <table class="priorities">
11
+ <tr>
12
+ <th>Pattern</th>
13
+ <th>Fairly</th>
14
+ <th></th>
15
+ </tr>
16
+ <% @priorities.each_with_index do |priority, i| %>
17
+ <tr class="line">
18
+ <td><input type="text" id="input-<%= i %>-pattern" name="priorities[][pattern]" value="<%= priority["pattern"] %>" /></td>
19
+ <td><input type="checkbox" id="input-<%= i %>-fairly" name="priorities[][fairly]" value="true" <%= "checked" if priority["fairly"] %> /></td>
20
+ <td>
21
+ <a href="#up" class="up">Up</a> |
22
+ <a href="#down" class="down">Down</a> |
23
+ <a href="#remove" class="remove">Remove</a>
24
+ </td>
25
+ </tr>
26
+ <% end %>
27
+ </table>
28
+
29
+ <a href="#add" class="add">Add</a>
30
+ <input type="submit" value="Save"/>
31
+
32
+ </form>
33
+
34
+ <script type="text/javascript" charset="utf-8">
35
+ function markDirty()
36
+ {
37
+ $("input[type=submit]").css({border:"3px orange solid"});
38
+ }
39
+
40
+ jQuery(function($) {
41
+ $("input").live("keypress", markDirty);
42
+ $("input[type=checkbox]").live("click", markDirty);
43
+
44
+ $("a.add").live("click", function(e) {
45
+ e.preventDefault();
46
+ var $table = $("table.priorities");
47
+ var $newRow = $table.find("tr.line:first").clone();
48
+ $newRow.find("input[type=text]").attr("value", "");
49
+ $newRow.find("input[type=checkbox]").attr("checked", false);
50
+ $newRow.appendTo($table);
51
+ markDirty();
52
+ });
53
+
54
+ $("a.remove").live("click", function(e) {
55
+ e.preventDefault();
56
+ var $link = $(this);
57
+ $link.parents("tr").remove();
58
+ markDirty();
59
+ });
60
+
61
+ $("a.up").live("click", function(e) {
62
+ e.preventDefault();
63
+ var $link = $(this);
64
+ var $row = $link.parents("tr");
65
+ $row.prev(".line").before($row);
66
+ markDirty();
67
+ });
68
+
69
+ $("a.down").live("click", function(e) {
70
+ e.preventDefault();
71
+ var $link = $(this);
72
+ var $row = $link.parents("tr");
73
+ $row.next(".line").after($row);
74
+ markDirty();
75
+ });
76
+
77
+ });
78
+ </script>
@@ -2,6 +2,7 @@
2
2
 
3
3
  require 'sinatra/base'
4
4
  require 'reqless'
5
+ require 'reqless/queue_patterns_helper'
5
6
 
6
7
  module Reqless
7
8
  # The Reqless web interface
@@ -78,6 +79,8 @@ module Reqless
78
79
  def tabs
79
80
  [
80
81
  { name: 'Queues' , path: '/queues' },
82
+ {:name => 'DynamicQueues', :path => '/dynamicqueues'},
83
+ {:name => 'QueuePriority', :path => '/queuepriority'},
81
84
  { name: 'Throttles', path: '/throttles'},
82
85
  { name: 'Workers' , path: '/workers' },
83
86
  { name: 'Track' , path: '/track' },
@@ -543,6 +546,77 @@ module Reqless
543
546
  end
544
547
  end
545
548
 
549
+ get "/dynamicqueues" do
550
+ @queues = []
551
+ real_queues = client.queues.counts.collect {|q| q['name'] }
552
+
553
+ queue_identifier_patterns = client.queue_patterns.get_queue_identifier_patterns
554
+ queue_identifier_patterns.each do |k, v|
555
+ expanded = QueuePatternsHelper.expand_queues(
556
+ ["@#{k}"], real_queues, queue_identifier_patterns
557
+ )
558
+ expanded = expanded.collect { |q| q.split(":").last }
559
+ view_data = {
560
+ 'name' => k,
561
+ 'value' => Array(v).join(", "),
562
+ 'expanded' => expanded.join(", ")
563
+ }
564
+ @queues << view_data
565
+ end
566
+
567
+ @queues.sort! do |a, b|
568
+ an = a['name']
569
+ bn = b['name']
570
+ if an == 'default'
571
+ 1
572
+ elsif bn == 'default'
573
+ -1
574
+ else
575
+ an <=> bn
576
+ end
577
+ end
578
+
579
+ erb :dynamic_queues, {:layout => true, :locals => { :title => 'Dynamic Queues' }}
580
+ end
581
+
582
+ post "/dynamicqueues" do
583
+ queues = params['queues']
584
+ dynamic_queues = {}
585
+ queues.each do |queue|
586
+ values = queue['value'].to_s.split(',').collect { |q| q.gsub(/\s/, '') }
587
+ dynamic_queues[queue['name']] = values
588
+ end
589
+
590
+ client.queue_patterns.set_queue_identifier_patterns(dynamic_queues)
591
+ redirect to("/dynamicqueues")
592
+ end
593
+
594
+ get "/queuepriority" do
595
+ # For the UI we always want the latest persisted data
596
+ @priorities = client.queue_patterns.get_queue_priority_patterns.map do |priority_pattern|
597
+ {
598
+ 'fairly' => priority_pattern.should_distribute_fairly,
599
+ 'pattern' => priority_pattern.pattern.join(', '),
600
+ }
601
+ end
602
+ erb :priorities, {:layout => true, :locals => { :title => 'Queue Priorities' }}
603
+ end
604
+
605
+ post "/queuepriority" do
606
+ priorities = params['priorities']
607
+ priority_patterns = priorities.map do |priority_pattern|
608
+ fairly = priority_pattern.fetch('fairly', 'false')
609
+ should_distribute_fairly = fairly == true || fairly == 'true'
610
+ Reqless::QueuePriorityPattern.new(
611
+ priority_pattern['pattern'].to_s.split(',').collect { |q| q.gsub(/\s/, '') },
612
+ should_distribute_fairly,
613
+ )
614
+ end
615
+ client.queue_patterns.set_queue_priority_patterns(priority_patterns)
616
+
617
+ redirect to("/queuepriority")
618
+ end
619
+
546
620
  # start the server if ruby file executed directly
547
621
  run! if app_file == $PROGRAM_NAME
548
622
  end
@@ -2,5 +2,5 @@
2
2
 
3
3
  module Reqless
4
4
  # VERSION is overwritten dynamically as part of the release process.
5
- VERSION = '0.0.3'
5
+ VERSION = '0.0.5'
6
6
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: reqless
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.3
4
+ version: 0.0.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dan Lecocq
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2024-08-27 00:00:00.000000000 Z
13
+ date: 2024-08-29 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: redis
@@ -303,9 +303,14 @@ files:
303
303
  - lib/reqless/config.rb
304
304
  - lib/reqless/failure_formatter.rb
305
305
  - lib/reqless/job.rb
306
+ - lib/reqless/job_reservers/delegating.rb
306
307
  - lib/reqless/job_reservers/ordered.rb
307
308
  - lib/reqless/job_reservers/round_robin.rb
308
309
  - lib/reqless/job_reservers/shuffled_round_robin.rb
310
+ - lib/reqless/job_reservers/strategies.rb
311
+ - lib/reqless/job_reservers/strategies/filtering.rb
312
+ - lib/reqless/job_reservers/strategies/ordering.rb
313
+ - lib/reqless/job_reservers/strategies/sources.rb
309
314
  - lib/reqless/lua/reqless-lib.lua
310
315
  - lib/reqless/lua/reqless.lua
311
316
  - lib/reqless/lua_script.rb
@@ -314,6 +319,7 @@ files:
314
319
  - lib/reqless/middleware/sentry.rb
315
320
  - lib/reqless/middleware/timeout.rb
316
321
  - lib/reqless/queue.rb
322
+ - lib/reqless/queue_patterns_helper.rb
317
323
  - lib/reqless/queue_priority_pattern.rb
318
324
  - lib/reqless/server.rb
319
325
  - lib/reqless/server/static/css/bootstrap-responsive.css
@@ -353,11 +359,13 @@ files:
353
359
  - lib/reqless/server/views/about.erb
354
360
  - lib/reqless/server/views/completed.erb
355
361
  - lib/reqless/server/views/config.erb
362
+ - lib/reqless/server/views/dynamic_queues.erb
356
363
  - lib/reqless/server/views/failed.erb
357
364
  - lib/reqless/server/views/failed_type.erb
358
365
  - lib/reqless/server/views/job.erb
359
366
  - lib/reqless/server/views/layout.erb
360
367
  - lib/reqless/server/views/overview.erb
368
+ - lib/reqless/server/views/priorities.erb
361
369
  - lib/reqless/server/views/queue.erb
362
370
  - lib/reqless/server/views/queues.erb
363
371
  - lib/reqless/server/views/tag.erb