reqless 0.0.2 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/reqless/lua/reqless-lib.lua +20 -1
- data/lib/reqless/lua/reqless.lua +19 -1
- data/lib/reqless/server/views/dynamic_queues.erb +64 -0
- data/lib/reqless/server/views/priorities.erb +78 -0
- data/lib/reqless/server.rb +125 -0
- data/lib/reqless/version.rb +2 -1
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f1042e561166bf47c7938f8d372d6ff1b69e6cdf999577b44e3c4a2ac3326cf1
|
4
|
+
data.tar.gz: 983e5f31909dca78405b16208064460f830048c9a277fdbc76f9a4b5b8ea194c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 89613e228c2b8bc014c25125eb65db6bd60816397b6e6df712e433d7901f4b21bd1b9444d73e8583d63209baeee513aca5e75f4c6a554008d9b78be0ac1bc9aa
|
7
|
+
data.tar.gz: fa149f59031bc460de0395bc1c5b768129f741ba4a646b6772049be60fb491e04df9584c8ba42293b88810c35180559b33292922fa855d9380d33873a9672978
|
@@ -1,4 +1,4 @@
|
|
1
|
-
-- Current SHA:
|
1
|
+
-- Current SHA: 54efb0679992da71b576cf16c043e9c9f985f426
|
2
2
|
-- This is a generated file
|
3
3
|
-- cjson can't tell an empty array from an empty object, so empty arrays end up
|
4
4
|
-- encoded as objects. This function makes empty arrays look like empty arrays.
|
@@ -2527,6 +2527,7 @@ function ReqlessQueue.counts(now, name)
|
|
2527
2527
|
end
|
2528
2528
|
local ReqlessQueuePatterns = {
|
2529
2529
|
default_identifiers_default_pattern = '["*"]',
|
2530
|
+
default_priority_pattern = '{"fairly": false, "pattern": ["default"]}',
|
2530
2531
|
ns = Reqless.ns .. "qp:",
|
2531
2532
|
}
|
2532
2533
|
ReqlessQueuePatterns.__index = ReqlessQueuePatterns
|
@@ -2599,6 +2600,10 @@ ReqlessQueuePatterns['getPriorityPatterns'] = function(now)
|
|
2599
2600
|
reply = redis.call('lrange', 'qmore:priority', 0, -1)
|
2600
2601
|
end
|
2601
2602
|
|
2603
|
+
if #reply == 0 then
|
2604
|
+
reply = {ReqlessQueuePatterns.default_priority_pattern}
|
2605
|
+
end
|
2606
|
+
|
2602
2607
|
return reply
|
2603
2608
|
end
|
2604
2609
|
|
@@ -2610,7 +2615,21 @@ ReqlessQueuePatterns['setPriorityPatterns'] = function(now, ...)
|
|
2610
2615
|
redis.call('del', key)
|
2611
2616
|
-- Clear out the legacy key
|
2612
2617
|
redis.call('del', 'qmore:priority')
|
2618
|
+
|
2613
2619
|
if #arg > 0 then
|
2620
|
+
-- Check for the default priority pattern and add one if none is given.
|
2621
|
+
local found_default = false
|
2622
|
+
for i = 1, #arg do
|
2623
|
+
local pattern = cjson.decode(arg[i])['pattern']
|
2624
|
+
if #pattern == 1 and pattern[1] == 'default' then
|
2625
|
+
found_default = true
|
2626
|
+
break
|
2627
|
+
end
|
2628
|
+
end
|
2629
|
+
if not found_default then
|
2630
|
+
table.insert(arg, ReqlessQueuePatterns.default_priority_pattern)
|
2631
|
+
end
|
2632
|
+
|
2614
2633
|
redis.call('rpush', key, unpack(arg))
|
2615
2634
|
end
|
2616
2635
|
end
|
data/lib/reqless/lua/reqless.lua
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
-- Current SHA:
|
1
|
+
-- Current SHA: 54efb0679992da71b576cf16c043e9c9f985f426
|
2
2
|
-- This is a generated file
|
3
3
|
local function cjsonArrayDegenerationWorkaround(array)
|
4
4
|
if #array == 0 then
|
@@ -1898,6 +1898,7 @@ function ReqlessQueue.counts(now, name)
|
|
1898
1898
|
end
|
1899
1899
|
local ReqlessQueuePatterns = {
|
1900
1900
|
default_identifiers_default_pattern = '["*"]',
|
1901
|
+
default_priority_pattern = '{"fairly": false, "pattern": ["default"]}',
|
1901
1902
|
ns = Reqless.ns .. "qp:",
|
1902
1903
|
}
|
1903
1904
|
ReqlessQueuePatterns.__index = ReqlessQueuePatterns
|
@@ -1961,6 +1962,10 @@ ReqlessQueuePatterns['getPriorityPatterns'] = function(now)
|
|
1961
1962
|
reply = redis.call('lrange', 'qmore:priority', 0, -1)
|
1962
1963
|
end
|
1963
1964
|
|
1965
|
+
if #reply == 0 then
|
1966
|
+
reply = {ReqlessQueuePatterns.default_priority_pattern}
|
1967
|
+
end
|
1968
|
+
|
1964
1969
|
return reply
|
1965
1970
|
end
|
1966
1971
|
|
@@ -1968,7 +1973,20 @@ ReqlessQueuePatterns['setPriorityPatterns'] = function(now, ...)
|
|
1968
1973
|
local key = ReqlessQueuePatterns.ns .. 'priorities'
|
1969
1974
|
redis.call('del', key)
|
1970
1975
|
redis.call('del', 'qmore:priority')
|
1976
|
+
|
1971
1977
|
if #arg > 0 then
|
1978
|
+
local found_default = false
|
1979
|
+
for i = 1, #arg do
|
1980
|
+
local pattern = cjson.decode(arg[i])['pattern']
|
1981
|
+
if #pattern == 1 and pattern[1] == 'default' then
|
1982
|
+
found_default = true
|
1983
|
+
break
|
1984
|
+
end
|
1985
|
+
end
|
1986
|
+
if not found_default then
|
1987
|
+
table.insert(arg, ReqlessQueuePatterns.default_priority_pattern)
|
1988
|
+
end
|
1989
|
+
|
1972
1990
|
redis.call('rpush', key, unpack(arg))
|
1973
1991
|
end
|
1974
1992
|
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
|
2
|
+
<div class="page-header">
|
3
|
+
<h1>Dynamic Queues</h1>
|
4
|
+
</div>
|
5
|
+
<p class="intro">
|
6
|
+
The list below shows the dynamic queues currently defined. When you start a worker with a dynamic queue key (@key_name), that key is looked up from the list below to determine the actual queues the worker should pull from. Wildcards (*) and negation (leading !) can be used to select the queues the worker should process. There is always a fallback key - @default, which workers will use if the key for that worker is empty. If both the key and the fallback are empty, the worker defaults to processing '*'
|
7
|
+
</p>
|
8
|
+
|
9
|
+
<form action="<%= u '/dynamicqueues' %>" method="POST" style="float:none; margin-top:10px">
|
10
|
+
|
11
|
+
<table class='queues'>
|
12
|
+
<tr>
|
13
|
+
<th>Name</th>
|
14
|
+
<th>Value</th>
|
15
|
+
<th>Expanded</th>
|
16
|
+
<th></th>
|
17
|
+
</tr>
|
18
|
+
<% @queues.each_with_index do |data, i| %>
|
19
|
+
<tr class="line">
|
20
|
+
<td><input type="text" id="input-<%= i %>-name" name="queues[][name]" value="<%= data['name'] %>" /></td>
|
21
|
+
<td><input type="text" id="input-<%= i %>-value" name="queues[][value]" value="<%= data['value'] %>" /></td>
|
22
|
+
<td class="expanded"><%= data['expanded'] %></td>
|
23
|
+
<td>
|
24
|
+
<a href="#remove" class="remove">Remove</a>
|
25
|
+
</td>
|
26
|
+
</tr>
|
27
|
+
<% end %>
|
28
|
+
</table>
|
29
|
+
|
30
|
+
<a href="#add" class="add">Add</a>
|
31
|
+
<input type="submit" value="Save"/>
|
32
|
+
|
33
|
+
</form>
|
34
|
+
|
35
|
+
<script type="text/javascript" charset="utf-8">
|
36
|
+
function markDirty()
|
37
|
+
{
|
38
|
+
$("input[type=submit]").css({border:"3px orange solid"});
|
39
|
+
}
|
40
|
+
|
41
|
+
jQuery(function($) {
|
42
|
+
|
43
|
+
$("input").live("keypress", markDirty);
|
44
|
+
|
45
|
+
$("a.add").live("click", function(e) {
|
46
|
+
e.preventDefault();
|
47
|
+
var $table = $("table.queues");
|
48
|
+
var $newRow = $table.find("tr.line:first").clone();
|
49
|
+
$newRow.find("input[type=text]").attr("value", "");
|
50
|
+
$newRow.find("td.expanded").html("")
|
51
|
+
$newRow.appendTo($table);
|
52
|
+
markDirty();
|
53
|
+
});
|
54
|
+
|
55
|
+
$("a.remove").live("click", function(e) {
|
56
|
+
e.preventDefault();
|
57
|
+
var $link = $(this);
|
58
|
+
$link.parents("tr").remove();
|
59
|
+
markDirty();
|
60
|
+
});
|
61
|
+
|
62
|
+
|
63
|
+
});
|
64
|
+
</script>
|
@@ -0,0 +1,78 @@
|
|
1
|
+
<div class="page-header">
|
2
|
+
<h1>Queue Priority</h1>
|
3
|
+
</div>
|
4
|
+
<p class="intro">
|
5
|
+
The list below orders queue name patterns by the priority you wish them to be executed in. The "Fairly" option allows you to indicate you want the queues within that pattern space be selected in a fair (random) manner, i.e. like resque-fairly. The 'default' pattern must always exist, and matches against all queues that aren't in any of the other patterns.
|
6
|
+
</p>
|
7
|
+
|
8
|
+
<form action="<%= u '/queuepriority' %>" method="POST" style="float:none; margin-top:10px">
|
9
|
+
|
10
|
+
<table class="priorities">
|
11
|
+
<tr>
|
12
|
+
<th>Pattern</th>
|
13
|
+
<th>Fairly</th>
|
14
|
+
<th></th>
|
15
|
+
</tr>
|
16
|
+
<% @priorities.each_with_index do |priority, i| %>
|
17
|
+
<tr class="line">
|
18
|
+
<td><input type="text" id="input-<%= i %>-pattern" name="priorities[][pattern]" value="<%= priority["pattern"] %>" /></td>
|
19
|
+
<td><input type="checkbox" id="input-<%= i %>-fairly" name="priorities[][fairly]" value="true" <%= "checked" if priority["fairly"] %> /></td>
|
20
|
+
<td>
|
21
|
+
<a href="#up" class="up">Up</a> |
|
22
|
+
<a href="#down" class="down">Down</a> |
|
23
|
+
<a href="#remove" class="remove">Remove</a>
|
24
|
+
</td>
|
25
|
+
</tr>
|
26
|
+
<% end %>
|
27
|
+
</table>
|
28
|
+
|
29
|
+
<a href="#add" class="add">Add</a>
|
30
|
+
<input type="submit" value="Save"/>
|
31
|
+
|
32
|
+
</form>
|
33
|
+
|
34
|
+
<script type="text/javascript" charset="utf-8">
|
35
|
+
function markDirty()
|
36
|
+
{
|
37
|
+
$("input[type=submit]").css({border:"3px orange solid"});
|
38
|
+
}
|
39
|
+
|
40
|
+
jQuery(function($) {
|
41
|
+
$("input").live("keypress", markDirty);
|
42
|
+
$("input[type=checkbox]").live("click", markDirty);
|
43
|
+
|
44
|
+
$("a.add").live("click", function(e) {
|
45
|
+
e.preventDefault();
|
46
|
+
var $table = $("table.priorities");
|
47
|
+
var $newRow = $table.find("tr.line:first").clone();
|
48
|
+
$newRow.find("input[type=text]").attr("value", "");
|
49
|
+
$newRow.find("input[type=checkbox]").attr("checked", false);
|
50
|
+
$newRow.appendTo($table);
|
51
|
+
markDirty();
|
52
|
+
});
|
53
|
+
|
54
|
+
$("a.remove").live("click", function(e) {
|
55
|
+
e.preventDefault();
|
56
|
+
var $link = $(this);
|
57
|
+
$link.parents("tr").remove();
|
58
|
+
markDirty();
|
59
|
+
});
|
60
|
+
|
61
|
+
$("a.up").live("click", function(e) {
|
62
|
+
e.preventDefault();
|
63
|
+
var $link = $(this);
|
64
|
+
var $row = $link.parents("tr");
|
65
|
+
$row.prev(".line").before($row);
|
66
|
+
markDirty();
|
67
|
+
});
|
68
|
+
|
69
|
+
$("a.down").live("click", function(e) {
|
70
|
+
e.preventDefault();
|
71
|
+
var $link = $(this);
|
72
|
+
var $row = $link.parents("tr");
|
73
|
+
$row.next(".line").after($row);
|
74
|
+
markDirty();
|
75
|
+
});
|
76
|
+
|
77
|
+
});
|
78
|
+
</script>
|
data/lib/reqless/server.rb
CHANGED
@@ -78,6 +78,8 @@ module Reqless
|
|
78
78
|
def tabs
|
79
79
|
[
|
80
80
|
{ name: 'Queues' , path: '/queues' },
|
81
|
+
{:name => 'DynamicQueues', :path => '/dynamicqueues'},
|
82
|
+
{:name => 'QueuePriority', :path => '/queuepriority'},
|
81
83
|
{ name: 'Throttles', path: '/throttles'},
|
82
84
|
{ name: 'Workers' , path: '/workers' },
|
83
85
|
{ name: 'Track' , path: '/track' },
|
@@ -156,6 +158,60 @@ module Reqless
|
|
156
158
|
formatted
|
157
159
|
end
|
158
160
|
end
|
161
|
+
|
162
|
+
# Returns a list of queues to use when searching for a job.
|
163
|
+
#
|
164
|
+
# A splat ("*") means you want every queue (in alpha order) - this
|
165
|
+
# can be useful for dynamically adding new queues.
|
166
|
+
#
|
167
|
+
# The splat can also be used as a wildcard within a queue name,
|
168
|
+
# e.g. "*high*", and negation can be indicated with a prefix of "!"
|
169
|
+
#
|
170
|
+
# An @key can be used to dynamically look up the queue list for key from redis.
|
171
|
+
# If no key is supplied, it defaults to the worker's hostname, and wildcards
|
172
|
+
# and negations can be used inside this dynamic queue list. Set the queue
|
173
|
+
# list for a key with set_dynamic_queue(key, ["q1", "q2"]
|
174
|
+
#
|
175
|
+
def expand_queues(queue_patterns, real_queues)
|
176
|
+
queue_patterns = queue_patterns.dup
|
177
|
+
real_queues = real_queues.dup
|
178
|
+
dynamic_queues = client.queue_patterns.get_queue_identifier_patterns
|
179
|
+
|
180
|
+
matched_queues = []
|
181
|
+
|
182
|
+
while q = queue_patterns.shift
|
183
|
+
q = q.to_s
|
184
|
+
negated = false
|
185
|
+
|
186
|
+
if q =~ /^(!)?@(.*)/
|
187
|
+
key = $2.strip
|
188
|
+
key = Socket.gethostname if key.size == 0
|
189
|
+
|
190
|
+
add_queues = dynamic_queues[key]
|
191
|
+
add_queues.map! { |q| q.gsub!(/^!/, '') || q.gsub!(/^/, '!') } if $1
|
192
|
+
|
193
|
+
queue_patterns.concat(add_queues)
|
194
|
+
next
|
195
|
+
end
|
196
|
+
|
197
|
+
if q =~ /^!/
|
198
|
+
negated = true
|
199
|
+
q = q[1..-1]
|
200
|
+
end
|
201
|
+
|
202
|
+
patstr = q.gsub(/\*/, ".*")
|
203
|
+
pattern = /^#{patstr}$/
|
204
|
+
if negated
|
205
|
+
matched_queues -= matched_queues.grep(pattern)
|
206
|
+
else
|
207
|
+
matches = real_queues.grep(/^#{pattern}$/)
|
208
|
+
matches = [q] if matches.size == 0 && q == patstr
|
209
|
+
matched_queues.concat(matches)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
return matched_queues.uniq.sort
|
214
|
+
end
|
159
215
|
end
|
160
216
|
|
161
217
|
get '/?' do
|
@@ -543,6 +599,75 @@ module Reqless
|
|
543
599
|
end
|
544
600
|
end
|
545
601
|
|
602
|
+
get "/dynamicqueues" do
|
603
|
+
@queues = []
|
604
|
+
real_queues = client.queues.counts.collect {|q| q['name'] }
|
605
|
+
|
606
|
+
dqueues = client.queue_patterns.get_queue_identifier_patterns
|
607
|
+
dqueues.each do |k, v|
|
608
|
+
expanded = expand_queues(["@#{k}"], real_queues)
|
609
|
+
expanded = expanded.collect { |q| q.split(":").last }
|
610
|
+
view_data = {
|
611
|
+
'name' => k,
|
612
|
+
'value' => Array(v).join(", "),
|
613
|
+
'expanded' => expanded.join(", ")
|
614
|
+
}
|
615
|
+
@queues << view_data
|
616
|
+
end
|
617
|
+
|
618
|
+
@queues.sort! do |a, b|
|
619
|
+
an = a['name']
|
620
|
+
bn = b['name']
|
621
|
+
if an == 'default'
|
622
|
+
1
|
623
|
+
elsif bn == 'default'
|
624
|
+
-1
|
625
|
+
else
|
626
|
+
an <=> bn
|
627
|
+
end
|
628
|
+
end
|
629
|
+
|
630
|
+
erb :dynamic_queues, {:layout => true, :locals => { :title => 'Dynamic Queues' }}
|
631
|
+
end
|
632
|
+
|
633
|
+
post "/dynamicqueues" do
|
634
|
+
queues = params['queues']
|
635
|
+
dynamic_queues = {}
|
636
|
+
queues.each do |queue|
|
637
|
+
values = queue['value'].to_s.split(',').collect { |q| q.gsub(/\s/, '') }
|
638
|
+
dynamic_queues[queue['name']] = values
|
639
|
+
end
|
640
|
+
|
641
|
+
client.queue_patterns.set_queue_identifier_patterns(dynamic_queues)
|
642
|
+
redirect to("/dynamicqueues")
|
643
|
+
end
|
644
|
+
|
645
|
+
get "/queuepriority" do
|
646
|
+
# For the UI we always want the latest persisted data
|
647
|
+
@priorities = client.queue_patterns.get_queue_priority_patterns.map do |priority_pattern|
|
648
|
+
{
|
649
|
+
'fairly' => priority_pattern.should_distribute_fairly,
|
650
|
+
'pattern' => priority_pattern.pattern.join(', '),
|
651
|
+
}
|
652
|
+
end
|
653
|
+
erb :priorities, {:layout => true, :locals => { :title => 'Queue Priorities' }}
|
654
|
+
end
|
655
|
+
|
656
|
+
post "/queuepriority" do
|
657
|
+
priorities = params['priorities']
|
658
|
+
priority_patterns = priorities.map do |priority_pattern|
|
659
|
+
fairly = priority_pattern.fetch('fairly', 'false')
|
660
|
+
should_distribute_fairly = fairly == true || fairly == 'true'
|
661
|
+
Reqless::QueuePriorityPattern.new(
|
662
|
+
priority_pattern['pattern'].to_s.split(',').collect { |q| q.gsub(/\s/, '') },
|
663
|
+
should_distribute_fairly,
|
664
|
+
)
|
665
|
+
end
|
666
|
+
client.queue_patterns.set_queue_priority_patterns(priority_patterns)
|
667
|
+
|
668
|
+
redirect to("/queuepriority")
|
669
|
+
end
|
670
|
+
|
546
671
|
# start the server if ruby file executed directly
|
547
672
|
run! if app_file == $PROGRAM_NAME
|
548
673
|
end
|
data/lib/reqless/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: reqless
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dan Lecocq
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2024-08-
|
13
|
+
date: 2024-08-28 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: redis
|
@@ -353,11 +353,13 @@ files:
|
|
353
353
|
- lib/reqless/server/views/about.erb
|
354
354
|
- lib/reqless/server/views/completed.erb
|
355
355
|
- lib/reqless/server/views/config.erb
|
356
|
+
- lib/reqless/server/views/dynamic_queues.erb
|
356
357
|
- lib/reqless/server/views/failed.erb
|
357
358
|
- lib/reqless/server/views/failed_type.erb
|
358
359
|
- lib/reqless/server/views/job.erb
|
359
360
|
- lib/reqless/server/views/layout.erb
|
360
361
|
- lib/reqless/server/views/overview.erb
|
362
|
+
- lib/reqless/server/views/priorities.erb
|
361
363
|
- lib/reqless/server/views/queue.erb
|
362
364
|
- lib/reqless/server/views/queues.erb
|
363
365
|
- lib/reqless/server/views/tag.erb
|