qyu 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +56 -0
- data/.rspec +3 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/LICENSE +21 -0
- data/README.md +90 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/server +17 -0
- data/bin/setup +8 -0
- data/examples/bin/simple +7 -0
- data/examples/config.rb +22 -0
- data/examples/simple/create_workflow.rb +18 -0
- data/examples/simple/enqueue_job.rb +8 -0
- data/examples/simple/worker.rb +32 -0
- data/lib/qyu.rb +74 -0
- data/lib/qyu/config.rb +35 -0
- data/lib/qyu/errors.rb +4 -0
- data/lib/qyu/errors/base.rb +8 -0
- data/lib/qyu/errors/could_not_fetch_task.rb +18 -0
- data/lib/qyu/errors/invalid_queue_name.rb +12 -0
- data/lib/qyu/errors/invalid_task_attributes.rb +12 -0
- data/lib/qyu/errors/job_not_found.rb +14 -0
- data/lib/qyu/errors/lock_already_acquired.rb +12 -0
- data/lib/qyu/errors/lock_not_acquired.rb +12 -0
- data/lib/qyu/errors/message_not_received.rb +12 -0
- data/lib/qyu/errors/not_implemented_error.rb +12 -0
- data/lib/qyu/errors/payload_validation_error.rb +12 -0
- data/lib/qyu/errors/task_not_found.rb +15 -0
- data/lib/qyu/errors/task_status_update_failed.rb +15 -0
- data/lib/qyu/errors/unknown_validation_option.rb +12 -0
- data/lib/qyu/errors/unsync_error.rb +12 -0
- data/lib/qyu/errors/workflow_descriptor_validation_error.rb +14 -0
- data/lib/qyu/errors/workflow_not_found.rb +15 -0
- data/lib/qyu/factory.rb +26 -0
- data/lib/qyu/models.rb +9 -0
- data/lib/qyu/models/concerns/workflow_descriptor_validator.rb +117 -0
- data/lib/qyu/models/enums/status.rb +44 -0
- data/lib/qyu/models/job.rb +174 -0
- data/lib/qyu/models/task.rb +218 -0
- data/lib/qyu/models/workflow.rb +85 -0
- data/lib/qyu/queue.rb +5 -0
- data/lib/qyu/queue/base.rb +46 -0
- data/lib/qyu/queue/memory/adapter.rb +90 -0
- data/lib/qyu/store.rb +5 -0
- data/lib/qyu/store/base.rb +106 -0
- data/lib/qyu/store/memory/adapter.rb +187 -0
- data/lib/qyu/ui.rb +56 -0
- data/lib/qyu/ui/helpers/pagination.rb +35 -0
- data/lib/qyu/ui/public/bootstrap.min.css +5 -0
- data/lib/qyu/ui/public/paper-dashboard.css +3315 -0
- data/lib/qyu/ui/public/script.js +28 -0
- data/lib/qyu/ui/public/style.css +6 -0
- data/lib/qyu/ui/views/footer.erb +18 -0
- data/lib/qyu/ui/views/helpers/pagination.erb +49 -0
- data/lib/qyu/ui/views/jobs.erb +58 -0
- data/lib/qyu/ui/views/kaminari/_first_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_gap.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_last_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_next_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_page.html.erb +9 -0
- data/lib/qyu/ui/views/kaminari/_paginator.html.erb +15 -0
- data/lib/qyu/ui/views/kaminari/_prev_page.html.erb +3 -0
- data/lib/qyu/ui/views/layout.erb +33 -0
- data/lib/qyu/ui/views/navbar.erb +29 -0
- data/lib/qyu/ui/views/pagination.erb +19 -0
- data/lib/qyu/ui/views/show_job.erb +55 -0
- data/lib/qyu/ui/views/sidebar.erb +17 -0
- data/lib/qyu/ui/views/task_row.erb +26 -0
- data/lib/qyu/utils.rb +17 -0
- data/lib/qyu/version.rb +3 -0
- data/lib/qyu/workers.rb +10 -0
- data/lib/qyu/workers/base.rb +126 -0
- data/lib/qyu/workers/concerns/callback.rb +38 -0
- data/lib/qyu/workers/concerns/failure_queue.rb +23 -0
- data/lib/qyu/workers/concerns/payload_validator.rb +124 -0
- data/lib/qyu/workers/sync.rb +63 -0
- data/qyu.gemspec +36 -0
- metadata +278 -0
@@ -0,0 +1,28 @@
|
|
1
|
+
(function(){
|
2
|
+
var collapser = function(){
|
3
|
+
var elem = this;
|
4
|
+
if ( elem.classList.contains('open') ){
|
5
|
+
var root = elem.dataset.root;
|
6
|
+
var level = elem.dataset.level;
|
7
|
+
document.querySelectorAll('tr[data-root="' + root + '"]').forEach(function(el){
|
8
|
+
var currentLevel = parseInt(el.dataset.level);
|
9
|
+
if (currentLevel > level) {
|
10
|
+
el.classList.add('collapse');
|
11
|
+
var parent = el.dataset.parent;
|
12
|
+
document.querySelector('a[data-id="' + parent + '"]').classList.remove('open');
|
13
|
+
}
|
14
|
+
});
|
15
|
+
elem.classList.remove('open');
|
16
|
+
} else {
|
17
|
+
var id = elem.dataset.id;
|
18
|
+
document.querySelectorAll('[data-parent="' + id + '"]').forEach(function(el){
|
19
|
+
el.classList.remove('collapse');
|
20
|
+
});
|
21
|
+
elem.classList.add('open');
|
22
|
+
}
|
23
|
+
}
|
24
|
+
|
25
|
+
document.querySelectorAll('.collapser').forEach(function(elem){
|
26
|
+
elem.addEventListener('click', collapser);
|
27
|
+
});
|
28
|
+
}).call(this);
|
@@ -0,0 +1,18 @@
|
|
1
|
+
<footer class="footer">
|
2
|
+
<div class="container-fluid">
|
3
|
+
<nav class="pull-left">
|
4
|
+
<ul>
|
5
|
+
<li>
|
6
|
+
<a href="https://github.com/FindHotel/qyu">
|
7
|
+
Qyu is a distributed task execution system for complex workflows
|
8
|
+
</a>
|
9
|
+
</li>
|
10
|
+
</ul>
|
11
|
+
</nav>
|
12
|
+
<div class="copyright pull-right">
|
13
|
+
<a href="https://www.findhotel.net/">
|
14
|
+
FindHotel B.V. © <%= Date.today.year %>
|
15
|
+
</a>
|
16
|
+
</div>
|
17
|
+
</div>
|
18
|
+
</footer>
|
@@ -0,0 +1,49 @@
|
|
1
|
+
<nav aria-label="Page navigation">
|
2
|
+
<ul class="pagination">
|
3
|
+
<% if collection.page > 1 %>
|
4
|
+
<li>
|
5
|
+
<a href="<%= url("/jobs?page=1") %>">
|
6
|
+
⇠ First
|
7
|
+
</a>
|
8
|
+
</li>
|
9
|
+
<li>
|
10
|
+
<a href="<%= url("/jobs?page=#{collection.page - 1}") %>" aria-label="Previous">
|
11
|
+
<span aria-hidden="true">«</span>
|
12
|
+
</a>
|
13
|
+
</li>
|
14
|
+
<% end %>
|
15
|
+
|
16
|
+
<% previous_pages_for(collection).each do |page_num| %>
|
17
|
+
<li>
|
18
|
+
<a href="<%= url("/jobs?page=#{page_num}") %>">
|
19
|
+
<%= page_num %>
|
20
|
+
</a>
|
21
|
+
</li>
|
22
|
+
<% end %>
|
23
|
+
<li class="active">
|
24
|
+
<a href="#">
|
25
|
+
<%= collection.page %> <span class="sr-only">(current)</span>
|
26
|
+
</a>
|
27
|
+
</li>
|
28
|
+
<% next_pages_for(collection).each do |page_num| %>
|
29
|
+
<li>
|
30
|
+
<a href="<%= url("/jobs?page=#{page_num}") %>">
|
31
|
+
<%= page_num %>
|
32
|
+
</a>
|
33
|
+
</li>
|
34
|
+
<% end %>
|
35
|
+
|
36
|
+
<% if collection.total_pages > collection.page %>
|
37
|
+
<li>
|
38
|
+
<a href="<%= url("/jobs?page=#{collection.page + 1}") %>" aria-label="Next">
|
39
|
+
<span aria-hidden="true">»</span>
|
40
|
+
</a>
|
41
|
+
</li>
|
42
|
+
<li>
|
43
|
+
<a href="<%= url("/jobs?page=#{collection.total_pages}" ) %>">
|
44
|
+
Last ⇢
|
45
|
+
</a>
|
46
|
+
</li>
|
47
|
+
<% end %>
|
48
|
+
</ul>
|
49
|
+
</nav>
|
@@ -0,0 +1,58 @@
|
|
1
|
+
<div class="row">
|
2
|
+
<div class="col-md-12">
|
3
|
+
<div class="card">
|
4
|
+
<div class="header">
|
5
|
+
<h4 class="title">Jobs</h4>
|
6
|
+
<!-- <p class="category">Last 24 Hours</p> -->
|
7
|
+
</div>
|
8
|
+
<div class="content">
|
9
|
+
<table class="table table-bordered">
|
10
|
+
<thead>
|
11
|
+
<tr>
|
12
|
+
<th>#</th>
|
13
|
+
<th>Description</th>
|
14
|
+
<th>Payload</th>
|
15
|
+
</tr>
|
16
|
+
</thead>
|
17
|
+
<tbody>
|
18
|
+
<% jobs.each do |job| %>
|
19
|
+
<tr>
|
20
|
+
<td>
|
21
|
+
<a href="<%= url("/jobs/#{job.id}") %>">
|
22
|
+
<p><%= job.id %></p>
|
23
|
+
<p>Created: <%= job.created_at %></p>
|
24
|
+
</a>
|
25
|
+
</td>
|
26
|
+
<td>
|
27
|
+
<strong>Starts:</strong>
|
28
|
+
<ul>
|
29
|
+
<% job.descriptor['starts'].each do |entry_point| %>
|
30
|
+
<li><%= entry_point %></li>
|
31
|
+
<% end %>
|
32
|
+
</ul>
|
33
|
+
<strong>Tasks:</strong>
|
34
|
+
<ul>
|
35
|
+
<% job.descriptor['tasks'].keys.each do |task| %>
|
36
|
+
<li><%= task %></li>
|
37
|
+
<% end %>
|
38
|
+
</ul>
|
39
|
+
</td>
|
40
|
+
<td>
|
41
|
+
<ul>
|
42
|
+
<% job.payload.each do |key, value| %>
|
43
|
+
<li><%= key %>: <%= value %></li>
|
44
|
+
<% end %>
|
45
|
+
</ul>
|
46
|
+
</td>
|
47
|
+
</tr>
|
48
|
+
<% end %>
|
49
|
+
</tbody>
|
50
|
+
</table>
|
51
|
+
|
52
|
+
<nav aria-label="Page navigation" class="text-center">
|
53
|
+
<%= erb :'helpers/pagination', locals: { collection: jobs } %>
|
54
|
+
</nav>
|
55
|
+
</div>
|
56
|
+
</div>
|
57
|
+
</div>
|
58
|
+
</div>
|
@@ -0,0 +1,9 @@
|
|
1
|
+
<% if page.current? %>
|
2
|
+
<li class='active'>
|
3
|
+
<%= content_tag :a, page, remote: remote, rel: (page.next? ? 'next' : (page.prev? ? 'prev' : nil)) %>
|
4
|
+
</li>
|
5
|
+
<% else %>
|
6
|
+
<li>
|
7
|
+
<%= link_to page, url, remote: remote, rel: (page.next? ? 'next' : (page.prev? ? 'prev' : nil)) %>
|
8
|
+
</li>
|
9
|
+
<% end %>
|
@@ -0,0 +1,15 @@
|
|
1
|
+
<%= paginator.render do -%>
|
2
|
+
<ul class="pagination">
|
3
|
+
<%= first_page_tag unless current_page.first? %>
|
4
|
+
<%= prev_page_tag unless current_page.first? %>
|
5
|
+
<% each_page do |page| -%>
|
6
|
+
<% if page.left_outer? || page.right_outer? || page.inside_window? -%>
|
7
|
+
<%= page_tag page %>
|
8
|
+
<% elsif !page.was_truncated? -%>
|
9
|
+
<%= gap_tag %>
|
10
|
+
<% end -%>
|
11
|
+
<% end -%>
|
12
|
+
<%= next_page_tag unless current_page.last? %>
|
13
|
+
<%= last_page_tag unless current_page.last? %>
|
14
|
+
</ul>
|
15
|
+
<% end -%>
|
@@ -0,0 +1,33 @@
|
|
1
|
+
<!DOCTYPE html>
|
2
|
+
<html lang="en">
|
3
|
+
<head>
|
4
|
+
<meta charset="utf-8" />
|
5
|
+
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
|
6
|
+
|
7
|
+
<title>Qyu Dashboard</title>
|
8
|
+
|
9
|
+
<meta content='width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=0' name='viewport' />
|
10
|
+
<meta name="viewport" content="width=device-width" />
|
11
|
+
|
12
|
+
<!-- Bootstrap core CSS -->
|
13
|
+
<link rel="stylesheet" type="text/css" href="<%= url("/bootstrap.min.css")%>">
|
14
|
+
<!-- Paper Dashboard core CSS -->
|
15
|
+
<link rel="stylesheet" type="text/css" href="<%= url("/paper-dashboard.css")%>">
|
16
|
+
<!-- custom CSS -->
|
17
|
+
<link rel="stylesheet" type="text/css" href="<%= url("/style.css")%>">
|
18
|
+
</head>
|
19
|
+
<body>
|
20
|
+
<div class="wrapper">
|
21
|
+
<div class="main-panel">
|
22
|
+
<%= erb :navbar %>
|
23
|
+
<div class="content">
|
24
|
+
<div class="container-fluid">
|
25
|
+
<%= yield %>
|
26
|
+
</div>
|
27
|
+
</div>
|
28
|
+
<%= erb :footer %>
|
29
|
+
</div>
|
30
|
+
</div>
|
31
|
+
<script src="<%= url("/script.js")%>"></script>
|
32
|
+
</body>
|
33
|
+
</html>
|
@@ -0,0 +1,29 @@
|
|
1
|
+
<nav class="navbar navbar-default">
|
2
|
+
<div class="container-fluid">
|
3
|
+
<div class="navbar-header">
|
4
|
+
<button type="button" class="navbar-toggle">
|
5
|
+
<span class="sr-only">Toggle navigation</span>
|
6
|
+
<span class="icon-bar bar1"></span>
|
7
|
+
<span class="icon-bar bar2"></span>
|
8
|
+
<span class="icon-bar bar3"></span>
|
9
|
+
</button>
|
10
|
+
<a class="navbar-brand" href="/">Qyu</a>
|
11
|
+
</div>
|
12
|
+
<div class="collapse navbar-collapse">
|
13
|
+
<ul class="nav navbar-nav navbar-right">
|
14
|
+
<li class="active">
|
15
|
+
<a href="#">
|
16
|
+
<i class="ti-panel"></i>
|
17
|
+
<p>Jobs</p>
|
18
|
+
</a>
|
19
|
+
</li>
|
20
|
+
<li>
|
21
|
+
<a href="#">
|
22
|
+
<i class="ti-settings"></i>
|
23
|
+
<p>Tasks</p>
|
24
|
+
</a>
|
25
|
+
</li>
|
26
|
+
</ul>
|
27
|
+
</div>
|
28
|
+
</div>
|
29
|
+
</nav>
|
@@ -0,0 +1,19 @@
|
|
1
|
+
<nav aria-label="Page navigation" class="text-center">
|
2
|
+
<ul class="pagination">
|
3
|
+
<li>
|
4
|
+
<a href="#" aria-label="Previous">
|
5
|
+
<span aria-hidden="true">«</span>
|
6
|
+
</a>
|
7
|
+
</li>
|
8
|
+
<li><a href="#">1</a></li>
|
9
|
+
<li><a href="#">2</a></li>
|
10
|
+
<li><a href="#">3</a></li>
|
11
|
+
<li><a href="#">4</a></li>
|
12
|
+
<li><a href="#">5</a></li>
|
13
|
+
<li>
|
14
|
+
<a href="#" aria-label="Next">
|
15
|
+
<span aria-hidden="true">»</span>
|
16
|
+
</a>
|
17
|
+
</li>
|
18
|
+
</ul>
|
19
|
+
</nav>
|
@@ -0,0 +1,55 @@
|
|
1
|
+
<div class="row">
|
2
|
+
<div class="col-md-12">
|
3
|
+
<div class="card">
|
4
|
+
<div class="header">
|
5
|
+
<h4 class="title">Job #<%= job.id %></h4>
|
6
|
+
<p class="category">
|
7
|
+
Tasks list: <%= total_count %> <%= 'task'.pluralize(total_count) %>
|
8
|
+
</p>
|
9
|
+
</div>
|
10
|
+
<div class="content">
|
11
|
+
<table class="table table-stripped">
|
12
|
+
<thead>
|
13
|
+
<tr>
|
14
|
+
<th>Task Name</th>
|
15
|
+
<th>Queued</th>
|
16
|
+
<th>Working</th>
|
17
|
+
<th>Completed</th>
|
18
|
+
<th>Failed</th>
|
19
|
+
<th>Invalid Payload</th>
|
20
|
+
</tr>
|
21
|
+
</thead>
|
22
|
+
<tbody>
|
23
|
+
<% task_statuses.each do |task_name, statuses| %>
|
24
|
+
<tr>
|
25
|
+
<td><%= task_name %></td>
|
26
|
+
<td><%= statuses['queued'] %></td>
|
27
|
+
<td><%= statuses['working'] %></td>
|
28
|
+
<td><%= statuses['completed'] %></td>
|
29
|
+
<td><%= statuses['failed'] %></td>
|
30
|
+
<td><%= statuses['invalid_payload'] %></td>
|
31
|
+
</tr>
|
32
|
+
<% end %>
|
33
|
+
</tbody>
|
34
|
+
</table>
|
35
|
+
<table class="table table-stripped">
|
36
|
+
<thead>
|
37
|
+
<tr>
|
38
|
+
<th>Name</th>
|
39
|
+
<th>Queue Name</th>
|
40
|
+
<th>Status</th>
|
41
|
+
<th>Payload</th>
|
42
|
+
<!-- <th>Locked Until</th> -->
|
43
|
+
<!-- <th>Locked By</th> -->
|
44
|
+
</tr>
|
45
|
+
</thead>
|
46
|
+
<tbody>
|
47
|
+
<% (tasks[nil] || []).each do |task| %>
|
48
|
+
<%= erb :task_row, locals: { tasks: tasks, task: task, level: 0, root_id: task.id } %>
|
49
|
+
<% end %>
|
50
|
+
</tbody>
|
51
|
+
</table>
|
52
|
+
</div>
|
53
|
+
</div>
|
54
|
+
</div>
|
55
|
+
</div>
|
@@ -0,0 +1,17 @@
|
|
1
|
+
<div class="sidebar" data-background-color="white" data-active-color="danger">
|
2
|
+
<div class="sidebar-wrapper">
|
3
|
+
<div class="logo">
|
4
|
+
<a href="/" class="simple-text">
|
5
|
+
Arc Yu
|
6
|
+
</a>
|
7
|
+
</div>
|
8
|
+
|
9
|
+
<ul class="nav">
|
10
|
+
<li class="active">
|
11
|
+
<a href="/">
|
12
|
+
<p>Jobs</p>
|
13
|
+
</a>
|
14
|
+
</li>
|
15
|
+
</ul>
|
16
|
+
</div>
|
17
|
+
</div>
|
@@ -0,0 +1,26 @@
|
|
1
|
+
<tr data-parent="<%= task.parent_task_id %>"
|
2
|
+
data-level="<%= level %>"
|
3
|
+
data-root="<%= root_id %>"
|
4
|
+
class="<%= 'collapse' if level > 0 %>">
|
5
|
+
<td>
|
6
|
+
<%= raw_html("• " * level) if level > 0 %>
|
7
|
+
<% if tasks[task.id] %>
|
8
|
+
<a href="#" class="collapser"
|
9
|
+
data-id="<%= task.id %>"
|
10
|
+
data-level="<%= level %>"
|
11
|
+
data-root="<%= root_id %>">
|
12
|
+
<%= task.name %>
|
13
|
+
</a>
|
14
|
+
<% else %>
|
15
|
+
<%= task.name %>
|
16
|
+
<% end %>
|
17
|
+
</td>
|
18
|
+
<td><%= task.queue_name %></td>
|
19
|
+
<td><%= task.status.status %></td>
|
20
|
+
<td><%= task.payload %></td>
|
21
|
+
<!-- <td><%# task.locked_until %></td> -->
|
22
|
+
<!-- <td><%# task.locked_by %></td> -->
|
23
|
+
</tr>
|
24
|
+
<% (tasks[task.id] || []).each do |t| %>
|
25
|
+
<%= erb :task_row, locals: { tasks: tasks, task: t, level: level + 1, root_id: root_id } %>
|
26
|
+
<% end %>
|
data/lib/qyu/utils.rb
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Utils
|
5
|
+
def self.seconds_after_time(seconds, start_time = Time.now)
|
6
|
+
start_time + seconds
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.uuid
|
10
|
+
SecureRandom.uuid
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.stringify_hash_keys(object)
|
14
|
+
object.map { |k, v| [k.to_s, v] }.to_h
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
data/lib/qyu/version.rb
ADDED
data/lib/qyu/workers.rb
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'qyu/workers/concerns/callback'
|
4
|
+
require 'qyu/workers/concerns/failure_queue'
|
5
|
+
require 'qyu/workers/concerns/payload_validator'
|
6
|
+
require 'qyu/workers/base'
|
7
|
+
require 'qyu/workers/sync'
|
8
|
+
|
9
|
+
Qyu::Worker = Qyu::Workers::Base
|
10
|
+
Qyu::SyncWorker = Qyu::Workers::Sync
|
@@ -0,0 +1,126 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Workers
|
5
|
+
class Base
|
6
|
+
include Concerns::Callback
|
7
|
+
include Concerns::PayloadValidator
|
8
|
+
include Concerns::FailureQueue
|
9
|
+
|
10
|
+
attr_reader :id
|
11
|
+
attr_accessor :processed_tasks
|
12
|
+
|
13
|
+
def initialize(&block)
|
14
|
+
@id = Qyu::Utils.uuid
|
15
|
+
@processed_tasks = 0
|
16
|
+
instance_exec(&block) if block_given?
|
17
|
+
end
|
18
|
+
|
19
|
+
def work(queue_name, blocking: true)
|
20
|
+
log(:info, "Worker started for queue '#{queue_name}'")
|
21
|
+
repeat = true
|
22
|
+
|
23
|
+
remaining_fetch_retries = 3
|
24
|
+
|
25
|
+
while repeat
|
26
|
+
run_callbacks(:execute) do
|
27
|
+
begin
|
28
|
+
fetched_task = fetch_task(queue_name)
|
29
|
+
validate_payload!(fetched_task)
|
30
|
+
log(:info, "Worker processed #{processed_tasks} tasks from queue `#{queue_name}`")
|
31
|
+
if fetched_task.acknowledgeable?
|
32
|
+
discard_completed_task(fetched_task)
|
33
|
+
elsif fetched_task.lock!
|
34
|
+
fetched_task.mark_working
|
35
|
+
begin
|
36
|
+
yield(fetched_task)
|
37
|
+
conclude_task(fetched_task)
|
38
|
+
rescue => ex
|
39
|
+
fail_task(fetched_task, ex)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
rescue Qyu::Errors::UnsyncError
|
43
|
+
rescue Qyu::Errors::CouldNotFetchTask => ex
|
44
|
+
if remaining_fetch_retries <= 0
|
45
|
+
acknowledge_message_with_task_id_not_found_in_store(ex)
|
46
|
+
else
|
47
|
+
sleep(remaining_fetch_retries)
|
48
|
+
remaining_fetch_retries -= 1
|
49
|
+
retry
|
50
|
+
end
|
51
|
+
rescue Qyu::Errors::PayloadValidationError
|
52
|
+
fetched_task.mark_invalid_payload
|
53
|
+
rescue => ex
|
54
|
+
log("Worker error: #{ex.class}: #{ex.message}")
|
55
|
+
log("Backtrace: #{ex.backtrace.join("\n")}")
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
repeat = blocking
|
60
|
+
run_garbage_collector
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
private
|
65
|
+
|
66
|
+
def fetch_task(queue_name)
|
67
|
+
fetched_task = Qyu::Task.fetch(queue_name)
|
68
|
+
@processed_tasks += 1
|
69
|
+
fetched_task
|
70
|
+
end
|
71
|
+
|
72
|
+
def discard_completed_task(fetched_task)
|
73
|
+
log(:debug, 'Fetched completed task. Discarding...')
|
74
|
+
fetched_task.acknowledge_message
|
75
|
+
end
|
76
|
+
|
77
|
+
def conclude_task(fetched_task)
|
78
|
+
Qyu.store.transaction do
|
79
|
+
log(:debug, 'Task finished. Creating next tasks.')
|
80
|
+
fetched_task.job.create_next_tasks(
|
81
|
+
fetched_task,
|
82
|
+
fetched_task.job.payload.merge(fetched_task.payload)
|
83
|
+
)
|
84
|
+
fetched_task.unlock!
|
85
|
+
fetched_task.mark_completed
|
86
|
+
end
|
87
|
+
fetched_task.acknowledge_message
|
88
|
+
end
|
89
|
+
|
90
|
+
def fail_task(fetched_task, exception)
|
91
|
+
unless exception.class == Qyu::Errors::UnsyncError
|
92
|
+
log("Worker error: #{exception.class}: #{exception.message}")
|
93
|
+
log("Backtrace: #{exception.backtrace.join("\n")}")
|
94
|
+
end
|
95
|
+
Qyu.store.transaction do
|
96
|
+
fetched_task.enqueue_in_failure_queue if @failure_queue
|
97
|
+
fetched_task.unlock!
|
98
|
+
fetched_task.mark_queued
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def acknowledge_message_with_task_id_not_found_in_store(exception)
|
103
|
+
# If a task is not found in the Store then there is no point attempting
|
104
|
+
# to fetch the message over and over again.
|
105
|
+
log("Worker error: #{exception.class}: #{exception.message}")
|
106
|
+
log("Backtrace: #{exception.backtrace.join("\n")}")
|
107
|
+
log("Original error: #{exception.original_error.class}: #{exception.original_error.message}")
|
108
|
+
log("Backtrace: #{exception.original_error.backtrace.join("\n")}")
|
109
|
+
if exception.original_error.class == Qyu::Errors::TaskNotFound &&
|
110
|
+
exception.queue_name &&
|
111
|
+
exception.message_id
|
112
|
+
Qyu::Task.acknowledge_message(exception.queue_name, exception.message_id)
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def log(level = :error, message)
|
117
|
+
Qyu.logger.public_send(level, "[#{id}] #{message}")
|
118
|
+
end
|
119
|
+
|
120
|
+
def run_garbage_collector
|
121
|
+
log(:debug, 'Running garbage collector')
|
122
|
+
GC.start
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|