qyu 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +5 -0
- data/CHANGELOG.md +17 -3
- data/README.md +9 -5
- data/lib/qyu/config.rb +1 -1
- data/lib/qyu/errors/invalid_worker_configuration_value.rb +12 -0
- data/lib/qyu/errors/missing_split_parameters.rb +12 -0
- data/lib/qyu/errors/payload_validation_error.rb +1 -1
- data/lib/qyu/factory.rb +1 -1
- data/lib/qyu/models/concerns/workflow_descriptor_validator.rb +6 -1
- data/lib/qyu/models/job.rb +35 -6
- data/lib/qyu/models/task.rb +24 -0
- data/lib/qyu/models/workflow.rb +8 -12
- data/lib/qyu/ui/views/footer.erb +3 -3
- data/lib/qyu/version.rb +1 -1
- data/lib/qyu/workers.rb +8 -5
- data/lib/qyu/workers/base.rb +20 -16
- data/lib/qyu/workers/concerns/split.rb +63 -0
- data/lib/qyu/workers/concerns/timeout.rb +32 -0
- data/lib/qyu/workers/split.rb +58 -0
- data/lib/qyu/workers/sync.rb +2 -0
- data/qyu.gemspec +11 -10
- metadata +40 -38
- data/examples/bin/simple +0 -7
- data/examples/config.rb +0 -22
- data/examples/simple/create_workflow.rb +0 -18
- data/examples/simple/enqueue_job.rb +0 -8
- data/examples/simple/worker.rb +0 -32
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: d7af2bf528a149f8caf81dfbc7bca11ee330e4bf9f8bf0aa4a8c0f2aad4e9315
|
4
|
+
data.tar.gz: 70863cdba21ff2873217b07155a8be979faa78992a6a0c979b7acd31a8bdef05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8bea1241c09b7fb824a5ff91eaacfb91c846953582ad927c5ba941f1a119350ebd7de0182eb6b7b6b63cb84cb1aebe928cb772c4ef8004fbaee41ee6c9f1443c
|
7
|
+
data.tar.gz: eea5679b5c8f5e82c5a7d6db58be46aabd00f0eeafa28e181512d4ad4a1a59820abf94d540d91e592bb12eed4de8722ce45a5a0a1b68cfc4c81cb5e90683e20a
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -4,11 +4,25 @@
|
|
4
4
|
|
5
5
|
---
|
6
6
|
|
7
|
-
[
|
8
|
-
## [
|
7
|
+
[UNRELEASED]: https://github.com/QyuTeam/qyu/compare/v1.1.0...HEAD
|
8
|
+
## [UNRELEASED]
|
9
|
+
### Added
|
10
|
+
-
|
11
|
+
|
12
|
+
[v1.1.0]: https://github.com/QyuTeam/qyu/compare/v1.0.2...v1.1.0
|
13
|
+
## [v1.1.0] March 29, 2018
|
14
|
+
### Added
|
15
|
+
- [FEATURE] `Qyu::SplitWorker` to simplify splitting and parallelization of input
|
16
|
+
- [FEATURE] `Qyu::SyncWorker` can now execute passed blocks if all synced tasks are successful
|
17
|
+
- [FEATURE] Introduced a `timeout` option for `Qyu::Worker` adding a timeout for each processing task
|
18
|
+
- [METHOD] `Qyu::Job#workflow`
|
19
|
+
- [METHOD] `Qyu::Task#descriptor`, `Qyu::Task#workflow_descriptor` and `Qyu::Task#workflow`
|
20
|
+
|
21
|
+
### Changed
|
22
|
+
- [DEPRECATION] `starts_parallel` is now favored over `starts_manually`
|
9
23
|
|
10
24
|
[v1.0.2]: https://github.com/QyuTeam/qyu/compare/v1.0.1...v1.0.2
|
11
|
-
## [v1.0.2]
|
25
|
+
## [v1.0.2] - March 27, 2018
|
12
26
|
### Added
|
13
27
|
- Added first `CHANGELOG.md`
|
14
28
|
- [TESTS] WorkflowDescriptorValidator tests
|
data/README.md
CHANGED
@@ -1,8 +1,9 @@
|
|
1
1
|
# Qyu 九
|
2
2
|
|
3
3
|
[](https://rubygems.org/gems/qyu)
|
4
|
-
[](https://travis-ci.org/QyuTeam/qyu)
|
5
5
|
[](https://codeclimate.com/github/QyuTeam/qyu/maintainability)
|
6
|
+
[](https://codecov.io/gh/QyuTeam/qyu)
|
6
7
|
[](https://rubyweekly.com/link/44495/web)
|
7
8
|
|
8
9
|
## Requirements:
|
@@ -61,6 +62,9 @@ w = Qyu::Worker.new do
|
|
61
62
|
|
62
63
|
# failure queue
|
63
64
|
failure_queue false
|
65
|
+
|
66
|
+
# timeout in seconds for each task processed by the worker
|
67
|
+
timeout 120
|
64
68
|
end
|
65
69
|
|
66
70
|
w.work('queue-name') do |task|
|
@@ -89,12 +93,12 @@ end
|
|
89
93
|
The memory queue and store is just for testing purposes. For production; use one of the following:
|
90
94
|
|
91
95
|
#### Stores
|
92
|
-
*ActiveRecord:* https://github.com/
|
93
|
-
*Redis:* https://github.com/
|
96
|
+
*ActiveRecord:* https://github.com/QyuTeam/qyu-store-activerecord
|
97
|
+
*Redis:* https://github.com/QyuTeam/qyu-store-redis
|
94
98
|
|
95
99
|
#### Queues
|
96
|
-
*Amazon SQS:* https://github.com/
|
97
|
-
*Redis:* https://github.com/
|
100
|
+
*Amazon SQS:* https://github.com/QyuTeam/qyu-queue-sqs
|
101
|
+
*Redis:* https://github.com/QyuTeam/qyu-queue-redis
|
98
102
|
|
99
103
|
## Glossary
|
100
104
|
|
data/lib/qyu/config.rb
CHANGED
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::InvalidWorkerConfigurationValue
|
6
|
+
class InvalidWorkerConfigurationValue < Base
|
7
|
+
def initialize(configuration_name, configuration_value = nil)
|
8
|
+
super("invalid worker configuration value #{configuration_name}: #{configuration_value}")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -5,7 +5,7 @@ module Qyu
|
|
5
5
|
# Qyu::Errors::PayloadValidationError
|
6
6
|
class PayloadValidationError < Base
|
7
7
|
def initialize(validation_errors_hash)
|
8
|
-
super("
|
8
|
+
super("validation failed for payload fields: #{validation_errors_hash}.")
|
9
9
|
end
|
10
10
|
end
|
11
11
|
end
|
data/lib/qyu/factory.rb
CHANGED
@@ -4,7 +4,10 @@ module Qyu
|
|
4
4
|
module Concerns
|
5
5
|
# Qyu::Concerns::WorkflowDescriptorValidator
|
6
6
|
class WorkflowDescriptorValidator
|
7
|
-
|
7
|
+
# TODO: starts_parallel is a the same as starts_manually. The latter will be removed in Qyu v2
|
8
|
+
ALLOWED_KEYS = %w(queue waits_for starts starts_parallel starts_manually starts_with_params).freeze
|
9
|
+
DEPRECATED_KEYS = %(starts_manually)
|
10
|
+
|
8
11
|
attr_reader :errors
|
9
12
|
|
10
13
|
def initialize(descriptor)
|
@@ -85,6 +88,7 @@ module Qyu
|
|
85
88
|
|
86
89
|
def validate_task_reference_formats(task_name)
|
87
90
|
validate_format(task_name, 'starts', Array) &&
|
91
|
+
validate_format(task_name, 'starts_parallel', Array) &&
|
88
92
|
validate_format(task_name, 'starts_manually', Array) &&
|
89
93
|
validate_format(task_name, 'starts_with_params', Hash) &&
|
90
94
|
validate_format(task_name, 'waits_for', Hash)
|
@@ -92,6 +96,7 @@ module Qyu
|
|
92
96
|
|
93
97
|
def validate_referenced_tasks(task_name)
|
94
98
|
validate_presence_of_reference_tasks(task_name, 'starts', Array) &&
|
99
|
+
validate_presence_of_reference_tasks(task_name, 'starts_parallel', Array) &&
|
95
100
|
validate_presence_of_reference_tasks(task_name, 'starts_manually', Array) &&
|
96
101
|
validate_presence_of_reference_tasks(task_name, 'starts_with_params', Hash) &&
|
97
102
|
validate_presence_of_reference_tasks(task_name, 'waits_for', Hash)
|
data/lib/qyu/models/job.rb
CHANGED
@@ -3,8 +3,15 @@
|
|
3
3
|
module Qyu
|
4
4
|
# Qyu::Job
|
5
5
|
class Job
|
6
|
-
attr_reader :descriptor, :payload, :id, :created_at, :updated_at
|
6
|
+
attr_reader :descriptor, :payload, :workflow, :id, :created_at, :updated_at
|
7
7
|
|
8
|
+
## Class Methods
|
9
|
+
|
10
|
+
# Creates a new job via a workflow name / object and a payload
|
11
|
+
#
|
12
|
+
# @param [String, Qyu::Workflow] workflow to run
|
13
|
+
# @param [Hash] payload
|
14
|
+
# @return [Qyu::Job]
|
8
15
|
def self.create(workflow:, payload:)
|
9
16
|
workflow = Workflow.find_by(name: workflow) if workflow.is_a?(String)
|
10
17
|
id = persist(workflow, payload)
|
@@ -12,10 +19,15 @@ module Qyu
|
|
12
19
|
new(id, workflow, payload, time, time)
|
13
20
|
end
|
14
21
|
|
22
|
+
# Finds Job by ID. Returns `nil` if job is not present in store
|
23
|
+
#
|
24
|
+
# @return [Qyu::Job, nil]
|
15
25
|
def self.find(id)
|
16
26
|
job_attrs = Qyu.store.find_job(id)
|
17
|
-
|
18
|
-
|
27
|
+
if job_attrs
|
28
|
+
new(id, job_attrs['workflow'], job_attrs['payload'],
|
29
|
+
job_attrs['created_at'], job_attrs['updated_at'])
|
30
|
+
end
|
19
31
|
end
|
20
32
|
|
21
33
|
def self.select(limit: 30, offset: 0, order: :asc)
|
@@ -26,18 +38,34 @@ module Qyu
|
|
26
38
|
end
|
27
39
|
end
|
28
40
|
|
41
|
+
# Counts job in state store
|
42
|
+
#
|
43
|
+
# @return [Integer] jobs count
|
29
44
|
def self.count
|
30
45
|
Qyu.store.count_jobs
|
31
46
|
end
|
32
47
|
|
48
|
+
# Deletes job from state store by ID
|
49
|
+
#
|
50
|
+
# @param [Object] id
|
51
|
+
# @return [Object] deleted job
|
33
52
|
def self.delete(id)
|
34
53
|
Qyu.store.delete_job(id)
|
35
54
|
end
|
36
55
|
|
56
|
+
# Clears completed jobs
|
57
|
+
#
|
58
|
+
# @return [Integer] cleared jobs count
|
37
59
|
def self.clear_completed
|
38
60
|
Qyu.store.clear_completed_jobs
|
39
61
|
end
|
40
62
|
|
63
|
+
## Instance Methods
|
64
|
+
|
65
|
+
# Starts job execution
|
66
|
+
# Enqueues all tasks scheduled to start at the beginning (`starts` key in workflow descriptor)
|
67
|
+
#
|
68
|
+
# #=> job.start
|
41
69
|
def start
|
42
70
|
descriptor['starts'].each do |task_name|
|
43
71
|
create_task(nil, task_name, payload)
|
@@ -59,8 +87,8 @@ module Qyu
|
|
59
87
|
descriptor['tasks'][task.name]['waits_for'].keys
|
60
88
|
end
|
61
89
|
|
62
|
-
def sync_condition(task,
|
63
|
-
descriptor['tasks'][task.name]['waits_for'][
|
90
|
+
def sync_condition(task, next_task_name)
|
91
|
+
descriptor['tasks'][task.name]['waits_for'][next_task_name]['condition']
|
64
92
|
end
|
65
93
|
|
66
94
|
def create_task(parent_task, task_name, payload)
|
@@ -110,6 +138,7 @@ module Qyu
|
|
110
138
|
all_task_names = []
|
111
139
|
all_task_names.concat(desc['starts'] || [])
|
112
140
|
all_task_names.concat((desc['starts_with_params'] || {}).keys)
|
141
|
+
all_task_names.concat(desc['starts_parallel'] || [])
|
113
142
|
all_task_names.concat(desc['starts_manually'] || [])
|
114
143
|
all_task_names.include?(tasks_path[-1])
|
115
144
|
end
|
@@ -141,7 +170,7 @@ module Qyu
|
|
141
170
|
private
|
142
171
|
|
143
172
|
def initialize(id, workflow, payload, created_at = nil, updated_at = nil)
|
144
|
-
@workflow = workflow
|
173
|
+
@workflow = Qyu::Workflow.new(workflow['id'], workflow['name'], workflow['descriptor'])
|
145
174
|
@descriptor = @workflow['descriptor']
|
146
175
|
@payload = payload
|
147
176
|
@id = id
|
data/lib/qyu/models/task.rb
CHANGED
@@ -171,10 +171,34 @@ module Qyu
|
|
171
171
|
Qyu.queue.enqueue_task_to_failed_queue(queue_name, id)
|
172
172
|
end
|
173
173
|
|
174
|
+
# Returns workflow specified in parent job
|
175
|
+
#
|
176
|
+
# @return [Qyu::Workflow] full workflow
|
177
|
+
def workflow
|
178
|
+
job.workflow
|
179
|
+
end
|
180
|
+
|
181
|
+
# Returns workflow descriptor from parent job
|
182
|
+
#
|
183
|
+
# @return [Hash] workflow descriptor
|
184
|
+
def workflow_descriptor
|
185
|
+
job.descriptor
|
186
|
+
end
|
187
|
+
|
188
|
+
# Returns parent job
|
189
|
+
#
|
190
|
+
# @return [Qyu::Job] parent job
|
174
191
|
def job
|
175
192
|
@job ||= Qyu::Job.find(job_id)
|
176
193
|
end
|
177
194
|
|
195
|
+
# Returns task descriptor
|
196
|
+
#
|
197
|
+
# @return [Hash] task descriptor
|
198
|
+
def descriptor
|
199
|
+
workflow_descriptor['tasks'][name]
|
200
|
+
end
|
201
|
+
|
178
202
|
def [](attribute)
|
179
203
|
public_send(attribute)
|
180
204
|
end
|
data/lib/qyu/models/workflow.rb
CHANGED
@@ -5,6 +5,14 @@ module Qyu
|
|
5
5
|
class Workflow
|
6
6
|
attr_reader :id, :name, :descriptor, :created_at, :updated_at
|
7
7
|
|
8
|
+
def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
|
9
|
+
@id = id
|
10
|
+
@name = name
|
11
|
+
@descriptor = descriptor
|
12
|
+
@created_at = created_at
|
13
|
+
@updated_at = updated_at
|
14
|
+
end
|
15
|
+
|
8
16
|
class << self
|
9
17
|
def create(name:, descriptor:)
|
10
18
|
validator = Qyu::Concerns::WorkflowDescriptorValidator.new(descriptor)
|
@@ -70,17 +78,5 @@ module Qyu
|
|
70
78
|
def [](attribute)
|
71
79
|
public_send(attribute)
|
72
80
|
end
|
73
|
-
|
74
|
-
private_class_method :new
|
75
|
-
|
76
|
-
private
|
77
|
-
|
78
|
-
def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
|
79
|
-
@id = id
|
80
|
-
@name = name
|
81
|
-
@descriptor = descriptor
|
82
|
-
@created_at = created_at
|
83
|
-
@updated_at = updated_at
|
84
|
-
end
|
85
81
|
end
|
86
82
|
end
|
data/lib/qyu/ui/views/footer.erb
CHANGED
@@ -3,15 +3,15 @@
|
|
3
3
|
<nav class="pull-left">
|
4
4
|
<ul>
|
5
5
|
<li>
|
6
|
-
<a href="https://github.com/
|
6
|
+
<a href="https://github.com/QyuTeam/qyu">
|
7
7
|
Qyu is a distributed task execution system for complex workflows
|
8
8
|
</a>
|
9
9
|
</li>
|
10
10
|
</ul>
|
11
11
|
</nav>
|
12
12
|
<div class="copyright pull-right">
|
13
|
-
<a href="https://
|
14
|
-
|
13
|
+
<a href="https://github.com/QyuTeam">
|
14
|
+
Qyu © <%= Date.today.year %>
|
15
15
|
</a>
|
16
16
|
</div>
|
17
17
|
</div>
|
data/lib/qyu/version.rb
CHANGED
data/lib/qyu/workers.rb
CHANGED
@@ -1,10 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
require
|
3
|
+
(
|
4
|
+
Dir["#{File.dirname(__FILE__)}/workers/concerns/*.rb"] +
|
5
|
+
Dir["#{File.dirname(__FILE__)}/workers/*.rb"]
|
6
|
+
).each do |path|
|
7
|
+
require path
|
8
|
+
end
|
9
|
+
|
8
10
|
|
9
11
|
Qyu::Worker = Qyu::Workers::Base
|
12
|
+
Qyu::SplitWorker = Qyu::Workers::Split
|
10
13
|
Qyu::SyncWorker = Qyu::Workers::Sync
|
data/lib/qyu/workers/base.rb
CHANGED
@@ -22,8 +22,9 @@ module Qyu
|
|
22
22
|
# t.unlock! && t.mark_finished && t.acknowledge_message
|
23
23
|
class Base
|
24
24
|
include Concerns::Callback
|
25
|
-
include Concerns::PayloadValidator
|
26
25
|
include Concerns::FailureQueue
|
26
|
+
include Concerns::PayloadValidator
|
27
|
+
include Concerns::Timeout
|
27
28
|
|
28
29
|
attr_reader :id
|
29
30
|
attr_accessor :processed_tasks
|
@@ -35,7 +36,7 @@ module Qyu
|
|
35
36
|
end
|
36
37
|
|
37
38
|
def work(queue_name, blocking: true)
|
38
|
-
log(:info, "
|
39
|
+
log(:info, "worker started for queue '#{queue_name}'")
|
39
40
|
repeat = true
|
40
41
|
|
41
42
|
remaining_fetch_retries = 3
|
@@ -45,13 +46,15 @@ module Qyu
|
|
45
46
|
begin
|
46
47
|
fetched_task = fetch_task(queue_name)
|
47
48
|
validate_payload!(fetched_task)
|
48
|
-
log(:info, "
|
49
|
+
log(:info, "worker processed #{processed_tasks} tasks from queue `#{queue_name}`")
|
49
50
|
if fetched_task.acknowledgeable?
|
50
51
|
discard_completed_task(fetched_task)
|
51
52
|
elsif fetched_task.lock!
|
52
53
|
fetched_task.mark_working
|
53
54
|
begin
|
54
|
-
|
55
|
+
Timeout::timeout(@timeout) do
|
56
|
+
yield(fetched_task)
|
57
|
+
end
|
55
58
|
conclude_task(fetched_task)
|
56
59
|
rescue => ex
|
57
60
|
fail_task(fetched_task, ex)
|
@@ -66,11 +69,12 @@ module Qyu
|
|
66
69
|
remaining_fetch_retries -= 1
|
67
70
|
retry
|
68
71
|
end
|
69
|
-
rescue Qyu::Errors::PayloadValidationError
|
72
|
+
rescue Qyu::Errors::PayloadValidationError => ex
|
73
|
+
log("invalid payload: #{ex.class}: #{ex.message}")
|
70
74
|
fetched_task.mark_invalid_payload
|
71
75
|
rescue => ex
|
72
|
-
log("
|
73
|
-
log("
|
76
|
+
log("worker error: #{ex.class}: #{ex.message}")
|
77
|
+
log("backtrace: #{ex.backtrace.join("\n")}")
|
74
78
|
end
|
75
79
|
end
|
76
80
|
|
@@ -88,13 +92,13 @@ module Qyu
|
|
88
92
|
end
|
89
93
|
|
90
94
|
def discard_completed_task(fetched_task)
|
91
|
-
log(:debug, '
|
95
|
+
log(:debug, 'fetched completed task and discarding it...')
|
92
96
|
fetched_task.acknowledge_message
|
93
97
|
end
|
94
98
|
|
95
99
|
def conclude_task(fetched_task)
|
96
100
|
Qyu.store.transaction do
|
97
|
-
log(:debug, '
|
101
|
+
log(:debug, 'task finished and creating next tasks.')
|
98
102
|
fetched_task.job.create_next_tasks(
|
99
103
|
fetched_task,
|
100
104
|
fetched_task.job.payload.merge(fetched_task.payload)
|
@@ -107,8 +111,8 @@ module Qyu
|
|
107
111
|
|
108
112
|
def fail_task(fetched_task, exception)
|
109
113
|
unless exception.class == Qyu::Errors::UnsyncError
|
110
|
-
log("
|
111
|
-
log("
|
114
|
+
log("worker error: #{exception.class}: #{exception.message}")
|
115
|
+
log("backtrace: #{exception.backtrace.join("\n")}")
|
112
116
|
end
|
113
117
|
Qyu.store.transaction do
|
114
118
|
fetched_task.enqueue_in_failure_queue if @failure_queue
|
@@ -120,10 +124,10 @@ module Qyu
|
|
120
124
|
def acknowledge_message_with_task_id_not_found_in_store(exception)
|
121
125
|
# If a task is not found in the Store then there is no point attempting
|
122
126
|
# to fetch the message over and over again.
|
123
|
-
log("
|
124
|
-
log("
|
125
|
-
log("
|
126
|
-
log("
|
127
|
+
log("worker error: #{exception.class}: #{exception.message}")
|
128
|
+
log("backtrace: #{exception.backtrace.join("\n")}")
|
129
|
+
log("original error: #{exception.original_error.class}: #{exception.original_error.message}")
|
130
|
+
log("backtrace: #{exception.original_error.backtrace.join("\n")}")
|
127
131
|
if exception.original_error.class == Qyu::Errors::TaskNotFound &&
|
128
132
|
exception.queue_name &&
|
129
133
|
exception.message_id
|
@@ -136,7 +140,7 @@ module Qyu
|
|
136
140
|
end
|
137
141
|
|
138
142
|
def run_garbage_collector
|
139
|
-
log(:debug, '
|
143
|
+
log(:debug, 'running garbage collector')
|
140
144
|
GC.start
|
141
145
|
end
|
142
146
|
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Workers
|
5
|
+
module Concerns
|
6
|
+
# Qyu::Workers::Concerns::Split
|
7
|
+
#
|
8
|
+
# Adds ability to split workers to specify slice size and splittable variable name
|
9
|
+
#
|
10
|
+
# Qyu::SplitWorker.new do
|
11
|
+
# slice_size 25
|
12
|
+
# payload_key 'array'
|
13
|
+
# end
|
14
|
+
#
|
15
|
+
module Split
|
16
|
+
|
17
|
+
@slice_size = 25
|
18
|
+
|
19
|
+
# Configures slice size
|
20
|
+
#
|
21
|
+
# slice_size 25
|
22
|
+
#
|
23
|
+
# @param slice_size [Integer]
|
24
|
+
def slice_size(slsz)
|
25
|
+
@slice_size = slsz
|
26
|
+
end
|
27
|
+
|
28
|
+
# Configures payload key with array to split
|
29
|
+
#
|
30
|
+
# payload_key 25
|
31
|
+
#
|
32
|
+
# @param payload_key [String]
|
33
|
+
def payload_key(var_name)
|
34
|
+
@payload_key = var_name
|
35
|
+
end
|
36
|
+
|
37
|
+
# Run with sampling a single element out of the array
|
38
|
+
#
|
39
|
+
# sample true
|
40
|
+
#
|
41
|
+
# @param sample [Boolean]
|
42
|
+
def sample(smpl)
|
43
|
+
@sample = smpl
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
def validate_split_parameters!
|
48
|
+
if @payload_key.nil?
|
49
|
+
raise Qyu::Errors::MissingSplitParameters.new('payload_key')
|
50
|
+
end
|
51
|
+
|
52
|
+
if @slice_size.nil?
|
53
|
+
raise Qyu::Errors::MissingSplitParameters.new('slice_size')
|
54
|
+
end
|
55
|
+
|
56
|
+
if @slice_size.zero?
|
57
|
+
raise Qyu::Errors::InvalidWorkerConfigurationValue.new('slice_size', 0)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'timeout'
|
4
|
+
|
5
|
+
module Qyu
|
6
|
+
module Workers
|
7
|
+
module Concerns
|
8
|
+
# Qyu::Workers::Concerns::Timeout
|
9
|
+
#
|
10
|
+
# Adds timeout to running tasks in a worker
|
11
|
+
#
|
12
|
+
# Qyu::Worker.new do
|
13
|
+
# timeout 0 # disabled (default)
|
14
|
+
# # or
|
15
|
+
# timeout 3600
|
16
|
+
# end
|
17
|
+
#
|
18
|
+
module Timeout
|
19
|
+
|
20
|
+
# Configures timeout
|
21
|
+
#
|
22
|
+
# timeout 0 # default
|
23
|
+
# timeout 3600
|
24
|
+
#
|
25
|
+
# @param [Integer]
|
26
|
+
def timeout(seconds)
|
27
|
+
@timeout = seconds.to_i
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Workers
|
5
|
+
# Qyu::Workers::Split
|
6
|
+
#
|
7
|
+
# Starts a worker to split a certain payload key into multiple jobs
|
8
|
+
#
|
9
|
+
# Qyu::SplitWorker.new do
|
10
|
+
# slice_size 25
|
11
|
+
# payload_key 'array'
|
12
|
+
# end
|
13
|
+
#
|
14
|
+
class Split < Base
|
15
|
+
include Qyu::Workers::Concerns::Split
|
16
|
+
|
17
|
+
attr_accessor :splittable
|
18
|
+
|
19
|
+
# Assign a splittable variable
|
20
|
+
# by being at the end of a block
|
21
|
+
# worker.work('queue') do
|
22
|
+
# # do anything
|
23
|
+
# # splittable variable must be at the end
|
24
|
+
# [1, 2, 3, 4, 5, 6, 6]
|
25
|
+
# end
|
26
|
+
#
|
27
|
+
# or by passing
|
28
|
+
# payload_key 'array'
|
29
|
+
# to worker initializer then just
|
30
|
+
# worker.work('queue')
|
31
|
+
# @param queue_name [String]
|
32
|
+
def work(queue_name)
|
33
|
+
validate_split_parameters!
|
34
|
+
|
35
|
+
super do |task|
|
36
|
+
if block_given?
|
37
|
+
@splittable = yield(task)
|
38
|
+
else
|
39
|
+
# or by passing
|
40
|
+
# payload_key 'array'
|
41
|
+
# to worker initializer
|
42
|
+
@splittable = task.payload[@payload_key]
|
43
|
+
end
|
44
|
+
|
45
|
+
@splittable.each_slice(@slice_size).with_index do |slice, i|
|
46
|
+
log(:debug, "Split started for queue '#{queue_name}'")
|
47
|
+
input = @sample ? slice.sample : slice
|
48
|
+
new_payload = task.payload.merge({ @payload_key => input })
|
49
|
+
task_names_to_start = task.descriptor['starts_parallel'] || task.descriptor['starts_manually']
|
50
|
+
task_names_to_start.each do |task_name_to_start|
|
51
|
+
task.job.create_task(task, task_name_to_start, new_payload)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
data/lib/qyu/workers/sync.rb
CHANGED
@@ -14,6 +14,8 @@ module Qyu
|
|
14
14
|
log(:debug, "Task: #{task_name}, Sync condition: #{sync_condition}")
|
15
15
|
if respond_to?(sync_condition['function'], true)
|
16
16
|
__send__(sync_condition['function'], job, task, task_name, sync_condition['param'])
|
17
|
+
# execute attached sync block only if codition passes (i.e. No errors raised)
|
18
|
+
yield(task) if block_given?
|
17
19
|
else
|
18
20
|
fail Qyu::Errors::NotImplementedError
|
19
21
|
end
|
data/qyu.gemspec
CHANGED
@@ -8,14 +8,15 @@ Gem::Specification.new do |spec|
|
|
8
8
|
spec.version = Qyu::VERSION
|
9
9
|
spec.authors = ['Elod Peter', 'Mohamed Osama']
|
10
10
|
spec.email = ['bejmuller@gmail.com', 'mohamed.o.alnagdy@gmail.com']
|
11
|
+
spec.license = "MIT"
|
11
12
|
|
12
13
|
spec.summary = 'Distributed task execution system for complex workflows'
|
13
|
-
spec.description = '
|
14
|
-
spec.homepage = 'https://github.com/
|
14
|
+
spec.description = 'Qyu makes use of a message queue and a state store to provide a reliable distributed task execution system for complex workflows'
|
15
|
+
spec.homepage = 'https://github.com/QyuTeam/qyu'
|
15
16
|
spec.required_ruby_version = '>= 2.4'
|
16
17
|
|
17
18
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
|
-
f.match(%r{^(test|spec|features)/})
|
19
|
+
f.match(%r{^(test|spec|features|examples)/})
|
19
20
|
end
|
20
21
|
spec.bindir = "exe"
|
21
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
@@ -26,13 +27,13 @@ Gem::Specification.new do |spec|
|
|
26
27
|
spec.add_runtime_dependency 'activesupport', '~> 5.1'
|
27
28
|
|
28
29
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
29
|
-
spec.add_development_dependency 'dotenv'
|
30
|
-
spec.add_development_dependency 'rack-test'
|
31
|
-
spec.add_development_dependency 'pry'
|
32
|
-
spec.add_development_dependency 'pry-byebug'
|
30
|
+
spec.add_development_dependency 'dotenv', '~> 2.0'
|
31
|
+
spec.add_development_dependency 'rack-test', '~> 1.0'
|
32
|
+
spec.add_development_dependency 'pry', '~> 0.11'
|
33
|
+
spec.add_development_dependency 'pry-byebug', '~> 3.0'
|
33
34
|
spec.add_development_dependency 'rake', '~> 12.0'
|
34
|
-
spec.add_development_dependency 'rspec', '~> 3.
|
35
|
-
spec.add_development_dependency '
|
36
|
-
spec.add_development_dependency 'sinatra'
|
35
|
+
spec.add_development_dependency 'rspec', '~> 3.5'
|
36
|
+
spec.add_development_dependency 'codecov', '~> 0.1'
|
37
|
+
spec.add_development_dependency 'sinatra', '~> 2.0'
|
37
38
|
spec.add_development_dependency 'timecop', '~> 0.9'
|
38
39
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: qyu
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elod Peter
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-03-
|
12
|
+
date: 2018-03-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activesupport
|
@@ -43,58 +43,58 @@ dependencies:
|
|
43
43
|
name: dotenv
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
|
-
- - "
|
46
|
+
- - "~>"
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version: '0'
|
48
|
+
version: '2.0'
|
49
49
|
type: :development
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
|
-
- - "
|
53
|
+
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: '0'
|
55
|
+
version: '2.0'
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
57
|
name: rack-test
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
59
59
|
requirements:
|
60
|
-
- - "
|
60
|
+
- - "~>"
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version: '0'
|
62
|
+
version: '1.0'
|
63
63
|
type: :development
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- - "
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
69
|
+
version: '1.0'
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
71
|
name: pry
|
72
72
|
requirement: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
|
-
- - "
|
74
|
+
- - "~>"
|
75
75
|
- !ruby/object:Gem::Version
|
76
|
-
version: '0'
|
76
|
+
version: '0.11'
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
|
-
- - "
|
81
|
+
- - "~>"
|
82
82
|
- !ruby/object:Gem::Version
|
83
|
-
version: '0'
|
83
|
+
version: '0.11'
|
84
84
|
- !ruby/object:Gem::Dependency
|
85
85
|
name: pry-byebug
|
86
86
|
requirement: !ruby/object:Gem::Requirement
|
87
87
|
requirements:
|
88
|
-
- - "
|
88
|
+
- - "~>"
|
89
89
|
- !ruby/object:Gem::Version
|
90
|
-
version: '0'
|
90
|
+
version: '3.0'
|
91
91
|
type: :development
|
92
92
|
prerelease: false
|
93
93
|
version_requirements: !ruby/object:Gem::Requirement
|
94
94
|
requirements:
|
95
|
-
- - "
|
95
|
+
- - "~>"
|
96
96
|
- !ruby/object:Gem::Version
|
97
|
-
version: '0'
|
97
|
+
version: '3.0'
|
98
98
|
- !ruby/object:Gem::Dependency
|
99
99
|
name: rake
|
100
100
|
requirement: !ruby/object:Gem::Requirement
|
@@ -115,42 +115,42 @@ dependencies:
|
|
115
115
|
requirements:
|
116
116
|
- - "~>"
|
117
117
|
- !ruby/object:Gem::Version
|
118
|
-
version: '3.
|
118
|
+
version: '3.5'
|
119
119
|
type: :development
|
120
120
|
prerelease: false
|
121
121
|
version_requirements: !ruby/object:Gem::Requirement
|
122
122
|
requirements:
|
123
123
|
- - "~>"
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: '3.
|
125
|
+
version: '3.5'
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
|
-
name:
|
127
|
+
name: codecov
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
129
129
|
requirements:
|
130
|
-
- - "
|
130
|
+
- - "~>"
|
131
131
|
- !ruby/object:Gem::Version
|
132
|
-
version: '0'
|
132
|
+
version: '0.1'
|
133
133
|
type: :development
|
134
134
|
prerelease: false
|
135
135
|
version_requirements: !ruby/object:Gem::Requirement
|
136
136
|
requirements:
|
137
|
-
- - "
|
137
|
+
- - "~>"
|
138
138
|
- !ruby/object:Gem::Version
|
139
|
-
version: '0'
|
139
|
+
version: '0.1'
|
140
140
|
- !ruby/object:Gem::Dependency
|
141
141
|
name: sinatra
|
142
142
|
requirement: !ruby/object:Gem::Requirement
|
143
143
|
requirements:
|
144
|
-
- - "
|
144
|
+
- - "~>"
|
145
145
|
- !ruby/object:Gem::Version
|
146
|
-
version: '0'
|
146
|
+
version: '2.0'
|
147
147
|
type: :development
|
148
148
|
prerelease: false
|
149
149
|
version_requirements: !ruby/object:Gem::Requirement
|
150
150
|
requirements:
|
151
|
-
- - "
|
151
|
+
- - "~>"
|
152
152
|
- !ruby/object:Gem::Version
|
153
|
-
version: '0'
|
153
|
+
version: '2.0'
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
155
|
name: timecop
|
156
156
|
requirement: !ruby/object:Gem::Requirement
|
@@ -165,7 +165,8 @@ dependencies:
|
|
165
165
|
- - "~>"
|
166
166
|
- !ruby/object:Gem::Version
|
167
167
|
version: '0.9'
|
168
|
-
description:
|
168
|
+
description: Qyu makes use of a message queue and a state store to provide a reliable
|
169
|
+
distributed task execution system for complex workflows
|
169
170
|
email:
|
170
171
|
- bejmuller@gmail.com
|
171
172
|
- mohamed.o.alnagdy@gmail.com
|
@@ -185,11 +186,6 @@ files:
|
|
185
186
|
- bin/console
|
186
187
|
- bin/server
|
187
188
|
- bin/setup
|
188
|
-
- examples/bin/simple
|
189
|
-
- examples/config.rb
|
190
|
-
- examples/simple/create_workflow.rb
|
191
|
-
- examples/simple/enqueue_job.rb
|
192
|
-
- examples/simple/worker.rb
|
193
189
|
- lib/qyu.rb
|
194
190
|
- lib/qyu/config.rb
|
195
191
|
- lib/qyu/errors.rb
|
@@ -197,10 +193,12 @@ files:
|
|
197
193
|
- lib/qyu/errors/could_not_fetch_task.rb
|
198
194
|
- lib/qyu/errors/invalid_queue_name.rb
|
199
195
|
- lib/qyu/errors/invalid_task_attributes.rb
|
196
|
+
- lib/qyu/errors/invalid_worker_configuration_value.rb
|
200
197
|
- lib/qyu/errors/job_not_found.rb
|
201
198
|
- lib/qyu/errors/lock_already_acquired.rb
|
202
199
|
- lib/qyu/errors/lock_not_acquired.rb
|
203
200
|
- lib/qyu/errors/message_not_received.rb
|
201
|
+
- lib/qyu/errors/missing_split_parameters.rb
|
204
202
|
- lib/qyu/errors/not_implemented_error.rb
|
205
203
|
- lib/qyu/errors/payload_validation_error.rb
|
206
204
|
- lib/qyu/errors/task_not_found.rb
|
@@ -251,10 +249,14 @@ files:
|
|
251
249
|
- lib/qyu/workers/concerns/callback.rb
|
252
250
|
- lib/qyu/workers/concerns/failure_queue.rb
|
253
251
|
- lib/qyu/workers/concerns/payload_validator.rb
|
252
|
+
- lib/qyu/workers/concerns/split.rb
|
253
|
+
- lib/qyu/workers/concerns/timeout.rb
|
254
|
+
- lib/qyu/workers/split.rb
|
254
255
|
- lib/qyu/workers/sync.rb
|
255
256
|
- qyu.gemspec
|
256
|
-
homepage: https://github.com/
|
257
|
-
licenses:
|
257
|
+
homepage: https://github.com/QyuTeam/qyu
|
258
|
+
licenses:
|
259
|
+
- MIT
|
258
260
|
metadata:
|
259
261
|
yard.run: yri
|
260
262
|
post_install_message:
|
@@ -273,7 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
273
275
|
version: '0'
|
274
276
|
requirements: []
|
275
277
|
rubyforge_project:
|
276
|
-
rubygems_version: 2.
|
278
|
+
rubygems_version: 2.7.4
|
277
279
|
signing_key:
|
278
280
|
specification_version: 4
|
279
281
|
summary: Distributed task execution system for complex workflows
|
data/examples/bin/simple
DELETED
data/examples/config.rb
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'bundler/setup'
|
5
|
-
Bundler.setup
|
6
|
-
|
7
|
-
require 'qyu'
|
8
|
-
logger = Logger.new(STDOUT)
|
9
|
-
logger.level = :info
|
10
|
-
|
11
|
-
Qyu.configure(
|
12
|
-
queue: {
|
13
|
-
type: :memory
|
14
|
-
},
|
15
|
-
store: {
|
16
|
-
type: :memory,
|
17
|
-
lease_period: 60
|
18
|
-
},
|
19
|
-
logger: logger
|
20
|
-
)
|
21
|
-
|
22
|
-
Qyu.test_connections
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative '../config'
|
4
|
-
|
5
|
-
descriptor = {
|
6
|
-
'starts' => %w(
|
7
|
-
print_hello
|
8
|
-
),
|
9
|
-
'tasks' => {
|
10
|
-
'print_hello' => {
|
11
|
-
'queue' => 'print-hello'
|
12
|
-
}
|
13
|
-
}
|
14
|
-
}
|
15
|
-
|
16
|
-
name = 'say-hello'
|
17
|
-
Qyu.logger.info "Creating workflow #{name}"
|
18
|
-
Qyu::Workflow.create(name: name, descriptor: descriptor)
|
data/examples/simple/worker.rb
DELETED
@@ -1,32 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require_relative '../config'
|
4
|
-
|
5
|
-
class SimpleWorker
|
6
|
-
def initialize
|
7
|
-
@worker = Qyu::Worker.new do
|
8
|
-
callback :execute, :before do
|
9
|
-
Qyu.logger.info 'Waiting for task..'
|
10
|
-
end
|
11
|
-
|
12
|
-
callback :execute, :after do
|
13
|
-
Qyu.logger.info 'Done'
|
14
|
-
end
|
15
|
-
|
16
|
-
# Payload validation
|
17
|
-
validates :times, presence: true, type: :integer
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
def run
|
22
|
-
# Consumes message from print-hello queue
|
23
|
-
@worker.work('print-hello') do |task|
|
24
|
-
task.payload['times'].times do |i|
|
25
|
-
Qyu.logger.info "#{i + 1}. Hello"
|
26
|
-
end
|
27
|
-
rescue StandardError => ex
|
28
|
-
Qyu.logger.error 'OMG :('
|
29
|
-
Qyu.logger.error ex.message
|
30
|
-
end
|
31
|
-
end
|
32
|
-
end
|