qyu 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +56 -0
- data/.rspec +3 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/LICENSE +21 -0
- data/README.md +90 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/server +17 -0
- data/bin/setup +8 -0
- data/examples/bin/simple +7 -0
- data/examples/config.rb +22 -0
- data/examples/simple/create_workflow.rb +18 -0
- data/examples/simple/enqueue_job.rb +8 -0
- data/examples/simple/worker.rb +32 -0
- data/lib/qyu.rb +74 -0
- data/lib/qyu/config.rb +35 -0
- data/lib/qyu/errors.rb +4 -0
- data/lib/qyu/errors/base.rb +8 -0
- data/lib/qyu/errors/could_not_fetch_task.rb +18 -0
- data/lib/qyu/errors/invalid_queue_name.rb +12 -0
- data/lib/qyu/errors/invalid_task_attributes.rb +12 -0
- data/lib/qyu/errors/job_not_found.rb +14 -0
- data/lib/qyu/errors/lock_already_acquired.rb +12 -0
- data/lib/qyu/errors/lock_not_acquired.rb +12 -0
- data/lib/qyu/errors/message_not_received.rb +12 -0
- data/lib/qyu/errors/not_implemented_error.rb +12 -0
- data/lib/qyu/errors/payload_validation_error.rb +12 -0
- data/lib/qyu/errors/task_not_found.rb +15 -0
- data/lib/qyu/errors/task_status_update_failed.rb +15 -0
- data/lib/qyu/errors/unknown_validation_option.rb +12 -0
- data/lib/qyu/errors/unsync_error.rb +12 -0
- data/lib/qyu/errors/workflow_descriptor_validation_error.rb +14 -0
- data/lib/qyu/errors/workflow_not_found.rb +15 -0
- data/lib/qyu/factory.rb +26 -0
- data/lib/qyu/models.rb +9 -0
- data/lib/qyu/models/concerns/workflow_descriptor_validator.rb +117 -0
- data/lib/qyu/models/enums/status.rb +44 -0
- data/lib/qyu/models/job.rb +174 -0
- data/lib/qyu/models/task.rb +218 -0
- data/lib/qyu/models/workflow.rb +85 -0
- data/lib/qyu/queue.rb +5 -0
- data/lib/qyu/queue/base.rb +46 -0
- data/lib/qyu/queue/memory/adapter.rb +90 -0
- data/lib/qyu/store.rb +5 -0
- data/lib/qyu/store/base.rb +106 -0
- data/lib/qyu/store/memory/adapter.rb +187 -0
- data/lib/qyu/ui.rb +56 -0
- data/lib/qyu/ui/helpers/pagination.rb +35 -0
- data/lib/qyu/ui/public/bootstrap.min.css +5 -0
- data/lib/qyu/ui/public/paper-dashboard.css +3315 -0
- data/lib/qyu/ui/public/script.js +28 -0
- data/lib/qyu/ui/public/style.css +6 -0
- data/lib/qyu/ui/views/footer.erb +18 -0
- data/lib/qyu/ui/views/helpers/pagination.erb +49 -0
- data/lib/qyu/ui/views/jobs.erb +58 -0
- data/lib/qyu/ui/views/kaminari/_first_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_gap.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_last_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_next_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_page.html.erb +9 -0
- data/lib/qyu/ui/views/kaminari/_paginator.html.erb +15 -0
- data/lib/qyu/ui/views/kaminari/_prev_page.html.erb +3 -0
- data/lib/qyu/ui/views/layout.erb +33 -0
- data/lib/qyu/ui/views/navbar.erb +29 -0
- data/lib/qyu/ui/views/pagination.erb +19 -0
- data/lib/qyu/ui/views/show_job.erb +55 -0
- data/lib/qyu/ui/views/sidebar.erb +17 -0
- data/lib/qyu/ui/views/task_row.erb +26 -0
- data/lib/qyu/utils.rb +17 -0
- data/lib/qyu/version.rb +3 -0
- data/lib/qyu/workers.rb +10 -0
- data/lib/qyu/workers/base.rb +126 -0
- data/lib/qyu/workers/concerns/callback.rb +38 -0
- data/lib/qyu/workers/concerns/failure_queue.rb +23 -0
- data/lib/qyu/workers/concerns/payload_validator.rb +124 -0
- data/lib/qyu/workers/sync.rb +63 -0
- data/qyu.gemspec +36 -0
- metadata +278 -0
data/lib/qyu/config.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
# Qyu::Config
|
5
|
+
class Config
|
6
|
+
attr_reader :queue, :store
|
7
|
+
|
8
|
+
class ServiceConfig
|
9
|
+
class << self
|
10
|
+
def register(adapter_class)
|
11
|
+
types[adapter_class::TYPE] = adapter_class
|
12
|
+
end
|
13
|
+
|
14
|
+
def valid?(config)
|
15
|
+
types[config[:type]].valid_config?(config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def types
|
19
|
+
@__types ||= {}
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class QueueConfig < ServiceConfig; end
|
25
|
+
class StoreConfig < ServiceConfig; end
|
26
|
+
|
27
|
+
def initialize(queue:, store:)
|
28
|
+
fail 'Invalid message queue configuration' unless QueueConfig.valid?(queue)
|
29
|
+
fail 'Invalid state store configuration' unless StoreConfig.valid?(store)
|
30
|
+
|
31
|
+
@queue = queue
|
32
|
+
@store = store
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/qyu/errors.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::CouldNotFetchTask
|
6
|
+
# TODO: rethink this...
|
7
|
+
class CouldNotFetchTask < Base
|
8
|
+
attr_reader :original_error, :queue_name, :message_id, :task_id
|
9
|
+
def initialize(queue_name, message_id, task_id, original_error)
|
10
|
+
super("Task cannot be fetched from queue=#{queue_name} with message_id=#{message_id} task_id=#{task_id}.")
|
11
|
+
@original_error = original_error
|
12
|
+
@task_id = task_id
|
13
|
+
@message_id = message_id
|
14
|
+
@queue_name = queue_name
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::JobNotFound
|
6
|
+
class JobNotFound < Base
|
7
|
+
attr_reader :original_error
|
8
|
+
def initialize(id, original_error)
|
9
|
+
super("Job not found with id=#{id}.")
|
10
|
+
@original_error = original_error
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::PayloadValidationError
|
6
|
+
class PayloadValidationError < Base
|
7
|
+
def initialize(validation_errors_hash)
|
8
|
+
super("Validation failed for payload fields: #{validation_errors_hash}.")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::TaskNotFound
|
6
|
+
class TaskNotFound < Base
|
7
|
+
attr_reader :task_id, :original_error
|
8
|
+
def initialize(task_id, original_error)
|
9
|
+
super("Task not found with id=#{task_id}.")
|
10
|
+
@original_error = original_error
|
11
|
+
@task_id = task_id
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::TaskStatusUpdateFailed
|
6
|
+
class TaskStatusUpdateFailed < Base
|
7
|
+
attr_reader :task_id, :status
|
8
|
+
def initialize(task_id, status)
|
9
|
+
super("Task status cannot be updated task_id=#{task_id} status=#{status}.")
|
10
|
+
@task_id = task_id
|
11
|
+
@status = status
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::WorkflowDescriptorValidatorationError
|
6
|
+
class WorkflowDescriptorValidatorationError < Base
|
7
|
+
attr_reader :validation_errors
|
8
|
+
def initialize(validation_errors)
|
9
|
+
super('Invalid Job descriptor.')
|
10
|
+
@validation_errors = validation_errors
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::WorkflowNotFound
|
6
|
+
class WorkflowNotFound < Base
|
7
|
+
attr_reader :key, :workflow_id
|
8
|
+
def initialize(key, workflow_id)
|
9
|
+
super("Workflow not found with #{key}=#{workflow_id}.")
|
10
|
+
@key = key
|
11
|
+
@workflow_id = workflow_id
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/qyu/factory.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
# Qyu::Factory
|
5
|
+
class Factory
|
6
|
+
class ServiceFactory
|
7
|
+
class << self
|
8
|
+
def register(adapter_class)
|
9
|
+
types[adapter_class::TYPE] = adapter_class
|
10
|
+
end
|
11
|
+
|
12
|
+
def types
|
13
|
+
@__types ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def get(config)
|
17
|
+
Qyu.logger.info "Got factory #{types[config[:type]]}"
|
18
|
+
types[config[:type]].new(config)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class QueueFactory < ServiceFactory; end
|
24
|
+
class StoreFactory < ServiceFactory; end
|
25
|
+
end
|
26
|
+
end
|
data/lib/qyu/models.rb
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Concerns
|
5
|
+
# Qyu::Concerns::WorkflowDescriptorValidator
|
6
|
+
class WorkflowDescriptorValidator
|
7
|
+
ALLOWED_KEYS = %w(queue waits_for starts starts_manually starts_with_params).freeze
|
8
|
+
attr_reader :errors
|
9
|
+
|
10
|
+
def initialize(descriptor)
|
11
|
+
@descriptor = descriptor
|
12
|
+
@errors = []
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
validate
|
17
|
+
@errors.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate
|
21
|
+
@errors << 'Descriptor type must be a Hash.' unless validate_descriptor_type
|
22
|
+
@errors << 'Entry points (starts) must be an Array.' unless validate_entry_points_type
|
23
|
+
@errors << 'Tasks must be a Hash.' unless validate_tasks_type
|
24
|
+
unless validate_entry_points_presence
|
25
|
+
@errors << 'There must be at least 1 entry point, and all entry points must exist in the tasks Hash.'
|
26
|
+
end
|
27
|
+
@errors << 'There must be at least 1 task in the tasks Hash.' unless validate_tasks_presence
|
28
|
+
|
29
|
+
tasks.keys.each do |task_name|
|
30
|
+
unless validate_queue_presence(task_name)
|
31
|
+
@errors << "#{task_name} must have a valid queue."
|
32
|
+
end
|
33
|
+
unless validate_task_keys(task_name)
|
34
|
+
@errors << "#{task_name} must only contain the following keys: #{ALLOWED_KEYS}."
|
35
|
+
end
|
36
|
+
unless validate_task_reference_formats(task_name)
|
37
|
+
@errors << "#{task_name} must follow the reference declaration format."
|
38
|
+
end
|
39
|
+
unless validate_task_references(task_name)
|
40
|
+
@errors << "#{task_name} must list existing tasks in its references."
|
41
|
+
end
|
42
|
+
unless validate_sync_condition_params(task_name)
|
43
|
+
@errors << "#{task_name} must pass the correct parameters to the sync task."
|
44
|
+
end
|
45
|
+
end
|
46
|
+
rescue => ex
|
47
|
+
Qyu.logger.error "Error while validation: #{ex.class}: #{ex.message}"
|
48
|
+
Qyu.logger.error "Backtrace: #{ex.backtrace.join("\n")}"
|
49
|
+
@errors << "#{ex.class}: #{ex.message}"
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def validate_descriptor_type
|
55
|
+
@descriptor.is_a?(Hash)
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_entry_points_type
|
59
|
+
entry_points.is_a?(Array)
|
60
|
+
end
|
61
|
+
|
62
|
+
def validate_tasks_type
|
63
|
+
tasks.is_a?(Hash)
|
64
|
+
end
|
65
|
+
|
66
|
+
def validate_entry_points_presence
|
67
|
+
!entry_points.empty? && \
|
68
|
+
entry_points.all? { |task_name| tasks.keys.include?(task_name) }
|
69
|
+
end
|
70
|
+
|
71
|
+
def validate_tasks_presence
|
72
|
+
!tasks.empty?
|
73
|
+
end
|
74
|
+
|
75
|
+
def validate_queue_presence(task_name)
|
76
|
+
tasks[task_name]['queue'].is_a?(String)
|
77
|
+
end
|
78
|
+
|
79
|
+
def validate_task_keys(task_name)
|
80
|
+
tasks[task_name].keys.all? { |key| ALLOWED_KEYS.include?(key) }
|
81
|
+
end
|
82
|
+
|
83
|
+
def validate_task_reference_formats(task_name)
|
84
|
+
(tasks[task_name]['starts'].nil? || tasks[task_name]['starts'].is_a?(Array)) &&
|
85
|
+
(tasks[task_name]['starts_manually'].nil? || tasks[task_name]['starts_manually'].is_a?(Array)) &&
|
86
|
+
(tasks[task_name]['starts_with_params'].nil? || tasks[task_name]['starts_with_params'].is_a?(Hash))
|
87
|
+
(tasks[task_name]['waits_for'].nil? || tasks[task_name]['waits_for'].is_a?(Hash))
|
88
|
+
end
|
89
|
+
|
90
|
+
def validate_task_references(task_name)
|
91
|
+
(tasks[task_name]['starts'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
|
92
|
+
(tasks[task_name]['starts_manually'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
|
93
|
+
(tasks[task_name]['starts_with_params'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
|
94
|
+
(tasks[task_name]['waits_for'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
|
95
|
+
end
|
96
|
+
|
97
|
+
def validate_sync_condition_params(task_name)
|
98
|
+
return true unless tasks[task_name]['starts_with_params']
|
99
|
+
tasks[task_name]['starts_with_params'].all? do |started_task_name, params_config|
|
100
|
+
params_config.all? do |param_name, _param_config|
|
101
|
+
tasks[started_task_name]['waits_for'].detect do |_t_name, sync_config|
|
102
|
+
sync_config['condition']['param'] == param_name
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def entry_points
|
109
|
+
@descriptor['starts']
|
110
|
+
end
|
111
|
+
|
112
|
+
def tasks
|
113
|
+
@descriptor['tasks']
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
class Status
|
5
|
+
COMPLETED = 'completed'
|
6
|
+
QUEUED = 'queued'
|
7
|
+
WORKING = 'working'
|
8
|
+
FAILED = 'failed'
|
9
|
+
INVALID_PAYLOAD = 'invalid_payload'
|
10
|
+
|
11
|
+
def self.find(id)
|
12
|
+
Qyu.store.find_task(@id)
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(id)
|
16
|
+
@id = id
|
17
|
+
end
|
18
|
+
|
19
|
+
def status
|
20
|
+
t = Qyu.store.find_task(@id)
|
21
|
+
t['status']
|
22
|
+
end
|
23
|
+
|
24
|
+
def completed?
|
25
|
+
status == COMPLETED
|
26
|
+
end
|
27
|
+
|
28
|
+
def queued?
|
29
|
+
status == QUEUED
|
30
|
+
end
|
31
|
+
|
32
|
+
def working?
|
33
|
+
status == WORKING
|
34
|
+
end
|
35
|
+
|
36
|
+
def failed?
|
37
|
+
status == FAILED
|
38
|
+
end
|
39
|
+
|
40
|
+
def invalid_payload?
|
41
|
+
status == INVALID_PAYLOAD
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|