qyu 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +56 -0
- data/.rspec +3 -0
- data/.travis.yml +5 -0
- data/CODE_OF_CONDUCT.md +74 -0
- data/Gemfile +6 -0
- data/LICENSE +21 -0
- data/README.md +90 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/server +17 -0
- data/bin/setup +8 -0
- data/examples/bin/simple +7 -0
- data/examples/config.rb +22 -0
- data/examples/simple/create_workflow.rb +18 -0
- data/examples/simple/enqueue_job.rb +8 -0
- data/examples/simple/worker.rb +32 -0
- data/lib/qyu.rb +74 -0
- data/lib/qyu/config.rb +35 -0
- data/lib/qyu/errors.rb +4 -0
- data/lib/qyu/errors/base.rb +8 -0
- data/lib/qyu/errors/could_not_fetch_task.rb +18 -0
- data/lib/qyu/errors/invalid_queue_name.rb +12 -0
- data/lib/qyu/errors/invalid_task_attributes.rb +12 -0
- data/lib/qyu/errors/job_not_found.rb +14 -0
- data/lib/qyu/errors/lock_already_acquired.rb +12 -0
- data/lib/qyu/errors/lock_not_acquired.rb +12 -0
- data/lib/qyu/errors/message_not_received.rb +12 -0
- data/lib/qyu/errors/not_implemented_error.rb +12 -0
- data/lib/qyu/errors/payload_validation_error.rb +12 -0
- data/lib/qyu/errors/task_not_found.rb +15 -0
- data/lib/qyu/errors/task_status_update_failed.rb +15 -0
- data/lib/qyu/errors/unknown_validation_option.rb +12 -0
- data/lib/qyu/errors/unsync_error.rb +12 -0
- data/lib/qyu/errors/workflow_descriptor_validation_error.rb +14 -0
- data/lib/qyu/errors/workflow_not_found.rb +15 -0
- data/lib/qyu/factory.rb +26 -0
- data/lib/qyu/models.rb +9 -0
- data/lib/qyu/models/concerns/workflow_descriptor_validator.rb +117 -0
- data/lib/qyu/models/enums/status.rb +44 -0
- data/lib/qyu/models/job.rb +174 -0
- data/lib/qyu/models/task.rb +218 -0
- data/lib/qyu/models/workflow.rb +85 -0
- data/lib/qyu/queue.rb +5 -0
- data/lib/qyu/queue/base.rb +46 -0
- data/lib/qyu/queue/memory/adapter.rb +90 -0
- data/lib/qyu/store.rb +5 -0
- data/lib/qyu/store/base.rb +106 -0
- data/lib/qyu/store/memory/adapter.rb +187 -0
- data/lib/qyu/ui.rb +56 -0
- data/lib/qyu/ui/helpers/pagination.rb +35 -0
- data/lib/qyu/ui/public/bootstrap.min.css +5 -0
- data/lib/qyu/ui/public/paper-dashboard.css +3315 -0
- data/lib/qyu/ui/public/script.js +28 -0
- data/lib/qyu/ui/public/style.css +6 -0
- data/lib/qyu/ui/views/footer.erb +18 -0
- data/lib/qyu/ui/views/helpers/pagination.erb +49 -0
- data/lib/qyu/ui/views/jobs.erb +58 -0
- data/lib/qyu/ui/views/kaminari/_first_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_gap.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_last_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_next_page.html.erb +3 -0
- data/lib/qyu/ui/views/kaminari/_page.html.erb +9 -0
- data/lib/qyu/ui/views/kaminari/_paginator.html.erb +15 -0
- data/lib/qyu/ui/views/kaminari/_prev_page.html.erb +3 -0
- data/lib/qyu/ui/views/layout.erb +33 -0
- data/lib/qyu/ui/views/navbar.erb +29 -0
- data/lib/qyu/ui/views/pagination.erb +19 -0
- data/lib/qyu/ui/views/show_job.erb +55 -0
- data/lib/qyu/ui/views/sidebar.erb +17 -0
- data/lib/qyu/ui/views/task_row.erb +26 -0
- data/lib/qyu/utils.rb +17 -0
- data/lib/qyu/version.rb +3 -0
- data/lib/qyu/workers.rb +10 -0
- data/lib/qyu/workers/base.rb +126 -0
- data/lib/qyu/workers/concerns/callback.rb +38 -0
- data/lib/qyu/workers/concerns/failure_queue.rb +23 -0
- data/lib/qyu/workers/concerns/payload_validator.rb +124 -0
- data/lib/qyu/workers/sync.rb +63 -0
- data/qyu.gemspec +36 -0
- metadata +278 -0
data/lib/qyu/config.rb
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
# Qyu::Config
|
5
|
+
class Config
|
6
|
+
attr_reader :queue, :store
|
7
|
+
|
8
|
+
class ServiceConfig
|
9
|
+
class << self
|
10
|
+
def register(adapter_class)
|
11
|
+
types[adapter_class::TYPE] = adapter_class
|
12
|
+
end
|
13
|
+
|
14
|
+
def valid?(config)
|
15
|
+
types[config[:type]].valid_config?(config)
|
16
|
+
end
|
17
|
+
|
18
|
+
def types
|
19
|
+
@__types ||= {}
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
class QueueConfig < ServiceConfig; end
|
25
|
+
class StoreConfig < ServiceConfig; end
|
26
|
+
|
27
|
+
def initialize(queue:, store:)
|
28
|
+
fail 'Invalid message queue configuration' unless QueueConfig.valid?(queue)
|
29
|
+
fail 'Invalid state store configuration' unless StoreConfig.valid?(store)
|
30
|
+
|
31
|
+
@queue = queue
|
32
|
+
@store = store
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/qyu/errors.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::CouldNotFetchTask
|
6
|
+
# TODO: rethink this...
|
7
|
+
class CouldNotFetchTask < Base
|
8
|
+
attr_reader :original_error, :queue_name, :message_id, :task_id
|
9
|
+
def initialize(queue_name, message_id, task_id, original_error)
|
10
|
+
super("Task cannot be fetched from queue=#{queue_name} with message_id=#{message_id} task_id=#{task_id}.")
|
11
|
+
@original_error = original_error
|
12
|
+
@task_id = task_id
|
13
|
+
@message_id = message_id
|
14
|
+
@queue_name = queue_name
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::JobNotFound
|
6
|
+
class JobNotFound < Base
|
7
|
+
attr_reader :original_error
|
8
|
+
def initialize(id, original_error)
|
9
|
+
super("Job not found with id=#{id}.")
|
10
|
+
@original_error = original_error
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::PayloadValidationError
|
6
|
+
class PayloadValidationError < Base
|
7
|
+
def initialize(validation_errors_hash)
|
8
|
+
super("Validation failed for payload fields: #{validation_errors_hash}.")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::TaskNotFound
|
6
|
+
class TaskNotFound < Base
|
7
|
+
attr_reader :task_id, :original_error
|
8
|
+
def initialize(task_id, original_error)
|
9
|
+
super("Task not found with id=#{task_id}.")
|
10
|
+
@original_error = original_error
|
11
|
+
@task_id = task_id
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::TaskStatusUpdateFailed
|
6
|
+
class TaskStatusUpdateFailed < Base
|
7
|
+
attr_reader :task_id, :status
|
8
|
+
def initialize(task_id, status)
|
9
|
+
super("Task status cannot be updated task_id=#{task_id} status=#{status}.")
|
10
|
+
@task_id = task_id
|
11
|
+
@status = status
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::WorkflowDescriptorValidatorationError
|
6
|
+
class WorkflowDescriptorValidatorationError < Base
|
7
|
+
attr_reader :validation_errors
|
8
|
+
def initialize(validation_errors)
|
9
|
+
super('Invalid Job descriptor.')
|
10
|
+
@validation_errors = validation_errors
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Errors
|
5
|
+
# Qyu::Errors::WorkflowNotFound
|
6
|
+
class WorkflowNotFound < Base
|
7
|
+
attr_reader :key, :workflow_id
|
8
|
+
def initialize(key, workflow_id)
|
9
|
+
super("Workflow not found with #{key}=#{workflow_id}.")
|
10
|
+
@key = key
|
11
|
+
@workflow_id = workflow_id
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
data/lib/qyu/factory.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
# Qyu::Factory
|
5
|
+
class Factory
|
6
|
+
class ServiceFactory
|
7
|
+
class << self
|
8
|
+
def register(adapter_class)
|
9
|
+
types[adapter_class::TYPE] = adapter_class
|
10
|
+
end
|
11
|
+
|
12
|
+
def types
|
13
|
+
@__types ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def get(config)
|
17
|
+
Qyu.logger.info "Got factory #{types[config[:type]]}"
|
18
|
+
types[config[:type]].new(config)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class QueueFactory < ServiceFactory; end
|
24
|
+
class StoreFactory < ServiceFactory; end
|
25
|
+
end
|
26
|
+
end
|
data/lib/qyu/models.rb
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
module Concerns
|
5
|
+
# Qyu::Concerns::WorkflowDescriptorValidator
|
6
|
+
class WorkflowDescriptorValidator
|
7
|
+
ALLOWED_KEYS = %w(queue waits_for starts starts_manually starts_with_params).freeze
|
8
|
+
attr_reader :errors
|
9
|
+
|
10
|
+
def initialize(descriptor)
|
11
|
+
@descriptor = descriptor
|
12
|
+
@errors = []
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
validate
|
17
|
+
@errors.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def validate
|
21
|
+
@errors << 'Descriptor type must be a Hash.' unless validate_descriptor_type
|
22
|
+
@errors << 'Entry points (starts) must be an Array.' unless validate_entry_points_type
|
23
|
+
@errors << 'Tasks must be a Hash.' unless validate_tasks_type
|
24
|
+
unless validate_entry_points_presence
|
25
|
+
@errors << 'There must be at least 1 entry point, and all entry points must exist in the tasks Hash.'
|
26
|
+
end
|
27
|
+
@errors << 'There must be at least 1 task in the tasks Hash.' unless validate_tasks_presence
|
28
|
+
|
29
|
+
tasks.keys.each do |task_name|
|
30
|
+
unless validate_queue_presence(task_name)
|
31
|
+
@errors << "#{task_name} must have a valid queue."
|
32
|
+
end
|
33
|
+
unless validate_task_keys(task_name)
|
34
|
+
@errors << "#{task_name} must only contain the following keys: #{ALLOWED_KEYS}."
|
35
|
+
end
|
36
|
+
unless validate_task_reference_formats(task_name)
|
37
|
+
@errors << "#{task_name} must follow the reference declaration format."
|
38
|
+
end
|
39
|
+
unless validate_task_references(task_name)
|
40
|
+
@errors << "#{task_name} must list existing tasks in its references."
|
41
|
+
end
|
42
|
+
unless validate_sync_condition_params(task_name)
|
43
|
+
@errors << "#{task_name} must pass the correct parameters to the sync task."
|
44
|
+
end
|
45
|
+
end
|
46
|
+
rescue => ex
|
47
|
+
Qyu.logger.error "Error while validation: #{ex.class}: #{ex.message}"
|
48
|
+
Qyu.logger.error "Backtrace: #{ex.backtrace.join("\n")}"
|
49
|
+
@errors << "#{ex.class}: #{ex.message}"
|
50
|
+
end
|
51
|
+
|
52
|
+
private
|
53
|
+
|
54
|
+
def validate_descriptor_type
|
55
|
+
@descriptor.is_a?(Hash)
|
56
|
+
end
|
57
|
+
|
58
|
+
def validate_entry_points_type
|
59
|
+
entry_points.is_a?(Array)
|
60
|
+
end
|
61
|
+
|
62
|
+
def validate_tasks_type
|
63
|
+
tasks.is_a?(Hash)
|
64
|
+
end
|
65
|
+
|
66
|
+
def validate_entry_points_presence
|
67
|
+
!entry_points.empty? && \
|
68
|
+
entry_points.all? { |task_name| tasks.keys.include?(task_name) }
|
69
|
+
end
|
70
|
+
|
71
|
+
def validate_tasks_presence
|
72
|
+
!tasks.empty?
|
73
|
+
end
|
74
|
+
|
75
|
+
def validate_queue_presence(task_name)
|
76
|
+
tasks[task_name]['queue'].is_a?(String)
|
77
|
+
end
|
78
|
+
|
79
|
+
def validate_task_keys(task_name)
|
80
|
+
tasks[task_name].keys.all? { |key| ALLOWED_KEYS.include?(key) }
|
81
|
+
end
|
82
|
+
|
83
|
+
def validate_task_reference_formats(task_name)
|
84
|
+
(tasks[task_name]['starts'].nil? || tasks[task_name]['starts'].is_a?(Array)) &&
|
85
|
+
(tasks[task_name]['starts_manually'].nil? || tasks[task_name]['starts_manually'].is_a?(Array)) &&
|
86
|
+
(tasks[task_name]['starts_with_params'].nil? || tasks[task_name]['starts_with_params'].is_a?(Hash))
|
87
|
+
(tasks[task_name]['waits_for'].nil? || tasks[task_name]['waits_for'].is_a?(Hash))
|
88
|
+
end
|
89
|
+
|
90
|
+
def validate_task_references(task_name)
|
91
|
+
(tasks[task_name]['starts'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
|
92
|
+
(tasks[task_name]['starts_manually'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
|
93
|
+
(tasks[task_name]['starts_with_params'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
|
94
|
+
(tasks[task_name]['waits_for'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
|
95
|
+
end
|
96
|
+
|
97
|
+
def validate_sync_condition_params(task_name)
|
98
|
+
return true unless tasks[task_name]['starts_with_params']
|
99
|
+
tasks[task_name]['starts_with_params'].all? do |started_task_name, params_config|
|
100
|
+
params_config.all? do |param_name, _param_config|
|
101
|
+
tasks[started_task_name]['waits_for'].detect do |_t_name, sync_config|
|
102
|
+
sync_config['condition']['param'] == param_name
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def entry_points
|
109
|
+
@descriptor['starts']
|
110
|
+
end
|
111
|
+
|
112
|
+
def tasks
|
113
|
+
@descriptor['tasks']
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Qyu
|
4
|
+
class Status
|
5
|
+
COMPLETED = 'completed'
|
6
|
+
QUEUED = 'queued'
|
7
|
+
WORKING = 'working'
|
8
|
+
FAILED = 'failed'
|
9
|
+
INVALID_PAYLOAD = 'invalid_payload'
|
10
|
+
|
11
|
+
def self.find(id)
|
12
|
+
Qyu.store.find_task(@id)
|
13
|
+
end
|
14
|
+
|
15
|
+
def initialize(id)
|
16
|
+
@id = id
|
17
|
+
end
|
18
|
+
|
19
|
+
def status
|
20
|
+
t = Qyu.store.find_task(@id)
|
21
|
+
t['status']
|
22
|
+
end
|
23
|
+
|
24
|
+
def completed?
|
25
|
+
status == COMPLETED
|
26
|
+
end
|
27
|
+
|
28
|
+
def queued?
|
29
|
+
status == QUEUED
|
30
|
+
end
|
31
|
+
|
32
|
+
def working?
|
33
|
+
status == WORKING
|
34
|
+
end
|
35
|
+
|
36
|
+
def failed?
|
37
|
+
status == FAILED
|
38
|
+
end
|
39
|
+
|
40
|
+
def invalid_payload?
|
41
|
+
status == INVALID_PAYLOAD
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|