qyu 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +56 -0
  3. data/.rspec +3 -0
  4. data/.travis.yml +5 -0
  5. data/CODE_OF_CONDUCT.md +74 -0
  6. data/Gemfile +6 -0
  7. data/LICENSE +21 -0
  8. data/README.md +90 -0
  9. data/Rakefile +6 -0
  10. data/bin/console +14 -0
  11. data/bin/server +17 -0
  12. data/bin/setup +8 -0
  13. data/examples/bin/simple +7 -0
  14. data/examples/config.rb +22 -0
  15. data/examples/simple/create_workflow.rb +18 -0
  16. data/examples/simple/enqueue_job.rb +8 -0
  17. data/examples/simple/worker.rb +32 -0
  18. data/lib/qyu.rb +74 -0
  19. data/lib/qyu/config.rb +35 -0
  20. data/lib/qyu/errors.rb +4 -0
  21. data/lib/qyu/errors/base.rb +8 -0
  22. data/lib/qyu/errors/could_not_fetch_task.rb +18 -0
  23. data/lib/qyu/errors/invalid_queue_name.rb +12 -0
  24. data/lib/qyu/errors/invalid_task_attributes.rb +12 -0
  25. data/lib/qyu/errors/job_not_found.rb +14 -0
  26. data/lib/qyu/errors/lock_already_acquired.rb +12 -0
  27. data/lib/qyu/errors/lock_not_acquired.rb +12 -0
  28. data/lib/qyu/errors/message_not_received.rb +12 -0
  29. data/lib/qyu/errors/not_implemented_error.rb +12 -0
  30. data/lib/qyu/errors/payload_validation_error.rb +12 -0
  31. data/lib/qyu/errors/task_not_found.rb +15 -0
  32. data/lib/qyu/errors/task_status_update_failed.rb +15 -0
  33. data/lib/qyu/errors/unknown_validation_option.rb +12 -0
  34. data/lib/qyu/errors/unsync_error.rb +12 -0
  35. data/lib/qyu/errors/workflow_descriptor_validation_error.rb +14 -0
  36. data/lib/qyu/errors/workflow_not_found.rb +15 -0
  37. data/lib/qyu/factory.rb +26 -0
  38. data/lib/qyu/models.rb +9 -0
  39. data/lib/qyu/models/concerns/workflow_descriptor_validator.rb +117 -0
  40. data/lib/qyu/models/enums/status.rb +44 -0
  41. data/lib/qyu/models/job.rb +174 -0
  42. data/lib/qyu/models/task.rb +218 -0
  43. data/lib/qyu/models/workflow.rb +85 -0
  44. data/lib/qyu/queue.rb +5 -0
  45. data/lib/qyu/queue/base.rb +46 -0
  46. data/lib/qyu/queue/memory/adapter.rb +90 -0
  47. data/lib/qyu/store.rb +5 -0
  48. data/lib/qyu/store/base.rb +106 -0
  49. data/lib/qyu/store/memory/adapter.rb +187 -0
  50. data/lib/qyu/ui.rb +56 -0
  51. data/lib/qyu/ui/helpers/pagination.rb +35 -0
  52. data/lib/qyu/ui/public/bootstrap.min.css +5 -0
  53. data/lib/qyu/ui/public/paper-dashboard.css +3315 -0
  54. data/lib/qyu/ui/public/script.js +28 -0
  55. data/lib/qyu/ui/public/style.css +6 -0
  56. data/lib/qyu/ui/views/footer.erb +18 -0
  57. data/lib/qyu/ui/views/helpers/pagination.erb +49 -0
  58. data/lib/qyu/ui/views/jobs.erb +58 -0
  59. data/lib/qyu/ui/views/kaminari/_first_page.html.erb +3 -0
  60. data/lib/qyu/ui/views/kaminari/_gap.html.erb +3 -0
  61. data/lib/qyu/ui/views/kaminari/_last_page.html.erb +3 -0
  62. data/lib/qyu/ui/views/kaminari/_next_page.html.erb +3 -0
  63. data/lib/qyu/ui/views/kaminari/_page.html.erb +9 -0
  64. data/lib/qyu/ui/views/kaminari/_paginator.html.erb +15 -0
  65. data/lib/qyu/ui/views/kaminari/_prev_page.html.erb +3 -0
  66. data/lib/qyu/ui/views/layout.erb +33 -0
  67. data/lib/qyu/ui/views/navbar.erb +29 -0
  68. data/lib/qyu/ui/views/pagination.erb +19 -0
  69. data/lib/qyu/ui/views/show_job.erb +55 -0
  70. data/lib/qyu/ui/views/sidebar.erb +17 -0
  71. data/lib/qyu/ui/views/task_row.erb +26 -0
  72. data/lib/qyu/utils.rb +17 -0
  73. data/lib/qyu/version.rb +3 -0
  74. data/lib/qyu/workers.rb +10 -0
  75. data/lib/qyu/workers/base.rb +126 -0
  76. data/lib/qyu/workers/concerns/callback.rb +38 -0
  77. data/lib/qyu/workers/concerns/failure_queue.rb +23 -0
  78. data/lib/qyu/workers/concerns/payload_validator.rb +124 -0
  79. data/lib/qyu/workers/sync.rb +63 -0
  80. data/qyu.gemspec +36 -0
  81. metadata +278 -0
@@ -0,0 +1,174 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ class Job
5
+ attr_reader :descriptor, :payload, :id, :created_at, :updated_at
6
+
7
+ def self.create(workflow:, payload:)
8
+ workflow = Workflow.find_by(name: workflow) if workflow.is_a?(String)
9
+ id = persist(workflow, payload)
10
+ time = Time.now
11
+ new(id, workflow, payload, time, time)
12
+ end
13
+
14
+ def self.find(id)
15
+ job_attrs = Qyu.store.find_job(id)
16
+ new(id, job_attrs['workflow'], job_attrs['payload'],
17
+ job_attrs['created_at'], job_attrs['updated_at'])
18
+ end
19
+
20
+ def self.select(limit: 30, offset: 0, order: :asc)
21
+ job_records = Qyu.store.select_jobs(limit, offset, order)
22
+ job_records.map do |record|
23
+ new(record['id'], record['workflow'], record['payload'],
24
+ record['created_at'], record['updated_at'])
25
+ end
26
+ end
27
+
28
+ def self.count
29
+ Qyu.store.count_jobs
30
+ end
31
+
32
+ def self.delete(id)
33
+ Qyu.store.delete_job(id)
34
+ end
35
+
36
+ def self.clear_completed
37
+ Qyu.store.clear_completed_jobs
38
+ end
39
+
40
+ def start
41
+ descriptor['starts'].each do |task_name|
42
+ create_task(nil, task_name, payload)
43
+ end
44
+ end
45
+
46
+ def queue_name(task_name)
47
+ descriptor['tasks'][task_name]['queue']
48
+ end
49
+
50
+ def next_task_names(src_task_name)
51
+ {
52
+ 'without_params' => descriptor['tasks'][src_task_name]['starts'],
53
+ 'with_params' => descriptor['tasks'][src_task_name]['starts_with_params']
54
+ }
55
+ end
56
+
57
+ def tasks_to_wait_for(task)
58
+ descriptor['tasks'][task.name]['waits_for'].keys
59
+ end
60
+
61
+ def sync_condition(task, task_name)
62
+ descriptor['tasks'][task.name]['waits_for'][task_name]['condition']
63
+ end
64
+
65
+ def create_task(parent_task, task_name, payload)
66
+ parent_task_id = parent_task.nil? ? nil : parent_task.id
67
+ Qyu.logger.debug "Task (ID=#{parent_task_id}) created a new task"
68
+ Qyu::Task.create(
69
+ queue_name: queue_name(task_name),
70
+ attributes: {
71
+ 'name' => task_name,
72
+ 'parent_task_id' => parent_task_id,
73
+ 'job_id' => id,
74
+ 'payload' => task_payload(payload, task_name)
75
+ })
76
+ end
77
+
78
+ def create_next_tasks(parent_task, payload)
79
+ Qyu.logger.debug "Creating next tasks for task (ID=#{parent_task.id})"
80
+ next_tasks = next_task_names(parent_task.name)
81
+ Qyu.logger.debug "Next task names: #{next_tasks}"
82
+
83
+ next_tasks['without_params']&.each do |next_task_name|
84
+ create_task(parent_task, next_task_name, payload)
85
+ end
86
+
87
+ next_tasks['with_params']&.each do |next_task_name, params|
88
+ updated_payload = payload.dup
89
+ params.each do |param_name, value_eqs|
90
+ f = value_eqs.keys[0]
91
+ x = value_eqs.values[0]
92
+ updated_payload[param_name] = calc_func_x(parent_task, f, x)
93
+ end
94
+ create_task(parent_task, next_task_name, updated_payload)
95
+ end
96
+ end
97
+
98
+ def find_task_ids_by_name(task_name)
99
+ Qyu.store.find_task_ids_by_job_id_and_name(id, task_name)
100
+ end
101
+
102
+ def find_task_ids_by_name_and_ancestor_task_id(task_name, ancestor_task_id)
103
+ ancestor_task_name = Qyu.store.find_task(ancestor_task_id)['name']
104
+ tasks_path = [task_name]
105
+ key_idx = 0
106
+
107
+ while tasks_path[-1] != ancestor_task_name
108
+ found_task = descriptor['tasks'].detect do |_, desc|
109
+ all_task_names = []
110
+ all_task_names.concat(desc['starts'] || [])
111
+ all_task_names.concat((desc['starts_with_params'] || {}).keys)
112
+ all_task_names.concat(desc['starts_manually'] || [])
113
+ all_task_names.include?(tasks_path[-1])
114
+ end
115
+ tasks_path << found_task[key_idx] if found_task
116
+ end
117
+
118
+ tasks_topdown_path = tasks_path.reverse
119
+ # remove topmost task (ancestor_task) from the path
120
+ tasks_topdown_path.shift
121
+
122
+ # traverse task tree from top down, and find the <task_name> "descendants" of <ancestor_task>
123
+ parent_task_ids = [ancestor_task_id]
124
+ tasks_topdown_path.each do |t_name|
125
+ parent_task_ids = Qyu.store.find_task_ids_by_job_id_name_and_parent_task_ids(id, t_name, parent_task_ids)
126
+ end
127
+ parent_task_ids
128
+ end
129
+
130
+ def task_status_counts
131
+ Qyu.store.task_status_counts(id)
132
+ end
133
+
134
+ def [](attribute)
135
+ public_send(attribute)
136
+ end
137
+
138
+ private_class_method :new
139
+
140
+ private
141
+
142
+ def initialize(id, workflow, payload, created_at = nil, updated_at = nil)
143
+ @workflow = workflow
144
+ @descriptor = @workflow['descriptor']
145
+ @payload = payload
146
+ @id = id
147
+ @created_at = created_at
148
+ @updated_at = updated_at
149
+ end
150
+
151
+ def self.persist(workflow, payload)
152
+ workflow = Qyu::Workflow.find_by(name: workflow) if workflow.is_a?(String)
153
+ Qyu.store.persist_job(workflow, payload)
154
+ end
155
+
156
+ def calc_func_x(task, func, x)
157
+ if func == 'count'
158
+ find_task_ids_by_name_and_ancestor_task_id(x, task.id).count
159
+ else
160
+ fail Qyu::Errors::NotImplementedError
161
+ end
162
+ end
163
+
164
+ def task_payload(payload, task_name)
165
+ shared_payload = payload.dup.reject { |k, _v| task_name?(k) }
166
+ shared_payload.merge!(payload[task_name]) if payload[task_name].is_a?(Hash)
167
+ shared_payload
168
+ end
169
+
170
+ def task_name?(string)
171
+ descriptor['tasks'].keys.include?(string)
172
+ end
173
+ end
174
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ # A Task represents a unit of work in a workflow.
5
+ # Conceptually a Task:
6
+ # - may not exist outside the context of a queue.
7
+ # - it is created ON the queue
8
+ # - it remains on the queue until it was successfully processed (or failed "enough" times)
9
+ class Task
10
+ attr_reader :queue_name, :payload, :status, :id, :job_id, :name, :parent_task_id,
11
+ :message_id, :created_at, :updated_at
12
+
13
+ LEASE_PERCENTAGE_THRESHOLD_BEFORE_RENEWAL = 0.8
14
+ POLL_INTERVAL = 0.5
15
+
16
+ # @returns Task
17
+ # by defintion Task.create does 2 things:
18
+ # - persists the Task in the Store
19
+ # - enqueues the Task to the Queue
20
+ # We have to make sure that a Task is unique in the Store. Because of this
21
+ # create first looks up if the task has already been persisted. If it exists then
22
+ # there is no need to persist it again, only to enqueue it.
23
+ # Double (or multiple) delivery of messages is allowed and handled at worker level.
24
+ # Possible scenario:
25
+ # A Job failed at some point. A few of its tasks completed successfully, others failed.
26
+ # Because of this, certain tasks haven't even been created.
27
+ # When we restart the job, the tasks will be recreated. If a task has already existed,
28
+ # and completed, then that state will be unchanged, and when the worker picks it up,
29
+ # will notice the completed state, acknowledge the message, and continue the next steps.
30
+ def self.create(queue_name: nil, attributes: nil)
31
+ fail Qyu::Errors::InvalidTaskAttributes unless valid_attributes?(attributes)
32
+ fail Qyu::Errors::InvalidQueueName unless valid_queue_name?(queue_name)
33
+ Qyu.logger.debug "find_or_persist queue_name=#{queue_name} and attributes=#{attributes}"
34
+ task_id = Qyu.store.find_or_persist_task(
35
+ attributes['name'],
36
+ queue_name,
37
+ attributes['payload'],
38
+ attributes['job_id'],
39
+ attributes['parent_task_id']
40
+ ) do |t_id|
41
+ Qyu.logger.debug "enqueue queue_name=#{queue_name} and task_id=#{t_id}"
42
+ Qyu.queue.enqueue_task(queue_name, t_id)
43
+ end
44
+
45
+ new(task_id, attributes, queue_name)
46
+ end
47
+
48
+ # @returns Task
49
+ def self.fetch(queue_name)
50
+ fail Qyu::Errors::InvalidQueueName unless valid_queue_name?(queue_name)
51
+ begin
52
+ message = Qyu.queue.fetch_next_message(queue_name)
53
+ task_id = message['task_id']
54
+ task_attrs = Qyu.store.find_task(task_id)
55
+ rescue => ex
56
+ message ||= {}
57
+ raise Qyu::Errors::CouldNotFetchTask.new(queue_name, message['id'], message['task_id'], ex)
58
+ end
59
+ new(task_id, task_attrs, queue_name, message['id'])
60
+ end
61
+
62
+ def self.select(job_id:)
63
+ Qyu.store.select_tasks_by_job_id(job_id).map do |task|
64
+ new(task['id'], task, task['queue_name'])
65
+ end
66
+ end
67
+
68
+ def self.valid_attributes?(_attributes)
69
+ true
70
+ end
71
+
72
+ def self.valid_queue_name?(queue_name)
73
+ !queue_name.nil? && queue_name != ''
74
+ end
75
+
76
+ def acknowledgeable?
77
+ @status.completed? || @status.invalid_payload?
78
+ end
79
+
80
+ def completed?
81
+ @status.completed?
82
+ end
83
+
84
+ def locked?
85
+ !@lease_token.nil? && !@locked_until.nil? && Time.now < @locked_until
86
+ end
87
+
88
+ def lock!
89
+ fail Qyu::Errors::LockAlreadyAcquired if locked?
90
+ Qyu.logger.debug "Task with ID=#{id} lock!"
91
+
92
+ @lease_token, @locked_until = Qyu.store.lock_task!(id, Qyu.config.store[:lease_period])
93
+ Qyu.logger.debug "lease_token = #{@lease_token} | locked_until = #{@locked_until}"
94
+ return false if @lease_token.nil?
95
+
96
+ schedule_renewal
97
+ true
98
+ end
99
+
100
+ def unlock!
101
+ fail Qyu::Errors::LockNotAcquired unless locked?
102
+ Qyu.logger.debug "Task with ID=#{id} unlocking!"
103
+
104
+ @lease_thread&.kill
105
+ success = Qyu.store.unlock_task!(id, @lease_token)
106
+ if success
107
+ @lease_token = nil
108
+ @locked_until = nil
109
+ end
110
+
111
+ success
112
+ end
113
+
114
+ def mark_queued
115
+ Qyu.store.update_status(id, Status::QUEUED)
116
+ Qyu.logger.debug "Task with ID=#{id} marked queued."
117
+ end
118
+
119
+ def mark_working
120
+ Qyu.store.update_status(id, Status::WORKING)
121
+ Qyu.logger.debug "Task with ID=#{id} marked working."
122
+ end
123
+
124
+ def mark_completed
125
+ Qyu.store.update_status(id, Status::COMPLETED)
126
+ Qyu.logger.info "Task with ID=#{id} marked completed."
127
+ end
128
+
129
+ def mark_failed
130
+ Qyu.store.update_status(id, Status::FAILED)
131
+ Qyu.logger.debug "Task with ID=#{id} marked failed."
132
+ end
133
+
134
+ def mark_invalid_payload
135
+ Qyu.store.update_status(id, Status::INVALID_PAYLOAD)
136
+ Qyu.logger.debug "Task with ID=#{id} has invalid payload."
137
+ end
138
+
139
+ def acknowledge_message
140
+ fail Qyu::Errors::MessageNotReceived if message_id.nil?
141
+ self.class.acknowledge_message(queue_name, message_id)
142
+ end
143
+
144
+ def self.acknowledge_message(queue_name, message_id)
145
+ Qyu.logger.debug "Acknowledging message with ID=#{message_id} from queue `#{queue_name}`"
146
+ Qyu.queue.acknowledge_message(queue_name, message_id)
147
+ end
148
+
149
+ def requeue
150
+ # TODO For FIFO queues (future use)
151
+ fail Qyu::Errors::MessageNotReceived if message_id.nil?
152
+ self.class.acknowledge_message(queue_name, message_id)
153
+ self.class.requeue(queue_name, id, message_id)
154
+ end
155
+
156
+ def self.requeue(queue_name, id, message_id)
157
+ # TODO For FIFO queues (future use)
158
+ Qyu.logger.debug "Re-enqueuing message with ID=#{message_id} in queue `#{queue_name}`"
159
+ Qyu.queue.enqueue_task(queue_name, id)
160
+ end
161
+
162
+ def enqueue_in_failure_queue
163
+ fail Qyu::Errors::MessageNotReceived if message_id.nil?
164
+ self.class.acknowledge_message(queue_name, message_id)
165
+ self.class.enqueue_in_failure_queue(queue_name, id, message_id)
166
+ end
167
+
168
+ def self.enqueue_in_failure_queue(queue_name, id, message_id)
169
+ Qyu.logger.debug "Enqueuing failed message with ID=#{message_id} in #{queue_name} failures queue"
170
+ Qyu.queue.enqueue_task_to_failed_queue(queue_name, id)
171
+ end
172
+
173
+ def job
174
+ @job ||= Qyu::Job.find(job_id)
175
+ end
176
+
177
+ def [](attribute)
178
+ public_send(attribute)
179
+ end
180
+
181
+ private
182
+
183
+ def initialize(id, attributes, queue_name, message_id = nil)
184
+ # puts "task initialized attrs: #{attributes}"
185
+ @status = Status.new(id)
186
+ @id = id
187
+ @job_id = attributes['job_id']
188
+ @parent_task_id = attributes['parent_task_id']
189
+ @payload = attributes['payload']
190
+ @queue_name = queue_name
191
+ @message_id = message_id
192
+ @name = attributes['name']
193
+ @created_at = attributes['created_at']
194
+ @updated_at = attributes['updated_at']
195
+
196
+ @locked_until = nil
197
+ @lease_thread = nil
198
+ @lease_token = nil
199
+ end
200
+
201
+ def schedule_renewal
202
+ Qyu.logger.debug 'scheduling renewal'
203
+ renewal_moment = Qyu::Utils.seconds_after_time(-1 * LEASE_PERCENTAGE_THRESHOLD_BEFORE_RENEWAL * Qyu.config.store[:lease_period], @locked_until)
204
+ Qyu.logger.debug "renewal moment: #{renewal_moment}"
205
+ @lease_thread = Thread.new do
206
+ Qyu.logger.debug 'lease thread entered'
207
+ while Time.now < renewal_moment
208
+ sleep(POLL_INTERVAL)
209
+ Qyu.logger.debug 'lease thread sleep'
210
+ end
211
+ Qyu.logger.debug 'lease thread time has come'
212
+ @locked_until = Qyu.store.renew_lock_lease(id, Qyu.config.store[:lease_period], @lease_token)
213
+ Qyu.logger.debug "lease thread locked until = #{@locked_until}"
214
+ schedule_renewal
215
+ end
216
+ end
217
+ end
218
+ end
@@ -0,0 +1,85 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ class Workflow
5
+ attr_reader :id, :name, :descriptor, :created_at, :updated_at
6
+
7
+ class << self
8
+ def create(name:, descriptor:)
9
+ validator = Qyu::Concerns::WorkflowDescriptorValidator.new(descriptor)
10
+ fail Qyu::Errors::WorkflowDescriptorValidatorationError, validator.errors unless validator.valid?
11
+ id = persist(name, descriptor)
12
+ time = Time.now
13
+ new(id, name, descriptor, time, time)
14
+ end
15
+
16
+ def find(id, raise_error: true)
17
+ workflow_attrs = Qyu.store.find_workflow(id)
18
+ raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow_attrs.nil? && raise_error
19
+ return nil if workflow_attrs.nil?
20
+ new(id, workflow_attrs['name'], workflow_attrs['descriptor'])
21
+ end
22
+
23
+ def find_by(name: nil, id: nil)
24
+ return find_by_name(name) if name
25
+ return find(id, raise_error: false) if id
26
+ end
27
+
28
+ def find_by!(name: nil, id: nil)
29
+ workflow = find_by(name: name, id: id)
30
+ raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow.nil? && id
31
+ raise Qyu::Errors::WorkflowNotFound.new(:id, id) if workflow.nil? && id
32
+ workflow
33
+ end
34
+
35
+ def select(limit: 30, offset: 0, order: :asc)
36
+ workflow_records = Qyu.store.select_workflows(limit, offset, order)
37
+ workflow_records.map do |record|
38
+ new(record['id'], record['name'], record['descriptor'])
39
+ end
40
+ end
41
+
42
+ def delete(id)
43
+ Qyu.store.delete_workflow(id)
44
+ end
45
+
46
+ def delete_by(name: nil, id: nil)
47
+ raise ArgumentError, 'specify either name or id' if (name && id) || (name.nil? && id.nil?)
48
+ Qyu.store.delete_workflow_by_name(name) if name
49
+ delete(id) if id
50
+ end
51
+
52
+ def count
53
+ Qyu.store.count_workflows
54
+ end
55
+
56
+ private
57
+
58
+ def persist(name, descriptor)
59
+ Qyu.store.persist_workflow(name, descriptor)
60
+ end
61
+
62
+ def find_by_name(name)
63
+ workflow_attrs = Qyu.store.find_workflow_by_name(name)
64
+ return nil unless workflow_attrs
65
+ new(workflow_attrs['id'], workflow_attrs['name'], workflow_attrs['descriptor'])
66
+ end
67
+ end
68
+
69
+ def [](attribute)
70
+ public_send(attribute)
71
+ end
72
+
73
+ private_class_method :new
74
+
75
+ private
76
+
77
+ def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
78
+ @id = id
79
+ @name = name
80
+ @descriptor = descriptor
81
+ @created_at = created_at
82
+ @updated_at = updated_at
83
+ end
84
+ end
85
+ end