qyu 1.0.2 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 856ddbf8f80f887606bc72bc83bc901f5c8516a2
4
- data.tar.gz: 026e1996fb30e4e3d8fe45febefabae331196a61
2
+ SHA256:
3
+ metadata.gz: d7af2bf528a149f8caf81dfbc7bca11ee330e4bf9f8bf0aa4a8c0f2aad4e9315
4
+ data.tar.gz: 70863cdba21ff2873217b07155a8be979faa78992a6a0c979b7acd31a8bdef05
5
5
  SHA512:
6
- metadata.gz: 56aef9a753f450f32e9e33fa398b6758bee60dcc62e7b79aa04527d623f40416a117bc343ac94c45f020c9c54da706ae6cefa86e687ae0a001e59df144f83131
7
- data.tar.gz: 8e7b894defa30e3e183d90db0be04ac8b88a6a58bc1b3b6d66adca8505d9b530a8cd6d029240b1a8b6b1252fed6adcd823ef8f365906a618a99378a3ee01ccf7
6
+ metadata.gz: 8bea1241c09b7fb824a5ff91eaacfb91c846953582ad927c5ba941f1a119350ebd7de0182eb6b7b6b63cb84cb1aebe928cb772c4ef8004fbaee41ee6c9f1443c
7
+ data.tar.gz: eea5679b5c8f5e82c5a7d6db58be46aabd00f0eeafa28e181512d4ad4a1a59820abf94d540d91e592bb12eed4de8722ce45a5a0a1b68cfc4c81cb5e90683e20a
@@ -2,4 +2,9 @@ sudo: false
2
2
  language: ruby
3
3
  rvm:
4
4
  - 2.4.2
5
+ - 2.5.0
6
+ env:
7
+ global:
8
+ - CI=true
9
+ - CODECOV_TOKEN=c2463f9a-6760-4aa5-b042-749c83b1c389
5
10
  before_install: gem install bundler -v 1.16.0
@@ -4,11 +4,25 @@
4
4
 
5
5
  ---
6
6
 
7
- [Unreleased]: https://github.com/QyuTeam/qyu/compare/v1.0.2...HEAD
8
- ## [Unreleased]
7
+ [UNRELEASED]: https://github.com/QyuTeam/qyu/compare/v1.1.0...HEAD
8
+ ## [UNRELEASED]
9
+ ### Added
10
+ -
11
+
12
+ [v1.1.0]: https://github.com/QyuTeam/qyu/compare/v1.0.2...v1.1.0
13
+ ## [v1.1.0] March 29, 2018
14
+ ### Added
15
+ - [FEATURE] `Qyu::SplitWorker` to simplify splitting and parallelization of input
16
+ - [FEATURE] `Qyu::SyncWorker` can now execute passed blocks if all synced tasks are successful
17
+ - [FEATURE] Introduced a `timeout` option for `Qyu::Worker` adding a timeout for each processing task
18
+ - [METHOD] `Qyu::Job#workflow`
19
+ - [METHOD] `Qyu::Task#descriptor`, `Qyu::Task#workflow_descriptor` and `Qyu::Task#workflow`
20
+
21
+ ### Changed
22
+ - [DEPRECATION] `starts_parallel` is now favored over `starts_manually`
9
23
 
10
24
  [v1.0.2]: https://github.com/QyuTeam/qyu/compare/v1.0.1...v1.0.2
11
- ## [v1.0.2]
25
+ ## [v1.0.2] - March 27, 2018
12
26
  ### Added
13
27
  - Added first `CHANGELOG.md`
14
28
  - [TESTS] WorkflowDescriptorValidator tests
data/README.md CHANGED
@@ -1,8 +1,9 @@
1
1
  # Qyu 九
2
2
 
3
3
  [![Gem Version](https://img.shields.io/gem/v/qyu.svg)](https://rubygems.org/gems/qyu)
4
- [![Build Status](https://travis-ci.org/FindHotel/qyu.svg)](https://travis-ci.org/FindHotel/qyu)
4
+ [![Build Status](https://travis-ci.org/QyuTeam/qyu.svg)](https://travis-ci.org/QyuTeam/qyu)
5
5
  [![Maintainability](https://api.codeclimate.com/v1/badges/88b8e0a8621d1da5c237/maintainability)](https://codeclimate.com/github/QyuTeam/qyu/maintainability)
6
+ [![codecov](https://codecov.io/gh/QyuTeam/qyu/branch/master/graph/badge.svg)](https://codecov.io/gh/QyuTeam/qyu)
6
7
  [![Press](https://img.shields.io/badge/RubyWeekly-Blog%20Post-green.svg)](https://rubyweekly.com/link/44495/web)
7
8
 
8
9
  ## Requirements:
@@ -61,6 +62,9 @@ w = Qyu::Worker.new do
61
62
 
62
63
  # failure queue
63
64
  failure_queue false
65
+
66
+ # timeout in seconds for each task processed by the worker
67
+ timeout 120
64
68
  end
65
69
 
66
70
  w.work('queue-name') do |task|
@@ -89,12 +93,12 @@ end
89
93
  The memory queue and store is just for testing purposes. For production; use one of the following:
90
94
 
91
95
  #### Stores
92
- *ActiveRecord:* https://github.com/FindHotel/qyu-store-activerecord
93
- *Redis:* https://github.com/FindHotel/qyu-store-redis
96
+ *ActiveRecord:* https://github.com/QyuTeam/qyu-store-activerecord
97
+ *Redis:* https://github.com/QyuTeam/qyu-store-redis
94
98
 
95
99
  #### Queues
96
- *Amazon SQS:* https://github.com/FindHotel/qyu-queue-sqs
97
- *Redis:* https://github.com/FindHotel/qyu-queue-redis
100
+ *Amazon SQS:* https://github.com/QyuTeam/qyu-queue-sqs
101
+ *Redis:* https://github.com/QyuTeam/qyu-queue-redis
98
102
 
99
103
  ## Glossary
100
104
 
@@ -25,7 +25,7 @@ module Qyu
25
25
 
26
26
  # Get registered services
27
27
  #
28
- # @return [Hash] registereds services
28
+ # @return [Hash] registered services
29
29
  def types
30
30
  @__types ||= {}
31
31
  end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ module Errors
5
+ # Qyu::Errors::InvalidWorkerConfigurationValue
6
+ class InvalidWorkerConfigurationValue < Base
7
+ def initialize(configuration_name, configuration_value = nil)
8
+ super("invalid worker configuration value #{configuration_name}: #{configuration_value}")
9
+ end
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,12 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ module Errors
5
+ # Qyu::Errors::MissingSplitParameters
6
+ class MissingSplitParameters < Base
7
+ def initialize(parameter_name)
8
+ super("Missing split parameter: #{parameter_name}")
9
+ end
10
+ end
11
+ end
12
+ end
@@ -5,7 +5,7 @@ module Qyu
5
5
  # Qyu::Errors::PayloadValidationError
6
6
  class PayloadValidationError < Base
7
7
  def initialize(validation_errors_hash)
8
- super("Validation failed for payload fields: #{validation_errors_hash}.")
8
+ super("validation failed for payload fields: #{validation_errors_hash}.")
9
9
  end
10
10
  end
11
11
  end
@@ -15,7 +15,7 @@ module Qyu
15
15
 
16
16
  # Get registered services
17
17
  #
18
- # @return [Hash] registereds services
18
+ # @return [Hash] registered services
19
19
  def types
20
20
  @__types ||= {}
21
21
  end
@@ -4,7 +4,10 @@ module Qyu
4
4
  module Concerns
5
5
  # Qyu::Concerns::WorkflowDescriptorValidator
6
6
  class WorkflowDescriptorValidator
7
- ALLOWED_KEYS = %w(queue waits_for starts starts_manually starts_with_params).freeze
7
+ # TODO: starts_parallel is a the same as starts_manually. The latter will be removed in Qyu v2
8
+ ALLOWED_KEYS = %w(queue waits_for starts starts_parallel starts_manually starts_with_params).freeze
9
+ DEPRECATED_KEYS = %(starts_manually)
10
+
8
11
  attr_reader :errors
9
12
 
10
13
  def initialize(descriptor)
@@ -85,6 +88,7 @@ module Qyu
85
88
 
86
89
  def validate_task_reference_formats(task_name)
87
90
  validate_format(task_name, 'starts', Array) &&
91
+ validate_format(task_name, 'starts_parallel', Array) &&
88
92
  validate_format(task_name, 'starts_manually', Array) &&
89
93
  validate_format(task_name, 'starts_with_params', Hash) &&
90
94
  validate_format(task_name, 'waits_for', Hash)
@@ -92,6 +96,7 @@ module Qyu
92
96
 
93
97
  def validate_referenced_tasks(task_name)
94
98
  validate_presence_of_reference_tasks(task_name, 'starts', Array) &&
99
+ validate_presence_of_reference_tasks(task_name, 'starts_parallel', Array) &&
95
100
  validate_presence_of_reference_tasks(task_name, 'starts_manually', Array) &&
96
101
  validate_presence_of_reference_tasks(task_name, 'starts_with_params', Hash) &&
97
102
  validate_presence_of_reference_tasks(task_name, 'waits_for', Hash)
@@ -3,8 +3,15 @@
3
3
  module Qyu
4
4
  # Qyu::Job
5
5
  class Job
6
- attr_reader :descriptor, :payload, :id, :created_at, :updated_at
6
+ attr_reader :descriptor, :payload, :workflow, :id, :created_at, :updated_at
7
7
 
8
+ ## Class Methods
9
+
10
+ # Creates a new job via a workflow name / object and a payload
11
+ #
12
+ # @param [String, Qyu::Workflow] workflow to run
13
+ # @param [Hash] payload
14
+ # @return [Qyu::Job]
8
15
  def self.create(workflow:, payload:)
9
16
  workflow = Workflow.find_by(name: workflow) if workflow.is_a?(String)
10
17
  id = persist(workflow, payload)
@@ -12,10 +19,15 @@ module Qyu
12
19
  new(id, workflow, payload, time, time)
13
20
  end
14
21
 
22
+ # Finds Job by ID. Returns `nil` if job is not present in store
23
+ #
24
+ # @return [Qyu::Job, nil]
15
25
  def self.find(id)
16
26
  job_attrs = Qyu.store.find_job(id)
17
- new(id, job_attrs['workflow'], job_attrs['payload'],
18
- job_attrs['created_at'], job_attrs['updated_at'])
27
+ if job_attrs
28
+ new(id, job_attrs['workflow'], job_attrs['payload'],
29
+ job_attrs['created_at'], job_attrs['updated_at'])
30
+ end
19
31
  end
20
32
 
21
33
  def self.select(limit: 30, offset: 0, order: :asc)
@@ -26,18 +38,34 @@ module Qyu
26
38
  end
27
39
  end
28
40
 
41
+ # Counts job in state store
42
+ #
43
+ # @return [Integer] jobs count
29
44
  def self.count
30
45
  Qyu.store.count_jobs
31
46
  end
32
47
 
48
+ # Deletes job from state store by ID
49
+ #
50
+ # @param [Object] id
51
+ # @return [Object] deleted job
33
52
  def self.delete(id)
34
53
  Qyu.store.delete_job(id)
35
54
  end
36
55
 
56
+ # Clears completed jobs
57
+ #
58
+ # @return [Integer] cleared jobs count
37
59
  def self.clear_completed
38
60
  Qyu.store.clear_completed_jobs
39
61
  end
40
62
 
63
+ ## Instance Methods
64
+
65
+ # Starts job execution
66
+ # Enqueues all tasks scheduled to start at the beginning (`starts` key in workflow descriptor)
67
+ #
68
+ # #=> job.start
41
69
  def start
42
70
  descriptor['starts'].each do |task_name|
43
71
  create_task(nil, task_name, payload)
@@ -59,8 +87,8 @@ module Qyu
59
87
  descriptor['tasks'][task.name]['waits_for'].keys
60
88
  end
61
89
 
62
- def sync_condition(task, task_name)
63
- descriptor['tasks'][task.name]['waits_for'][task_name]['condition']
90
+ def sync_condition(task, next_task_name)
91
+ descriptor['tasks'][task.name]['waits_for'][next_task_name]['condition']
64
92
  end
65
93
 
66
94
  def create_task(parent_task, task_name, payload)
@@ -110,6 +138,7 @@ module Qyu
110
138
  all_task_names = []
111
139
  all_task_names.concat(desc['starts'] || [])
112
140
  all_task_names.concat((desc['starts_with_params'] || {}).keys)
141
+ all_task_names.concat(desc['starts_parallel'] || [])
113
142
  all_task_names.concat(desc['starts_manually'] || [])
114
143
  all_task_names.include?(tasks_path[-1])
115
144
  end
@@ -141,7 +170,7 @@ module Qyu
141
170
  private
142
171
 
143
172
  def initialize(id, workflow, payload, created_at = nil, updated_at = nil)
144
- @workflow = workflow
173
+ @workflow = Qyu::Workflow.new(workflow['id'], workflow['name'], workflow['descriptor'])
145
174
  @descriptor = @workflow['descriptor']
146
175
  @payload = payload
147
176
  @id = id
@@ -171,10 +171,34 @@ module Qyu
171
171
  Qyu.queue.enqueue_task_to_failed_queue(queue_name, id)
172
172
  end
173
173
 
174
+ # Returns workflow specified in parent job
175
+ #
176
+ # @return [Qyu::Workflow] full workflow
177
+ def workflow
178
+ job.workflow
179
+ end
180
+
181
+ # Returns workflow descriptor from parent job
182
+ #
183
+ # @return [Hash] workflow descriptor
184
+ def workflow_descriptor
185
+ job.descriptor
186
+ end
187
+
188
+ # Returns parent job
189
+ #
190
+ # @return [Qyu::Job] parent job
174
191
  def job
175
192
  @job ||= Qyu::Job.find(job_id)
176
193
  end
177
194
 
195
+ # Returns task descriptor
196
+ #
197
+ # @return [Hash] task descriptor
198
+ def descriptor
199
+ workflow_descriptor['tasks'][name]
200
+ end
201
+
178
202
  def [](attribute)
179
203
  public_send(attribute)
180
204
  end
@@ -5,6 +5,14 @@ module Qyu
5
5
  class Workflow
6
6
  attr_reader :id, :name, :descriptor, :created_at, :updated_at
7
7
 
8
+ def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
9
+ @id = id
10
+ @name = name
11
+ @descriptor = descriptor
12
+ @created_at = created_at
13
+ @updated_at = updated_at
14
+ end
15
+
8
16
  class << self
9
17
  def create(name:, descriptor:)
10
18
  validator = Qyu::Concerns::WorkflowDescriptorValidator.new(descriptor)
@@ -70,17 +78,5 @@ module Qyu
70
78
  def [](attribute)
71
79
  public_send(attribute)
72
80
  end
73
-
74
- private_class_method :new
75
-
76
- private
77
-
78
- def initialize(id, name, descriptor, created_at = nil, updated_at = nil)
79
- @id = id
80
- @name = name
81
- @descriptor = descriptor
82
- @created_at = created_at
83
- @updated_at = updated_at
84
- end
85
81
  end
86
82
  end
@@ -3,15 +3,15 @@
3
3
  <nav class="pull-left">
4
4
  <ul>
5
5
  <li>
6
- <a href="https://github.com/FindHotel/qyu">
6
+ <a href="https://github.com/QyuTeam/qyu">
7
7
  Qyu is a distributed task execution system for complex workflows
8
8
  </a>
9
9
  </li>
10
10
  </ul>
11
11
  </nav>
12
12
  <div class="copyright pull-right">
13
- <a href="https://www.findhotel.net/">
14
- FindHotel B.V. &copy; <%= Date.today.year %>
13
+ <a href="https://github.com/QyuTeam">
14
+ Qyu &copy; <%= Date.today.year %>
15
15
  </a>
16
16
  </div>
17
17
  </div>
@@ -1,3 +1,3 @@
1
1
  module Qyu
2
- VERSION = '1.0.2'
2
+ VERSION = '1.1.0'
3
3
  end
@@ -1,10 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'qyu/workers/concerns/callback'
4
- require 'qyu/workers/concerns/failure_queue'
5
- require 'qyu/workers/concerns/payload_validator'
6
- require 'qyu/workers/base'
7
- require 'qyu/workers/sync'
3
+ (
4
+ Dir["#{File.dirname(__FILE__)}/workers/concerns/*.rb"] +
5
+ Dir["#{File.dirname(__FILE__)}/workers/*.rb"]
6
+ ).each do |path|
7
+ require path
8
+ end
9
+
8
10
 
9
11
  Qyu::Worker = Qyu::Workers::Base
12
+ Qyu::SplitWorker = Qyu::Workers::Split
10
13
  Qyu::SyncWorker = Qyu::Workers::Sync
@@ -22,8 +22,9 @@ module Qyu
22
22
  # t.unlock! && t.mark_finished && t.acknowledge_message
23
23
  class Base
24
24
  include Concerns::Callback
25
- include Concerns::PayloadValidator
26
25
  include Concerns::FailureQueue
26
+ include Concerns::PayloadValidator
27
+ include Concerns::Timeout
27
28
 
28
29
  attr_reader :id
29
30
  attr_accessor :processed_tasks
@@ -35,7 +36,7 @@ module Qyu
35
36
  end
36
37
 
37
38
  def work(queue_name, blocking: true)
38
- log(:info, "Worker started for queue '#{queue_name}'")
39
+ log(:info, "worker started for queue '#{queue_name}'")
39
40
  repeat = true
40
41
 
41
42
  remaining_fetch_retries = 3
@@ -45,13 +46,15 @@ module Qyu
45
46
  begin
46
47
  fetched_task = fetch_task(queue_name)
47
48
  validate_payload!(fetched_task)
48
- log(:info, "Worker processed #{processed_tasks} tasks from queue `#{queue_name}`")
49
+ log(:info, "worker processed #{processed_tasks} tasks from queue `#{queue_name}`")
49
50
  if fetched_task.acknowledgeable?
50
51
  discard_completed_task(fetched_task)
51
52
  elsif fetched_task.lock!
52
53
  fetched_task.mark_working
53
54
  begin
54
- yield(fetched_task)
55
+ Timeout::timeout(@timeout) do
56
+ yield(fetched_task)
57
+ end
55
58
  conclude_task(fetched_task)
56
59
  rescue => ex
57
60
  fail_task(fetched_task, ex)
@@ -66,11 +69,12 @@ module Qyu
66
69
  remaining_fetch_retries -= 1
67
70
  retry
68
71
  end
69
- rescue Qyu::Errors::PayloadValidationError
72
+ rescue Qyu::Errors::PayloadValidationError => ex
73
+ log("invalid payload: #{ex.class}: #{ex.message}")
70
74
  fetched_task.mark_invalid_payload
71
75
  rescue => ex
72
- log("Worker error: #{ex.class}: #{ex.message}")
73
- log("Backtrace: #{ex.backtrace.join("\n")}")
76
+ log("worker error: #{ex.class}: #{ex.message}")
77
+ log("backtrace: #{ex.backtrace.join("\n")}")
74
78
  end
75
79
  end
76
80
 
@@ -88,13 +92,13 @@ module Qyu
88
92
  end
89
93
 
90
94
  def discard_completed_task(fetched_task)
91
- log(:debug, 'Fetched completed task. Discarding...')
95
+ log(:debug, 'fetched completed task and discarding it...')
92
96
  fetched_task.acknowledge_message
93
97
  end
94
98
 
95
99
  def conclude_task(fetched_task)
96
100
  Qyu.store.transaction do
97
- log(:debug, 'Task finished. Creating next tasks.')
101
+ log(:debug, 'task finished and creating next tasks.')
98
102
  fetched_task.job.create_next_tasks(
99
103
  fetched_task,
100
104
  fetched_task.job.payload.merge(fetched_task.payload)
@@ -107,8 +111,8 @@ module Qyu
107
111
 
108
112
  def fail_task(fetched_task, exception)
109
113
  unless exception.class == Qyu::Errors::UnsyncError
110
- log("Worker error: #{exception.class}: #{exception.message}")
111
- log("Backtrace: #{exception.backtrace.join("\n")}")
114
+ log("worker error: #{exception.class}: #{exception.message}")
115
+ log("backtrace: #{exception.backtrace.join("\n")}")
112
116
  end
113
117
  Qyu.store.transaction do
114
118
  fetched_task.enqueue_in_failure_queue if @failure_queue
@@ -120,10 +124,10 @@ module Qyu
120
124
  def acknowledge_message_with_task_id_not_found_in_store(exception)
121
125
  # If a task is not found in the Store then there is no point attempting
122
126
  # to fetch the message over and over again.
123
- log("Worker error: #{exception.class}: #{exception.message}")
124
- log("Backtrace: #{exception.backtrace.join("\n")}")
125
- log("Original error: #{exception.original_error.class}: #{exception.original_error.message}")
126
- log("Backtrace: #{exception.original_error.backtrace.join("\n")}")
127
+ log("worker error: #{exception.class}: #{exception.message}")
128
+ log("backtrace: #{exception.backtrace.join("\n")}")
129
+ log("original error: #{exception.original_error.class}: #{exception.original_error.message}")
130
+ log("backtrace: #{exception.original_error.backtrace.join("\n")}")
127
131
  if exception.original_error.class == Qyu::Errors::TaskNotFound &&
128
132
  exception.queue_name &&
129
133
  exception.message_id
@@ -136,7 +140,7 @@ module Qyu
136
140
  end
137
141
 
138
142
  def run_garbage_collector
139
- log(:debug, 'Running garbage collector')
143
+ log(:debug, 'running garbage collector')
140
144
  GC.start
141
145
  end
142
146
  end
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ module Workers
5
+ module Concerns
6
+ # Qyu::Workers::Concerns::Split
7
+ #
8
+ # Adds ability to split workers to specify slice size and splittable variable name
9
+ #
10
+ # Qyu::SplitWorker.new do
11
+ # slice_size 25
12
+ # payload_key 'array'
13
+ # end
14
+ #
15
+ module Split
16
+
17
+ @slice_size = 25
18
+
19
+ # Configures slice size
20
+ #
21
+ # slice_size 25
22
+ #
23
+ # @param slice_size [Integer]
24
+ def slice_size(slsz)
25
+ @slice_size = slsz
26
+ end
27
+
28
+ # Configures payload key with array to split
29
+ #
30
+ # payload_key 25
31
+ #
32
+ # @param payload_key [String]
33
+ def payload_key(var_name)
34
+ @payload_key = var_name
35
+ end
36
+
37
+ # Run with sampling a single element out of the array
38
+ #
39
+ # sample true
40
+ #
41
+ # @param sample [Boolean]
42
+ def sample(smpl)
43
+ @sample = smpl
44
+ end
45
+
46
+ private
47
+ def validate_split_parameters!
48
+ if @payload_key.nil?
49
+ raise Qyu::Errors::MissingSplitParameters.new('payload_key')
50
+ end
51
+
52
+ if @slice_size.nil?
53
+ raise Qyu::Errors::MissingSplitParameters.new('slice_size')
54
+ end
55
+
56
+ if @slice_size.zero?
57
+ raise Qyu::Errors::InvalidWorkerConfigurationValue.new('slice_size', 0)
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,32 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'timeout'
4
+
5
+ module Qyu
6
+ module Workers
7
+ module Concerns
8
+ # Qyu::Workers::Concerns::Timeout
9
+ #
10
+ # Adds timeout to running tasks in a worker
11
+ #
12
+ # Qyu::Worker.new do
13
+ # timeout 0 # disabled (default)
14
+ # # or
15
+ # timeout 3600
16
+ # end
17
+ #
18
+ module Timeout
19
+
20
+ # Configures timeout
21
+ #
22
+ # timeout 0 # default
23
+ # timeout 3600
24
+ #
25
+ # @param [Integer]
26
+ def timeout(seconds)
27
+ @timeout = seconds.to_i
28
+ end
29
+ end
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,58 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Qyu
4
+ module Workers
5
+ # Qyu::Workers::Split
6
+ #
7
+ # Starts a worker to split a certain payload key into multiple jobs
8
+ #
9
+ # Qyu::SplitWorker.new do
10
+ # slice_size 25
11
+ # payload_key 'array'
12
+ # end
13
+ #
14
+ class Split < Base
15
+ include Qyu::Workers::Concerns::Split
16
+
17
+ attr_accessor :splittable
18
+
19
+ # Assign a splittable variable
20
+ # by being at the end of a block
21
+ # worker.work('queue') do
22
+ # # do anything
23
+ # # splittable variable must be at the end
24
+ # [1, 2, 3, 4, 5, 6, 6]
25
+ # end
26
+ #
27
+ # or by passing
28
+ # payload_key 'array'
29
+ # to worker initializer then just
30
+ # worker.work('queue')
31
+ # @param queue_name [String]
32
+ def work(queue_name)
33
+ validate_split_parameters!
34
+
35
+ super do |task|
36
+ if block_given?
37
+ @splittable = yield(task)
38
+ else
39
+ # or by passing
40
+ # payload_key 'array'
41
+ # to worker initializer
42
+ @splittable = task.payload[@payload_key]
43
+ end
44
+
45
+ @splittable.each_slice(@slice_size).with_index do |slice, i|
46
+ log(:debug, "Split started for queue '#{queue_name}'")
47
+ input = @sample ? slice.sample : slice
48
+ new_payload = task.payload.merge({ @payload_key => input })
49
+ task_names_to_start = task.descriptor['starts_parallel'] || task.descriptor['starts_manually']
50
+ task_names_to_start.each do |task_name_to_start|
51
+ task.job.create_task(task, task_name_to_start, new_payload)
52
+ end
53
+ end
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -14,6 +14,8 @@ module Qyu
14
14
  log(:debug, "Task: #{task_name}, Sync condition: #{sync_condition}")
15
15
  if respond_to?(sync_condition['function'], true)
16
16
  __send__(sync_condition['function'], job, task, task_name, sync_condition['param'])
17
+ # execute attached sync block only if codition passes (i.e. No errors raised)
18
+ yield(task) if block_given?
17
19
  else
18
20
  fail Qyu::Errors::NotImplementedError
19
21
  end
@@ -8,14 +8,15 @@ Gem::Specification.new do |spec|
8
8
  spec.version = Qyu::VERSION
9
9
  spec.authors = ['Elod Peter', 'Mohamed Osama']
10
10
  spec.email = ['bejmuller@gmail.com', 'mohamed.o.alnagdy@gmail.com']
11
+ spec.license = "MIT"
11
12
 
12
13
  spec.summary = 'Distributed task execution system for complex workflows'
13
- spec.description = 'Distributed task execution system for complex workflows'
14
- spec.homepage = 'https://github.com/FindHotel/qyu'
14
+ spec.description = 'Qyu makes use of a message queue and a state store to provide a reliable distributed task execution system for complex workflows'
15
+ spec.homepage = 'https://github.com/QyuTeam/qyu'
15
16
  spec.required_ruby_version = '>= 2.4'
16
17
 
17
18
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
- f.match(%r{^(test|spec|features)/})
19
+ f.match(%r{^(test|spec|features|examples)/})
19
20
  end
20
21
  spec.bindir = "exe"
21
22
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -26,13 +27,13 @@ Gem::Specification.new do |spec|
26
27
  spec.add_runtime_dependency 'activesupport', '~> 5.1'
27
28
 
28
29
  spec.add_development_dependency 'bundler', '~> 1.16'
29
- spec.add_development_dependency 'dotenv'
30
- spec.add_development_dependency 'rack-test'
31
- spec.add_development_dependency 'pry'
32
- spec.add_development_dependency 'pry-byebug'
30
+ spec.add_development_dependency 'dotenv', '~> 2.0'
31
+ spec.add_development_dependency 'rack-test', '~> 1.0'
32
+ spec.add_development_dependency 'pry', '~> 0.11'
33
+ spec.add_development_dependency 'pry-byebug', '~> 3.0'
33
34
  spec.add_development_dependency 'rake', '~> 12.0'
34
- spec.add_development_dependency 'rspec', '~> 3.7'
35
- spec.add_development_dependency 'simplecov'
36
- spec.add_development_dependency 'sinatra'
35
+ spec.add_development_dependency 'rspec', '~> 3.5'
36
+ spec.add_development_dependency 'codecov', '~> 0.1'
37
+ spec.add_development_dependency 'sinatra', '~> 2.0'
37
38
  spec.add_development_dependency 'timecop', '~> 0.9'
38
39
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: qyu
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.0.2
4
+ version: 1.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elod Peter
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: exe
11
11
  cert_chain: []
12
- date: 2018-03-27 00:00:00.000000000 Z
12
+ date: 2018-03-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: activesupport
@@ -43,58 +43,58 @@ dependencies:
43
43
  name: dotenv
44
44
  requirement: !ruby/object:Gem::Requirement
45
45
  requirements:
46
- - - ">="
46
+ - - "~>"
47
47
  - !ruby/object:Gem::Version
48
- version: '0'
48
+ version: '2.0'
49
49
  type: :development
50
50
  prerelease: false
51
51
  version_requirements: !ruby/object:Gem::Requirement
52
52
  requirements:
53
- - - ">="
53
+ - - "~>"
54
54
  - !ruby/object:Gem::Version
55
- version: '0'
55
+ version: '2.0'
56
56
  - !ruby/object:Gem::Dependency
57
57
  name: rack-test
58
58
  requirement: !ruby/object:Gem::Requirement
59
59
  requirements:
60
- - - ">="
60
+ - - "~>"
61
61
  - !ruby/object:Gem::Version
62
- version: '0'
62
+ version: '1.0'
63
63
  type: :development
64
64
  prerelease: false
65
65
  version_requirements: !ruby/object:Gem::Requirement
66
66
  requirements:
67
- - - ">="
67
+ - - "~>"
68
68
  - !ruby/object:Gem::Version
69
- version: '0'
69
+ version: '1.0'
70
70
  - !ruby/object:Gem::Dependency
71
71
  name: pry
72
72
  requirement: !ruby/object:Gem::Requirement
73
73
  requirements:
74
- - - ">="
74
+ - - "~>"
75
75
  - !ruby/object:Gem::Version
76
- version: '0'
76
+ version: '0.11'
77
77
  type: :development
78
78
  prerelease: false
79
79
  version_requirements: !ruby/object:Gem::Requirement
80
80
  requirements:
81
- - - ">="
81
+ - - "~>"
82
82
  - !ruby/object:Gem::Version
83
- version: '0'
83
+ version: '0.11'
84
84
  - !ruby/object:Gem::Dependency
85
85
  name: pry-byebug
86
86
  requirement: !ruby/object:Gem::Requirement
87
87
  requirements:
88
- - - ">="
88
+ - - "~>"
89
89
  - !ruby/object:Gem::Version
90
- version: '0'
90
+ version: '3.0'
91
91
  type: :development
92
92
  prerelease: false
93
93
  version_requirements: !ruby/object:Gem::Requirement
94
94
  requirements:
95
- - - ">="
95
+ - - "~>"
96
96
  - !ruby/object:Gem::Version
97
- version: '0'
97
+ version: '3.0'
98
98
  - !ruby/object:Gem::Dependency
99
99
  name: rake
100
100
  requirement: !ruby/object:Gem::Requirement
@@ -115,42 +115,42 @@ dependencies:
115
115
  requirements:
116
116
  - - "~>"
117
117
  - !ruby/object:Gem::Version
118
- version: '3.7'
118
+ version: '3.5'
119
119
  type: :development
120
120
  prerelease: false
121
121
  version_requirements: !ruby/object:Gem::Requirement
122
122
  requirements:
123
123
  - - "~>"
124
124
  - !ruby/object:Gem::Version
125
- version: '3.7'
125
+ version: '3.5'
126
126
  - !ruby/object:Gem::Dependency
127
- name: simplecov
127
+ name: codecov
128
128
  requirement: !ruby/object:Gem::Requirement
129
129
  requirements:
130
- - - ">="
130
+ - - "~>"
131
131
  - !ruby/object:Gem::Version
132
- version: '0'
132
+ version: '0.1'
133
133
  type: :development
134
134
  prerelease: false
135
135
  version_requirements: !ruby/object:Gem::Requirement
136
136
  requirements:
137
- - - ">="
137
+ - - "~>"
138
138
  - !ruby/object:Gem::Version
139
- version: '0'
139
+ version: '0.1'
140
140
  - !ruby/object:Gem::Dependency
141
141
  name: sinatra
142
142
  requirement: !ruby/object:Gem::Requirement
143
143
  requirements:
144
- - - ">="
144
+ - - "~>"
145
145
  - !ruby/object:Gem::Version
146
- version: '0'
146
+ version: '2.0'
147
147
  type: :development
148
148
  prerelease: false
149
149
  version_requirements: !ruby/object:Gem::Requirement
150
150
  requirements:
151
- - - ">="
151
+ - - "~>"
152
152
  - !ruby/object:Gem::Version
153
- version: '0'
153
+ version: '2.0'
154
154
  - !ruby/object:Gem::Dependency
155
155
  name: timecop
156
156
  requirement: !ruby/object:Gem::Requirement
@@ -165,7 +165,8 @@ dependencies:
165
165
  - - "~>"
166
166
  - !ruby/object:Gem::Version
167
167
  version: '0.9'
168
- description: Distributed task execution system for complex workflows
168
+ description: Qyu makes use of a message queue and a state store to provide a reliable
169
+ distributed task execution system for complex workflows
169
170
  email:
170
171
  - bejmuller@gmail.com
171
172
  - mohamed.o.alnagdy@gmail.com
@@ -185,11 +186,6 @@ files:
185
186
  - bin/console
186
187
  - bin/server
187
188
  - bin/setup
188
- - examples/bin/simple
189
- - examples/config.rb
190
- - examples/simple/create_workflow.rb
191
- - examples/simple/enqueue_job.rb
192
- - examples/simple/worker.rb
193
189
  - lib/qyu.rb
194
190
  - lib/qyu/config.rb
195
191
  - lib/qyu/errors.rb
@@ -197,10 +193,12 @@ files:
197
193
  - lib/qyu/errors/could_not_fetch_task.rb
198
194
  - lib/qyu/errors/invalid_queue_name.rb
199
195
  - lib/qyu/errors/invalid_task_attributes.rb
196
+ - lib/qyu/errors/invalid_worker_configuration_value.rb
200
197
  - lib/qyu/errors/job_not_found.rb
201
198
  - lib/qyu/errors/lock_already_acquired.rb
202
199
  - lib/qyu/errors/lock_not_acquired.rb
203
200
  - lib/qyu/errors/message_not_received.rb
201
+ - lib/qyu/errors/missing_split_parameters.rb
204
202
  - lib/qyu/errors/not_implemented_error.rb
205
203
  - lib/qyu/errors/payload_validation_error.rb
206
204
  - lib/qyu/errors/task_not_found.rb
@@ -251,10 +249,14 @@ files:
251
249
  - lib/qyu/workers/concerns/callback.rb
252
250
  - lib/qyu/workers/concerns/failure_queue.rb
253
251
  - lib/qyu/workers/concerns/payload_validator.rb
252
+ - lib/qyu/workers/concerns/split.rb
253
+ - lib/qyu/workers/concerns/timeout.rb
254
+ - lib/qyu/workers/split.rb
254
255
  - lib/qyu/workers/sync.rb
255
256
  - qyu.gemspec
256
- homepage: https://github.com/FindHotel/qyu
257
- licenses: []
257
+ homepage: https://github.com/QyuTeam/qyu
258
+ licenses:
259
+ - MIT
258
260
  metadata:
259
261
  yard.run: yri
260
262
  post_install_message:
@@ -273,7 +275,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
273
275
  version: '0'
274
276
  requirements: []
275
277
  rubyforge_project:
276
- rubygems_version: 2.6.8
278
+ rubygems_version: 2.7.4
277
279
  signing_key:
278
280
  specification_version: 4
279
281
  summary: Distributed task execution system for complex workflows
@@ -1,7 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require_relative '../simple/create_workflow'
4
- require_relative '../simple/enqueue_job'
5
- require_relative '../simple/worker'
6
-
7
- SimpleWorker.new.run
@@ -1,22 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'rubygems'
4
- require 'bundler/setup'
5
- Bundler.setup
6
-
7
- require 'qyu'
8
- logger = Logger.new(STDOUT)
9
- logger.level = :info
10
-
11
- Qyu.configure(
12
- queue: {
13
- type: :memory
14
- },
15
- store: {
16
- type: :memory,
17
- lease_period: 60
18
- },
19
- logger: logger
20
- )
21
-
22
- Qyu.test_connections
@@ -1,18 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../config'
4
-
5
- descriptor = {
6
- 'starts' => %w(
7
- print_hello
8
- ),
9
- 'tasks' => {
10
- 'print_hello' => {
11
- 'queue' => 'print-hello'
12
- }
13
- }
14
- }
15
-
16
- name = 'say-hello'
17
- Qyu.logger.info "Creating workflow #{name}"
18
- Qyu::Workflow.create(name: name, descriptor: descriptor)
@@ -1,8 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../config'
4
-
5
- payload = { 'times' => 5 }
6
- Qyu.logger.info "Enqueuing job with payload #{payload}"
7
- job = Qyu::Job.create(workflow: 'say-hello', payload: payload)
8
- job.start
@@ -1,32 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require_relative '../config'
4
-
5
- class SimpleWorker
6
- def initialize
7
- @worker = Qyu::Worker.new do
8
- callback :execute, :before do
9
- Qyu.logger.info 'Waiting for task..'
10
- end
11
-
12
- callback :execute, :after do
13
- Qyu.logger.info 'Done'
14
- end
15
-
16
- # Payload validation
17
- validates :times, presence: true, type: :integer
18
- end
19
- end
20
-
21
- def run
22
- # Consumes message from print-hello queue
23
- @worker.work('print-hello') do |task|
24
- task.payload['times'].times do |i|
25
- Qyu.logger.info "#{i + 1}. Hello"
26
- end
27
- rescue StandardError => ex
28
- Qyu.logger.error 'OMG :('
29
- Qyu.logger.error ex.message
30
- end
31
- end
32
- end