cloudtasker 0.2.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/.rubocop.yml +5 -0
  4. data/.travis.yml +10 -1
  5. data/Appraisals +25 -0
  6. data/CHANGELOG.md +29 -0
  7. data/Gemfile.lock +27 -4
  8. data/README.md +571 -6
  9. data/Rakefile +6 -0
  10. data/app/controllers/cloudtasker/application_controller.rb +2 -0
  11. data/app/controllers/cloudtasker/worker_controller.rb +24 -2
  12. data/cloudtasker.gemspec +5 -3
  13. data/docs/BATCH_JOBS.md +66 -0
  14. data/docs/CRON_JOBS.md +65 -0
  15. data/docs/UNIQUE_JOBS.md +127 -0
  16. data/exe/cloudtasker +15 -0
  17. data/gemfiles/.bundle/config +2 -0
  18. data/gemfiles/google_cloud_tasks_1.0.gemfile +9 -0
  19. data/gemfiles/google_cloud_tasks_1.0.gemfile.lock +263 -0
  20. data/gemfiles/google_cloud_tasks_1.1.gemfile +9 -0
  21. data/gemfiles/google_cloud_tasks_1.1.gemfile.lock +263 -0
  22. data/gemfiles/google_cloud_tasks_1.2.gemfile +9 -0
  23. data/gemfiles/google_cloud_tasks_1.2.gemfile.lock +263 -0
  24. data/gemfiles/google_cloud_tasks_1.3.gemfile +9 -0
  25. data/gemfiles/google_cloud_tasks_1.3.gemfile.lock +264 -0
  26. data/gemfiles/rails_4.0.gemfile +10 -0
  27. data/gemfiles/rails_4.1.gemfile +9 -0
  28. data/gemfiles/rails_4.2.gemfile +9 -0
  29. data/gemfiles/rails_5.0.gemfile +9 -0
  30. data/gemfiles/rails_5.1.gemfile +9 -0
  31. data/gemfiles/rails_5.2.gemfile +9 -0
  32. data/gemfiles/rails_5.2.gemfile.lock +247 -0
  33. data/gemfiles/rails_6.0.gemfile +9 -0
  34. data/gemfiles/rails_6.0.gemfile.lock +263 -0
  35. data/lib/cloudtasker.rb +19 -1
  36. data/lib/cloudtasker/backend/google_cloud_task.rb +139 -0
  37. data/lib/cloudtasker/backend/memory_task.rb +190 -0
  38. data/lib/cloudtasker/backend/redis_task.rb +249 -0
  39. data/lib/cloudtasker/batch/batch_progress.rb +19 -1
  40. data/lib/cloudtasker/batch/job.rb +85 -23
  41. data/lib/cloudtasker/cli.rb +194 -0
  42. data/lib/cloudtasker/cloud_task.rb +91 -0
  43. data/lib/cloudtasker/config.rb +64 -2
  44. data/lib/cloudtasker/cron/job.rb +2 -2
  45. data/lib/cloudtasker/cron/schedule.rb +25 -11
  46. data/lib/cloudtasker/dead_worker_error.rb +6 -0
  47. data/lib/cloudtasker/local_server.rb +74 -0
  48. data/lib/cloudtasker/railtie.rb +10 -0
  49. data/lib/cloudtasker/redis_client.rb +2 -2
  50. data/lib/cloudtasker/testing.rb +133 -0
  51. data/lib/cloudtasker/unique_job/job.rb +1 -1
  52. data/lib/cloudtasker/unique_job/lock/base_lock.rb +1 -1
  53. data/lib/cloudtasker/unique_job/lock/until_executed.rb +3 -1
  54. data/lib/cloudtasker/unique_job/lock/while_executing.rb +3 -1
  55. data/lib/cloudtasker/version.rb +1 -1
  56. data/lib/cloudtasker/worker.rb +61 -17
  57. data/lib/cloudtasker/{task.rb → worker_handler.rb} +10 -77
  58. data/lib/cloudtasker/worker_logger.rb +155 -0
  59. data/lib/tasks/setup_queue.rake +10 -0
  60. metadata +70 -6
@@ -24,9 +24,11 @@ module Cloudtasker
24
24
  def execute
25
25
  job.lock!
26
26
  yield
27
- job.unlock!
28
27
  rescue LockError
29
28
  conflict_instance.on_execute { yield }
29
+ ensure
30
+ # Unlock the job on any error to avoid deadlocks.
31
+ job.unlock!
30
32
  end
31
33
  end
32
34
  end
@@ -13,9 +13,11 @@ module Cloudtasker
13
13
  def execute
14
14
  job.lock!
15
15
  yield
16
- job.unlock!
17
16
  rescue LockError
18
17
  conflict_instance.on_execute { yield }
18
+ ensure
19
+ # Unlock the job on any error to avoid deadlocks.
20
+ job.unlock!
19
21
  end
20
22
  end
21
23
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Cloudtasker
4
- VERSION = '0.2.0'
4
+ VERSION = '0.7.0'
5
5
  end
@@ -6,7 +6,7 @@ module Cloudtasker
6
6
  # Add class method to including class
7
7
  def self.included(base)
8
8
  base.extend(ClassMethods)
9
- base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued
9
+ base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
10
10
  end
11
11
 
12
12
  #
@@ -32,8 +32,9 @@ module Cloudtasker
32
32
  # @return [Cloudtasker::Worker, nil] The instantiated worker.
33
33
  #
34
34
  def self.from_hash(hash)
35
- # Symbolize payload keys
35
+ # Symbolize metadata keys and stringify job arguments
36
36
  payload = JSON.parse(hash.to_json, symbolize_names: true)
37
+ payload[:job_args] = JSON.parse(hash[:job_args].to_json)
37
38
 
38
39
  # Extract worker parameters
39
40
  klass_name = payload&.dig(:worker)
@@ -44,7 +45,7 @@ module Cloudtasker
44
45
  return nil unless worker_klass.include?(self)
45
46
 
46
47
  # Return instantiated worker
47
- worker_klass.new(payload.slice(:job_args, :job_id, :job_meta))
48
+ worker_klass.new(payload.slice(:job_args, :job_id, :job_meta, :job_retries))
48
49
  rescue NameError
49
50
  nil
50
51
  end
@@ -54,12 +55,12 @@ module Cloudtasker
54
55
  #
55
56
  # Set the worker runtime options.
56
57
  #
57
- # @param [Hash] opts The worker options
58
+ # @param [Hash] opts The worker options.
58
59
  #
59
- # @return [<Type>] <description>
60
+ # @return [Hash] The options set.
60
61
  #
61
62
  def cloudtasker_options(opts = {})
62
- opt_list = opts&.map { |k, v| [k.to_s, v] } || [] # stringify
63
+ opt_list = opts&.map { |k, v| [k.to_sym, v] } || [] # symbolize
63
64
  @cloudtasker_options_hash = Hash[opt_list]
64
65
  end
65
66
 
@@ -69,7 +70,7 @@ module Cloudtasker
69
70
  # @return [Hash] The worker runtime options.
70
71
  #
71
72
  def cloudtasker_options_hash
72
- @cloudtasker_options_hash
73
+ @cloudtasker_options_hash || {}
73
74
  end
74
75
 
75
76
  #
@@ -77,7 +78,7 @@ module Cloudtasker
77
78
  #
78
79
  # @param [Array<any>] *args List of worker arguments
79
80
  #
80
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
81
+ # @return [Cloudtasker::CloudTask] The Google Task response
81
82
  #
82
83
  def perform_async(*args)
83
84
  perform_in(nil, *args)
@@ -89,7 +90,7 @@ module Cloudtasker
89
90
  # @param [Integer, nil] interval The delay in seconds.
90
91
  # @param [Array<any>] *args List of worker arguments.
91
92
  #
92
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
93
+ # @return [Cloudtasker::CloudTask] The Google Task response
93
94
  #
94
95
  def perform_in(interval, *args)
95
96
  new(job_args: args).schedule(interval: interval)
@@ -101,11 +102,20 @@ module Cloudtasker
101
102
  # @param [Time, Integer] time_at The time at which the job should run.
102
103
  # @param [Array<any>] *args List of worker arguments
103
104
  #
104
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
105
+ # @return [Cloudtasker::CloudTask] The Google Task response
105
106
  #
106
107
  def perform_at(time_at, *args)
107
108
  new(job_args: args).schedule(time_at: time_at)
108
109
  end
110
+
111
+ #
112
+ # Return the numbeer of times this worker will be retried.
113
+ #
114
+ # @return [Integer] The number of retries.
115
+ #
116
+ def max_retries
117
+ cloudtasker_options_hash[:max_retries] || Cloudtasker.config.max_retries
118
+ end
109
119
  end
110
120
 
111
121
  #
@@ -114,10 +124,20 @@ module Cloudtasker
114
124
  # @param [Array<any>] job_args The list of perform args.
115
125
  # @param [String] job_id A unique ID identifying this job.
116
126
  #
117
- def initialize(job_args: [], job_id: nil, job_meta: {})
127
+ def initialize(job_args: [], job_id: nil, job_meta: {}, job_retries: 0)
118
128
  @job_args = job_args
119
129
  @job_id = job_id || SecureRandom.uuid
120
130
  @job_meta = MetaStore.new(job_meta)
131
+ @job_retries = job_retries || 0
132
+ end
133
+
134
+ #
135
+ # Return the Cloudtasker logger instance.
136
+ #
137
+ # @return [Logger, any] The cloudtasker logger.
138
+ #
139
+ def logger
140
+ @logger ||= WorkerLogger.new(self)
121
141
  end
122
142
 
123
143
  #
@@ -126,9 +146,22 @@ module Cloudtasker
126
146
  # @return [Any] The result of the perform.
127
147
  #
128
148
  def execute
129
- Cloudtasker.config.server_middleware.invoke(self) do
130
- perform(*job_args)
149
+ logger.info('Starting job...')
150
+ resp = Cloudtasker.config.server_middleware.invoke(self) do
151
+ begin
152
+ perform(*job_args)
153
+ rescue StandardError => e
154
+ try(:on_error, e)
155
+ return raise(e) unless job_dead?
156
+
157
+ # Flag job as dead
158
+ logger.info('Job dead')
159
+ try(:on_dead, e)
160
+ raise(DeadWorkerError, e)
161
+ end
131
162
  end
163
+ logger.info('Job done')
164
+ resp
132
165
  end
133
166
 
134
167
  #
@@ -138,11 +171,11 @@ module Cloudtasker
138
171
  #
139
172
  # @param [Time, Integer] interval The time at which the job should run
140
173
  #
141
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
174
+ # @return [Cloudtasker::CloudTask] The Google Task response
142
175
  #
143
176
  def schedule(interval: nil, time_at: nil)
144
177
  Cloudtasker.config.client_middleware.invoke(self) do
145
- Task.new(self).schedule(interval: interval, time_at: time_at)
178
+ WorkerHandler.new(self).schedule(interval: interval, time_at: time_at)
146
179
  end
147
180
  end
148
181
 
@@ -155,7 +188,7 @@ module Cloudtasker
155
188
  #
156
189
  # @param [Integer] interval Delay to wait before processing the job again (in seconds).
157
190
  #
158
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
191
+ # @return [Cloudtasker::CloudTask] The Google Task response
159
192
  #
160
193
  def reenqueue(interval)
161
194
  @job_reenqueued = true
@@ -182,7 +215,8 @@ module Cloudtasker
182
215
  worker: self.class.to_s,
183
216
  job_id: job_id,
184
217
  job_args: job_args,
185
- job_meta: job_meta.to_h
218
+ job_meta: job_meta.to_h,
219
+ job_retries: job_retries
186
220
  }
187
221
  end
188
222
 
@@ -207,5 +241,15 @@ module Cloudtasker
207
241
  def ==(other)
208
242
  other.is_a?(self.class) && other.job_id == job_id
209
243
  end
244
+
245
+ #
246
+ # Return true if the job has excceeded its maximum number
247
+ # of retries
248
+ #
249
+ # @return [Boolean] True if the job is dead
250
+ #
251
+ def job_dead?
252
+ job_retries >= Cloudtasker.config.max_retries
253
+ end
210
254
  end
211
255
  end
@@ -3,39 +3,13 @@
3
3
  require 'google/cloud/tasks'
4
4
 
5
5
  module Cloudtasker
6
- # Build, serialize and schedule tasks on GCP Cloud Task
7
- class Task
6
+ # Build, serialize and schedule tasks on the processing backend.
7
+ class WorkerHandler
8
8
  attr_reader :worker, :job_args
9
9
 
10
10
  # Alrogith used to sign the verification token
11
11
  JWT_ALG = 'HS256'
12
12
 
13
- # TODO: Move to a dedicated CloudTask class
14
- #
15
- # Find a Cloud task
16
- #
17
- # @param [String] id The ID of the task.
18
- #
19
- # @return [Google::Cloud::Tasks::V2beta3::Task] The cloud task.
20
- #
21
- def self.find(id)
22
- client.get_task(id)
23
- rescue Google::Gax::RetryError
24
- nil
25
- end
26
-
27
- # TODO: Move to a dedicated CloudTask class
28
- #
29
- # Delete a Cloud task
30
- #
31
- # @param [String] id The ID of the task.
32
- #
33
- def self.delete(id)
34
- client.delete_task(id)
35
- rescue Google::Gax::RetryError
36
- nil
37
- end
38
-
39
13
  #
40
14
  # Execute a task worker from a task payload
41
15
  #
@@ -48,15 +22,6 @@ module Cloudtasker
48
22
  worker.execute
49
23
  end
50
24
 
51
- #
52
- # Return the Google Cloud Task client.
53
- #
54
- # @return [Google::Cloud::Tasks] The Google Cloud Task client.
55
- #
56
- def self.client
57
- @client ||= ::Google::Cloud::Tasks.new(version: :v2beta3)
58
- end
59
-
60
25
  #
61
26
  # Prepare a new cloud task.
62
27
  #
@@ -66,37 +31,6 @@ module Cloudtasker
66
31
  @worker = worker
67
32
  end
68
33
 
69
- #
70
- # Return the Google Cloud Task client.
71
- #
72
- # @return [Google::Cloud::Tasks] The Google Cloud Task client.
73
- #
74
- def client
75
- self.class.client
76
- end
77
-
78
- #
79
- # Return the cloudtasker configuration. See Cloudtasker#configure.
80
- #
81
- # @return [Cloudtasker::Config] The library configuration.
82
- #
83
- def config
84
- Cloudtasker.config
85
- end
86
-
87
- #
88
- # Return the fully qualified path for the Cloud Task queue.
89
- #
90
- # @return [String] The queue path.
91
- #
92
- def queue_path
93
- client.queue_path(
94
- config.gcp_project_id,
95
- config.gcp_location_id,
96
- config.gcp_queue_id
97
- )
98
- end
99
-
100
34
  #
101
35
  # Return the full task configuration sent to Cloud Task
102
36
  #
@@ -106,7 +40,7 @@ module Cloudtasker
106
40
  {
107
41
  http_request: {
108
42
  http_method: 'POST',
109
- url: config.processor_url,
43
+ url: Cloudtasker.config.processor_url,
110
44
  headers: {
111
45
  'Content-Type' => 'application/json',
112
46
  'Authorization' => "Bearer #{Authenticator.verification_token}"
@@ -133,7 +67,7 @@ module Cloudtasker
133
67
  worker: worker.class.to_s,
134
68
  job_id: worker.job_id,
135
69
  job_args: worker.job_args,
136
- job_meta: worker.job_meta
70
+ job_meta: worker.job_meta.to_h
137
71
  }
138
72
  end
139
73
 
@@ -142,16 +76,15 @@ module Cloudtasker
142
76
  # before running a task.
143
77
  #
144
78
  # @param [Integer, nil] interval The time to wait.
79
+ # @param [Integer, nil] time_at The time at which the job should run.
145
80
  #
146
- # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
81
+ # @return [Integer, nil] The Unix timestamp.
147
82
  #
148
83
  def schedule_time(interval: nil, time_at: nil)
149
84
  return nil unless interval || time_at
150
85
 
151
- # Generate protobuf timestamp
152
- timestamp = Google::Protobuf::Timestamp.new
153
- timestamp.seconds = (time_at || Time.now).to_i + interval.to_i
154
- timestamp
86
+ # Generate the complete Unix timestamp
87
+ (time_at || Time.now).to_i + interval.to_i
155
88
  end
156
89
 
157
90
  #
@@ -160,7 +93,7 @@ module Cloudtasker
160
93
  # @param [Integer, nil] interval How to wait before running the task.
161
94
  # Leave to `nil` to run now.
162
95
  #
163
- # @return [Google::Cloud::Tasks::V2beta3::Task] The Google Task response
96
+ # @return [Cloudtasker::CloudTask] The Google Task response
164
97
  #
165
98
  def schedule(interval: nil, time_at: nil)
166
99
  # Generate task payload
@@ -169,7 +102,7 @@ module Cloudtasker
169
102
  ).compact
170
103
 
171
104
  # Create and return remote task
172
- client.create_task(queue_path, task)
105
+ CloudTask.create(task)
173
106
  end
174
107
  end
175
108
  end
@@ -0,0 +1,155 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Cloudtasker
4
+ # Add contextual information to logs generated
5
+ # by workers
6
+ class WorkerLogger
7
+ attr_accessor :worker
8
+
9
+ class << self
10
+ attr_accessor :log_context_processor
11
+ end
12
+
13
+ # Only log the job meta information by default (exclude arguments)
14
+ DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta) }
15
+
16
+ #
17
+ # Build a new instance of the class.
18
+ #
19
+ # @param [Cloudtasker::Worker] worker The worker.
20
+ #
21
+ def initialize(worker)
22
+ @worker = worker
23
+ end
24
+
25
+ #
26
+ # Return the Proc responsible for formatting the log payload.
27
+ #
28
+ # @return [Proc] The context processor.
29
+ #
30
+ def context_processor
31
+ @context_processor ||= worker.class.cloudtasker_options_hash[:log_context_processor] ||
32
+ self.class.log_context_processor ||
33
+ DEFAULT_CONTEXT_PROCESSOR
34
+ end
35
+
36
+ #
37
+ # The block to pass to log messages.
38
+ #
39
+ # @return [Proc] The log block.
40
+ #
41
+ def log_block
42
+ @log_block ||= proc { context_processor.call(worker) }
43
+ end
44
+
45
+ #
46
+ # Return the Cloudtasker logger.
47
+ #
48
+ # @return [Logger, any] The cloudtasker logger.
49
+ #
50
+ def logger
51
+ Cloudtasker.logger
52
+ end
53
+
54
+ #
55
+ # Format main log message.
56
+ #
57
+ # @param [String] msg The message to log.
58
+ #
59
+ # @return [String] The formatted log message
60
+ #
61
+ def formatted_message(msg)
62
+ "[Cloudtasker][#{worker.job_id}] #{msg}"
63
+ end
64
+
65
+ #
66
+ # Log an info message.
67
+ #
68
+ # @param [String] msg The message to log.
69
+ # @param [Proc] &block Optional context block.
70
+ #
71
+ def info(msg, &block)
72
+ log_message(:info, msg, &block)
73
+ end
74
+
75
+ #
76
+ # Log an error message.
77
+ #
78
+ # @param [String] msg The message to log.
79
+ # @param [Proc] &block Optional context block.
80
+ #
81
+ def error(msg, &block)
82
+ log_message(:error, msg, &block)
83
+ end
84
+
85
+ #
86
+ # Log an fatal message.
87
+ #
88
+ # @param [String] msg The message to log.
89
+ # @param [Proc] &block Optional context block.
90
+ #
91
+ def fatal(msg, &block)
92
+ log_message(:fatal, msg, &block)
93
+ end
94
+
95
+ #
96
+ # Log an debut message.
97
+ #
98
+ # @param [String] msg The message to log.
99
+ # @param [Proc] &block Optional context block.
100
+ #
101
+ def debug(msg, &block)
102
+ log_message(:debug, msg, &block)
103
+ end
104
+
105
+ #
106
+ # Delegate all methods to the underlying logger.
107
+ #
108
+ # @param [String, Symbol] name The method to delegate.
109
+ # @param [Array<any>] *args The list of method arguments.
110
+ # @param [Proc] &block Block passed to the method.
111
+ #
112
+ # @return [Any] The method return value
113
+ #
114
+ def method_missing(name, *args, &block)
115
+ if logger.respond_to?(name)
116
+ logger.send(name, *args, &block)
117
+ else
118
+ super
119
+ end
120
+ end
121
+
122
+ #
123
+ # Check if the class respond to a certain method.
124
+ #
125
+ # @param [String, Symbol] name The name of the method.
126
+ # @param [Boolean] include_private Whether to check private methods or not. Default to false.
127
+ #
128
+ # @return [Boolean] Return true if the class respond to this method.
129
+ #
130
+ def respond_to_missing?(name, include_private = false)
131
+ logger.respond_to?(name) || super
132
+ end
133
+
134
+ private
135
+
136
+ #
137
+ # Log a message for the provided log level.
138
+ #
139
+ # @param [String, Symbol] level The log level
140
+ # @param [String] msg The message to log.
141
+ # @param [Proc] &block Optional context block.
142
+ #
143
+ def log_message(level, msg, &block)
144
+ payload_block = block || log_block
145
+
146
+ # ActiveSupport::Logger does not support passing a payload through a block on top
147
+ # of a message.
148
+ if defined?(ActiveSupport::Logger) && logger.is_a?(ActiveSupport::Logger)
149
+ logger.send(level) { "#{formatted_message(msg)} -- #{payload_block.call}" }
150
+ else
151
+ logger.send(level, formatted_message(msg), &payload_block)
152
+ end
153
+ end
154
+ end
155
+ end