cloudtasker 0.11.rc1 → 0.12.rc2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +12 -1
- data/CHANGELOG.md +22 -0
- data/README.md +94 -0
- data/app/controllers/cloudtasker/worker_controller.rb +4 -17
- data/cloudtasker.gemspec +1 -0
- data/docs/BATCH_JOBS.md +1 -1
- data/lib/active_job/queue_adapters/cloudtasker_adapter.rb +82 -0
- data/lib/cloudtasker/backend/redis_task.rb +17 -9
- data/lib/cloudtasker/batch/job.rb +19 -6
- data/lib/cloudtasker/batch/middleware.rb +2 -0
- data/lib/cloudtasker/cli.rb +1 -0
- data/lib/cloudtasker/cron/schedule.rb +17 -8
- data/lib/cloudtasker/engine.rb +5 -1
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +45 -7
- data/lib/cloudtasker/worker_handler.rb +26 -0
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11c4d8d88b554792e888929324e79521426f5484973253a44c50a97849a5842c
|
4
|
+
data.tar.gz: 1f10788d0ced509c82eea06ddfbd0b1d47a5731507c8bac5c597f41cc13b2d98
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: abe6437f179edd589afb88ed970c7870377c836446ba6b4169ffd60565be15fcf92557dd71965795579f58ba19af63a9225ad55cf8c97bb8ab5612e31d5c8bdf
|
7
|
+
data.tar.gz: 23d17b1fade5f62594250b239b07e603bc8d8f0862465dd28ec3efc6b260cb86774e88d3b9aead4704eff1d071a3dda2f35d9d68302c6dcef56a1d8b707caffe
|
data/.rubocop.yml
CHANGED
@@ -13,6 +13,8 @@ Metrics/ModuleLength:
|
|
13
13
|
|
14
14
|
Metrics/AbcSize:
|
15
15
|
Max: 20
|
16
|
+
Exclude:
|
17
|
+
- 'spec/support/*'
|
16
18
|
|
17
19
|
Metrics/LineLength:
|
18
20
|
Max: 120
|
@@ -20,6 +22,10 @@ Metrics/LineLength:
|
|
20
22
|
Metrics/MethodLength:
|
21
23
|
Max: 20
|
22
24
|
|
25
|
+
RSpec/DescribeClass:
|
26
|
+
Exclude:
|
27
|
+
- 'spec/integration/**/*_spec.rb'
|
28
|
+
|
23
29
|
RSpec/ExpectInHook:
|
24
30
|
Enabled: false
|
25
31
|
|
@@ -43,4 +49,9 @@ Metrics/ParameterLists:
|
|
43
49
|
CountKeywordArgs: false
|
44
50
|
|
45
51
|
RSpec/MessageSpies:
|
46
|
-
Enabled: false
|
52
|
+
Enabled: false
|
53
|
+
|
54
|
+
RSpec/MultipleExpectations:
|
55
|
+
Exclude:
|
56
|
+
- 'examples/**/*'
|
57
|
+
- 'spec/integration/**/*'
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,27 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## Latest RC [v0.12.rc1](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc1) (2021-03-11)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc1)
|
6
|
+
|
7
|
+
**Improvements:**
|
8
|
+
- ActiveJob: do not double log errors (ActiveJob has its own error logging)
|
9
|
+
- Error logging: Use worker logger so as to include context (job args etc.)
|
10
|
+
- Error logging: Do not log exception and stack trace separately, combine them instead.
|
11
|
+
- Batch callbacks: Retry jobs when completion callback fails
|
12
|
+
- Redis: Use Redis Sets instead of key pattern matching for listing methods (Cron jobs and Local Server)
|
13
|
+
|
14
|
+
**Fixed bugs:**
|
15
|
+
- Retries: Enforce job retry limit on job processing. There was an edge case where jobs could be retried indefinitely on batch callback errors.
|
16
|
+
|
17
|
+
## [v0.11.0](https://github.com/keypup-io/cloudtasker/tree/v0.11.0) (2021-03-11)
|
18
|
+
|
19
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.11.0)
|
20
|
+
|
21
|
+
**Improvements:**
|
22
|
+
- Worker: drop job (return 205 response) when worker arguments are not available (e.g. arguments were stored in Redis and the latter was flushed)
|
23
|
+
- Rails: add ActiveJob adapter (thanks @vovimayhem)
|
24
|
+
|
3
25
|
## [v0.10.1](https://github.com/keypup-io/cloudtasker/tree/v0.10.1) (2020-10-05)
|
4
26
|
|
5
27
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.10.1)
|
data/README.md
CHANGED
@@ -12,10 +12,13 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
|
|
12
12
|
|
13
13
|
A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
|
14
14
|
|
15
|
+
**Maturity**: This gem is production-ready. We at Keypup have already processed millions of jobs using Cloudtasker and all related extensions (cron, batch and unique jobs). We are planning to release the official `v1.0.0` somewhere in 2021, in case we've missed any edge-case bug.
|
16
|
+
|
15
17
|
## Summary
|
16
18
|
|
17
19
|
1. [Installation](#installation)
|
18
20
|
2. [Get started with Rails](#get-started-with-rails)
|
21
|
+
2. [Get started with Rails & ActiveJob](#get-started-with-rails--activejob)
|
19
22
|
3. [Configuring Cloudtasker](#configuring-cloudtasker)
|
20
23
|
1. [Cloud Tasks authentication & permissions](#cloud-tasks-authentication--permissions)
|
21
24
|
2. [Cloudtasker initializer](#cloudtasker-initializer)
|
@@ -132,6 +135,95 @@ That's it! Your job was picked up by the Cloudtasker local server and sent for p
|
|
132
135
|
|
133
136
|
Now jump to the next section to configure your app to use Google Cloud Tasks as a backend.
|
134
137
|
|
138
|
+
## Get started with Rails & ActiveJob
|
139
|
+
**Note**: ActiveJob is supported since `0.11.0`
|
140
|
+
**Note**: Cloudtasker extensions (cron, batch and unique jobs) are not available when using cloudtasker via ActiveJob.
|
141
|
+
|
142
|
+
Cloudtasker is pre-integrated with ActiveJob. Follow the steps below to get started.
|
143
|
+
|
144
|
+
Install redis on your machine (this is required by the Cloudtasker local processing server)
|
145
|
+
```bash
|
146
|
+
# E.g. using brew
|
147
|
+
brew install redis
|
148
|
+
```
|
149
|
+
|
150
|
+
Add the following initializer
|
151
|
+
```ruby
|
152
|
+
# config/initializers/cloudtasker.rb
|
153
|
+
|
154
|
+
Cloudtasker.configure do |config|
|
155
|
+
#
|
156
|
+
# Adapt the server port to be the one used by your Rails web process
|
157
|
+
#
|
158
|
+
config.processor_host = 'http://localhost:3000'
|
159
|
+
|
160
|
+
#
|
161
|
+
# If you do not have any Rails secret_key_base defined, uncomment the following
|
162
|
+
# This secret is used to authenticate jobs sent to the processing endpoint
|
163
|
+
# of your application.
|
164
|
+
#
|
165
|
+
# config.secret = 'some-long-token'
|
166
|
+
end
|
167
|
+
```
|
168
|
+
|
169
|
+
Configure ActiveJob to use Cloudtasker. You can also configure ActiveJob per environment via the config/environments/:env.rb files
|
170
|
+
```ruby
|
171
|
+
# config/application.rb
|
172
|
+
|
173
|
+
require_relative 'boot'
|
174
|
+
require 'rails/all'
|
175
|
+
|
176
|
+
Bundler.require(*Rails.groups)
|
177
|
+
|
178
|
+
module Dummy
|
179
|
+
class Application < Rails::Application
|
180
|
+
# Initialize configuration defaults for originally generated Rails version.
|
181
|
+
config.load_defaults 6.0
|
182
|
+
|
183
|
+
# Settings in config/environments/* take precedence over those specified here.
|
184
|
+
# Application configuration can go into files in config/initializers
|
185
|
+
# -- all .rb files in that directory are automatically loaded after loading
|
186
|
+
# the framework and any gems in your application.
|
187
|
+
|
188
|
+
# Use cloudtasker as the ActiveJob backend:
|
189
|
+
config.active_job.queue_adapter = :cloudtasker
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
```
|
194
|
+
|
195
|
+
Define your first job:
|
196
|
+
```ruby
|
197
|
+
# app/jobs/example_job.rb
|
198
|
+
|
199
|
+
class ExampleJob < ApplicationJob
|
200
|
+
queue_as :default
|
201
|
+
|
202
|
+
def perform(some_arg)
|
203
|
+
logger.info("Job run with #{some_arg}. This is working!")
|
204
|
+
end
|
205
|
+
end
|
206
|
+
```
|
207
|
+
|
208
|
+
Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` to your foreman config as a `worker` process)
|
209
|
+
```bash
|
210
|
+
# In one terminal
|
211
|
+
> rails s -p 3000
|
212
|
+
|
213
|
+
# In another terminal
|
214
|
+
> cloudtasker
|
215
|
+
```
|
216
|
+
|
217
|
+
Open a Rails console and enqueue some jobs
|
218
|
+
```ruby
|
219
|
+
# Process job as soon as possible
|
220
|
+
ExampleJob.perform_later('foo')
|
221
|
+
|
222
|
+
# Process job in 60 seconds
|
223
|
+
ExampleJob.set(wait: 60).perform_later('foo')
|
224
|
+
```
|
225
|
+
|
226
|
+
|
135
227
|
## Configuring Cloudtasker
|
136
228
|
|
137
229
|
### Cloud Tasks authentication & permissions
|
@@ -361,6 +453,8 @@ CriticalWorker.schedule(args: [1], queue: :important)
|
|
361
453
|
```
|
362
454
|
|
363
455
|
## Extensions
|
456
|
+
**Note**: Extensions are not available when using cloudtasker via ActiveJob.
|
457
|
+
|
364
458
|
Cloudtasker comes with three optional features:
|
365
459
|
- Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
|
366
460
|
- Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
|
@@ -19,32 +19,19 @@ module Cloudtasker
|
|
19
19
|
# Process payload
|
20
20
|
WorkerHandler.execute_from_payload!(payload)
|
21
21
|
head :no_content
|
22
|
-
rescue DeadWorkerError, MissingWorkerArgumentsError
|
22
|
+
rescue DeadWorkerError, MissingWorkerArgumentsError
|
23
23
|
# 205: job will NOT be retried
|
24
|
-
log_error(e)
|
25
24
|
head :reset_content
|
26
|
-
rescue InvalidWorkerError
|
25
|
+
rescue InvalidWorkerError
|
27
26
|
# 404: Job will be retried
|
28
|
-
log_error(e)
|
29
27
|
head :not_found
|
30
|
-
rescue StandardError
|
31
|
-
#
|
32
|
-
log_error(e)
|
28
|
+
rescue StandardError
|
29
|
+
# 422: Job will be retried
|
33
30
|
head :unprocessable_entity
|
34
31
|
end
|
35
32
|
|
36
33
|
private
|
37
34
|
|
38
|
-
#
|
39
|
-
# Log an error via cloudtasker logger.
|
40
|
-
#
|
41
|
-
# @param [Exception] e The error to log
|
42
|
-
#
|
43
|
-
def log_error(error)
|
44
|
-
Cloudtasker.logger.error(error)
|
45
|
-
Cloudtasker.logger.error(error.backtrace.join("\n"))
|
46
|
-
end
|
47
|
-
|
48
35
|
#
|
49
36
|
# Parse the request body and return the actual job
|
50
37
|
# payload.
|
data/cloudtasker.gemspec
CHANGED
@@ -41,6 +41,7 @@ Gem::Specification.new do |spec|
|
|
41
41
|
spec.add_development_dependency 'github_changelog_generator'
|
42
42
|
spec.add_development_dependency 'rake', '>= 12.3.3'
|
43
43
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
44
|
+
spec.add_development_dependency 'rspec-json_expectations', '~> 2.2'
|
44
45
|
spec.add_development_dependency 'rubocop', '0.76.0'
|
45
46
|
spec.add_development_dependency 'rubocop-rspec', '1.37.0'
|
46
47
|
spec.add_development_dependency 'semantic_logger'
|
data/docs/BATCH_JOBS.md
CHANGED
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# ActiveJob docs: http://guides.rubyonrails.org/active_job_basics.html
|
4
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
5
|
+
|
6
|
+
module ActiveJob
|
7
|
+
module QueueAdapters
|
8
|
+
# == Cloudtasker adapter for Active Job
|
9
|
+
#
|
10
|
+
# To use Cloudtasker set the queue_adapter config to +:cloudtasker+.
|
11
|
+
#
|
12
|
+
# Rails.application.config.active_job.queue_adapter = :cloudtasker
|
13
|
+
class CloudtaskerAdapter
|
14
|
+
SERIALIZATION_FILTERED_KEYS = [
|
15
|
+
'executions', # Given by the worker at processing
|
16
|
+
'provider_job_id', # Also given by the worker at processing
|
17
|
+
'priority' # Not used
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
# Enqueues the given ActiveJob instance for execution
|
21
|
+
#
|
22
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
23
|
+
#
|
24
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
25
|
+
#
|
26
|
+
def enqueue(job)
|
27
|
+
build_worker(job).schedule
|
28
|
+
end
|
29
|
+
|
30
|
+
# Enqueues the given ActiveJob instance for execution at a given time
|
31
|
+
#
|
32
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
33
|
+
# @param precise_timestamp [Integer] The timestamp at which the job must be executed
|
34
|
+
#
|
35
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
36
|
+
#
|
37
|
+
def enqueue_at(job, precise_timestamp)
|
38
|
+
build_worker(job).schedule(time_at: Time.at(precise_timestamp))
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def build_worker(job)
|
44
|
+
job_serialization = job.serialize.except(*SERIALIZATION_FILTERED_KEYS)
|
45
|
+
|
46
|
+
JobWrapper.new(
|
47
|
+
job_id: job_serialization.delete('job_id'),
|
48
|
+
job_queue: job_serialization.delete('queue_name'),
|
49
|
+
job_args: [job_serialization]
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
# == Job Wrapper for the Cloudtasker adapter
|
54
|
+
#
|
55
|
+
# Executes jobs scheduled by the Cloudtasker ActiveJob adapter
|
56
|
+
class JobWrapper #:nodoc:
|
57
|
+
include Cloudtasker::Worker
|
58
|
+
|
59
|
+
# Executes the given serialized ActiveJob call.
|
60
|
+
# - See https://api.rubyonrails.org/classes/ActiveJob/Core.html#method-i-serialize
|
61
|
+
#
|
62
|
+
# @param [Hash] job_serialization The serialized ActiveJob call
|
63
|
+
#
|
64
|
+
# @return [any] The execution of the ActiveJob call
|
65
|
+
#
|
66
|
+
def perform(job_serialization, *_extra_options)
|
67
|
+
job_executions = job_retries < 1 ? 0 : (job_retries + 1)
|
68
|
+
|
69
|
+
job_serialization.merge!(
|
70
|
+
'job_id' => job_id,
|
71
|
+
'queue_name' => job_queue,
|
72
|
+
'provider_job_id' => task_id,
|
73
|
+
'executions' => job_executions,
|
74
|
+
'priority' => nil
|
75
|
+
)
|
76
|
+
|
77
|
+
Base.execute job_serialization
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -23,14 +23,12 @@ module Cloudtasker
|
|
23
23
|
#
|
24
24
|
# Return a namespaced key.
|
25
25
|
#
|
26
|
-
# @param [String, Symbol] val The key to namespace
|
26
|
+
# @param [String, Symbol, nil] val The key to namespace
|
27
27
|
#
|
28
28
|
# @return [String] The namespaced key.
|
29
29
|
#
|
30
|
-
def self.key(val)
|
31
|
-
|
32
|
-
|
33
|
-
[to_s.underscore, val.to_s].join('/')
|
30
|
+
def self.key(val = nil)
|
31
|
+
[to_s.underscore, val].compact.map(&:to_s).join('/')
|
34
32
|
end
|
35
33
|
|
36
34
|
#
|
@@ -39,9 +37,17 @@ module Cloudtasker
|
|
39
37
|
# @return [Array<Cloudtasker::Backend::RedisTask>] All the tasks.
|
40
38
|
#
|
41
39
|
def self.all
|
42
|
-
redis.
|
43
|
-
|
44
|
-
|
40
|
+
if redis.exists?(key)
|
41
|
+
# Use Schedule Set if available
|
42
|
+
redis.smembers(key).map { |id| find(id) }
|
43
|
+
else
|
44
|
+
# Fallback to redis key matching and migrate tasks
|
45
|
+
# to use Task Set instead.
|
46
|
+
redis.search(key('*')).map do |gid|
|
47
|
+
task_id = gid.sub(key(''), '')
|
48
|
+
redis.sadd(key, task_id)
|
49
|
+
find(task_id)
|
50
|
+
end
|
45
51
|
end
|
46
52
|
end
|
47
53
|
|
@@ -82,6 +88,7 @@ module Cloudtasker
|
|
82
88
|
|
83
89
|
# Save job
|
84
90
|
redis.write(key(id), payload)
|
91
|
+
redis.sadd(key, id)
|
85
92
|
new(payload.merge(id: id))
|
86
93
|
end
|
87
94
|
|
@@ -105,6 +112,7 @@ module Cloudtasker
|
|
105
112
|
# @param [String] id The task id.
|
106
113
|
#
|
107
114
|
def self.delete(id)
|
115
|
+
redis.srem(key, id)
|
108
116
|
redis.del(key(id))
|
109
117
|
end
|
110
118
|
|
@@ -176,7 +184,7 @@ module Cloudtasker
|
|
176
184
|
# Remove the task from the queue.
|
177
185
|
#
|
178
186
|
def destroy
|
179
|
-
|
187
|
+
self.class.delete(id)
|
180
188
|
end
|
181
189
|
|
182
190
|
#
|
@@ -13,6 +13,10 @@ module Cloudtasker
|
|
13
13
|
# List of statuses triggering a completion callback
|
14
14
|
COMPLETION_STATUSES = %w[completed dead].freeze
|
15
15
|
|
16
|
+
# These callbacks do not need to raise errors on their own
|
17
|
+
# because the jobs will be either retried or dropped
|
18
|
+
IGNORED_ERRORED_CALLBACKS = %i[on_child_error on_child_dead].freeze
|
19
|
+
|
16
20
|
#
|
17
21
|
# Return the cloudtasker redis client
|
18
22
|
#
|
@@ -62,6 +66,10 @@ module Cloudtasker
|
|
62
66
|
# @return [Cloudtasker::Batch::Job] The attached batch.
|
63
67
|
#
|
64
68
|
def self.for(worker)
|
69
|
+
# Load extension if not loaded already on the worker class
|
70
|
+
worker.class.include(Extension::Worker) unless worker.class <= Extension::Worker
|
71
|
+
|
72
|
+
# Add batch capability
|
65
73
|
worker.batch = new(worker)
|
66
74
|
end
|
67
75
|
|
@@ -246,8 +254,8 @@ module Cloudtasker
|
|
246
254
|
end
|
247
255
|
|
248
256
|
#
|
249
|
-
# Run worker callback
|
250
|
-
#
|
257
|
+
# Run worker callback. The error and dead callbacks get
|
258
|
+
# silenced should they raise an error.
|
251
259
|
#
|
252
260
|
# @param [String, Symbol] callback The callback to run.
|
253
261
|
# @param [Array<any>] *args The callback arguments.
|
@@ -257,9 +265,14 @@ module Cloudtasker
|
|
257
265
|
def run_worker_callback(callback, *args)
|
258
266
|
worker.try(callback, *args)
|
259
267
|
rescue StandardError => e
|
260
|
-
|
261
|
-
|
262
|
-
|
268
|
+
# There is no point in retrying jobs due to failure callbacks failing
|
269
|
+
# Only completion callbacks will trigger a re-run of the job because
|
270
|
+
# these do matter for batch completion
|
271
|
+
raise(e) unless IGNORED_ERRORED_CALLBACKS.include?(callback)
|
272
|
+
|
273
|
+
# Log error instead
|
274
|
+
worker.logger.error(e)
|
275
|
+
worker.logger.error("Callback #{callback} failed to run. Skipping to preserve error flow.")
|
263
276
|
end
|
264
277
|
|
265
278
|
#
|
@@ -271,7 +284,7 @@ module Cloudtasker
|
|
271
284
|
|
272
285
|
# Propagate event
|
273
286
|
parent_batch&.on_child_complete(self, status)
|
274
|
-
|
287
|
+
|
275
288
|
# The batch tree is complete. Cleanup the tree.
|
276
289
|
cleanup unless parent_batch
|
277
290
|
end
|
data/lib/cloudtasker/cli.rb
CHANGED
@@ -21,14 +21,12 @@ module Cloudtasker
|
|
21
21
|
#
|
22
22
|
# Return a namespaced key.
|
23
23
|
#
|
24
|
-
# @param [String, Symbol] val The key to namespace
|
24
|
+
# @param [String, Symbol, nil] val The key to namespace
|
25
25
|
#
|
26
26
|
# @return [String] The namespaced key.
|
27
27
|
#
|
28
|
-
def self.key(val)
|
29
|
-
|
30
|
-
|
31
|
-
[to_s.underscore, val.to_s].join('/')
|
28
|
+
def self.key(val = nil)
|
29
|
+
[to_s.underscore, val].compact.map(&:to_s).join('/')
|
32
30
|
end
|
33
31
|
|
34
32
|
#
|
@@ -37,8 +35,17 @@ module Cloudtasker
|
|
37
35
|
# @return [Array<Cloudtasker::Batch::Schedule>] The list of stored schedules.
|
38
36
|
#
|
39
37
|
def self.all
|
40
|
-
redis.
|
41
|
-
|
38
|
+
if redis.exists?(key)
|
39
|
+
# Use Schedule Set if available
|
40
|
+
redis.smembers(key).map { |id| find(id) }
|
41
|
+
else
|
42
|
+
# Fallback to redis key matching and migrate schedules
|
43
|
+
# to use Schedule Set instead.
|
44
|
+
redis.search(key('*')).map do |gid|
|
45
|
+
schedule_id = gid.sub(key(''), '')
|
46
|
+
redis.sadd(key, schedule_id)
|
47
|
+
find(schedule_id)
|
48
|
+
end
|
42
49
|
end
|
43
50
|
end
|
44
51
|
|
@@ -90,7 +97,7 @@ module Cloudtasker
|
|
90
97
|
end
|
91
98
|
|
92
99
|
#
|
93
|
-
#
|
100
|
+
# Delete a schedule by id.
|
94
101
|
#
|
95
102
|
# @param [String] id The schedule id.
|
96
103
|
#
|
@@ -101,6 +108,7 @@ module Cloudtasker
|
|
101
108
|
|
102
109
|
# Delete task and stored schedule
|
103
110
|
CloudTask.delete(schedule.task_id) if schedule.task_id
|
111
|
+
redis.srem(key, schedule.id)
|
104
112
|
redis.del(schedule.gid)
|
105
113
|
end
|
106
114
|
end
|
@@ -270,6 +278,7 @@ module Cloudtasker
|
|
270
278
|
|
271
279
|
# Save schedule
|
272
280
|
config_was_changed = config_changed?
|
281
|
+
redis.sadd(self.class.key, id)
|
273
282
|
redis.write(gid, to_h)
|
274
283
|
|
275
284
|
# Stop there if backend does not need update
|
data/lib/cloudtasker/engine.rb
CHANGED
@@ -5,10 +5,14 @@ module Cloudtasker
|
|
5
5
|
class Engine < ::Rails::Engine
|
6
6
|
isolate_namespace Cloudtasker
|
7
7
|
|
8
|
-
|
8
|
+
config.before_initialize do
|
9
|
+
# Mount cloudtasker processing endpoint
|
9
10
|
Rails.application.routes.append do
|
10
11
|
mount Cloudtasker::Engine, at: '/cloudtasker'
|
11
12
|
end
|
13
|
+
|
14
|
+
# Add ActiveJob adapter
|
15
|
+
require 'active_job/queue_adapters/cloudtasker_adapter' if defined?(::ActiveJob::Railtie)
|
12
16
|
end
|
13
17
|
|
14
18
|
config.generators do |g|
|
data/lib/cloudtasker/version.rb
CHANGED
data/lib/cloudtasker/worker.rb
CHANGED
@@ -311,13 +311,25 @@ module Cloudtasker
|
|
311
311
|
end
|
312
312
|
|
313
313
|
#
|
314
|
-
# Return true if the job
|
315
|
-
#
|
314
|
+
# Return true if the job must declared dead upon raising
|
315
|
+
# an error.
|
316
|
+
#
|
317
|
+
# @return [Boolean] True if the job must die on error.
|
318
|
+
#
|
319
|
+
def job_must_die?
|
320
|
+
job_retries >= job_max_retries
|
321
|
+
end
|
322
|
+
|
323
|
+
#
|
324
|
+
# Return true if the job has strictly excceeded its maximum number
|
325
|
+
# of retries.
|
326
|
+
#
|
327
|
+
# Used a preemptive filter when running the job.
|
316
328
|
#
|
317
329
|
# @return [Boolean] True if the job is dead
|
318
330
|
#
|
319
331
|
def job_dead?
|
320
|
-
job_retries
|
332
|
+
job_retries > job_max_retries
|
321
333
|
end
|
322
334
|
|
323
335
|
#
|
@@ -332,11 +344,35 @@ module Cloudtasker
|
|
332
344
|
(perform_ended_at - perform_started_at).ceil(3)
|
333
345
|
end
|
334
346
|
|
347
|
+
#
|
348
|
+
# Run worker callback.
|
349
|
+
#
|
350
|
+
# @param [String, Symbol] callback The callback to run.
|
351
|
+
# @param [Array<any>] *args The callback arguments.
|
352
|
+
#
|
353
|
+
# @return [any] The callback return value
|
354
|
+
#
|
355
|
+
def run_callback(callback, *args)
|
356
|
+
try(callback, *args)
|
357
|
+
end
|
358
|
+
|
335
359
|
#=============================
|
336
360
|
# Private
|
337
361
|
#=============================
|
338
362
|
private
|
339
363
|
|
364
|
+
#
|
365
|
+
# Flag the worker as dead by invoking the on_dead hook
|
366
|
+
# and raising a DeadWorkerError
|
367
|
+
#
|
368
|
+
# @param [Exception, nil] error An optional exception to be passed to the DeadWorkerError.
|
369
|
+
#
|
370
|
+
def flag_as_dead(error = nil)
|
371
|
+
run_callback(:on_dead, error || DeadWorkerError.new)
|
372
|
+
ensure
|
373
|
+
raise(DeadWorkerError, error)
|
374
|
+
end
|
375
|
+
|
340
376
|
#
|
341
377
|
# Execute the worker perform method through the middleware chain.
|
342
378
|
#
|
@@ -346,6 +382,9 @@ module Cloudtasker
|
|
346
382
|
self.perform_started_at = Time.now
|
347
383
|
|
348
384
|
Cloudtasker.config.server_middleware.invoke(self) do
|
385
|
+
# Immediately abort the job if it is already dead
|
386
|
+
flag_as_dead if job_dead?
|
387
|
+
|
349
388
|
begin
|
350
389
|
# Abort if arguments are missing. This may happen with redis arguments storage
|
351
390
|
# if Cloud Tasks times out on a job but the job still succeeds
|
@@ -356,12 +395,11 @@ module Cloudtasker
|
|
356
395
|
# Perform the job
|
357
396
|
perform(*job_args)
|
358
397
|
rescue StandardError => e
|
359
|
-
|
360
|
-
return raise(e) unless
|
398
|
+
run_callback(:on_error, e)
|
399
|
+
return raise(e) unless job_must_die?
|
361
400
|
|
362
401
|
# Flag job as dead
|
363
|
-
|
364
|
-
raise(DeadWorkerError, e)
|
402
|
+
flag_as_dead(e)
|
365
403
|
end
|
366
404
|
end
|
367
405
|
ensure
|
@@ -45,6 +45,28 @@ module Cloudtasker
|
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
|
+
#
|
49
|
+
# Log error on execution failure.
|
50
|
+
#
|
51
|
+
# @param [Cloudtasker::Worker, nil] worker The worker.
|
52
|
+
# @param [Exception] error The error to log.
|
53
|
+
#
|
54
|
+
# @void
|
55
|
+
#
|
56
|
+
def self.log_execution_error(worker, error)
|
57
|
+
# ActiveJob has its own error logging. No need to double log the error.
|
58
|
+
# Note: we use string matching instead of class matching as
|
59
|
+
# ActiveJob::QueueAdapters::CloudtaskerAdapter::JobWrapper might not be loaded
|
60
|
+
return if worker.class.to_s =~ /^ActiveJob::/
|
61
|
+
|
62
|
+
# Choose logger to use based on context
|
63
|
+
# Worker will be nil on InvalidWorkerError - in that case we use generic logging
|
64
|
+
logger = worker&.logger || Cloudtasker.logger
|
65
|
+
|
66
|
+
# Log error
|
67
|
+
logger.error(error)
|
68
|
+
end
|
69
|
+
|
48
70
|
#
|
49
71
|
# Execute a task worker from a task payload
|
50
72
|
#
|
@@ -88,6 +110,10 @@ module Cloudtasker
|
|
88
110
|
rescue DeadWorkerError, MissingWorkerArgumentsError => e
|
89
111
|
# Delete stored args payload if job is dead
|
90
112
|
redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
|
113
|
+
log_execution_error(worker, e)
|
114
|
+
raise(e)
|
115
|
+
rescue StandardError => e
|
116
|
+
log_execution_error(worker, e)
|
91
117
|
raise(e)
|
92
118
|
end
|
93
119
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudtasker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.12.rc2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arnaud Lachaume
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -178,6 +178,20 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '3.0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec-json_expectations
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '2.2'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '2.2'
|
181
195
|
- !ruby/object:Gem::Dependency
|
182
196
|
name: rubocop
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
@@ -343,6 +357,7 @@ files:
|
|
343
357
|
- gemfiles/semantic_logger_4.7.0.gemfile
|
344
358
|
- gemfiles/semantic_logger_4.7.2.gemfile
|
345
359
|
- gemfiles/semantic_logger_4.7.gemfile
|
360
|
+
- lib/active_job/queue_adapters/cloudtasker_adapter.rb
|
346
361
|
- lib/cloudtasker.rb
|
347
362
|
- lib/cloudtasker/authentication_error.rb
|
348
363
|
- lib/cloudtasker/authenticator.rb
|