cloudtasker 0.10.rc8 → 0.11.rc3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +7 -3
- data/.rubocop.yml +4 -0
- data/Appraisals +16 -0
- data/CHANGELOG.md +49 -4
- data/README.md +148 -33
- data/app/controllers/cloudtasker/worker_controller.rb +15 -4
- data/cloudtasker.gemspec +2 -0
- data/gemfiles/semantic_logger_3.4.gemfile +7 -0
- data/gemfiles/semantic_logger_4.6.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.0.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.2.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.gemfile +7 -0
- data/lib/active_job/queue_adapters/cloudtasker_adapter.rb +82 -0
- data/lib/cloudtasker.rb +1 -0
- data/lib/cloudtasker/backend/memory_task.rb +3 -0
- data/lib/cloudtasker/batch/job.rb +4 -0
- data/lib/cloudtasker/batch/middleware.rb +2 -0
- data/lib/cloudtasker/cli.rb +1 -0
- data/lib/cloudtasker/engine.rb +5 -1
- data/lib/cloudtasker/missing_worker_arguments_error.rb +6 -0
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +22 -1
- data/lib/cloudtasker/worker_handler.rb +1 -1
- data/lib/cloudtasker/worker_logger.rb +1 -1
- metadata +37 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 60e17f257b39bf8d5651792bf7f4d27a6aa22307694881a3ee002d8228f719d7
|
4
|
+
data.tar.gz: ac58dfb7f5b65f2c76f437d5baa25b10ef8f3a7aa6aa95c3e1a815e10391fa80
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a3ed460f1298b3d3d70ef6bf761b2430a572c9dd50b4dc1f7f9a3cb8efed45aada429710da8b422e1a5d8a36c5a621f483f4401bf2f829674a94687c9e042334
|
7
|
+
data.tar.gz: d4d44ddee20abacedd2a5d06ed53d73af052419c73396706d822c5a70561ec4038e4817b0f6520024de33b869f92ded70b019b63d31a834f94a67114482f9abb
|
data/.github/workflows/test.yml
CHANGED
@@ -2,9 +2,9 @@ name: Test
|
|
2
2
|
|
3
3
|
on:
|
4
4
|
push:
|
5
|
-
branches: [ master ]
|
5
|
+
branches: [ master, 0.9-stable ]
|
6
6
|
pull_request:
|
7
|
-
branches: [ master ]
|
7
|
+
branches: [ master, 0.9-stable ]
|
8
8
|
|
9
9
|
jobs:
|
10
10
|
build:
|
@@ -21,6 +21,10 @@ jobs:
|
|
21
21
|
- 'google-cloud-tasks-1.3'
|
22
22
|
- 'rails-5.2'
|
23
23
|
- 'rails-6.0'
|
24
|
+
- 'semantic_logger-3.4'
|
25
|
+
- 'semantic_logger-4.6'
|
26
|
+
- 'semantic_logger-4.7.0'
|
27
|
+
- 'semantic_logger-4.7.2'
|
24
28
|
steps:
|
25
29
|
- name: Setup System
|
26
30
|
run: sudo apt-get install libsqlite3-dev
|
@@ -38,4 +42,4 @@ jobs:
|
|
38
42
|
bundle install --jobs 4 --retry 3
|
39
43
|
bundle exec rubocop
|
40
44
|
bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
|
41
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
|
45
|
+
bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
|
data/.rubocop.yml
CHANGED
data/Appraisals
CHANGED
@@ -23,3 +23,19 @@ end
|
|
23
23
|
appraise 'rails-6.0' do
|
24
24
|
gem 'rails', '6.0'
|
25
25
|
end
|
26
|
+
|
27
|
+
appraise 'semantic_logger-3.4' do
|
28
|
+
gem 'semantic_logger', '3.4.1'
|
29
|
+
end
|
30
|
+
|
31
|
+
appraise 'semantic_logger-4.6' do
|
32
|
+
gem 'semantic_logger', '4.6.1'
|
33
|
+
end
|
34
|
+
|
35
|
+
appraise 'semantic_logger-4.7.0' do
|
36
|
+
gem 'semantic_logger', '4.7.0'
|
37
|
+
end
|
38
|
+
|
39
|
+
appraise 'semantic_logger-4.7.2' do
|
40
|
+
gem 'semantic_logger', '4.7.2'
|
41
|
+
end
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,54 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## Latest RC: [v0.11.rc2](https://github.com/keypup-io/cloudtasker/tree/v0.11.rc2) (2020-11-23)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.11.rc2)
|
6
|
+
|
7
|
+
**Improvements:**
|
8
|
+
- Worker: drop job (return 205 response) when worker arguments are not available (e.g. arguments were stored in Redis and the latter was flushed)
|
9
|
+
- Rails: add ActiveJob adapter (thanks @vovimayhem)
|
10
|
+
|
11
|
+
## [v0.10.1](https://github.com/keypup-io/cloudtasker/tree/v0.10.1) (2020-10-05)
|
12
|
+
|
13
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.10.1)
|
14
|
+
|
15
|
+
**Fixed bugs:**
|
16
|
+
- Local server: delete dead task from local server queue
|
17
|
+
- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
|
18
|
+
- Worker: fix configuration of `max_retries` at worker level
|
19
|
+
|
20
|
+
## [v0.10.0](https://github.com/keypup-io/cloudtasker/tree/v0.10.0) (2020-09-02)
|
21
|
+
|
22
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.10.0)
|
23
|
+
|
24
|
+
**Improvements:**
|
25
|
+
- Logging: Add worker name in log messages
|
26
|
+
- Logging: Add job duration in log messages
|
27
|
+
- Logging: Add Cloud Cloud Task ID in log messages
|
28
|
+
- Unique Job: Support TTL for lock keys. This feature prevents queues from being dead-locked when a critical crash occurs while processing a unique job.
|
29
|
+
- Worker: support payload storage in Redis instead of sending the payload to Google Cloud Tasks. This is useful when job arguments are expected to exceed 100kb, which is the limit set by Google Cloud Tasks
|
30
|
+
|
31
|
+
**Fixed bugs:**
|
32
|
+
- Local processing error: improve error handling and retries around network interruptions
|
33
|
+
- Redis client: prevent deadlocks in high concurrency scenario by slowing down poll time and enforcing lock expiration
|
34
|
+
- Redis client: use connecion pool with Redis to prevent race conditions
|
35
|
+
- Google API: improve error handling on job creation
|
36
|
+
- Google API: use the `X-CloudTasks-TaskRetryCount` instead of `X-CloudTasks-TaskExecutionCount` to detect how many retries Google Cloud Tasks has performed. Using `X-CloudTasks-TaskRetryCount` is theoretically less accurate than using `X-CloudTasks-TaskExecutionCount` because it includes the number of "app unreachable" retries but `X-CloudTasks-TaskExecutionCount` is currently bugged and remains at zero all the time. See [this issue](https://github.com/keypup-io/cloudtasker/issues/6)
|
37
|
+
|
38
|
+
## [v0.9.4](https://github.com/keypup-io/cloudtasker/tree/v0.9.4) (2020-10-05)
|
39
|
+
|
40
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.9.4)
|
41
|
+
|
42
|
+
**Fixed bugs:**
|
43
|
+
- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
|
44
|
+
|
45
|
+
## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
|
46
|
+
|
47
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
|
48
|
+
|
49
|
+
**Fixed bugs:**
|
50
|
+
- Google Cloud Tasks: lock version to `~> 1.0` (Google recently released a v2 which changes its bindings completely). An [issue](https://github.com/keypup-io/cloudtasker/issues/11) has been raised to upgrade Cloudtasker to `google-cloud-tasks` `v2`.
|
51
|
+
|
3
52
|
## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
|
4
53
|
|
5
54
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
|
@@ -71,7 +120,3 @@ For Sinatra applications please update your Cloudtasker controller according to
|
|
71
120
|
## [v0.1.0](https://github.com/keypup-io/cloudtasker/tree/v0.1.0) (2019-11-17)
|
72
121
|
|
73
122
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/c137feb1ceaaaa4e2fecac0d1f0b4c73151ae002...v0.1.0)
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
|
data/README.md
CHANGED
@@ -12,10 +12,13 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
|
|
12
12
|
|
13
13
|
A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
|
14
14
|
|
15
|
+
**Maturity**: This gem is production-ready. We at Keypup have already processed millions of jobs using Cloudtasker and all related extensions (cron, batch and unique jobs). I'm waiting till the end of 2020 before releasing the official `v1.0.0` in case we've missed any edge-case bug.
|
16
|
+
|
15
17
|
## Summary
|
16
18
|
|
17
19
|
1. [Installation](#installation)
|
18
20
|
2. [Get started with Rails](#get-started-with-rails)
|
21
|
+
2. [Get started with Rails & ActiveJob](#get-started-with-rails--activejob)
|
19
22
|
3. [Configuring Cloudtasker](#configuring-cloudtasker)
|
20
23
|
1. [Cloud Tasks authentication & permissions](#cloud-tasks-authentication--permissions)
|
21
24
|
2. [Cloudtasker initializer](#cloudtasker-initializer)
|
@@ -75,7 +78,7 @@ Cloudtasker.configure do |config|
|
|
75
78
|
# Adapt the server port to be the one used by your Rails web process
|
76
79
|
#
|
77
80
|
config.processor_host = 'http://localhost:3000'
|
78
|
-
|
81
|
+
|
79
82
|
#
|
80
83
|
# If you do not have any Rails secret_key_base defined, uncomment the following
|
81
84
|
# This secret is used to authenticate jobs sent to the processing endpoint
|
@@ -132,6 +135,95 @@ That's it! Your job was picked up by the Cloudtasker local server and sent for p
|
|
132
135
|
|
133
136
|
Now jump to the next section to configure your app to use Google Cloud Tasks as a backend.
|
134
137
|
|
138
|
+
## Get started with Rails & ActiveJob
|
139
|
+
**Note**: ActiveJob is supported since `0.11.rc2`
|
140
|
+
**Note**: Cloudtasker extensions (cron, batch and unique jobs) are not available when using cloudtasker via ActiveJob.
|
141
|
+
|
142
|
+
Cloudtasker is pre-integrated with ActiveJob. Follow the steps below to get started.
|
143
|
+
|
144
|
+
Install redis on your machine (this is required by the Cloudtasker local processing server)
|
145
|
+
```bash
|
146
|
+
# E.g. using brew
|
147
|
+
brew install redis
|
148
|
+
```
|
149
|
+
|
150
|
+
Add the following initializer
|
151
|
+
```ruby
|
152
|
+
# config/initializers/cloudtasker.rb
|
153
|
+
|
154
|
+
Cloudtasker.configure do |config|
|
155
|
+
#
|
156
|
+
# Adapt the server port to be the one used by your Rails web process
|
157
|
+
#
|
158
|
+
config.processor_host = 'http://localhost:3000'
|
159
|
+
|
160
|
+
#
|
161
|
+
# If you do not have any Rails secret_key_base defined, uncomment the following
|
162
|
+
# This secret is used to authenticate jobs sent to the processing endpoint
|
163
|
+
# of your application.
|
164
|
+
#
|
165
|
+
# config.secret = 'some-long-token'
|
166
|
+
end
|
167
|
+
```
|
168
|
+
|
169
|
+
Configure ActiveJob to use Cloudtasker. You can also configure ActiveJob per environment via the config/environments/:env.rb files
|
170
|
+
```ruby
|
171
|
+
# config/application.rb
|
172
|
+
|
173
|
+
require_relative 'boot'
|
174
|
+
require 'rails/all'
|
175
|
+
|
176
|
+
Bundler.require(*Rails.groups)
|
177
|
+
|
178
|
+
module Dummy
|
179
|
+
class Application < Rails::Application
|
180
|
+
# Initialize configuration defaults for originally generated Rails version.
|
181
|
+
config.load_defaults 6.0
|
182
|
+
|
183
|
+
# Settings in config/environments/* take precedence over those specified here.
|
184
|
+
# Application configuration can go into files in config/initializers
|
185
|
+
# -- all .rb files in that directory are automatically loaded after loading
|
186
|
+
# the framework and any gems in your application.
|
187
|
+
|
188
|
+
# Use cloudtasker as the ActiveJob backend:
|
189
|
+
config.active_job.queue_adapter = :cloudtasker
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
```
|
194
|
+
|
195
|
+
Define your first job:
|
196
|
+
```ruby
|
197
|
+
# app/jobs/example_job.rb
|
198
|
+
|
199
|
+
class ExampleJob < ApplicationJob
|
200
|
+
queue_as :default
|
201
|
+
|
202
|
+
def perform(some_arg)
|
203
|
+
logger.info("Job run with #{some_arg}. This is working!")
|
204
|
+
end
|
205
|
+
end
|
206
|
+
```
|
207
|
+
|
208
|
+
Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` to your foreman config as a `worker` process)
|
209
|
+
```bash
|
210
|
+
# In one terminal
|
211
|
+
> rails s -p 3000
|
212
|
+
|
213
|
+
# In another terminal
|
214
|
+
> cloudtasker
|
215
|
+
```
|
216
|
+
|
217
|
+
Open a Rails console and enqueue some jobs
|
218
|
+
```ruby
|
219
|
+
# Process job as soon as possible
|
220
|
+
ExampleJob.perform_later('foo')
|
221
|
+
|
222
|
+
# Process job in 60 seconds
|
223
|
+
ExampleJob.set(wait: 60).perform_later('foo')
|
224
|
+
```
|
225
|
+
|
226
|
+
|
135
227
|
## Configuring Cloudtasker
|
136
228
|
|
137
229
|
### Cloud Tasks authentication & permissions
|
@@ -158,14 +250,14 @@ The gem can be configured through an initializer. See below all the available co
|
|
158
250
|
Cloudtasker.configure do |config|
|
159
251
|
#
|
160
252
|
# If you do not have any Rails secret_key_base defined, uncomment the following.
|
161
|
-
# This secret is used to authenticate jobs sent to the processing endpoint
|
253
|
+
# This secret is used to authenticate jobs sent to the processing endpoint
|
162
254
|
# of your application.
|
163
255
|
#
|
164
256
|
# Default with Rails: Rails.application.credentials.secret_key_base
|
165
257
|
#
|
166
258
|
# config.secret = 'some-long-token'
|
167
259
|
|
168
|
-
#
|
260
|
+
#
|
169
261
|
# Specify the details of your Google Cloud Task location.
|
170
262
|
#
|
171
263
|
# This not required in development using the Cloudtasker local server.
|
@@ -189,21 +281,21 @@ Cloudtasker.configure do |config|
|
|
189
281
|
#
|
190
282
|
# Specific queues can be created in Cloud Tasks using the gcloud SDK or
|
191
283
|
# via the `rake cloudtasker:setup_queue name=<queue_name>` task.
|
192
|
-
#
|
284
|
+
#
|
193
285
|
config.gcp_queue_prefix = 'my-app'
|
194
286
|
|
195
|
-
#
|
287
|
+
#
|
196
288
|
# Specify the publicly accessible host for your application
|
197
289
|
#
|
198
290
|
# > E.g. in development, using the cloudtasker local server
|
199
291
|
# config.processor_host = 'http://localhost:3000'
|
200
|
-
#
|
292
|
+
#
|
201
293
|
# > E.g. in development, using `config.mode = :production` and ngrok
|
202
294
|
# config.processor_host = 'https://111111.ngrok.io'
|
203
295
|
#
|
204
296
|
config.processor_host = 'https://app.mydomain.com'
|
205
297
|
|
206
|
-
#
|
298
|
+
#
|
207
299
|
# Specify the mode of operation:
|
208
300
|
# - :development => jobs will be pushed to Redis and picked up by the Cloudtasker local server
|
209
301
|
# - :production => jobs will be pushed to Google Cloud Tasks. Requires a publicly accessible domain.
|
@@ -212,20 +304,20 @@ Cloudtasker.configure do |config|
|
|
212
304
|
#
|
213
305
|
# config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
|
214
306
|
|
215
|
-
#
|
307
|
+
#
|
216
308
|
# Specify the logger to use
|
217
|
-
#
|
309
|
+
#
|
218
310
|
# Default with Rails: Rails.logger
|
219
311
|
# Default without Rails: Logger.new(STDOUT)
|
220
|
-
#
|
312
|
+
#
|
221
313
|
# config.logger = MyLogger.new(STDOUT)
|
222
314
|
|
223
|
-
#
|
315
|
+
#
|
224
316
|
# Specify how many retries are allowed on jobs. This number of retries excludes any
|
225
317
|
# connectivity error due to the application being down or unreachable.
|
226
|
-
#
|
318
|
+
#
|
227
319
|
# Default: 25
|
228
|
-
#
|
320
|
+
#
|
229
321
|
# config.max_retries = 10
|
230
322
|
|
231
323
|
#
|
@@ -250,7 +342,7 @@ Cloudtasker.configure do |config|
|
|
250
342
|
# You can set this configuration parameter to a KB value if you want to store jobs
|
251
343
|
# args in redis only if the JSONified arguments payload exceeds that threshold.
|
252
344
|
#
|
253
|
-
# Supported since: v0.10.
|
345
|
+
# Supported since: v0.10.0
|
254
346
|
#
|
255
347
|
# Default: false
|
256
348
|
#
|
@@ -262,7 +354,7 @@ Cloudtasker.configure do |config|
|
|
262
354
|
end
|
263
355
|
```
|
264
356
|
|
265
|
-
If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
|
357
|
+
If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
|
266
358
|
|
267
359
|
Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
|
268
360
|
```bash
|
@@ -361,10 +453,12 @@ CriticalWorker.schedule(args: [1], queue: :important)
|
|
361
453
|
```
|
362
454
|
|
363
455
|
## Extensions
|
456
|
+
**Note**: Extensions are not available when using cloudtasker via ActiveJob.
|
457
|
+
|
364
458
|
Cloudtasker comes with three optional features:
|
365
459
|
- Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
|
366
460
|
- Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
|
367
|
-
- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
|
461
|
+
- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
|
368
462
|
|
369
463
|
## Working locally
|
370
464
|
|
@@ -381,9 +475,9 @@ You can configure your application to use the Cloudtasker local server using the
|
|
381
475
|
|
382
476
|
Cloudtasker.configure do |config|
|
383
477
|
# ... other options
|
384
|
-
|
478
|
+
|
385
479
|
# Push jobs to redis and let the Cloudtasker local server collect them
|
386
|
-
# This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
|
480
|
+
# This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
|
387
481
|
# to a non-development environment
|
388
482
|
config.mode = :development
|
389
483
|
end
|
@@ -427,7 +521,7 @@ Cloudtasker.configure do |config|
|
|
427
521
|
|
428
522
|
# Use your ngrok domain as the processor host
|
429
523
|
config.processor_host = 'https://your-tunnel-id.ngrok.io'
|
430
|
-
|
524
|
+
|
431
525
|
# Force Cloudtasker to use Google Cloud Tasks in development
|
432
526
|
config.mode = :production
|
433
527
|
end
|
@@ -508,7 +602,7 @@ end
|
|
508
602
|
See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
|
509
603
|
|
510
604
|
### Searching logs: Job ID vs Task ID
|
511
|
-
**Note**: `task_id` field is available in logs starting with `0.10.
|
605
|
+
**Note**: `task_id` field is available in logs starting with `0.10.0`
|
512
606
|
|
513
607
|
Job instances are assigned two different different IDs for tracking and logging purpose: `job_id` and `task_id`. These IDs are found in each log entry to facilitate search.
|
514
608
|
|
@@ -571,26 +665,24 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
|
|
571
665
|
|
572
666
|
Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
|
573
667
|
|
574
|
-
**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `v0.10.
|
668
|
+
**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `v0.10.0` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
|
575
669
|
|
576
670
|
E.g. Set max number of retries globally via the cloudtasker initializer.
|
577
671
|
```ruby
|
578
672
|
# config/initializers/cloudtasker.rb
|
579
673
|
|
580
674
|
Cloudtasker.configure do |config|
|
581
|
-
#
|
675
|
+
#
|
582
676
|
# Specify how many retries are allowed on jobs. This number of retries excludes any
|
583
677
|
# connectivity error that would be due to the application being down or unreachable.
|
584
|
-
#
|
678
|
+
#
|
585
679
|
# Default: 25
|
586
|
-
#
|
680
|
+
#
|
587
681
|
config.max_retries = 10
|
588
682
|
end
|
589
683
|
```
|
590
684
|
|
591
685
|
E.g. Set max number of retries to 3 on a given worker
|
592
|
-
|
593
|
-
E.g.
|
594
686
|
```ruby
|
595
687
|
# app/workers/some_error_worker.rb
|
596
688
|
|
@@ -600,7 +692,30 @@ class SomeErrorWorker
|
|
600
692
|
# This will override the global setting
|
601
693
|
cloudtasker_options max_retries: 3
|
602
694
|
|
603
|
-
def perform
|
695
|
+
def perform
|
696
|
+
raise(ArgumentError)
|
697
|
+
end
|
698
|
+
end
|
699
|
+
```
|
700
|
+
|
701
|
+
E.g. Evaluate the number of max retries at runtime (Supported since: v0.10.1)
|
702
|
+
```ruby
|
703
|
+
# app/workers/some_error_worker.rb
|
704
|
+
|
705
|
+
class SomeErrorWorker
|
706
|
+
include Cloudtasker::Worker
|
707
|
+
|
708
|
+
# Return the number of max retries based on
|
709
|
+
# worker arguments.
|
710
|
+
#
|
711
|
+
# If this method returns nil then max_retries
|
712
|
+
# will delegate to the class `max_retries` setting or Cloudtasker
|
713
|
+
# `max_retries` configuration otion.
|
714
|
+
def max_retries(arg1, arg2)
|
715
|
+
arg1 == 'foo' ? 13 : nil
|
716
|
+
end
|
717
|
+
|
718
|
+
def perform(arg1, arg2)
|
604
719
|
raise(ArgumentError)
|
605
720
|
end
|
606
721
|
end
|
@@ -618,7 +733,7 @@ require 'cloudtasker/testing'
|
|
618
733
|
# Mode 1 (default): Push jobs to Google Cloud Tasks (env != development) or Redis (env == development)
|
619
734
|
Cloudtasker::Testing.enable!
|
620
735
|
|
621
|
-
# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
|
736
|
+
# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
|
622
737
|
# Cloudtasker::Worker.drain_all (process all jobs) or MyWorker.drain (process jobs for specific worker)
|
623
738
|
Cloudtasker::Testing.fake!
|
624
739
|
|
@@ -682,9 +797,9 @@ Below are examples of rspec tests. It is assumed that `Cloudtasker::Testing.fake
|
|
682
797
|
**Example 1**: Testing that a job is scheduled
|
683
798
|
```ruby
|
684
799
|
describe 'worker scheduling'
|
685
|
-
subject(:enqueue_job) { MyWorker.perform_async(1,2) }
|
686
|
-
|
687
|
-
it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
|
800
|
+
subject(:enqueue_job) { MyWorker.perform_async(1,2) }
|
801
|
+
|
802
|
+
it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
|
688
803
|
end
|
689
804
|
```
|
690
805
|
|
@@ -692,7 +807,7 @@ end
|
|
692
807
|
```ruby
|
693
808
|
describe 'worker calls api'
|
694
809
|
subject { Cloudtasker::Testing.inline! { MyApiWorker.perform_async(1,2) } }
|
695
|
-
|
810
|
+
|
696
811
|
before { expect(MyApi).to receive(:fetch).and_return([]) }
|
697
812
|
it { is_expected.to be_truthy }
|
698
813
|
end
|
@@ -772,7 +887,7 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
|
|
772
887
|
Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
|
773
888
|
|
774
889
|
#### Option 1: Use Cloudtasker optional support for payload storage in Redis
|
775
|
-
**Supported since**: `0.10.
|
890
|
+
**Supported since**: `0.10.0`
|
776
891
|
|
777
892
|
Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
|
778
893
|
|
@@ -19,21 +19,32 @@ module Cloudtasker
|
|
19
19
|
# Process payload
|
20
20
|
WorkerHandler.execute_from_payload!(payload)
|
21
21
|
head :no_content
|
22
|
-
rescue DeadWorkerError
|
22
|
+
rescue DeadWorkerError, MissingWorkerArgumentsError => e
|
23
23
|
# 205: job will NOT be retried
|
24
|
+
log_error(e)
|
24
25
|
head :reset_content
|
25
|
-
rescue InvalidWorkerError
|
26
|
+
rescue InvalidWorkerError => e
|
26
27
|
# 404: Job will be retried
|
28
|
+
log_error(e)
|
27
29
|
head :not_found
|
28
30
|
rescue StandardError => e
|
29
31
|
# 404: Job will be retried
|
30
|
-
|
31
|
-
Cloudtasker.logger.error(e.backtrace.join("\n"))
|
32
|
+
log_error(e)
|
32
33
|
head :unprocessable_entity
|
33
34
|
end
|
34
35
|
|
35
36
|
private
|
36
37
|
|
38
|
+
#
|
39
|
+
# Log an error via cloudtasker logger.
|
40
|
+
#
|
41
|
+
# @param [Exception] e The error to log
|
42
|
+
#
|
43
|
+
def log_error(error)
|
44
|
+
Cloudtasker.logger.error(error)
|
45
|
+
Cloudtasker.logger.error(error.backtrace.join("\n"))
|
46
|
+
end
|
47
|
+
|
37
48
|
#
|
38
49
|
# Parse the request body and return the actual job
|
39
50
|
# payload.
|
data/cloudtasker.gemspec
CHANGED
@@ -41,8 +41,10 @@ Gem::Specification.new do |spec|
|
|
41
41
|
spec.add_development_dependency 'github_changelog_generator'
|
42
42
|
spec.add_development_dependency 'rake', '>= 12.3.3'
|
43
43
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
44
|
+
spec.add_development_dependency 'rspec-json_expectations', '~> 2.2'
|
44
45
|
spec.add_development_dependency 'rubocop', '0.76.0'
|
45
46
|
spec.add_development_dependency 'rubocop-rspec', '1.37.0'
|
47
|
+
spec.add_development_dependency 'semantic_logger'
|
46
48
|
spec.add_development_dependency 'timecop'
|
47
49
|
spec.add_development_dependency 'webmock'
|
48
50
|
|
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# ActiveJob docs: http://guides.rubyonrails.org/active_job_basics.html
|
4
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
5
|
+
|
6
|
+
module ActiveJob
|
7
|
+
module QueueAdapters
|
8
|
+
# == Cloudtasker adapter for Active Job
|
9
|
+
#
|
10
|
+
# To use Cloudtasker set the queue_adapter config to +:cloudtasker+.
|
11
|
+
#
|
12
|
+
# Rails.application.config.active_job.queue_adapter = :cloudtasker
|
13
|
+
class CloudtaskerAdapter
|
14
|
+
SERIALIZATION_FILTERED_KEYS = [
|
15
|
+
'executions', # Given by the worker at processing
|
16
|
+
'provider_job_id', # Also given by the worker at processing
|
17
|
+
'priority' # Not used
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
# Enqueues the given ActiveJob instance for execution
|
21
|
+
#
|
22
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
23
|
+
#
|
24
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
25
|
+
#
|
26
|
+
def enqueue(job)
|
27
|
+
build_worker(job).schedule
|
28
|
+
end
|
29
|
+
|
30
|
+
# Enqueues the given ActiveJob instance for execution at a given time
|
31
|
+
#
|
32
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
33
|
+
# @param precise_timestamp [Integer] The timestamp at which the job must be executed
|
34
|
+
#
|
35
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
36
|
+
#
|
37
|
+
def enqueue_at(job, precise_timestamp)
|
38
|
+
build_worker(job).schedule(time_at: Time.at(precise_timestamp))
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def build_worker(job)
|
44
|
+
job_serialization = job.serialize.except(*SERIALIZATION_FILTERED_KEYS)
|
45
|
+
|
46
|
+
JobWrapper.new(
|
47
|
+
job_id: job_serialization.delete('job_id'),
|
48
|
+
job_queue: job_serialization.delete('queue_name'),
|
49
|
+
job_args: [job_serialization]
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
# == Job Wrapper for the Cloudtasker adapter
|
54
|
+
#
|
55
|
+
# Executes jobs scheduled by the Cloudtasker ActiveJob adapter
|
56
|
+
class JobWrapper #:nodoc:
|
57
|
+
include Cloudtasker::Worker
|
58
|
+
|
59
|
+
# Executes the given serialized ActiveJob call.
|
60
|
+
# - See https://api.rubyonrails.org/classes/ActiveJob/Core.html#method-i-serialize
|
61
|
+
#
|
62
|
+
# @param [Hash] job_serialization The serialized ActiveJob call
|
63
|
+
#
|
64
|
+
# @return [any] The execution of the ActiveJob call
|
65
|
+
#
|
66
|
+
def perform(job_serialization, *_extra_options)
|
67
|
+
job_executions = job_retries < 1 ? 0 : (job_retries + 1)
|
68
|
+
|
69
|
+
job_serialization.merge!(
|
70
|
+
'job_id' => job_id,
|
71
|
+
'queue_name' => job_queue,
|
72
|
+
'provider_job_id' => task_id,
|
73
|
+
'executions' => job_executions,
|
74
|
+
'priority' => nil
|
75
|
+
)
|
76
|
+
|
77
|
+
Base.execute job_serialization
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/cloudtasker.rb
CHANGED
@@ -8,6 +8,7 @@ require 'cloudtasker/config'
|
|
8
8
|
require 'cloudtasker/authentication_error'
|
9
9
|
require 'cloudtasker/dead_worker_error'
|
10
10
|
require 'cloudtasker/invalid_worker_error'
|
11
|
+
require 'cloudtasker/missing_worker_arguments_error'
|
11
12
|
require 'cloudtasker/max_task_size_exceeded_error'
|
12
13
|
|
13
14
|
require 'cloudtasker/middleware/chain'
|
@@ -166,6 +166,9 @@ module Cloudtasker
|
|
166
166
|
# Delete task
|
167
167
|
self.class.delete(id)
|
168
168
|
resp
|
169
|
+
rescue DeadWorkerError => e
|
170
|
+
self.class.delete(id)
|
171
|
+
raise(e) if self.class.inline_mode?
|
169
172
|
rescue StandardError => e
|
170
173
|
self.job_retries += 1
|
171
174
|
raise(e) if self.class.inline_mode?
|
@@ -62,6 +62,10 @@ module Cloudtasker
|
|
62
62
|
# @return [Cloudtasker::Batch::Job] The attached batch.
|
63
63
|
#
|
64
64
|
def self.for(worker)
|
65
|
+
# Load extension if not loaded already on the worker class
|
66
|
+
worker.class.include(Extension::Worker) unless worker.class <= Extension::Worker
|
67
|
+
|
68
|
+
# Add batch capability
|
65
69
|
worker.batch = new(worker)
|
66
70
|
end
|
67
71
|
|
data/lib/cloudtasker/cli.rb
CHANGED
data/lib/cloudtasker/engine.rb
CHANGED
@@ -5,10 +5,14 @@ module Cloudtasker
|
|
5
5
|
class Engine < ::Rails::Engine
|
6
6
|
isolate_namespace Cloudtasker
|
7
7
|
|
8
|
-
|
8
|
+
config.before_initialize do
|
9
|
+
# Mount cloudtasker processing endpoint
|
9
10
|
Rails.application.routes.append do
|
10
11
|
mount Cloudtasker::Engine, at: '/cloudtasker'
|
11
12
|
end
|
13
|
+
|
14
|
+
# Add ActiveJob adapter
|
15
|
+
require 'active_job/queue_adapters/cloudtasker_adapter' if defined?(::ActiveJob::Railtie)
|
12
16
|
end
|
13
17
|
|
14
18
|
config.generators do |g|
|
data/lib/cloudtasker/version.rb
CHANGED
data/lib/cloudtasker/worker.rb
CHANGED
@@ -296,6 +296,20 @@ module Cloudtasker
|
|
296
296
|
other.is_a?(self.class) && other.job_id == job_id
|
297
297
|
end
|
298
298
|
|
299
|
+
#
|
300
|
+
# Return the max number of retries allowed for this job.
|
301
|
+
#
|
302
|
+
# The order of precedence for retry lookup is:
|
303
|
+
# - Worker `max_retries` method
|
304
|
+
# - Class `max_retries` option
|
305
|
+
# - Cloudtasker `max_retries` config option
|
306
|
+
#
|
307
|
+
# @return [Integer] The number of retries
|
308
|
+
#
|
309
|
+
def job_max_retries
|
310
|
+
@job_max_retries ||= (try(:max_retries, *job_args) || self.class.max_retries)
|
311
|
+
end
|
312
|
+
|
299
313
|
#
|
300
314
|
# Return true if the job has excceeded its maximum number
|
301
315
|
# of retries
|
@@ -303,7 +317,7 @@ module Cloudtasker
|
|
303
317
|
# @return [Boolean] True if the job is dead
|
304
318
|
#
|
305
319
|
def job_dead?
|
306
|
-
job_retries >=
|
320
|
+
job_retries >= job_max_retries
|
307
321
|
end
|
308
322
|
|
309
323
|
#
|
@@ -333,6 +347,13 @@ module Cloudtasker
|
|
333
347
|
|
334
348
|
Cloudtasker.config.server_middleware.invoke(self) do
|
335
349
|
begin
|
350
|
+
# Abort if arguments are missing. This may happen with redis arguments storage
|
351
|
+
# if Cloud Tasks times out on a job but the job still succeeds
|
352
|
+
if job_args.empty? && [0, -1].exclude?(method(:perform).arity)
|
353
|
+
raise(MissingWorkerArgumentsError, 'worker arguments are missing')
|
354
|
+
end
|
355
|
+
|
356
|
+
# Perform the job
|
336
357
|
perform(*job_args)
|
337
358
|
rescue StandardError => e
|
338
359
|
try(:on_error, e)
|
@@ -85,7 +85,7 @@ module Cloudtasker
|
|
85
85
|
redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
|
86
86
|
|
87
87
|
resp
|
88
|
-
rescue DeadWorkerError => e
|
88
|
+
rescue DeadWorkerError, MissingWorkerArgumentsError => e
|
89
89
|
# Delete stored args payload if job is dead
|
90
90
|
redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
|
91
91
|
raise(e)
|
@@ -142,7 +142,7 @@ module Cloudtasker
|
|
142
142
|
#
|
143
143
|
def log_message(level, msg, &block)
|
144
144
|
# Merge log-specific context into worker-specific context
|
145
|
-
payload_block = -> { log_block.call.merge(block&.call || {}) }
|
145
|
+
payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
|
146
146
|
|
147
147
|
# ActiveSupport::Logger does not support passing a payload through a block on top
|
148
148
|
# of a message.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudtasker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.rc3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arnaud Lachaume
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -178,6 +178,20 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '3.0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec-json_expectations
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '2.2'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '2.2'
|
181
195
|
- !ruby/object:Gem::Dependency
|
182
196
|
name: rubocop
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +220,20 @@ dependencies:
|
|
206
220
|
- - '='
|
207
221
|
- !ruby/object:Gem::Version
|
208
222
|
version: 1.37.0
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: semantic_logger
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - ">="
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: '0'
|
209
237
|
- !ruby/object:Gem::Dependency
|
210
238
|
name: timecop
|
211
239
|
requirement: !ruby/object:Gem::Requirement
|
@@ -324,6 +352,12 @@ files:
|
|
324
352
|
- gemfiles/rails_5.2.gemfile.lock
|
325
353
|
- gemfiles/rails_6.0.gemfile
|
326
354
|
- gemfiles/rails_6.0.gemfile.lock
|
355
|
+
- gemfiles/semantic_logger_3.4.gemfile
|
356
|
+
- gemfiles/semantic_logger_4.6.gemfile
|
357
|
+
- gemfiles/semantic_logger_4.7.0.gemfile
|
358
|
+
- gemfiles/semantic_logger_4.7.2.gemfile
|
359
|
+
- gemfiles/semantic_logger_4.7.gemfile
|
360
|
+
- lib/active_job/queue_adapters/cloudtasker_adapter.rb
|
327
361
|
- lib/cloudtasker.rb
|
328
362
|
- lib/cloudtasker/authentication_error.rb
|
329
363
|
- lib/cloudtasker/authenticator.rb
|
@@ -351,6 +385,7 @@ files:
|
|
351
385
|
- lib/cloudtasker/max_task_size_exceeded_error.rb
|
352
386
|
- lib/cloudtasker/meta_store.rb
|
353
387
|
- lib/cloudtasker/middleware/chain.rb
|
388
|
+
- lib/cloudtasker/missing_worker_arguments_error.rb
|
354
389
|
- lib/cloudtasker/redis_client.rb
|
355
390
|
- lib/cloudtasker/testing.rb
|
356
391
|
- lib/cloudtasker/unique_job.rb
|