cloudtasker 0.10.0 → 0.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/test.yml +4 -0
- data/.rubocop.yml +4 -0
- data/Appraisals +16 -0
- data/CHANGELOG.md +24 -4
- data/README.md +144 -29
- data/app/controllers/cloudtasker/worker_controller.rb +15 -4
- data/cloudtasker.gemspec +2 -0
- data/docs/BATCH_JOBS.md +1 -1
- data/gemfiles/semantic_logger_3.4.gemfile +7 -0
- data/gemfiles/semantic_logger_4.6.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.0.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.2.gemfile +7 -0
- data/gemfiles/semantic_logger_4.7.gemfile +7 -0
- data/lib/active_job/queue_adapters/cloudtasker_adapter.rb +82 -0
- data/lib/cloudtasker.rb +1 -0
- data/lib/cloudtasker/backend/memory_task.rb +3 -0
- data/lib/cloudtasker/batch/job.rb +4 -0
- data/lib/cloudtasker/batch/middleware.rb +2 -0
- data/lib/cloudtasker/cli.rb +1 -0
- data/lib/cloudtasker/engine.rb +5 -1
- data/lib/cloudtasker/missing_worker_arguments_error.rb +6 -0
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +22 -1
- data/lib/cloudtasker/worker_handler.rb +1 -1
- data/lib/cloudtasker/worker_logger.rb +1 -1
- metadata +37 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f4e53fe9c99a8c6a74e064d7b7c562cdb671682aaed9bb6112a77c7fd453124
|
4
|
+
data.tar.gz: 3b2e6aeca4fde352acd7b0db6fe469221514bf7fd753c07c3d9540d881bf1c2f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4be2ce9466e7ffd2cc274d57d79eedad96ae56d699632f88bfdf6384413aa9bfa19dff8a124926bd2b41229ce53f3126b2bd04646cc9125136bc93fdffb0642a
|
7
|
+
data.tar.gz: f6a3f7fa152f5ebedbe25945e86538691ee658c5bc02b840fd3d1f337ba0fcd78db6a84c52bcd6ade01af51335ee81d457152bfa9767d3ac9541b5d31ef48126
|
data/.github/workflows/test.yml
CHANGED
@@ -21,6 +21,10 @@ jobs:
|
|
21
21
|
- 'google-cloud-tasks-1.3'
|
22
22
|
- 'rails-5.2'
|
23
23
|
- 'rails-6.0'
|
24
|
+
- 'semantic_logger-3.4'
|
25
|
+
- 'semantic_logger-4.6'
|
26
|
+
- 'semantic_logger-4.7.0'
|
27
|
+
- 'semantic_logger-4.7.2'
|
24
28
|
steps:
|
25
29
|
- name: Setup System
|
26
30
|
run: sudo apt-get install libsqlite3-dev
|
data/.rubocop.yml
CHANGED
data/Appraisals
CHANGED
@@ -23,3 +23,19 @@ end
|
|
23
23
|
appraise 'rails-6.0' do
|
24
24
|
gem 'rails', '6.0'
|
25
25
|
end
|
26
|
+
|
27
|
+
appraise 'semantic_logger-3.4' do
|
28
|
+
gem 'semantic_logger', '3.4.1'
|
29
|
+
end
|
30
|
+
|
31
|
+
appraise 'semantic_logger-4.6' do
|
32
|
+
gem 'semantic_logger', '4.6.1'
|
33
|
+
end
|
34
|
+
|
35
|
+
appraise 'semantic_logger-4.7.0' do
|
36
|
+
gem 'semantic_logger', '4.7.0'
|
37
|
+
end
|
38
|
+
|
39
|
+
appraise 'semantic_logger-4.7.2' do
|
40
|
+
gem 'semantic_logger', '4.7.2'
|
41
|
+
end
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,22 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
+
## [v0.11.0](https://github.com/keypup-io/cloudtasker/tree/v0.11.0) (2020-11-23)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.11.0)
|
6
|
+
|
7
|
+
**Improvements:**
|
8
|
+
- Worker: drop job (return 205 response) when worker arguments are not available (e.g. arguments were stored in Redis and the latter was flushed)
|
9
|
+
- Rails: add ActiveJob adapter (thanks @vovimayhem)
|
10
|
+
|
11
|
+
## [v0.10.1](https://github.com/keypup-io/cloudtasker/tree/v0.10.1) (2020-10-05)
|
12
|
+
|
13
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.10.1)
|
14
|
+
|
15
|
+
**Fixed bugs:**
|
16
|
+
- Local server: delete dead task from local server queue
|
17
|
+
- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
|
18
|
+
- Worker: fix configuration of `max_retries` at worker level
|
19
|
+
|
3
20
|
## [v0.10.0](https://github.com/keypup-io/cloudtasker/tree/v0.10.0) (2020-09-02)
|
4
21
|
|
5
22
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.10.0)
|
@@ -18,6 +35,13 @@
|
|
18
35
|
- Google API: improve error handling on job creation
|
19
36
|
- Google API: use the `X-CloudTasks-TaskRetryCount` instead of `X-CloudTasks-TaskExecutionCount` to detect how many retries Google Cloud Tasks has performed. Using `X-CloudTasks-TaskRetryCount` is theoretically less accurate than using `X-CloudTasks-TaskExecutionCount` because it includes the number of "app unreachable" retries but `X-CloudTasks-TaskExecutionCount` is currently bugged and remains at zero all the time. See [this issue](https://github.com/keypup-io/cloudtasker/issues/6)
|
20
37
|
|
38
|
+
## [v0.9.4](https://github.com/keypup-io/cloudtasker/tree/v0.9.4) (2020-10-05)
|
39
|
+
|
40
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.9.4)
|
41
|
+
|
42
|
+
**Fixed bugs:**
|
43
|
+
- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
|
44
|
+
|
21
45
|
## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
|
22
46
|
|
23
47
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
|
@@ -96,7 +120,3 @@ For Sinatra applications please update your Cloudtasker controller according to
|
|
96
120
|
## [v0.1.0](https://github.com/keypup-io/cloudtasker/tree/v0.1.0) (2019-11-17)
|
97
121
|
|
98
122
|
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/c137feb1ceaaaa4e2fecac0d1f0b4c73151ae002...v0.1.0)
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
|
data/README.md
CHANGED
@@ -12,10 +12,13 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
|
|
12
12
|
|
13
13
|
A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
|
14
14
|
|
15
|
+
**Maturity**: This gem is production-ready. We at Keypup have already processed millions of jobs using Cloudtasker and all related extensions (cron, batch and unique jobs). I'm waiting till the end of 2020 before releasing the official `v1.0.0` in case we've missed any edge-case bug.
|
16
|
+
|
15
17
|
## Summary
|
16
18
|
|
17
19
|
1. [Installation](#installation)
|
18
20
|
2. [Get started with Rails](#get-started-with-rails)
|
21
|
+
2. [Get started with Rails & ActiveJob](#get-started-with-rails--activejob)
|
19
22
|
3. [Configuring Cloudtasker](#configuring-cloudtasker)
|
20
23
|
1. [Cloud Tasks authentication & permissions](#cloud-tasks-authentication--permissions)
|
21
24
|
2. [Cloudtasker initializer](#cloudtasker-initializer)
|
@@ -75,7 +78,7 @@ Cloudtasker.configure do |config|
|
|
75
78
|
# Adapt the server port to be the one used by your Rails web process
|
76
79
|
#
|
77
80
|
config.processor_host = 'http://localhost:3000'
|
78
|
-
|
81
|
+
|
79
82
|
#
|
80
83
|
# If you do not have any Rails secret_key_base defined, uncomment the following
|
81
84
|
# This secret is used to authenticate jobs sent to the processing endpoint
|
@@ -132,6 +135,95 @@ That's it! Your job was picked up by the Cloudtasker local server and sent for p
|
|
132
135
|
|
133
136
|
Now jump to the next section to configure your app to use Google Cloud Tasks as a backend.
|
134
137
|
|
138
|
+
## Get started with Rails & ActiveJob
|
139
|
+
**Note**: ActiveJob is supported since `0.11.0`
|
140
|
+
**Note**: Cloudtasker extensions (cron, batch and unique jobs) are not available when using cloudtasker via ActiveJob.
|
141
|
+
|
142
|
+
Cloudtasker is pre-integrated with ActiveJob. Follow the steps below to get started.
|
143
|
+
|
144
|
+
Install redis on your machine (this is required by the Cloudtasker local processing server)
|
145
|
+
```bash
|
146
|
+
# E.g. using brew
|
147
|
+
brew install redis
|
148
|
+
```
|
149
|
+
|
150
|
+
Add the following initializer
|
151
|
+
```ruby
|
152
|
+
# config/initializers/cloudtasker.rb
|
153
|
+
|
154
|
+
Cloudtasker.configure do |config|
|
155
|
+
#
|
156
|
+
# Adapt the server port to be the one used by your Rails web process
|
157
|
+
#
|
158
|
+
config.processor_host = 'http://localhost:3000'
|
159
|
+
|
160
|
+
#
|
161
|
+
# If you do not have any Rails secret_key_base defined, uncomment the following
|
162
|
+
# This secret is used to authenticate jobs sent to the processing endpoint
|
163
|
+
# of your application.
|
164
|
+
#
|
165
|
+
# config.secret = 'some-long-token'
|
166
|
+
end
|
167
|
+
```
|
168
|
+
|
169
|
+
Configure ActiveJob to use Cloudtasker. You can also configure ActiveJob per environment via the config/environments/:env.rb files
|
170
|
+
```ruby
|
171
|
+
# config/application.rb
|
172
|
+
|
173
|
+
require_relative 'boot'
|
174
|
+
require 'rails/all'
|
175
|
+
|
176
|
+
Bundler.require(*Rails.groups)
|
177
|
+
|
178
|
+
module Dummy
|
179
|
+
class Application < Rails::Application
|
180
|
+
# Initialize configuration defaults for originally generated Rails version.
|
181
|
+
config.load_defaults 6.0
|
182
|
+
|
183
|
+
# Settings in config/environments/* take precedence over those specified here.
|
184
|
+
# Application configuration can go into files in config/initializers
|
185
|
+
# -- all .rb files in that directory are automatically loaded after loading
|
186
|
+
# the framework and any gems in your application.
|
187
|
+
|
188
|
+
# Use cloudtasker as the ActiveJob backend:
|
189
|
+
config.active_job.queue_adapter = :cloudtasker
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
```
|
194
|
+
|
195
|
+
Define your first job:
|
196
|
+
```ruby
|
197
|
+
# app/jobs/example_job.rb
|
198
|
+
|
199
|
+
class ExampleJob < ApplicationJob
|
200
|
+
queue_as :default
|
201
|
+
|
202
|
+
def perform(some_arg)
|
203
|
+
logger.info("Job run with #{some_arg}. This is working!")
|
204
|
+
end
|
205
|
+
end
|
206
|
+
```
|
207
|
+
|
208
|
+
Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` to your foreman config as a `worker` process)
|
209
|
+
```bash
|
210
|
+
# In one terminal
|
211
|
+
> rails s -p 3000
|
212
|
+
|
213
|
+
# In another terminal
|
214
|
+
> cloudtasker
|
215
|
+
```
|
216
|
+
|
217
|
+
Open a Rails console and enqueue some jobs
|
218
|
+
```ruby
|
219
|
+
# Process job as soon as possible
|
220
|
+
ExampleJob.perform_later('foo')
|
221
|
+
|
222
|
+
# Process job in 60 seconds
|
223
|
+
ExampleJob.set(wait: 60).perform_later('foo')
|
224
|
+
```
|
225
|
+
|
226
|
+
|
135
227
|
## Configuring Cloudtasker
|
136
228
|
|
137
229
|
### Cloud Tasks authentication & permissions
|
@@ -158,14 +250,14 @@ The gem can be configured through an initializer. See below all the available co
|
|
158
250
|
Cloudtasker.configure do |config|
|
159
251
|
#
|
160
252
|
# If you do not have any Rails secret_key_base defined, uncomment the following.
|
161
|
-
# This secret is used to authenticate jobs sent to the processing endpoint
|
253
|
+
# This secret is used to authenticate jobs sent to the processing endpoint
|
162
254
|
# of your application.
|
163
255
|
#
|
164
256
|
# Default with Rails: Rails.application.credentials.secret_key_base
|
165
257
|
#
|
166
258
|
# config.secret = 'some-long-token'
|
167
259
|
|
168
|
-
#
|
260
|
+
#
|
169
261
|
# Specify the details of your Google Cloud Task location.
|
170
262
|
#
|
171
263
|
# This not required in development using the Cloudtasker local server.
|
@@ -189,21 +281,21 @@ Cloudtasker.configure do |config|
|
|
189
281
|
#
|
190
282
|
# Specific queues can be created in Cloud Tasks using the gcloud SDK or
|
191
283
|
# via the `rake cloudtasker:setup_queue name=<queue_name>` task.
|
192
|
-
#
|
284
|
+
#
|
193
285
|
config.gcp_queue_prefix = 'my-app'
|
194
286
|
|
195
|
-
#
|
287
|
+
#
|
196
288
|
# Specify the publicly accessible host for your application
|
197
289
|
#
|
198
290
|
# > E.g. in development, using the cloudtasker local server
|
199
291
|
# config.processor_host = 'http://localhost:3000'
|
200
|
-
#
|
292
|
+
#
|
201
293
|
# > E.g. in development, using `config.mode = :production` and ngrok
|
202
294
|
# config.processor_host = 'https://111111.ngrok.io'
|
203
295
|
#
|
204
296
|
config.processor_host = 'https://app.mydomain.com'
|
205
297
|
|
206
|
-
#
|
298
|
+
#
|
207
299
|
# Specify the mode of operation:
|
208
300
|
# - :development => jobs will be pushed to Redis and picked up by the Cloudtasker local server
|
209
301
|
# - :production => jobs will be pushed to Google Cloud Tasks. Requires a publicly accessible domain.
|
@@ -212,20 +304,20 @@ Cloudtasker.configure do |config|
|
|
212
304
|
#
|
213
305
|
# config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
|
214
306
|
|
215
|
-
#
|
307
|
+
#
|
216
308
|
# Specify the logger to use
|
217
|
-
#
|
309
|
+
#
|
218
310
|
# Default with Rails: Rails.logger
|
219
311
|
# Default without Rails: Logger.new(STDOUT)
|
220
|
-
#
|
312
|
+
#
|
221
313
|
# config.logger = MyLogger.new(STDOUT)
|
222
314
|
|
223
|
-
#
|
315
|
+
#
|
224
316
|
# Specify how many retries are allowed on jobs. This number of retries excludes any
|
225
317
|
# connectivity error due to the application being down or unreachable.
|
226
|
-
#
|
318
|
+
#
|
227
319
|
# Default: 25
|
228
|
-
#
|
320
|
+
#
|
229
321
|
# config.max_retries = 10
|
230
322
|
|
231
323
|
#
|
@@ -262,7 +354,7 @@ Cloudtasker.configure do |config|
|
|
262
354
|
end
|
263
355
|
```
|
264
356
|
|
265
|
-
If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
|
357
|
+
If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
|
266
358
|
|
267
359
|
Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
|
268
360
|
```bash
|
@@ -361,10 +453,12 @@ CriticalWorker.schedule(args: [1], queue: :important)
|
|
361
453
|
```
|
362
454
|
|
363
455
|
## Extensions
|
456
|
+
**Note**: Extensions are not available when using cloudtasker via ActiveJob.
|
457
|
+
|
364
458
|
Cloudtasker comes with three optional features:
|
365
459
|
- Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
|
366
460
|
- Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
|
367
|
-
- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
|
461
|
+
- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
|
368
462
|
|
369
463
|
## Working locally
|
370
464
|
|
@@ -381,9 +475,9 @@ You can configure your application to use the Cloudtasker local server using the
|
|
381
475
|
|
382
476
|
Cloudtasker.configure do |config|
|
383
477
|
# ... other options
|
384
|
-
|
478
|
+
|
385
479
|
# Push jobs to redis and let the Cloudtasker local server collect them
|
386
|
-
# This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
|
480
|
+
# This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
|
387
481
|
# to a non-development environment
|
388
482
|
config.mode = :development
|
389
483
|
end
|
@@ -427,7 +521,7 @@ Cloudtasker.configure do |config|
|
|
427
521
|
|
428
522
|
# Use your ngrok domain as the processor host
|
429
523
|
config.processor_host = 'https://your-tunnel-id.ngrok.io'
|
430
|
-
|
524
|
+
|
431
525
|
# Force Cloudtasker to use Google Cloud Tasks in development
|
432
526
|
config.mode = :production
|
433
527
|
end
|
@@ -578,19 +672,17 @@ E.g. Set max number of retries globally via the cloudtasker initializer.
|
|
578
672
|
# config/initializers/cloudtasker.rb
|
579
673
|
|
580
674
|
Cloudtasker.configure do |config|
|
581
|
-
#
|
675
|
+
#
|
582
676
|
# Specify how many retries are allowed on jobs. This number of retries excludes any
|
583
677
|
# connectivity error that would be due to the application being down or unreachable.
|
584
|
-
#
|
678
|
+
#
|
585
679
|
# Default: 25
|
586
|
-
#
|
680
|
+
#
|
587
681
|
config.max_retries = 10
|
588
682
|
end
|
589
683
|
```
|
590
684
|
|
591
685
|
E.g. Set max number of retries to 3 on a given worker
|
592
|
-
|
593
|
-
E.g.
|
594
686
|
```ruby
|
595
687
|
# app/workers/some_error_worker.rb
|
596
688
|
|
@@ -600,7 +692,30 @@ class SomeErrorWorker
|
|
600
692
|
# This will override the global setting
|
601
693
|
cloudtasker_options max_retries: 3
|
602
694
|
|
603
|
-
def perform
|
695
|
+
def perform
|
696
|
+
raise(ArgumentError)
|
697
|
+
end
|
698
|
+
end
|
699
|
+
```
|
700
|
+
|
701
|
+
E.g. Evaluate the number of max retries at runtime (Supported since: v0.10.1)
|
702
|
+
```ruby
|
703
|
+
# app/workers/some_error_worker.rb
|
704
|
+
|
705
|
+
class SomeErrorWorker
|
706
|
+
include Cloudtasker::Worker
|
707
|
+
|
708
|
+
# Return the number of max retries based on
|
709
|
+
# worker arguments.
|
710
|
+
#
|
711
|
+
# If this method returns nil then max_retries
|
712
|
+
# will delegate to the class `max_retries` setting or Cloudtasker
|
713
|
+
# `max_retries` configuration otion.
|
714
|
+
def max_retries(arg1, arg2)
|
715
|
+
arg1 == 'foo' ? 13 : nil
|
716
|
+
end
|
717
|
+
|
718
|
+
def perform(arg1, arg2)
|
604
719
|
raise(ArgumentError)
|
605
720
|
end
|
606
721
|
end
|
@@ -618,7 +733,7 @@ require 'cloudtasker/testing'
|
|
618
733
|
# Mode 1 (default): Push jobs to Google Cloud Tasks (env != development) or Redis (env == development)
|
619
734
|
Cloudtasker::Testing.enable!
|
620
735
|
|
621
|
-
# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
|
736
|
+
# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
|
622
737
|
# Cloudtasker::Worker.drain_all (process all jobs) or MyWorker.drain (process jobs for specific worker)
|
623
738
|
Cloudtasker::Testing.fake!
|
624
739
|
|
@@ -682,9 +797,9 @@ Below are examples of rspec tests. It is assumed that `Cloudtasker::Testing.fake
|
|
682
797
|
**Example 1**: Testing that a job is scheduled
|
683
798
|
```ruby
|
684
799
|
describe 'worker scheduling'
|
685
|
-
subject(:enqueue_job) { MyWorker.perform_async(1,2) }
|
686
|
-
|
687
|
-
it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
|
800
|
+
subject(:enqueue_job) { MyWorker.perform_async(1,2) }
|
801
|
+
|
802
|
+
it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
|
688
803
|
end
|
689
804
|
```
|
690
805
|
|
@@ -692,7 +807,7 @@ end
|
|
692
807
|
```ruby
|
693
808
|
describe 'worker calls api'
|
694
809
|
subject { Cloudtasker::Testing.inline! { MyApiWorker.perform_async(1,2) } }
|
695
|
-
|
810
|
+
|
696
811
|
before { expect(MyApi).to receive(:fetch).and_return([]) }
|
697
812
|
it { is_expected.to be_truthy }
|
698
813
|
end
|
@@ -19,21 +19,32 @@ module Cloudtasker
|
|
19
19
|
# Process payload
|
20
20
|
WorkerHandler.execute_from_payload!(payload)
|
21
21
|
head :no_content
|
22
|
-
rescue DeadWorkerError
|
22
|
+
rescue DeadWorkerError, MissingWorkerArgumentsError => e
|
23
23
|
# 205: job will NOT be retried
|
24
|
+
log_error(e)
|
24
25
|
head :reset_content
|
25
|
-
rescue InvalidWorkerError
|
26
|
+
rescue InvalidWorkerError => e
|
26
27
|
# 404: Job will be retried
|
28
|
+
log_error(e)
|
27
29
|
head :not_found
|
28
30
|
rescue StandardError => e
|
29
31
|
# 404: Job will be retried
|
30
|
-
|
31
|
-
Cloudtasker.logger.error(e.backtrace.join("\n"))
|
32
|
+
log_error(e)
|
32
33
|
head :unprocessable_entity
|
33
34
|
end
|
34
35
|
|
35
36
|
private
|
36
37
|
|
38
|
+
#
|
39
|
+
# Log an error via cloudtasker logger.
|
40
|
+
#
|
41
|
+
# @param [Exception] e The error to log
|
42
|
+
#
|
43
|
+
def log_error(error)
|
44
|
+
Cloudtasker.logger.error(error)
|
45
|
+
Cloudtasker.logger.error(error.backtrace.join("\n"))
|
46
|
+
end
|
47
|
+
|
37
48
|
#
|
38
49
|
# Parse the request body and return the actual job
|
39
50
|
# payload.
|
data/cloudtasker.gemspec
CHANGED
@@ -41,8 +41,10 @@ Gem::Specification.new do |spec|
|
|
41
41
|
spec.add_development_dependency 'github_changelog_generator'
|
42
42
|
spec.add_development_dependency 'rake', '>= 12.3.3'
|
43
43
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
44
|
+
spec.add_development_dependency 'rspec-json_expectations', '~> 2.2'
|
44
45
|
spec.add_development_dependency 'rubocop', '0.76.0'
|
45
46
|
spec.add_development_dependency 'rubocop-rspec', '1.37.0'
|
47
|
+
spec.add_development_dependency 'semantic_logger'
|
46
48
|
spec.add_development_dependency 'timecop'
|
47
49
|
spec.add_development_dependency 'webmock'
|
48
50
|
|
data/docs/BATCH_JOBS.md
CHANGED
@@ -0,0 +1,82 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# ActiveJob docs: http://guides.rubyonrails.org/active_job_basics.html
|
4
|
+
# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
|
5
|
+
|
6
|
+
module ActiveJob
|
7
|
+
module QueueAdapters
|
8
|
+
# == Cloudtasker adapter for Active Job
|
9
|
+
#
|
10
|
+
# To use Cloudtasker set the queue_adapter config to +:cloudtasker+.
|
11
|
+
#
|
12
|
+
# Rails.application.config.active_job.queue_adapter = :cloudtasker
|
13
|
+
class CloudtaskerAdapter
|
14
|
+
SERIALIZATION_FILTERED_KEYS = [
|
15
|
+
'executions', # Given by the worker at processing
|
16
|
+
'provider_job_id', # Also given by the worker at processing
|
17
|
+
'priority' # Not used
|
18
|
+
].freeze
|
19
|
+
|
20
|
+
# Enqueues the given ActiveJob instance for execution
|
21
|
+
#
|
22
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
23
|
+
#
|
24
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
25
|
+
#
|
26
|
+
def enqueue(job)
|
27
|
+
build_worker(job).schedule
|
28
|
+
end
|
29
|
+
|
30
|
+
# Enqueues the given ActiveJob instance for execution at a given time
|
31
|
+
#
|
32
|
+
# @param job [ActiveJob::Base] The ActiveJob instance
|
33
|
+
# @param precise_timestamp [Integer] The timestamp at which the job must be executed
|
34
|
+
#
|
35
|
+
# @return [Cloudtasker::CloudTask] The Google Task response
|
36
|
+
#
|
37
|
+
def enqueue_at(job, precise_timestamp)
|
38
|
+
build_worker(job).schedule(time_at: Time.at(precise_timestamp))
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def build_worker(job)
|
44
|
+
job_serialization = job.serialize.except(*SERIALIZATION_FILTERED_KEYS)
|
45
|
+
|
46
|
+
JobWrapper.new(
|
47
|
+
job_id: job_serialization.delete('job_id'),
|
48
|
+
job_queue: job_serialization.delete('queue_name'),
|
49
|
+
job_args: [job_serialization]
|
50
|
+
)
|
51
|
+
end
|
52
|
+
|
53
|
+
# == Job Wrapper for the Cloudtasker adapter
|
54
|
+
#
|
55
|
+
# Executes jobs scheduled by the Cloudtasker ActiveJob adapter
|
56
|
+
class JobWrapper #:nodoc:
|
57
|
+
include Cloudtasker::Worker
|
58
|
+
|
59
|
+
# Executes the given serialized ActiveJob call.
|
60
|
+
# - See https://api.rubyonrails.org/classes/ActiveJob/Core.html#method-i-serialize
|
61
|
+
#
|
62
|
+
# @param [Hash] job_serialization The serialized ActiveJob call
|
63
|
+
#
|
64
|
+
# @return [any] The execution of the ActiveJob call
|
65
|
+
#
|
66
|
+
def perform(job_serialization, *_extra_options)
|
67
|
+
job_executions = job_retries < 1 ? 0 : (job_retries + 1)
|
68
|
+
|
69
|
+
job_serialization.merge!(
|
70
|
+
'job_id' => job_id,
|
71
|
+
'queue_name' => job_queue,
|
72
|
+
'provider_job_id' => task_id,
|
73
|
+
'executions' => job_executions,
|
74
|
+
'priority' => nil
|
75
|
+
)
|
76
|
+
|
77
|
+
Base.execute job_serialization
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
data/lib/cloudtasker.rb
CHANGED
@@ -8,6 +8,7 @@ require 'cloudtasker/config'
|
|
8
8
|
require 'cloudtasker/authentication_error'
|
9
9
|
require 'cloudtasker/dead_worker_error'
|
10
10
|
require 'cloudtasker/invalid_worker_error'
|
11
|
+
require 'cloudtasker/missing_worker_arguments_error'
|
11
12
|
require 'cloudtasker/max_task_size_exceeded_error'
|
12
13
|
|
13
14
|
require 'cloudtasker/middleware/chain'
|
@@ -166,6 +166,9 @@ module Cloudtasker
|
|
166
166
|
# Delete task
|
167
167
|
self.class.delete(id)
|
168
168
|
resp
|
169
|
+
rescue DeadWorkerError => e
|
170
|
+
self.class.delete(id)
|
171
|
+
raise(e) if self.class.inline_mode?
|
169
172
|
rescue StandardError => e
|
170
173
|
self.job_retries += 1
|
171
174
|
raise(e) if self.class.inline_mode?
|
@@ -62,6 +62,10 @@ module Cloudtasker
|
|
62
62
|
# @return [Cloudtasker::Batch::Job] The attached batch.
|
63
63
|
#
|
64
64
|
def self.for(worker)
|
65
|
+
# Load extension if not loaded already on the worker class
|
66
|
+
worker.class.include(Extension::Worker) unless worker.class <= Extension::Worker
|
67
|
+
|
68
|
+
# Add batch capability
|
65
69
|
worker.batch = new(worker)
|
66
70
|
end
|
67
71
|
|
data/lib/cloudtasker/cli.rb
CHANGED
data/lib/cloudtasker/engine.rb
CHANGED
@@ -5,10 +5,14 @@ module Cloudtasker
|
|
5
5
|
class Engine < ::Rails::Engine
|
6
6
|
isolate_namespace Cloudtasker
|
7
7
|
|
8
|
-
|
8
|
+
config.before_initialize do
|
9
|
+
# Mount cloudtasker processing endpoint
|
9
10
|
Rails.application.routes.append do
|
10
11
|
mount Cloudtasker::Engine, at: '/cloudtasker'
|
11
12
|
end
|
13
|
+
|
14
|
+
# Add ActiveJob adapter
|
15
|
+
require 'active_job/queue_adapters/cloudtasker_adapter' if defined?(::ActiveJob::Railtie)
|
12
16
|
end
|
13
17
|
|
14
18
|
config.generators do |g|
|
data/lib/cloudtasker/version.rb
CHANGED
data/lib/cloudtasker/worker.rb
CHANGED
@@ -296,6 +296,20 @@ module Cloudtasker
|
|
296
296
|
other.is_a?(self.class) && other.job_id == job_id
|
297
297
|
end
|
298
298
|
|
299
|
+
#
|
300
|
+
# Return the max number of retries allowed for this job.
|
301
|
+
#
|
302
|
+
# The order of precedence for retry lookup is:
|
303
|
+
# - Worker `max_retries` method
|
304
|
+
# - Class `max_retries` option
|
305
|
+
# - Cloudtasker `max_retries` config option
|
306
|
+
#
|
307
|
+
# @return [Integer] The number of retries
|
308
|
+
#
|
309
|
+
def job_max_retries
|
310
|
+
@job_max_retries ||= (try(:max_retries, *job_args) || self.class.max_retries)
|
311
|
+
end
|
312
|
+
|
299
313
|
#
|
300
314
|
# Return true if the job has excceeded its maximum number
|
301
315
|
# of retries
|
@@ -303,7 +317,7 @@ module Cloudtasker
|
|
303
317
|
# @return [Boolean] True if the job is dead
|
304
318
|
#
|
305
319
|
def job_dead?
|
306
|
-
job_retries >=
|
320
|
+
job_retries >= job_max_retries
|
307
321
|
end
|
308
322
|
|
309
323
|
#
|
@@ -333,6 +347,13 @@ module Cloudtasker
|
|
333
347
|
|
334
348
|
Cloudtasker.config.server_middleware.invoke(self) do
|
335
349
|
begin
|
350
|
+
# Abort if arguments are missing. This may happen with redis arguments storage
|
351
|
+
# if Cloud Tasks times out on a job but the job still succeeds
|
352
|
+
if job_args.empty? && [0, -1].exclude?(method(:perform).arity)
|
353
|
+
raise(MissingWorkerArgumentsError, 'worker arguments are missing')
|
354
|
+
end
|
355
|
+
|
356
|
+
# Perform the job
|
336
357
|
perform(*job_args)
|
337
358
|
rescue StandardError => e
|
338
359
|
try(:on_error, e)
|
@@ -85,7 +85,7 @@ module Cloudtasker
|
|
85
85
|
redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
|
86
86
|
|
87
87
|
resp
|
88
|
-
rescue DeadWorkerError => e
|
88
|
+
rescue DeadWorkerError, MissingWorkerArgumentsError => e
|
89
89
|
# Delete stored args payload if job is dead
|
90
90
|
redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
|
91
91
|
raise(e)
|
@@ -142,7 +142,7 @@ module Cloudtasker
|
|
142
142
|
#
|
143
143
|
def log_message(level, msg, &block)
|
144
144
|
# Merge log-specific context into worker-specific context
|
145
|
-
payload_block = -> { log_block.call.merge(block&.call || {}) }
|
145
|
+
payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
|
146
146
|
|
147
147
|
# ActiveSupport::Logger does not support passing a payload through a block on top
|
148
148
|
# of a message.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudtasker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.11.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arnaud Lachaume
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -178,6 +178,20 @@ dependencies:
|
|
178
178
|
- - "~>"
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '3.0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: rspec-json_expectations
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - "~>"
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '2.2'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - "~>"
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '2.2'
|
181
195
|
- !ruby/object:Gem::Dependency
|
182
196
|
name: rubocop
|
183
197
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +220,20 @@ dependencies:
|
|
206
220
|
- - '='
|
207
221
|
- !ruby/object:Gem::Version
|
208
222
|
version: 1.37.0
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: semantic_logger
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - ">="
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: '0'
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - ">="
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: '0'
|
209
237
|
- !ruby/object:Gem::Dependency
|
210
238
|
name: timecop
|
211
239
|
requirement: !ruby/object:Gem::Requirement
|
@@ -324,6 +352,12 @@ files:
|
|
324
352
|
- gemfiles/rails_5.2.gemfile.lock
|
325
353
|
- gemfiles/rails_6.0.gemfile
|
326
354
|
- gemfiles/rails_6.0.gemfile.lock
|
355
|
+
- gemfiles/semantic_logger_3.4.gemfile
|
356
|
+
- gemfiles/semantic_logger_4.6.gemfile
|
357
|
+
- gemfiles/semantic_logger_4.7.0.gemfile
|
358
|
+
- gemfiles/semantic_logger_4.7.2.gemfile
|
359
|
+
- gemfiles/semantic_logger_4.7.gemfile
|
360
|
+
- lib/active_job/queue_adapters/cloudtasker_adapter.rb
|
327
361
|
- lib/cloudtasker.rb
|
328
362
|
- lib/cloudtasker/authentication_error.rb
|
329
363
|
- lib/cloudtasker/authenticator.rb
|
@@ -351,6 +385,7 @@ files:
|
|
351
385
|
- lib/cloudtasker/max_task_size_exceeded_error.rb
|
352
386
|
- lib/cloudtasker/meta_store.rb
|
353
387
|
- lib/cloudtasker/middleware/chain.rb
|
388
|
+
- lib/cloudtasker/missing_worker_arguments_error.rb
|
354
389
|
- lib/cloudtasker/redis_client.rb
|
355
390
|
- lib/cloudtasker/testing.rb
|
356
391
|
- lib/cloudtasker/unique_job.rb
|