cloudtasker 0.3.0 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +33 -0
- data/Gemfile.lock +19 -2
- data/README.md +149 -14
- data/Rakefile +6 -0
- data/app/controllers/cloudtasker/worker_controller.rb +8 -6
- data/cloudtasker.gemspec +3 -2
- data/docs/BATCH_JOBS.md +27 -2
- data/docs/CRON_JOBS.md +20 -14
- data/exe/cloudtasker +13 -1
- data/lib/cloudtasker.rb +0 -1
- data/lib/cloudtasker/backend/google_cloud_task.rb +41 -8
- data/lib/cloudtasker/backend/memory_task.rb +5 -3
- data/lib/cloudtasker/backend/redis_task.rb +18 -9
- data/lib/cloudtasker/batch/batch_progress.rb +11 -2
- data/lib/cloudtasker/batch/job.rb +24 -9
- data/lib/cloudtasker/cli.rb +6 -5
- data/lib/cloudtasker/cloud_task.rb +4 -2
- data/lib/cloudtasker/config.rb +14 -8
- data/lib/cloudtasker/cron/job.rb +2 -2
- data/lib/cloudtasker/cron/schedule.rb +37 -21
- data/lib/cloudtasker/local_server.rb +44 -22
- data/lib/cloudtasker/redis_client.rb +7 -8
- data/lib/cloudtasker/unique_job/job.rb +2 -2
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker/worker.rb +46 -10
- data/lib/cloudtasker/worker_handler.rb +5 -3
- data/lib/cloudtasker/worker_logger.rb +1 -1
- data/lib/cloudtasker/worker_wrapper.rb +52 -0
- data/lib/tasks/setup_queue.rake +12 -2
- metadata +20 -5
- data/lib/cloudtasker/railtie.rb +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e2a110c5354118a009e8620c887eb0c8bb9ff5c7aa63fefcf242ac9a649bd0e1
|
4
|
+
data.tar.gz: 58772d851865727f326bc30dfa3b94f170fe602eff264ce8661f2abd175033e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: af8b0e59a08d7e65bcc46f03135c17932b69816c2282bc651b1ae60887bde3ea0e52af051accb5606f1c2d042dd15dc224313a6aa7b8e6222aeef01806147464
|
7
|
+
data.tar.gz: 8b9d7e921aca496913a36357a43e8e3c0f77e059ac31439b3ccbe749afde946117a9faca70de11ee50ed6a9ff97b3439889b90bdb30db534ba4ac7690a0e3bfb
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# Changelog
|
2
|
+
|
3
|
+
## [v0.7.0](https://github.com/keypup-io/cloudtasker/tree/v0.7.0) (2019-11-25)
|
4
|
+
|
5
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.6.0...v0.7.0)
|
6
|
+
|
7
|
+
## [v0.6.0](https://github.com/keypup-io/cloudtasker/tree/v0.6.0) (2019-11-25)
|
8
|
+
|
9
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.5.0...v0.6.0)
|
10
|
+
|
11
|
+
## [v0.5.0](https://github.com/keypup-io/cloudtasker/tree/v0.5.0) (2019-11-25)
|
12
|
+
|
13
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.4.0...v0.5.0)
|
14
|
+
|
15
|
+
## [v0.4.0](https://github.com/keypup-io/cloudtasker/tree/v0.4.0) (2019-11-25)
|
16
|
+
|
17
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.3.0...v0.4.0)
|
18
|
+
|
19
|
+
## [v0.3.0](https://github.com/keypup-io/cloudtasker/tree/v0.3.0) (2019-11-25)
|
20
|
+
|
21
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.2.0...v0.3.0)
|
22
|
+
|
23
|
+
## [v0.2.0](https://github.com/keypup-io/cloudtasker/tree/v0.2.0) (2019-11-18)
|
24
|
+
|
25
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.1.0...v0.2.0)
|
26
|
+
|
27
|
+
## [v0.1.0](https://github.com/keypup-io/cloudtasker/tree/v0.1.0) (2019-11-17)
|
28
|
+
|
29
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/c137feb1ceaaaa4e2fecac0d1f0b4c73151ae002...v0.1.0)
|
30
|
+
|
31
|
+
|
32
|
+
|
33
|
+
\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
cloudtasker (0.
|
4
|
+
cloudtasker (0.8.0)
|
5
5
|
activesupport
|
6
6
|
fugit
|
7
7
|
google-cloud-tasks
|
@@ -84,9 +84,19 @@ GEM
|
|
84
84
|
tzinfo
|
85
85
|
faraday (0.17.0)
|
86
86
|
multipart-post (>= 1.2, < 3)
|
87
|
+
faraday-http-cache (2.0.0)
|
88
|
+
faraday (~> 0.8)
|
87
89
|
fugit (1.3.3)
|
88
90
|
et-orbi (~> 1.1, >= 1.1.8)
|
89
91
|
raabro (~> 1.1)
|
92
|
+
github_changelog_generator (1.15.0)
|
93
|
+
activesupport
|
94
|
+
faraday-http-cache
|
95
|
+
multi_json
|
96
|
+
octokit (~> 4.6)
|
97
|
+
rainbow (>= 2.2.1)
|
98
|
+
rake (>= 10.0)
|
99
|
+
retriable (~> 3.0)
|
90
100
|
globalid (0.4.2)
|
91
101
|
activesupport (>= 4.2.0)
|
92
102
|
google-cloud-tasks (1.3.1)
|
@@ -100,7 +110,7 @@ GEM
|
|
100
110
|
googleauth (~> 0.9)
|
101
111
|
grpc (~> 1.24)
|
102
112
|
rly (~> 0.2.3)
|
103
|
-
google-protobuf (3.
|
113
|
+
google-protobuf (3.11.0-universal-darwin)
|
104
114
|
googleapis-common-protos (1.3.9)
|
105
115
|
google-protobuf (~> 3.0)
|
106
116
|
googleapis-common-protos-types (~> 1.0)
|
@@ -143,6 +153,8 @@ GEM
|
|
143
153
|
nio4r (2.5.2)
|
144
154
|
nokogiri (1.10.5)
|
145
155
|
mini_portile2 (~> 2.4.0)
|
156
|
+
octokit (4.14.0)
|
157
|
+
sawyer (~> 0.8.0, >= 0.5.3)
|
146
158
|
os (1.0.1)
|
147
159
|
parallel (1.18.0)
|
148
160
|
parser (2.6.5.0)
|
@@ -181,6 +193,7 @@ GEM
|
|
181
193
|
rainbow (3.0.0)
|
182
194
|
rake (10.5.0)
|
183
195
|
redis (4.1.3)
|
196
|
+
retriable (3.1.2)
|
184
197
|
rly (0.2.3)
|
185
198
|
rspec (3.9.0)
|
186
199
|
rspec-core (~> 3.9.0)
|
@@ -214,6 +227,9 @@ GEM
|
|
214
227
|
rubocop (>= 0.68.1)
|
215
228
|
ruby-progressbar (1.10.1)
|
216
229
|
safe_yaml (1.0.5)
|
230
|
+
sawyer (0.8.2)
|
231
|
+
addressable (>= 2.3.5)
|
232
|
+
faraday (> 0.8, < 2.0)
|
217
233
|
signet (0.12.0)
|
218
234
|
addressable (~> 2.3)
|
219
235
|
faraday (~> 0.9)
|
@@ -249,6 +265,7 @@ DEPENDENCIES
|
|
249
265
|
appraisal
|
250
266
|
bundler (~> 2.0)
|
251
267
|
cloudtasker!
|
268
|
+
github_changelog_generator
|
252
269
|
rails
|
253
270
|
rake (~> 10.0)
|
254
271
|
rspec (~> 3.0)
|
data/README.md
CHANGED
@@ -10,6 +10,30 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
|
|
10
10
|
|
11
11
|
A local processing server is also available in development. This local server processes jobs in lieu of Cloud Tasks and allow you to work offline.
|
12
12
|
|
13
|
+
## Summary
|
14
|
+
|
15
|
+
1. [Installation](#installation)
|
16
|
+
2. [Get started with Rails](#get-started-with-rails)
|
17
|
+
3. [Configuring Cloudtasker](#configuring-cloudtasker)
|
18
|
+
1. [Cloud Tasks authentication & permissions](#cloud-tasks-authentication--permissions)
|
19
|
+
2. [Cloudtasker initializer](#cloudtasker-initializer)
|
20
|
+
4. [Enqueuing jobs](#enqueuing-jobs)
|
21
|
+
5. [Managing worker queues](#managing-worker-queues)
|
22
|
+
1. [Creating queues](#creating-queues)
|
23
|
+
2. [Assigning queues to workers](#assigning-queues-to-workers)
|
24
|
+
6. [Extensions](#extensions)
|
25
|
+
7. [Working locally](#working-locally)
|
26
|
+
1. [Option 1: Cloudtasker local server](#option-1-cloudtasker-local-server)
|
27
|
+
2. [Option 2: Using ngrok](#option-2-using-ngrok)
|
28
|
+
8. [Logging](#logging)
|
29
|
+
1. [Configuring a logger](#configuring-a-logger)
|
30
|
+
2. [Logging context](#logging-context)
|
31
|
+
9. [Error Handling](#error-handling)
|
32
|
+
1. [HTTP Error codes](#http-error-codes)
|
33
|
+
2. [Error callbacks](#error-callbacks)
|
34
|
+
3. [Max retries](#max-retries)
|
35
|
+
10. [Best practices building workers](#best-practices-building-workers)
|
36
|
+
|
13
37
|
## Installation
|
14
38
|
|
15
39
|
Add this line to your application's Gemfile:
|
@@ -77,7 +101,7 @@ Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` t
|
|
77
101
|
> cloudtasker
|
78
102
|
```
|
79
103
|
|
80
|
-
Open a Rails console and enqueue
|
104
|
+
Open a Rails console and enqueue some jobs
|
81
105
|
```ruby
|
82
106
|
# Process job as soon as possible
|
83
107
|
DummyWorker.perform_async('foo')
|
@@ -136,13 +160,31 @@ Cloudtasker.configure do |config|
|
|
136
160
|
# config.secret = 'some-long-token'
|
137
161
|
|
138
162
|
#
|
139
|
-
# Specify the details of your Google Cloud Task
|
163
|
+
# Specify the details of your Google Cloud Task location.
|
140
164
|
#
|
141
165
|
# This not required in development using the Cloudtasker local server.
|
142
166
|
#
|
143
167
|
config.gcp_location_id = 'us-central1' # defaults to 'us-east1'
|
144
168
|
config.gcp_project_id = 'my-gcp-project'
|
145
|
-
|
169
|
+
|
170
|
+
#
|
171
|
+
# Specify the namespace for your Cloud Task queues.
|
172
|
+
#
|
173
|
+
# The gem assumes that a least a default queue named 'my-app-default'
|
174
|
+
# exists in Cloud Tasks. You can create this default queue using the
|
175
|
+
# gcloud SDK or via the `rake cloudtasker:setup_queue` task if you use Rails.
|
176
|
+
#
|
177
|
+
# Workers can be scheduled on different queues. The name of the queue
|
178
|
+
# in Cloud Tasks is always assumed to be prefixed with the prefix below.
|
179
|
+
#
|
180
|
+
# E.g.
|
181
|
+
# Setting `cloudtasker_options queue: 'critical'` on a worker means that
|
182
|
+
# the worker will be pushed to 'my-app-critical' in Cloud Tasks.
|
183
|
+
#
|
184
|
+
# Specific queues can be created in Cloud Tasks using the gcloud SDK or
|
185
|
+
# via the `rake cloudtasker:setup_queue name=<queue_name>` task.
|
186
|
+
#
|
187
|
+
config.gcp_queue_prefix = 'my-app'
|
146
188
|
|
147
189
|
#
|
148
190
|
# Specify the publicly accessible host for your application
|
@@ -194,7 +236,7 @@ Cloudtasker.configure do |config|
|
|
194
236
|
end
|
195
237
|
```
|
196
238
|
|
197
|
-
If
|
239
|
+
If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
|
198
240
|
|
199
241
|
Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
|
200
242
|
```bash
|
@@ -214,10 +256,15 @@ MyWorker.perform_in(5 * 60, arg1, arg2)
|
|
214
256
|
# or with Rails
|
215
257
|
MyWorker.perform_in(5.minutes, arg1, arg2)
|
216
258
|
|
217
|
-
# Worker will be processed on specific date
|
259
|
+
# Worker will be processed on a specific date
|
218
260
|
MyWorker.perform_at(Time.parse('2025-01-01 00:50:00Z'), arg1, arg2)
|
219
261
|
# also with Rails
|
220
262
|
MyWorker.perform_at(3.days.from_now, arg1, arg2)
|
263
|
+
|
264
|
+
# With all options, including which queue to run the worker on.
|
265
|
+
MyWorker.schedule(args: [arg1, arg2], time_at: Time.parse('2025-01-01 00:50:00Z'), queue: 'critical')
|
266
|
+
# or
|
267
|
+
MyWorker.schedule(args: [arg1, arg2], time_in: 5 * 60, queue: 'critical')
|
221
268
|
```
|
222
269
|
|
223
270
|
Cloudtasker also provides a helper for re-enqueuing jobs. Re-enqueued jobs keep the same worker id. Some middlewares may rely on this to track the fact that that a job didn't actually complete (e.g. Cloustasker batch). This is optional and you can always fallback to using exception management (raise an error) to retry/re-enqueue jobs.
|
@@ -241,6 +288,52 @@ class FetchResourceWorker
|
|
241
288
|
end
|
242
289
|
```
|
243
290
|
|
291
|
+
## Managing worker queues
|
292
|
+
|
293
|
+
Cloudtasker allows you to manage several queues and distribute workers across them based on job priority. By default jobs are pushed to the `default` queue, which is `<gcp_queue_prefix>-default` in Cloud Tasks.
|
294
|
+
|
295
|
+
### Creating queues
|
296
|
+
|
297
|
+
More queues can be created using the gcloud sdk or the `cloudtasker:setup_queue` rake task.
|
298
|
+
|
299
|
+
E.g. Create a `critical` queue with a concurrency of 5 via the gcloud SDK
|
300
|
+
```bash
|
301
|
+
gcloud tasks queues create <gcp_queue_prefix>-critical --max-concurrent-dispatches=5
|
302
|
+
```
|
303
|
+
|
304
|
+
E.g. Create a `real-time` queue with a concurrency of 15 via the rake task (Rails only)
|
305
|
+
```bash
|
306
|
+
rake cloudtasker:setup_queue name=real-time concurrency=15
|
307
|
+
```
|
308
|
+
|
309
|
+
When running the Cloudtasker local processing server, you can specify the concurrency for each queue using:
|
310
|
+
```bash
|
311
|
+
cloudtasker -q critical,5 -q important,4 -q default,3
|
312
|
+
```
|
313
|
+
|
314
|
+
### Assigning queues to workers
|
315
|
+
|
316
|
+
Queues can be assigned to workers via the `cloudtasker_options` directive on the worker class:
|
317
|
+
|
318
|
+
```ruby
|
319
|
+
# app/workers/critical_worker.rb
|
320
|
+
|
321
|
+
class CriticalWorker
|
322
|
+
include Cloudtasker::Worker
|
323
|
+
|
324
|
+
cloudtasker_options queue: :critical
|
325
|
+
|
326
|
+
def perform(some_arg)
|
327
|
+
logger.info("This is a critical job run with arg=#{some_arg}.")
|
328
|
+
end
|
329
|
+
end
|
330
|
+
```
|
331
|
+
|
332
|
+
Queues can also be assigned at runtime when scheduling a job:
|
333
|
+
```ruby
|
334
|
+
CriticalWorker.schedule(args: [1], queue: :important)
|
335
|
+
```
|
336
|
+
|
244
337
|
## Extensions
|
245
338
|
Cloudtasker comes with three optional features:
|
246
339
|
- Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
|
@@ -272,8 +365,6 @@ end
|
|
272
365
|
|
273
366
|
The Cloudtasker server can then be started using:
|
274
367
|
```bash
|
275
|
-
cloudtasker
|
276
|
-
# or
|
277
368
|
bundle exec cloudtasker
|
278
369
|
```
|
279
370
|
|
@@ -284,13 +375,18 @@ web: rails s
|
|
284
375
|
worker: cloudtasker
|
285
376
|
```
|
286
377
|
|
378
|
+
Note that the local development server runs with `5` concurrent threads by default. You can tune the number of threads per queue by running `cloudtasker` the following options:
|
379
|
+
```bash
|
380
|
+
cloudtasker -q critical,5 -q important,4 -q default,3
|
381
|
+
```
|
382
|
+
|
287
383
|
### Option 2: Using ngrok
|
288
384
|
|
289
385
|
Want to test your application end to end with Google Cloud Task? Then [ngrok](https://ngrok.io) is the way to go.
|
290
386
|
|
291
387
|
First start your ngrok tunnel and take note of the :
|
292
388
|
```bash
|
293
|
-
ngrok
|
389
|
+
ngrok http 3000
|
294
390
|
```
|
295
391
|
|
296
392
|
Take note of your ngrok domain and configure Cloudtasker to use Google Cloud Task in development via ngrok.
|
@@ -299,9 +395,9 @@ Take note of your ngrok domain and configure Cloudtasker to use Google Cloud Tas
|
|
299
395
|
|
300
396
|
Cloudtasker.configure do |config|
|
301
397
|
# Specify your Google Cloud Task queue configuration
|
302
|
-
|
303
|
-
|
304
|
-
|
398
|
+
config.gcp_location_id = 'us-central1'
|
399
|
+
config.gcp_project_id = 'my-gcp-project'
|
400
|
+
config.gcp_queue_prefix = 'my-app'
|
305
401
|
|
306
402
|
# Use your ngrok domain as the processor host
|
307
403
|
config.processor_host = 'https://your-tunnel-id.ngrok.io'
|
@@ -394,7 +490,7 @@ Jobs failing will automatically return an HTTP error to Cloud Task and trigger a
|
|
394
490
|
Jobs failing will automatically return the following HTTP error code to Cloud Tasks, based on the actual reason:
|
395
491
|
|
396
492
|
| Code | Description |
|
397
|
-
|
493
|
+
|------|-------------|
|
398
494
|
| 205 | The job is dead and has been removed from the queue |
|
399
495
|
| 404 | The job has specified an incorrect worker class. |
|
400
496
|
| 422 | An error happened during the execution of the worker (`perform` method) |
|
@@ -566,6 +662,45 @@ Rails.cache.write(payload_id, data)
|
|
566
662
|
BigPayloadWorker.perform_async(payload_id)
|
567
663
|
```
|
568
664
|
|
665
|
+
### Sizing the concurrency of your queues
|
666
|
+
|
667
|
+
When defining the max concurrency concurrency of your queues (`max_concurrent_dispatches` in Cloud Tasks) you must keep in mind the maximum number of threads that your application provides. Otherwise your application threads may eventually get exhausted and your users will experience outages if all your web threads are busy running jobs.
|
668
|
+
|
669
|
+
#### With server based applications
|
670
|
+
|
671
|
+
Let's consider an application deployed in production with 3 instances, each having `RAILS_MAX_THREADS` set to `20`. This gives us a total of `60` threads available.
|
672
|
+
|
673
|
+
Now let's say that we distribute jobs across two queues: `default` and `critical`. We can set the concurrency of each each queue depending on the profile of the application:
|
674
|
+
|
675
|
+
E.g. 1: The application serves requests from web users and runs backgrounds jobs in a balanced way
|
676
|
+
```
|
677
|
+
concurrency for default queue: 20
|
678
|
+
concurrency for critical queue: 10
|
679
|
+
|
680
|
+
Total threads consumed by jobs at most: 30
|
681
|
+
Total threads always available to web users at worst: 30
|
682
|
+
```
|
683
|
+
|
684
|
+
E.g. 2: The application is a micro-service API heavily focused on running jobs (e.g. data processing)
|
685
|
+
```
|
686
|
+
concurrency for default queue: 35
|
687
|
+
concurrency for critical queue: 15
|
688
|
+
|
689
|
+
Total threads consumed by jobs at most: 50
|
690
|
+
Total threads always available to web users at worst: 10
|
691
|
+
```
|
692
|
+
|
693
|
+
Also always ensure that your total number of threads does not exceed the available number of database connections (if you use any).
|
694
|
+
|
695
|
+
#### With serverless applications
|
696
|
+
|
697
|
+
In a serverless context your application will be scaled up/down based on traffic. When we say 'traffic' this includes requests from Cloud Tasks to run jobs.
|
698
|
+
|
699
|
+
Because your application is auto-scaled - and assuming you haven't set a maximum - your job processing capacity if theoretically unlimited. The main limiting factor in a serverless context becomes external constraints such as the number of database connections available.
|
700
|
+
|
701
|
+
To size the concurrency of your queues you should therefore take the most limiting factor - which is often the database connection pool size for RDBMS databases - and use the calculations of the previous section with this limiting factor as the capping parameter instead of the threads.
|
702
|
+
|
703
|
+
|
569
704
|
## Development
|
570
705
|
|
571
706
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -574,7 +709,7 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
574
709
|
|
575
710
|
## Contributing
|
576
711
|
|
577
|
-
Bug reports and pull requests are welcome on GitHub at https://github.com/
|
712
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/keypup-io/cloudtasker. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
578
713
|
|
579
714
|
## License
|
580
715
|
|
@@ -582,7 +717,7 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
582
717
|
|
583
718
|
## Code of Conduct
|
584
719
|
|
585
|
-
Everyone interacting in the Cloudtasker project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/
|
720
|
+
Everyone interacting in the Cloudtasker project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/keypup-io/cloudtasker/blob/master/CODE_OF_CONDUCT.md).
|
586
721
|
|
587
722
|
## Author
|
588
723
|
|
data/Rakefile
CHANGED
@@ -2,7 +2,13 @@
|
|
2
2
|
|
3
3
|
require 'bundler/gem_tasks'
|
4
4
|
require 'rspec/core/rake_task'
|
5
|
+
require 'github_changelog_generator/task'
|
5
6
|
|
6
7
|
RSpec::Core::RakeTask.new(:spec)
|
7
8
|
|
8
9
|
task default: :spec
|
10
|
+
|
11
|
+
GitHubChangelogGenerator::RakeTask.new :changelog do |config|
|
12
|
+
config.user = 'keypup-io'
|
13
|
+
config.project = 'cloudtasker'
|
14
|
+
end
|
@@ -16,13 +16,15 @@ module Cloudtasker
|
|
16
16
|
# Run a worker from a Cloud Task payload
|
17
17
|
#
|
18
18
|
def run
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
19
|
+
# Build payload
|
20
|
+
payload = request.params
|
21
|
+
.slice(:worker, :job_id, :job_args, :job_meta, :job_queue)
|
22
|
+
.merge(job_retries: job_retries)
|
23
|
+
|
24
|
+
# Process payload
|
25
|
+
WorkerHandler.execute_from_payload!(payload)
|
24
26
|
head :no_content
|
25
|
-
rescue
|
27
|
+
rescue DeadWorkerError
|
26
28
|
# 205: job will NOT be retried
|
27
29
|
head :reset_content
|
28
30
|
rescue InvalidWorkerError
|
data/cloudtasker.gemspec
CHANGED
@@ -10,8 +10,8 @@ Gem::Specification.new do |spec|
|
|
10
10
|
spec.authors = ['Arnaud Lachaume']
|
11
11
|
spec.email = ['arnaud.lachaume@keypup.io']
|
12
12
|
|
13
|
-
spec.summary = '
|
14
|
-
spec.description = '
|
13
|
+
spec.summary = 'Background jobs for Ruby using Google Cloud Tasks (beta)'
|
14
|
+
spec.description = 'Background jobs for Ruby using Google Cloud Tasks (beta)'
|
15
15
|
spec.homepage = 'https://github.com/keypup-io/cloudtasker'
|
16
16
|
spec.license = 'MIT'
|
17
17
|
|
@@ -38,6 +38,7 @@ Gem::Specification.new do |spec|
|
|
38
38
|
|
39
39
|
spec.add_development_dependency 'appraisal'
|
40
40
|
spec.add_development_dependency 'bundler', '~> 2.0'
|
41
|
+
spec.add_development_dependency 'github_changelog_generator'
|
41
42
|
spec.add_development_dependency 'rake', '~> 10.0'
|
42
43
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
43
44
|
spec.add_development_dependency 'rubocop', '0.76.0'
|
data/docs/BATCH_JOBS.md
CHANGED
@@ -9,7 +9,7 @@ The Cloudtasker batch job extension allows to add sub-jobs to regular jobs. This
|
|
9
9
|
You can enable batch jobs by adding the following to your cloudtasker initializer:
|
10
10
|
```ruby
|
11
11
|
# The batch job extension is optional and must be explicitly required
|
12
|
-
require 'cloudtasker/
|
12
|
+
require 'cloudtasker/batch'
|
13
13
|
|
14
14
|
Cloudtasker.configure do |config|
|
15
15
|
# Specify your redis url.
|
@@ -59,8 +59,33 @@ The following callbacks are available on your workers to track the progress of t
|
|
59
59
|
| `on_child_dead` | `The child job` | Invoked when a child has exhausted all of its retries |s
|
60
60
|
| `on_batch_complete` | none | Invoked when all chidren have finished or died |
|
61
61
|
|
62
|
+
## Queue management
|
63
|
+
|
64
|
+
Jobs added to a batch inherit the queue of the parent. It is possible to specify a different queue when adding a job to a batch using `add_to_queue` batch method.
|
65
|
+
|
66
|
+
E.g.
|
67
|
+
|
68
|
+
```ruby
|
69
|
+
def perform
|
70
|
+
batch.add_to_queue(:critical, SubWorker, arg1, arg2, arg3)
|
71
|
+
end
|
72
|
+
```
|
73
|
+
|
62
74
|
## Batch completion
|
63
75
|
|
64
76
|
Batches complete when all children have successfully completed or died (all retries exhausted).
|
65
77
|
|
66
|
-
Jobs that fail in a batch will be retried based on the `max_retries` setting configured globally or on the worker itself. The batch will be considered `pending` while workers retry. Therefore it may be a good idea to reduce the number of retries on your workers using `cloudtasker_options max_retries: 5` to ensure your batches don't hang for too long.
|
78
|
+
Jobs that fail in a batch will be retried based on the `max_retries` setting configured globally or on the worker itself. The batch will be considered `pending` while workers retry. Therefore it may be a good idea to reduce the number of retries on your workers using `cloudtasker_options max_retries: 5` to ensure your batches don't hang for too long.
|
79
|
+
|
80
|
+
## Batch progress tracking
|
81
|
+
|
82
|
+
You can access progression statistics in callback using `batch.progress`. See the [BatchProgress](../lib/cloudtasker/batch/batch_progress.rb) class for more details.
|
83
|
+
|
84
|
+
E.g.
|
85
|
+
```ruby
|
86
|
+
def on_batch_node_complete(_child_job)
|
87
|
+
logger.info("Total: #{batch.progress.total}")
|
88
|
+
logger.info("Completed: #{batch.progress.completed}")
|
89
|
+
logger.info("Progress: #{batch.progress.percent.to_i}%")
|
90
|
+
end
|
91
|
+
```
|