cloudtasker 0.2.0 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +4 -0
  3. data/.rubocop.yml +5 -0
  4. data/.travis.yml +10 -1
  5. data/Appraisals +25 -0
  6. data/Gemfile.lock +10 -4
  7. data/README.md +550 -4
  8. data/app/controllers/cloudtasker/application_controller.rb +2 -0
  9. data/app/controllers/cloudtasker/worker_controller.rb +22 -2
  10. data/cloudtasker.gemspec +4 -3
  11. data/docs/BATCH_JOBS.md +66 -0
  12. data/docs/CRON_JOBS.md +63 -0
  13. data/docs/UNIQUE_JOBS.md +127 -0
  14. data/exe/cloudtasker +15 -0
  15. data/gemfiles/.bundle/config +2 -0
  16. data/gemfiles/google_cloud_tasks_1.0.gemfile +9 -0
  17. data/gemfiles/google_cloud_tasks_1.0.gemfile.lock +263 -0
  18. data/gemfiles/google_cloud_tasks_1.1.gemfile +9 -0
  19. data/gemfiles/google_cloud_tasks_1.1.gemfile.lock +263 -0
  20. data/gemfiles/google_cloud_tasks_1.2.gemfile +9 -0
  21. data/gemfiles/google_cloud_tasks_1.2.gemfile.lock +263 -0
  22. data/gemfiles/google_cloud_tasks_1.3.gemfile +9 -0
  23. data/gemfiles/google_cloud_tasks_1.3.gemfile.lock +264 -0
  24. data/gemfiles/rails_4.0.gemfile +10 -0
  25. data/gemfiles/rails_4.1.gemfile +9 -0
  26. data/gemfiles/rails_4.2.gemfile +9 -0
  27. data/gemfiles/rails_5.0.gemfile +9 -0
  28. data/gemfiles/rails_5.1.gemfile +9 -0
  29. data/gemfiles/rails_5.2.gemfile +9 -0
  30. data/gemfiles/rails_5.2.gemfile.lock +247 -0
  31. data/gemfiles/rails_6.0.gemfile +9 -0
  32. data/gemfiles/rails_6.0.gemfile.lock +263 -0
  33. data/lib/cloudtasker.rb +19 -1
  34. data/lib/cloudtasker/backend/google_cloud_task.rb +139 -0
  35. data/lib/cloudtasker/backend/memory_task.rb +190 -0
  36. data/lib/cloudtasker/backend/redis_task.rb +248 -0
  37. data/lib/cloudtasker/batch/batch_progress.rb +19 -1
  38. data/lib/cloudtasker/batch/job.rb +81 -20
  39. data/lib/cloudtasker/cli.rb +194 -0
  40. data/lib/cloudtasker/cloud_task.rb +91 -0
  41. data/lib/cloudtasker/config.rb +64 -2
  42. data/lib/cloudtasker/cron/job.rb +2 -2
  43. data/lib/cloudtasker/cron/schedule.rb +15 -5
  44. data/lib/cloudtasker/dead_worker_error.rb +6 -0
  45. data/lib/cloudtasker/local_server.rb +74 -0
  46. data/lib/cloudtasker/railtie.rb +10 -0
  47. data/lib/cloudtasker/testing.rb +133 -0
  48. data/lib/cloudtasker/unique_job/job.rb +1 -1
  49. data/lib/cloudtasker/unique_job/lock/base_lock.rb +1 -1
  50. data/lib/cloudtasker/unique_job/lock/until_executed.rb +3 -1
  51. data/lib/cloudtasker/unique_job/lock/while_executing.rb +3 -1
  52. data/lib/cloudtasker/version.rb +1 -1
  53. data/lib/cloudtasker/worker.rb +59 -16
  54. data/lib/cloudtasker/{task.rb → worker_handler.rb} +10 -77
  55. data/lib/cloudtasker/worker_logger.rb +155 -0
  56. data/lib/tasks/setup_queue.rake +10 -0
  57. metadata +55 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 18e57cdddeab550d4ec1bacf9be3fedb298bc54b115a6d47675e9989a68062a0
4
- data.tar.gz: 37dbae4384a908554ad9b20de3265a360b5389227a3384b3e9557b8ed5434961
3
+ metadata.gz: 3bfe3a94e07d56d772542ad9077272c8d04438f78f9640a1345a78c47e7dd6f9
4
+ data.tar.gz: c0af1564db1aab1f3a4b5927677dcb9a372010e3ea7903ca4d3d03f49efbbeb4
5
5
  SHA512:
6
- metadata.gz: 9dfc4bef540a4e9a2c9c508295ba70acf18b4b08ada13d991d6f0327511d6f8af9e572c6e46bd68126755e3b756ae6f7fe8ad46caf6cd51248583c482fdc2083
7
- data.tar.gz: fafbb8f13932eefc1e56e26fd820276f9003d9654ed4092b0286a1e66f5e903076557a7e3999e322a8fe4deab1af76bc73e1a75f8cc97a20e865ff04fecb9841
6
+ metadata.gz: 20e8f8b0a9330f439614bb6dca9437d297a068af5c576627646dc9a3583664b96a9589f3ea755d6c91d581c7fa5889baca560ca52cb5f3970980475e34f9ce11
7
+ data.tar.gz: 4c8f074f7f9c53ea4144d07a8c38fb107703c602e0b295a2f53c5d0443fb98f688580c1b31c2af7f505e41a39eed8a76105bce820b7a1a8f20454bafa0a1e497
data/.gitignore CHANGED
@@ -3,9 +3,13 @@
3
3
  /_yardoc/
4
4
  /coverage/
5
5
  /doc/
6
+ /examples/rails/log/*.log
7
+ /examples/rails/tmp/
8
+ /log/
6
9
  /pkg/
7
10
  /spec/reports/
8
11
  /spec/dummy/log/
12
+ /spec/dummy/tmp/
9
13
  /tmp/
10
14
 
11
15
  # rspec failure tracking
@@ -24,4 +24,9 @@ RSpec/ScatteredSetup:
24
24
  Metrics/BlockLength:
25
25
  Exclude:
26
26
  - cloudtasker.gemspec
27
+ - 'spec/**/*'
28
+
29
+ Style/Documentation:
30
+ Exclude:
31
+ - 'examples/**/*'
27
32
  - 'spec/**/*'
@@ -1,7 +1,16 @@
1
1
  ---
2
- sudo: false
3
2
  language: ruby
4
3
  cache: bundler
5
4
  rvm:
5
+ - 2.3
6
+ - 2.4
6
7
  - 2.5.5
7
8
  before_install: gem install bundler -v 2.0.2
9
+ before_script: rubocop
10
+ gemfile:
11
+ - gemfiles/google_cloud_tasks_1.0.gemfile
12
+ - gemfiles/google_cloud_tasks_1.1.gemfile
13
+ - gemfiles/google_cloud_tasks_1.2.gemfile
14
+ - gemfiles/google_cloud_tasks_1.3.gemfile
15
+ - gemfiles/rails_5.2.gemfile
16
+ - gemfiles/rails_6.0.gemfile
@@ -0,0 +1,25 @@
1
+ # frozen_string_literal: true
2
+
3
+ appraise 'google-cloud-tasks-1.0' do
4
+ gem 'google-cloud-tasks', '1.0'
5
+ end
6
+
7
+ appraise 'google-cloud-tasks-1.1' do
8
+ gem 'google-cloud-tasks', '1.1'
9
+ end
10
+
11
+ appraise 'google-cloud-tasks-1.2' do
12
+ gem 'google-cloud-tasks', '1.2'
13
+ end
14
+
15
+ appraise 'google-cloud-tasks-1.3' do
16
+ gem 'google-cloud-tasks', '1.3'
17
+ end
18
+
19
+ appraise 'rails-5.2' do
20
+ gem 'rails', '5.2'
21
+ end
22
+
23
+ appraise 'rails-6.0' do
24
+ gem 'rails', '6.0'
25
+ end
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- cloudtasker (0.2.0)
4
+ cloudtasker (0.3.0)
5
5
  activesupport
6
6
  fugit
7
7
  google-cloud-tasks
@@ -68,6 +68,10 @@ GEM
68
68
  zeitwerk (~> 2.1, >= 2.1.8)
69
69
  addressable (2.7.0)
70
70
  public_suffix (>= 2.0.2, < 5.0)
71
+ appraisal (2.2.0)
72
+ bundler
73
+ rake
74
+ thor (>= 0.14.0)
71
75
  ast (2.4.0)
72
76
  builder (3.2.3)
73
77
  concurrent-ruby (1.1.5)
@@ -85,9 +89,10 @@ GEM
85
89
  raabro (~> 1.1)
86
90
  globalid (0.4.2)
87
91
  activesupport (>= 4.2.0)
88
- google-cloud-tasks (1.2.0)
92
+ google-cloud-tasks (1.3.1)
89
93
  google-gax (~> 1.8)
90
94
  googleapis-common-protos (>= 1.3.9, < 2.0)
95
+ googleapis-common-protos-types (>= 1.0.4, < 2.0)
91
96
  grpc-google-iam-v1 (~> 0.6.9)
92
97
  google-gax (1.8.1)
93
98
  google-protobuf (~> 3.9)
@@ -109,7 +114,7 @@ GEM
109
114
  multi_json (~> 1.11)
110
115
  os (>= 0.9, < 2.0)
111
116
  signet (~> 0.12)
112
- grpc (1.24.0-universal-darwin)
117
+ grpc (1.25.0-universal-darwin)
113
118
  google-protobuf (~> 3.8)
114
119
  googleapis-common-protos-types (~> 1.0)
115
120
  grpc-google-iam-v1 (0.6.9)
@@ -127,7 +132,7 @@ GEM
127
132
  mini_mime (>= 0.1.1)
128
133
  marcel (0.3.3)
129
134
  mimemagic (~> 0.3.2)
130
- memoist (0.16.0)
135
+ memoist (0.16.1)
131
136
  method_source (0.9.2)
132
137
  mimemagic (0.3.3)
133
138
  mini_mime (1.0.2)
@@ -241,6 +246,7 @@ PLATFORMS
241
246
  ruby
242
247
 
243
248
  DEPENDENCIES
249
+ appraisal
244
250
  bundler (~> 2.0)
245
251
  cloudtasker!
246
252
  rails
data/README.md CHANGED
@@ -1,8 +1,14 @@
1
1
  # Cloudtasker
2
2
 
3
- Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/cloudtasker`. To experiment with that code, run `bin/console` for an interactive prompt.
3
+ Background jobs for Ruby using Google Cloud Tasks.
4
4
 
5
- TODO: Delete this and the text above, and describe your gem
5
+ Cloudtasker provides an easy to manage interface to Google Cloud Tasks for background job processing. Workers can be defined programmatically using the Cloudtasker DSL and enqueued for processing using a simple to use API.
6
+
7
+ Cloudtasker is particularly suited for serverless applications only responding to HTTP requests and where running a dedicated job processing is not an option (e.g. deploy via [Cloud Run](https://cloud.google.com/run)). All jobs enqueued in Cloud Tasks via Cloudtasker eventually gets processed by your application via HTTP requests.
8
+
9
+ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOBS.md), [batch jobs](docs/BATCH_JOBS.md) and [unique jobs](docs/UNIQUE_JOBS.md).
10
+
11
+ A local processing server is also available in development. This local server processes jobs in lieu of Cloud Tasks and allow you to work offline.
6
12
 
7
13
  ## Installation
8
14
 
@@ -20,9 +26,545 @@ Or install it yourself as:
20
26
 
21
27
  $ gem install cloudtasker
22
28
 
23
- ## Usage
29
+ ## Get started with Rails
30
+
31
+ Cloudtasker is pre-integrated with Rails. Follow the steps below to get started.
32
+
33
+ Install redis on your machine (this is required by the Cloudtasker local processing server)
34
+ ```bash
35
+ # E.g. using brew
36
+ brew install redis
37
+ ```
38
+
39
+ Add the following initializer
40
+ ```ruby
41
+ # config/initializers/cloudtasker.rb
42
+
43
+ Cloudtasker.configure do |config|
44
+ #
45
+ # Adapt the server port to be the one used by your Rails web process
46
+ #
47
+ config.processor_host = 'http://localhost:3000'
48
+
49
+ #
50
+ # If you do not have any Rails secret_key_base defined, uncomment the following
51
+ # This secret is used to authenticate jobs sent to the processing endpoint
52
+ # of your application.
53
+ #
54
+ # config.secret = 'some-long-token'
55
+ end
56
+ ```
57
+
58
+ Define your first worker:
59
+ ```ruby
60
+ # app/workers/dummy_worker.rb
61
+
62
+ class DummyWorker
63
+ include Cloudtasker::Worker
64
+
65
+ def perform(some_arg)
66
+ logger.info("Job run with #{some_arg}. This is working!")
67
+ end
68
+ end
69
+ ```
70
+
71
+ Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` to your foreman config as a `worker` process)
72
+ ```bash
73
+ # In one terminal
74
+ > rails s -p 3000
75
+
76
+ # In another terminal
77
+ > cloudtasker
78
+ ```
79
+
80
+ Open a Rails console and enqueue your job
81
+ ```ruby
82
+ # Process job as soon as possible
83
+ DummyWorker.perform_async('foo')
84
+
85
+ # Process job in 60 seconds
86
+ DummyWorker.perform_in(10, 'foo')
87
+ ```
88
+
89
+ Your Rails logs should display the following:
90
+ ```log
91
+ Started POST "/cloudtasker/run" for ::1 at 2019-11-22 09:20:09 +0100
92
+
93
+ Processing by Cloudtasker::WorkerController#run as */*
94
+ Parameters: {"worker"=>"DummyWorker", "job_id"=>"d76040a1-367e-4e3b-854e-e05a74d5f773", "job_args"=>["foo"], "job_meta"=>{}}
95
+
96
+ I, [2019-11-22T09:20:09.319336 #49257] INFO -- [Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Starting job...: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}}
97
+ I, [2019-11-22T09:20:09.319938 #49257] INFO -- [Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Job run with foo. This is working!: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}}
98
+ I, [2019-11-22T09:20:09.320966 #49257] INFO -- [Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Job done: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}}
99
+ ```
100
+
101
+ That's it! Your job was picked up by the Cloudtasker local server and sent for processing to your Rails web process.
102
+
103
+ Now jump to the next section to configure your app to use Google Cloud Tasks as a backend.
104
+
105
+ ## Configuring Cloudtasker
106
+
107
+ ### Cloud Tasks authentication & permissions
108
+
109
+ The Google Cloud library authenticates via the Google Cloud SDK by default. If you do not have it setup then we recommend you [install it](https://cloud.google.com/sdk/docs/quickstarts).
110
+
111
+ Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-bigquery/AUTHENTICATION.md).
112
+
113
+ In order to function properly Cloudtasker requires the authenticated account to have the following IAM permissions:
114
+ - `cloudtasks.tasks.get`
115
+ - `cloudtasks.tasks.create`
116
+ - `cloudtasks.tasks.delete`
117
+
118
+ To get started quickly you can add the `roles/cloudtasks.queueAdmin` role to your account via the [IAM Console](https://console.cloud.google.com/iam-admin/iam). This is not required if your account is a project admin account.
119
+
120
+
121
+ ### Cloudtasker initializer
122
+
123
+ The gem can be configured through an initializer. See below all the available configuration options.
124
+
125
+ ```ruby
126
+ # config/initializers/cloudtasker.rb
127
+
128
+ Cloudtasker.configure do |config|
129
+ #
130
+ # If you do not have any Rails secret_key_base defined, uncomment the following.
131
+ # This secret is used to authenticate jobs sent to the processing endpoint
132
+ # of your application.
133
+ #
134
+ # Default with Rails: Rails.application.credentials.secret_key_base
135
+ #
136
+ # config.secret = 'some-long-token'
137
+
138
+ #
139
+ # Specify the details of your Google Cloud Task queue.
140
+ #
141
+ # This not required in development using the Cloudtasker local server.
142
+ #
143
+ config.gcp_location_id = 'us-central1' # defaults to 'us-east1'
144
+ config.gcp_project_id = 'my-gcp-project'
145
+ config.gcp_queue_id = 'my-queue'
146
+
147
+ #
148
+ # Specify the publicly accessible host for your application
149
+ #
150
+ # > E.g. in development, using the cloudtasker local server
151
+ # config.processor_host = 'http://localhost:3000'
152
+ #
153
+ # > E.g. in development, using `config.mode = :production` and ngrok
154
+ # config.processor_host = 'https://111111.ngrok.io'
155
+ #
156
+ config.processor_host = 'https://app.mydomain.com'
157
+
158
+ #
159
+ # Specify the mode of operation:
160
+ # - :development => jobs will be pushed to Redis and picked up by the Cloudtasker local server
161
+ # - :production => jobs will be pushed to Google Cloud Tasks. Requires a publicly accessible domain.
162
+ #
163
+ # Defaults to :development unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set to something else.
164
+ #
165
+ # config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
166
+
167
+ #
168
+ # Specify the logger to use
169
+ #
170
+ # Default with Rails: Rails.logger
171
+ # Default without Rails: Logger.new(STDOUT)
172
+ #
173
+ # config.logger = MyLogger.new(STDOUT)
174
+
175
+ #
176
+ # Specify how many retries are allowed on jobs. This number of retries excludes any
177
+ # connectivity error that would be due to the application being down or unreachable.
178
+ #
179
+ # Default: 25
180
+ #
181
+ # config.max_retries = 10
182
+
183
+ #
184
+ # Specify the redis connection hash.
185
+ #
186
+ # This is ONLY required in development for the Cloudtasker local server and in
187
+ # all environments if you use any cloudtasker extension (unique jobs, cron jobs or batch jobs)
188
+ #
189
+ # See https://github.com/redis/redis-rb for examples of configuration hashes.
190
+ #
191
+ # Default: redis-rb connects to redis://127.0.0.1:6379/0
192
+ #
193
+ # config.redis = { url: 'redis://localhost:6379/5' }
194
+ end
195
+ ```
196
+
197
+ If your queue does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
198
+
199
+ Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
200
+ ```bash
201
+ bundle exec rake cloudtasker:setup_queue
202
+ ```
203
+
204
+ ## Enqueuing jobs
205
+
206
+ Cloudtasker provides multiple ways of enqueuing jobs.
207
+
208
+ ```ruby
209
+ # Worker will be processed as soon as possible
210
+ MyWorker.perform_async(arg1, arg2)
211
+
212
+ # Worker will be processed in 5 minutes
213
+ MyWorker.perform_in(5 * 60, arg1, arg2)
214
+ # or with Rails
215
+ MyWorker.perform_in(5.minutes, arg1, arg2)
216
+
217
+ # Worker will be processed on specific date
218
+ MyWorker.perform_at(Time.parse('2025-01-01 00:50:00Z'), arg1, arg2)
219
+ # also with Rails
220
+ MyWorker.perform_at(3.days.from_now, arg1, arg2)
221
+ ```
222
+
223
+ Cloudtasker also provides a helper for re-enqueuing jobs. Re-enqueued jobs keep the same worker id. Some middlewares may rely on this to track the fact that that a job didn't actually complete (e.g. Cloustasker batch). This is optional and you can always fallback to using exception management (raise an error) to retry/re-enqueue jobs.
224
+
225
+ E.g.
226
+ ```ruby
227
+ # app/workers/fetch_resource_worker.rb
228
+
229
+ class FetchResourceWorker
230
+ include Cloudtasker::Worker
231
+
232
+ def perform(id)
233
+ # ...do some logic...
234
+ if some_condition
235
+ # Stop and re-enqueue the job to be run again in 10 seconds.
236
+ return reenqueue(10)
237
+ else
238
+ # ...keep going...
239
+ end
240
+ end
241
+ end
242
+ ```
243
+
244
+ ## Extensions
245
+ Cloudtasker comes with three optional features:
246
+ - Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
247
+ - Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
248
+ - Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
249
+
250
+ ## Working locally
251
+
252
+ Cloudtasker pushes jobs to Google Cloud Tasks, which in turn sends jobs for processing to your application via HTTP POST requests to the `/cloudtasker/run` endpoint of the publicly accessible domain of your application.
253
+
254
+ When working locally on your application it is usually not possible to have a public domain. So what are the options?
255
+
256
+ ### Option 1: Cloudtasker local server
257
+ The Cloudtasker local server is a ruby daemon that looks for jobs pushed to Redis and sends them to your application via HTTP POST requests. The server mimics the way Google Cloud Tasks works, but locally!
258
+
259
+ You can configure your applicatiion to use the Cloudtasker local server using the following initializer:
260
+ ```ruby
261
+ # config/initializers/cloudtasker.rb
262
+
263
+ Cloudtasker.configure do |config|
264
+ # ... other options
265
+
266
+ # Push jobs to redis and let the Cloudtasker local server collect them
267
+ # This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
268
+ # to a non-development environment
269
+ config.mode = :development
270
+ end
271
+ ```
272
+
273
+ The Cloudtasker server can then be started using:
274
+ ```bash
275
+ cloudtasker
276
+ # or
277
+ bundle exec cloudtasker
278
+ ```
279
+
280
+ You can as well define a Procfile to manage the cloudtasker process via foreman. Then use `foreman start` to launch both your Rails server and the Cloudtasker local server.
281
+ ```yaml
282
+ # Procfile
283
+ web: rails s
284
+ worker: cloudtasker
285
+ ```
286
+
287
+ ### Option 2: Using ngrok
288
+
289
+ Want to test your application end to end with Google Cloud Task? Then [ngrok](https://ngrok.io) is the way to go.
290
+
291
+ First start your ngrok tunnel and take note of the :
292
+ ```bash
293
+ ngrok tls 3000
294
+ ```
295
+
296
+ Take note of your ngrok domain and configure Cloudtasker to use Google Cloud Task in development via ngrok.
297
+ ```ruby
298
+ # config/initializers/cloudtasker.rb
299
+
300
+ Cloudtasker.configure do |config|
301
+ # Specify your Google Cloud Task queue configuration
302
+ # config.gcp_location_id = 'us-central1'
303
+ # config.gcp_project_id = 'my-gcp-project'
304
+ # config.gcp_queue_id = 'my-queue'
305
+
306
+ # Use your ngrok domain as the processor host
307
+ config.processor_host = 'https://your-tunnel-id.ngrok.io'
308
+
309
+ # Force Cloudtasker to use Google Cloud Tasks in development
310
+ config.mode = :production
311
+ end
312
+ ```
313
+
314
+ Finally start Rails to accept jobs from Google Cloud Tasks
315
+ ```bash
316
+ rails s
317
+ ```
318
+
319
+ ## Logging
320
+ There are several options available to configure logging and logging context.
321
+
322
+ ### Configuring a logger
323
+ Cloudtasker uses `Rails.logger` if Rails is available and falls back on a plain ruby logger `Logger.new(STDOUT)` if not.
324
+
325
+ It is also possible to configure your own logger. For example you can setup Cloudtasker with [semantic_logger](http://rocketjob.github.io/semantic_logger) by doing the following your initializer:
326
+ ```ruby
327
+ # config/initializers/cloudtasker.rb
328
+
329
+ Cloudtasker.configure do |config|
330
+ config.logger = SemanticLogger[Cloudtasker]
331
+ end
332
+ ```
24
333
 
25
- TODO: Write usage instructions here
334
+ ### Logging context
335
+ Cloudtasker provides worker contextual information to the worker `logger` method inside your worker methods.
336
+
337
+ For example:
338
+ ```ruby
339
+ # app/workers/dummy_worker.rb
340
+
341
+ class DummyWorker
342
+ include Cloudtasker::Worker
343
+
344
+ def perform(some_arg)
345
+ logger.info("Job run with #{some_arg}. This is working!")
346
+ end
347
+ end
348
+ ```
349
+
350
+ Will generate the following log with context `{:worker=> ..., :job_id=> ..., :job_meta=> ...}`
351
+ ```log
352
+ [Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Job run with foo. This is working!: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}}
353
+ ```
354
+
355
+ The way contextual information is displayed depends on the logger itself. For example with [semantic_logger](http://rocketjob.github.io/semantic_logger) contextual information might not appear in the log message but show up as payload data on the log entry itself (e.g. using the fluentd adapter).
356
+
357
+ Contextual information can be customised globally and locally using a log context_processor. By default the `Cloudtasker::WorkerLogger` is configured the following way:
358
+ ```ruby
359
+ Cloudtasker::WorkerLogger.log_context_processor = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta) }
360
+ ```
361
+
362
+ You can decide to add a global identifier for your worker logs using the following:
363
+ ```ruby
364
+ # config/initializers/cloudtasker.rb
365
+
366
+ Cloudtasker::WorkerLogger.log_context_processor = lambda { |worker|
367
+ worker.to_h.slice(:worker, :job_id, :job_meta).merge(app: 'my-app')
368
+ }
369
+ ```
370
+
371
+ You could also decide to log all available context (including arguments passed to perform) for specific workers only:
372
+ ```ruby
373
+ # app/workers/full_context_worker.rb
374
+
375
+ class FullContextWorker
376
+ include Cloudtasker::Worker
377
+
378
+ cloudtasker_options log_context_processor: ->(worker) { worker.to_h }
379
+
380
+ def perform(some_arg)
381
+ logger.info("This log entry will have full context!")
382
+ end
383
+ end
384
+ ```
385
+
386
+ See the [Cloudtasker::Worker class](blob/master/lib/cloudtasker/worker.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
387
+
388
+ ## Error Handling
389
+
390
+ Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of retries Cloud Task will do depends on the configuration of your queue in Cloud Tasks.
391
+
392
+ ### HTTP Error codes
393
+
394
+ Jobs failing will automatically return the following HTTP error code to Cloud Tasks, based on the actual reason:
395
+
396
+ | Code | Description |
397
+ |------|-------------|-----------|
398
+ | 205 | The job is dead and has been removed from the queue |
399
+ | 404 | The job has specified an incorrect worker class. |
400
+ | 422 | An error happened during the execution of the worker (`perform` method) |
401
+
402
+ ### Error callbacks
403
+
404
+ Workers can implement the `on_error(error)` and `on_dead(error)` callbacks to do things when a job fails during its execution:
405
+
406
+ E.g.
407
+ ```ruby
408
+ # app/workers/handle_error_worker.rb
409
+
410
+ class HandleErrorWorker
411
+ include Cloudtasker::Worker
412
+
413
+ def perform
414
+ raise(ArgumentError)
415
+ end
416
+
417
+ # The runtime error is passed as an argument.
418
+ def on_error(error)
419
+ logger.error("The following error happened: #{error}")
420
+ end
421
+
422
+ # The job has been retried too many times and will be removed
423
+ # from the queue.
424
+ def on_dead(error)
425
+ logger.error("The job died with the following error: #{error}")
426
+ end
427
+ end
428
+ ```
429
+
430
+ ### Max retries
431
+
432
+ By default jobs are retried 25 times - using an exponential backoff - before being declared dead. This number of retries can be customized locally on workers and/or globally via the Cloudtasker initializer.
433
+
434
+ Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
435
+
436
+ E.g. Set max number of retries globally via the cloudtasker initializer.
437
+ ```ruby
438
+ # config/initializers/cloudtasker.rb
439
+
440
+ Cloudtasker.configure do |config|
441
+ #
442
+ # Specify how many retries are allowed on jobs. This number of retries excludes any
443
+ # connectivity error that would be due to the application being down or unreachable.
444
+ #
445
+ # Default: 25
446
+ #
447
+ config.max_retries = 10
448
+ end
449
+ ```
450
+
451
+ E.g. Set max number of retries to 3 on a given worker
452
+
453
+ E.g.
454
+ ```ruby
455
+ # app/workers/some_error_worker.rb
456
+
457
+ class SomeErrorWorker
458
+ include Cloudtasker::Worker
459
+
460
+ # This will override the global setting
461
+ cloudtasker_options max_retries: 3
462
+
463
+ def perform()
464
+ raise(ArgumentError)
465
+ end
466
+ end
467
+ ```
468
+
469
+
470
+
471
+ ## Best practices building workers
472
+
473
+ Below are recommendations and notes about creating workers.
474
+
475
+ ### Use primitive arguments
476
+ Pushing a job via `MyWorker.perform_async(arg1, arg2)` will serialize all arguments as JSON. Cloudtasker does not do any magic marshalling and therefore passing user-defined class instance as arguments is likely to make your jobs fail because of JSON serialization/deserialization.
477
+
478
+ When defining your worker `perform` method, use primitive arguments (integers, strings, hashes).
479
+
480
+ Don't do that:
481
+ ```ruby
482
+ # app/workers/user_email_worker.rb
483
+
484
+ class UserEmailWorker
485
+ include Cloudtasker::Worker
486
+
487
+ def perform(user)
488
+ user.reload.send_email
489
+ end
490
+ end
491
+ ```
492
+
493
+ Do that:
494
+ ```ruby
495
+ # app/workers/user_email_worker.rb
496
+
497
+ class UserEmailWorker
498
+ include Cloudtasker::Worker
499
+
500
+ def perform(user_id)
501
+ User.find_by(id: user_id)&.send_email
502
+ end
503
+ end
504
+ ```
505
+
506
+ ### Assume hash arguments are stringified
507
+ Because of JSON serialization/deserialization hashes passed to `perform_*` methods will eventually be passed as stringified hashes to the worker `perform` method.
508
+
509
+ ```ruby
510
+ # Enqueuing a job with:
511
+ MyWorker.perform_async({ foo: 'bar', 'baz' => { key: 'value' } })
512
+
513
+ # will be processed as
514
+ MyWorker.new.perform({ 'foo' => 'bar', 'baz' => { 'key' => 'value' } })
515
+ ```
516
+
517
+ ### Be careful with default arguments
518
+ Default arguments passed to the `perform` method are not actually considered as job arguments. Default arguments will therefore be ignored in contextual logging and by extensions relying on arguments such as the `unique-job` extension.
519
+
520
+ Consider the following worker:
521
+ ```ruby
522
+ # app/workers/user_email_worker.rb
523
+
524
+ class UserEmailWorker
525
+ include Cloudtasker::Worker
526
+
527
+ cloudtasker_options lock: :until_executed
528
+
529
+ def perform(user_id, time_at = Time.now.iso8601)
530
+ User.find_by(id: user_id)&.send_email(Time.parse(time_at))
531
+ end
532
+ end
533
+ ```
534
+
535
+ If you enqueue this worker by omitting the second argument `MyWorker.perform_async(123)` then:
536
+ - The `time_at` argument will not be included in contextual logging
537
+ - The `time_at` argument will be ignored by the `unique-job` extension, meaning that job uniqueness will be only based on the `user_id` argument.
538
+
539
+ ### Handling big job payloads
540
+ Keep in mind that jobs are pushed to Google Cloud Tasks via API and then delivered to your application via API as well. Therefore any excessive job payload will slow down the enqueuing of jobs and create additional processing when receiving the job.
541
+
542
+ If you feel that a job payload is going to get big, prefer to store the payload using a datastore (e.g. Redis) and pass a reference to the job to retrieve the payload inside your job `perform` method.
543
+
544
+ E.g. Define a job like this
545
+ ```ruby
546
+ # app/workers/big_payload_worker.rb
547
+
548
+ class BigPayloadWorker
549
+ include Cloudtasker::Worker
550
+
551
+ def perform(payload_id)
552
+ data = Rails.cache.fetch(payload_id)
553
+ # ...do some processing...
554
+ end
555
+ end
556
+ ```
557
+
558
+ Then enqueue your job like this:
559
+ ```ruby
560
+ # Fetch and store the payload
561
+ data = ApiClient.fetch_thousands_of_records
562
+ payload_id = SecureRandom.uuid
563
+ Rails.cache.write(payload_id, data)
564
+
565
+ # Enqueue the processing job
566
+ BigPayloadWorker.perform_async(payload_id)
567
+ ```
26
568
 
27
569
  ## Development
28
570
 
@@ -41,3 +583,7 @@ The gem is available as open source under the terms of the [MIT License](https:/
41
583
  ## Code of Conduct
42
584
 
43
585
  Everyone interacting in the Cloudtasker project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/alachaum/cloudtasker/blob/master/CODE_OF_CONDUCT.md).
586
+
587
+ ## Author
588
+
589
+ Provided with :heart: by [keypup.io](https://keypup.io/)