cloudtasker 0.13.0 → 0.13.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/lint_rubocop.yml +2 -7
- data/.github/workflows/test_ruby_2.6.yml +37 -0
- data/.github/workflows/test_ruby_2.7.yml +21 -27
- data/.github/workflows/test_ruby_3.x.yml +21 -26
- data/.rubocop.yml +4 -1
- data/Appraisals +32 -32
- data/CHANGELOG.md +11 -2
- data/README.md +10 -11
- data/gemfiles/google_cloud_tasks_1.0.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_1.1.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_1.2.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_1.3.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_1.4.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_1.5.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_2.0.gemfile +1 -1
- data/gemfiles/google_cloud_tasks_2.1.gemfile +1 -1
- data/gemfiles/rails_5.2.gemfile +1 -1
- data/gemfiles/rails_6.0.gemfile +1 -1
- data/gemfiles/rails_6.1.gemfile +1 -1
- data/gemfiles/rails_7.0.gemfile +1 -1
- data/gemfiles/semantic_logger_3.4.gemfile +1 -1
- data/gemfiles/semantic_logger_4.6.gemfile +1 -1
- data/gemfiles/semantic_logger_4.7.0.gemfile +1 -1
- data/gemfiles/semantic_logger_4.7.2.gemfile +1 -1
- data/lib/cloudtasker/backend/google_cloud_task_v1.rb +24 -6
- data/lib/cloudtasker/backend/google_cloud_task_v2.rb +25 -6
- data/lib/cloudtasker/backend/redis_task.rb +4 -4
- data/lib/cloudtasker/batch/batch_progress.rb +17 -3
- data/lib/cloudtasker/batch/job.rb +36 -17
- data/lib/cloudtasker/config.rb +2 -16
- data/lib/cloudtasker/cron/schedule.rb +3 -3
- data/lib/cloudtasker/version.rb +1 -1
- data/lib/cloudtasker.rb +2 -1
- metadata +4 -4
- data/.github/workflows/test_ruby_2.5_2.6.yml +0 -44
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: acd6db925df54f7f8b10459c94e76fbd4f4316b706bf7918afaac9857161335d
|
4
|
+
data.tar.gz: 690a9de1bc62150eb9d6e76dc8689c28a7811996dd036edb3351dce88ee2e082
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc66650e5d56d77fe026acbe01dd74eaf44526d2218849731cfc8dae65b8bb932016940370b2e478904f45d23197b222c400f788ee9b368a4493746f1a14cb38
|
7
|
+
data.tar.gz: 4385ec200c8d5a2b6a1e7fdf9b700a294184894ab4d1c97f83d31a0c7adf636cb02694f1b85394f030e54fff66a0a78de1b01a4ebbb8dd1bf041c708ca64137e
|
@@ -8,13 +8,8 @@ jobs:
|
|
8
8
|
steps:
|
9
9
|
- uses: actions/checkout@v2
|
10
10
|
- uses: zhulik/redis-action@1.1.0
|
11
|
-
-
|
12
|
-
uses: ruby/setup-ruby@v1
|
11
|
+
- uses: ruby/setup-ruby@v1
|
13
12
|
with:
|
14
13
|
ruby-version: '3.0.1'
|
15
14
|
bundler-cache: true
|
16
|
-
-
|
17
|
-
run: |
|
18
|
-
gem install bundler
|
19
|
-
bundle install --jobs 4 --retry 3
|
20
|
-
bundle exec rubocop
|
15
|
+
- run: bundle exec rubocop
|
@@ -0,0 +1,37 @@
|
|
1
|
+
name: Ruby 2.6
|
2
|
+
|
3
|
+
on: [push, pull_request]
|
4
|
+
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ubuntu-latest
|
8
|
+
strategy:
|
9
|
+
matrix:
|
10
|
+
ruby:
|
11
|
+
- '2.6'
|
12
|
+
appraisal:
|
13
|
+
- 'google_cloud_tasks_1.0'
|
14
|
+
- 'google_cloud_tasks_1.1'
|
15
|
+
- 'google_cloud_tasks_1.2'
|
16
|
+
- 'google_cloud_tasks_1.3'
|
17
|
+
- 'google_cloud_tasks_1.4'
|
18
|
+
- 'google_cloud_tasks_1.5'
|
19
|
+
- 'google_cloud_tasks_2.0'
|
20
|
+
- 'google_cloud_tasks_2.1'
|
21
|
+
- 'rails_5.2'
|
22
|
+
- 'rails_6.0'
|
23
|
+
- 'rails_6.1'
|
24
|
+
- 'semantic_logger_3.4'
|
25
|
+
- 'semantic_logger_4.6'
|
26
|
+
- 'semantic_logger_4.7.0'
|
27
|
+
- 'semantic_logger_4.7.2'
|
28
|
+
env:
|
29
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
|
30
|
+
steps:
|
31
|
+
- uses: actions/checkout@v2
|
32
|
+
- uses: zhulik/redis-action@1.1.0
|
33
|
+
- uses: ruby/setup-ruby@v1
|
34
|
+
with:
|
35
|
+
ruby-version: ${{ matrix.ruby }}
|
36
|
+
bundler-cache: true
|
37
|
+
- run: bundle exec rspec
|
@@ -8,37 +8,31 @@ jobs:
|
|
8
8
|
strategy:
|
9
9
|
matrix:
|
10
10
|
ruby:
|
11
|
-
- '2.7
|
11
|
+
- '2.7'
|
12
12
|
appraisal:
|
13
|
-
- '
|
14
|
-
- '
|
15
|
-
- '
|
16
|
-
- '
|
17
|
-
- '
|
18
|
-
- '
|
19
|
-
- '
|
20
|
-
- '
|
21
|
-
- '
|
22
|
-
- '
|
23
|
-
- '
|
24
|
-
- '
|
25
|
-
- '
|
26
|
-
- '
|
27
|
-
- '
|
28
|
-
- '
|
13
|
+
- 'google_cloud_tasks_1.0'
|
14
|
+
- 'google_cloud_tasks_1.1'
|
15
|
+
- 'google_cloud_tasks_1.2'
|
16
|
+
- 'google_cloud_tasks_1.3'
|
17
|
+
- 'google_cloud_tasks_1.4'
|
18
|
+
- 'google_cloud_tasks_1.5'
|
19
|
+
- 'google_cloud_tasks_2.0'
|
20
|
+
- 'google_cloud_tasks_2.1'
|
21
|
+
- 'rails_5.2'
|
22
|
+
- 'rails_6.0'
|
23
|
+
- 'rails_6.1'
|
24
|
+
- 'rails_7.0'
|
25
|
+
- 'semantic_logger_3.4'
|
26
|
+
- 'semantic_logger_4.6'
|
27
|
+
- 'semantic_logger_4.7.0'
|
28
|
+
- 'semantic_logger_4.7.2'
|
29
|
+
env:
|
30
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
|
29
31
|
steps:
|
30
32
|
- uses: actions/checkout@v2
|
31
33
|
- uses: zhulik/redis-action@1.1.0
|
32
|
-
-
|
33
|
-
uses: ruby/setup-ruby@v1
|
34
|
+
- uses: ruby/setup-ruby@v1
|
34
35
|
with:
|
35
36
|
ruby-version: ${{ matrix.ruby }}
|
36
37
|
bundler-cache: true
|
37
|
-
-
|
38
|
-
env:
|
39
|
-
APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
|
40
|
-
run: |
|
41
|
-
gem install bundler
|
42
|
-
bundle install --jobs 4 --retry 3
|
43
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
|
44
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
|
38
|
+
- run: bundle exec rspec
|
@@ -8,36 +8,31 @@ jobs:
|
|
8
8
|
strategy:
|
9
9
|
matrix:
|
10
10
|
ruby:
|
11
|
-
- '3.0
|
12
|
-
|
11
|
+
- '3.0'
|
12
|
+
- '3.1'
|
13
|
+
- '3.2'
|
13
14
|
appraisal:
|
14
|
-
- '
|
15
|
-
- '
|
16
|
-
- '
|
17
|
-
- '
|
18
|
-
- '
|
19
|
-
- '
|
20
|
-
- '
|
21
|
-
- '
|
22
|
-
- '
|
23
|
-
- '
|
24
|
-
- '
|
25
|
-
- '
|
26
|
-
- '
|
27
|
-
- '
|
15
|
+
- 'google_cloud_tasks_1.0'
|
16
|
+
- 'google_cloud_tasks_1.1'
|
17
|
+
- 'google_cloud_tasks_1.2'
|
18
|
+
- 'google_cloud_tasks_1.3'
|
19
|
+
- 'google_cloud_tasks_1.4'
|
20
|
+
- 'google_cloud_tasks_1.5'
|
21
|
+
- 'google_cloud_tasks_2.0'
|
22
|
+
- 'google_cloud_tasks_2.1'
|
23
|
+
- 'rails_6.1'
|
24
|
+
- 'rails_7.0'
|
25
|
+
- 'semantic_logger_3.4'
|
26
|
+
- 'semantic_logger_4.6'
|
27
|
+
- 'semantic_logger_4.7.0'
|
28
|
+
- 'semantic_logger_4.7.2'
|
29
|
+
env:
|
30
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
|
28
31
|
steps:
|
29
32
|
- uses: actions/checkout@v2
|
30
33
|
- uses: zhulik/redis-action@1.1.0
|
31
|
-
-
|
32
|
-
uses: ruby/setup-ruby@v1
|
34
|
+
- uses: ruby/setup-ruby@v1
|
33
35
|
with:
|
34
36
|
ruby-version: ${{ matrix.ruby }}
|
35
37
|
bundler-cache: true
|
36
|
-
-
|
37
|
-
env:
|
38
|
-
APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
|
39
|
-
run: |
|
40
|
-
gem install bundler
|
41
|
-
bundle install --jobs 4 --retry 3
|
42
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
|
43
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
|
38
|
+
- run: bundle exec rspec
|
data/.rubocop.yml
CHANGED
data/Appraisals
CHANGED
@@ -1,85 +1,85 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
appraise '
|
4
|
-
gem 'activesupport', '6.1' # ruby 2.
|
3
|
+
appraise 'google_cloud_tasks_1.0' do
|
4
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
5
5
|
gem 'google-cloud-tasks', '~> 1.0.0'
|
6
6
|
end
|
7
7
|
|
8
|
-
appraise '
|
9
|
-
gem 'activesupport', '6.1' # ruby 2.
|
8
|
+
appraise 'google_cloud_tasks_1.1' do
|
9
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
10
10
|
gem 'google-cloud-tasks', '~> 1.1.0'
|
11
11
|
end
|
12
12
|
|
13
|
-
appraise '
|
14
|
-
gem 'activesupport', '6.1' # ruby 2.
|
13
|
+
appraise 'google_cloud_tasks_1.2' do
|
14
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
15
15
|
gem 'google-cloud-tasks', '~> 1.2.0'
|
16
16
|
end
|
17
17
|
|
18
|
-
appraise '
|
19
|
-
gem 'activesupport', '6.1' # ruby 2.
|
18
|
+
appraise 'google_cloud_tasks_1.3' do
|
19
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
20
20
|
gem 'google-cloud-tasks', '~> 1.3.0'
|
21
21
|
end
|
22
22
|
|
23
|
-
appraise '
|
24
|
-
gem 'activesupport', '6.1' # ruby 2.
|
23
|
+
appraise 'google_cloud_tasks_1.4' do
|
24
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
25
25
|
gem 'google-cloud-tasks', '~> 1.4.0'
|
26
26
|
end
|
27
27
|
|
28
|
-
appraise '
|
29
|
-
gem 'activesupport', '6.1' # ruby 2.
|
28
|
+
appraise 'google_cloud_tasks_1.5' do
|
29
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
30
30
|
gem 'google-cloud-tasks', '~> 1.5.0'
|
31
31
|
end
|
32
32
|
|
33
|
-
appraise '
|
34
|
-
gem 'activesupport', '6.1' # ruby 2.
|
33
|
+
appraise 'google_cloud_tasks_2.0' do
|
34
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
35
35
|
gem 'google-cloud-tasks', '~> 2.0.0'
|
36
36
|
end
|
37
37
|
|
38
|
-
appraise '
|
39
|
-
gem 'activesupport', '6.1' # ruby 2.
|
38
|
+
appraise 'google_cloud_tasks_2.1' do
|
39
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
40
40
|
gem 'google-cloud-tasks', '~> 2.1.0'
|
41
41
|
end
|
42
42
|
|
43
43
|
if RUBY_VERSION < '3'
|
44
|
-
appraise '
|
45
|
-
gem 'rails', '5.2'
|
44
|
+
appraise 'rails_5.2' do
|
45
|
+
gem 'rails', '~> 5.2.0'
|
46
46
|
gem 'rspec-rails'
|
47
47
|
end
|
48
48
|
|
49
|
-
appraise '
|
50
|
-
gem 'rails', '6.0'
|
49
|
+
appraise 'rails_6.0' do
|
50
|
+
gem 'rails', '~> 6.0.0'
|
51
51
|
gem 'rspec-rails'
|
52
52
|
end
|
53
53
|
end
|
54
54
|
|
55
|
-
appraise '
|
56
|
-
gem 'rails', '6.1'
|
55
|
+
appraise 'rails_6.1' do
|
56
|
+
gem 'rails', '~> 6.1.0'
|
57
57
|
gem 'rspec-rails'
|
58
58
|
end
|
59
59
|
|
60
60
|
if RUBY_VERSION >= '2.7'
|
61
|
-
appraise '
|
62
|
-
gem 'rails', '7.0'
|
61
|
+
appraise 'rails_7.0' do
|
62
|
+
gem 'rails', '~> 7.0.0'
|
63
63
|
gem 'rspec-rails'
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
-
appraise '
|
68
|
-
gem 'activesupport', '6.1' # ruby 2.
|
67
|
+
appraise 'semantic_logger_3.4' do
|
68
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
69
69
|
gem 'semantic_logger', '3.4.1'
|
70
70
|
end
|
71
71
|
|
72
|
-
appraise '
|
73
|
-
gem 'activesupport', '6.1' # ruby 2.
|
72
|
+
appraise 'semantic_logger_4.6' do
|
73
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
74
74
|
gem 'semantic_logger', '4.6.1'
|
75
75
|
end
|
76
76
|
|
77
|
-
appraise '
|
78
|
-
gem 'activesupport', '6.1' # ruby 2.
|
77
|
+
appraise 'semantic_logger_4.7.0' do
|
78
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
79
79
|
gem 'semantic_logger', '4.7.0'
|
80
80
|
end
|
81
81
|
|
82
|
-
appraise '
|
83
|
-
gem 'activesupport', '6.1' # ruby 2.
|
82
|
+
appraise 'semantic_logger_4.7.2' do
|
83
|
+
gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
|
84
84
|
gem 'semantic_logger', '4.7.2'
|
85
85
|
end
|
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,17 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
## [v0.13.
|
3
|
+
## [v0.13.1](https://github.com/keypup-io/cloudtasker/tree/v0.13.1) (2023-06-19)
|
4
4
|
|
5
|
-
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.
|
5
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.13.0...v0.13.1)
|
6
|
+
|
7
|
+
**Fixed bugs:**
|
8
|
+
- Batch jobs: Gracefully handle errors (e.g. Redis connection error) while setting up child jobs. Previously this could lead to ghost child jobs preventing the parent batch from finishing (child job is registered but is never enqueued)
|
9
|
+
- Protobuf: fix formatting of protobuf payload by removig `nil` values and formatting the `dispatch_deadline` as a proper `Google::Protobuf::Duration` (instead of `integer`). Fixes [#94](https://github.com/keypup-io/cloudtasker/issues/94)
|
10
|
+
|
11
|
+
|
12
|
+
## [v0.13.0](https://github.com/keypup-io/cloudtasker/tree/v0.13.0) (2022-03-11)
|
13
|
+
|
14
|
+
[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.12.2...v0.13.0)
|
6
15
|
|
7
16
|
**Improvements:**
|
8
17
|
- Dependencies: add support for google-cloud-tasks v2 and while keeping backward compatibility with v1. For existing projects, you may need to run `bundle update google-cloud-tasks` after upgrading cloudtasker to update google-cloud-tasks to the latest version.
|
data/README.md
CHANGED
@@ -12,8 +12,6 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
|
|
12
12
|
|
13
13
|
A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
|
14
14
|
|
15
|
-
**Maturity**: This gem is production-ready. We at Keypup have already processed millions of jobs using Cloudtasker and all related extensions (cron, batch and unique jobs). We are planning to release the official `v1.0.0` somewhere in 2022, in case we've missed any edge-case bug.
|
16
|
-
|
17
15
|
## Summary
|
18
16
|
|
19
17
|
1. [Installation](#installation)
|
@@ -51,9 +49,6 @@ Add this line to your application's Gemfile:
|
|
51
49
|
|
52
50
|
```ruby
|
53
51
|
gem 'cloudtasker'
|
54
|
-
|
55
|
-
# For Ruby 3 support, please use v0.13.rc1
|
56
|
-
gem 'cloudtasker, '~> 0.13.rc1'
|
57
52
|
```
|
58
53
|
|
59
54
|
And then execute:
|
@@ -235,15 +230,16 @@ Open a Rails console and enqueue some jobs
|
|
235
230
|
|
236
231
|
The Google Cloud library authenticates via the Google Cloud SDK by default. If you do not have it setup then we recommend you [install it](https://cloud.google.com/sdk/docs/quickstarts).
|
237
232
|
|
238
|
-
Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/
|
233
|
+
Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/main/AUTHENTICATION.md).
|
239
234
|
|
240
235
|
In order to function properly Cloudtasker requires the authenticated account to have the following IAM permissions:
|
241
236
|
- `cloudtasks.tasks.get`
|
242
237
|
- `cloudtasks.tasks.create`
|
243
238
|
- `cloudtasks.tasks.delete`
|
244
239
|
|
245
|
-
To get started quickly you can add the `roles/cloudtasks.
|
240
|
+
To get started quickly you can add the `roles/cloudtasks.admin` role to your account via the [IAM Console](https://console.cloud.google.com/iam-admin/iam). This is not required if your account is a project admin account.
|
246
241
|
|
242
|
+
The GCP project ID and region values are not loaded automatically by the Google Cloud library, and must be explicitly defined in the initializer when using Google Cloud Tasks.
|
247
243
|
|
248
244
|
### Cloudtasker initializer
|
249
245
|
|
@@ -263,16 +259,19 @@ Cloudtasker.configure do |config|
|
|
263
259
|
# config.secret = 'some-long-token'
|
264
260
|
|
265
261
|
#
|
266
|
-
# Specify the details of your Google Cloud Task location.
|
267
|
-
#
|
268
|
-
# This
|
269
|
-
#
|
262
|
+
# Specify the details of your Google Cloud Task location.
|
263
|
+
#
|
264
|
+
# This is required when the mode of operation is set to :production
|
265
|
+
#
|
270
266
|
config.gcp_location_id = 'us-central1' # defaults to 'us-east1'
|
271
267
|
config.gcp_project_id = 'my-gcp-project'
|
272
268
|
|
273
269
|
#
|
274
270
|
# Specify the namespace for your Cloud Task queues.
|
275
271
|
#
|
272
|
+
# Specifying a namespace is optional but strongly recommended to keep
|
273
|
+
# queues organised, especially in a micro-service environment.
|
274
|
+
#
|
276
275
|
# The gem assumes that a least a default queue named 'my-app-default'
|
277
276
|
# exists in Cloud Tasks. You can create this default queue using the
|
278
277
|
# gcloud SDK or via the `rake cloudtasker:setup_queue` task if you use Rails.
|
data/gemfiles/rails_5.2.gemfile
CHANGED
data/gemfiles/rails_6.0.gemfile
CHANGED
data/gemfiles/rails_6.1.gemfile
CHANGED
data/gemfiles/rails_7.0.gemfile
CHANGED
@@ -70,7 +70,7 @@ module Cloudtasker
|
|
70
70
|
client.queue_path(
|
71
71
|
config.gcp_project_id,
|
72
72
|
config.gcp_location_id,
|
73
|
-
[config.gcp_queue_prefix, queue_name].join('-')
|
73
|
+
[config.gcp_queue_prefix, queue_name].map(&:presence).compact.join('-')
|
74
74
|
)
|
75
75
|
end
|
76
76
|
|
@@ -82,13 +82,27 @@ module Cloudtasker
|
|
82
82
|
#
|
83
83
|
# @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
|
84
84
|
#
|
85
|
-
def self.
|
85
|
+
def self.format_protobuf_time(schedule_time)
|
86
86
|
return nil unless schedule_time
|
87
87
|
|
88
88
|
# Generate protobuf timestamp
|
89
89
|
Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
|
90
90
|
end
|
91
91
|
|
92
|
+
#
|
93
|
+
# Return a protobuf duration.
|
94
|
+
#
|
95
|
+
# @param [Integer, nil] duration A duration in seconds.
|
96
|
+
#
|
97
|
+
# @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
|
98
|
+
#
|
99
|
+
def self.format_protobuf_duration(duration)
|
100
|
+
return nil unless duration
|
101
|
+
|
102
|
+
# Generate protobuf timestamp
|
103
|
+
Google::Protobuf::Duration.new.tap { |e| e.seconds = duration.to_i }
|
104
|
+
end
|
105
|
+
|
92
106
|
#
|
93
107
|
# Format the job payload sent to Cloud Tasks.
|
94
108
|
#
|
@@ -99,8 +113,11 @@ module Cloudtasker
|
|
99
113
|
def self.format_task_payload(payload)
|
100
114
|
payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
|
101
115
|
|
102
|
-
# Format schedule time to Google
|
103
|
-
payload[:schedule_time] =
|
116
|
+
# Format schedule time to Google::Protobuf::Timestamp
|
117
|
+
payload[:schedule_time] = format_protobuf_time(payload[:schedule_time])
|
118
|
+
|
119
|
+
# Format dispatch_deadline to Google::Protobuf::Duration
|
120
|
+
payload[:dispatch_deadline] = format_protobuf_duration(payload[:dispatch_deadline])
|
104
121
|
|
105
122
|
# Encode job content to support UTF-8. Google Cloud Task
|
106
123
|
# expect content to be ASCII-8BIT compatible (binary)
|
@@ -109,7 +126,7 @@ module Cloudtasker
|
|
109
126
|
payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
|
110
127
|
payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
|
111
128
|
|
112
|
-
payload
|
129
|
+
payload.compact
|
113
130
|
end
|
114
131
|
|
115
132
|
#
|
@@ -186,7 +203,7 @@ module Cloudtasker
|
|
186
203
|
.match(%r{/queues/([^/]+)})
|
187
204
|
&.captures
|
188
205
|
&.first
|
189
|
-
&.sub(
|
206
|
+
&.sub(/^#{self.class.config.gcp_queue_prefix}-/, '')
|
190
207
|
end
|
191
208
|
|
192
209
|
#
|
@@ -199,6 +216,7 @@ module Cloudtasker
|
|
199
216
|
id: gcp_task.name,
|
200
217
|
http_request: gcp_task.to_h[:http_request],
|
201
218
|
schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
|
219
|
+
dispatch_deadline: gcp_task.to_h.dig(:dispatch_deadline, :seconds).to_i,
|
202
220
|
retries: gcp_task.to_h[:response_count],
|
203
221
|
queue: relative_queue
|
204
222
|
}
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'google/cloud/tasks'
|
4
|
+
require 'google/protobuf/duration_pb'
|
4
5
|
require 'google/protobuf/timestamp_pb'
|
5
6
|
require 'retriable'
|
6
7
|
|
@@ -71,7 +72,7 @@ module Cloudtasker
|
|
71
72
|
client.queue_path(
|
72
73
|
project: config.gcp_project_id,
|
73
74
|
location: config.gcp_location_id,
|
74
|
-
queue: [config.gcp_queue_prefix, queue_name].join('-')
|
75
|
+
queue: [config.gcp_queue_prefix, queue_name].map(&:presence).compact.join('-')
|
75
76
|
)
|
76
77
|
end
|
77
78
|
|
@@ -83,13 +84,27 @@ module Cloudtasker
|
|
83
84
|
#
|
84
85
|
# @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
|
85
86
|
#
|
86
|
-
def self.
|
87
|
+
def self.format_protobuf_time(schedule_time)
|
87
88
|
return nil unless schedule_time
|
88
89
|
|
89
90
|
# Generate protobuf timestamp
|
90
91
|
Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
|
91
92
|
end
|
92
93
|
|
94
|
+
#
|
95
|
+
# Return a protobuf duration.
|
96
|
+
#
|
97
|
+
# @param [Integer, nil] duration A duration in seconds.
|
98
|
+
#
|
99
|
+
# @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
|
100
|
+
#
|
101
|
+
def self.format_protobuf_duration(duration)
|
102
|
+
return nil unless duration
|
103
|
+
|
104
|
+
# Generate protobuf timestamp
|
105
|
+
Google::Protobuf::Duration.new.tap { |e| e.seconds = duration.to_i }
|
106
|
+
end
|
107
|
+
|
93
108
|
#
|
94
109
|
# Format the job payload sent to Cloud Tasks.
|
95
110
|
#
|
@@ -100,8 +115,11 @@ module Cloudtasker
|
|
100
115
|
def self.format_task_payload(payload)
|
101
116
|
payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
|
102
117
|
|
103
|
-
# Format schedule time to Google
|
104
|
-
payload[:schedule_time] =
|
118
|
+
# Format schedule time to Google::Protobuf::Timestamp
|
119
|
+
payload[:schedule_time] = format_protobuf_time(payload[:schedule_time])
|
120
|
+
|
121
|
+
# Format dispatch_deadline to Google::Protobuf::Duration
|
122
|
+
payload[:dispatch_deadline] = format_protobuf_duration(payload[:dispatch_deadline])
|
105
123
|
|
106
124
|
# Encode job content to support UTF-8.
|
107
125
|
# Google Cloud Task expect content to be ASCII-8BIT compatible (binary)
|
@@ -110,7 +128,7 @@ module Cloudtasker
|
|
110
128
|
payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
|
111
129
|
payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
|
112
130
|
|
113
|
-
payload
|
131
|
+
payload.compact
|
114
132
|
end
|
115
133
|
|
116
134
|
#
|
@@ -188,7 +206,7 @@ module Cloudtasker
|
|
188
206
|
.match(%r{/queues/([^/]+)})
|
189
207
|
&.captures
|
190
208
|
&.first
|
191
|
-
&.sub(
|
209
|
+
&.sub(/^#{self.class.config.gcp_queue_prefix}-/, '')
|
192
210
|
end
|
193
211
|
|
194
212
|
#
|
@@ -201,6 +219,7 @@ module Cloudtasker
|
|
201
219
|
id: gcp_task.name,
|
202
220
|
http_request: gcp_task.to_h[:http_request],
|
203
221
|
schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
|
222
|
+
dispatch_deadline: gcp_task.to_h.dig(:dispatch_deadline, :seconds).to_i,
|
204
223
|
retries: gcp_task.to_h[:response_count],
|
205
224
|
queue: relative_queue
|
206
225
|
}
|
@@ -45,7 +45,7 @@ module Cloudtasker
|
|
45
45
|
# to use Task Set instead.
|
46
46
|
redis.search(key('*')).map do |gid|
|
47
47
|
task_id = gid.sub(key(''), '')
|
48
|
-
redis.sadd(key, task_id)
|
48
|
+
redis.sadd(key, [task_id])
|
49
49
|
find(task_id)
|
50
50
|
end
|
51
51
|
end
|
@@ -88,7 +88,7 @@ module Cloudtasker
|
|
88
88
|
|
89
89
|
# Save job
|
90
90
|
redis.write(key(id), payload)
|
91
|
-
redis.sadd(key, id)
|
91
|
+
redis.sadd(key, [id])
|
92
92
|
new(**payload.merge(id: id))
|
93
93
|
end
|
94
94
|
|
@@ -112,7 +112,7 @@ module Cloudtasker
|
|
112
112
|
# @param [String] id The task id.
|
113
113
|
#
|
114
114
|
def self.delete(id)
|
115
|
-
redis.srem(key, id)
|
115
|
+
redis.srem(key, [id])
|
116
116
|
redis.del(key(id))
|
117
117
|
end
|
118
118
|
|
@@ -186,7 +186,7 @@ module Cloudtasker
|
|
186
186
|
queue: queue,
|
187
187
|
dispatch_deadline: dispatch_deadline
|
188
188
|
)
|
189
|
-
redis.sadd(self.class.key, id)
|
189
|
+
redis.sadd(self.class.key, [id])
|
190
190
|
end
|
191
191
|
|
192
192
|
#
|
@@ -92,12 +92,26 @@ module Cloudtasker
|
|
92
92
|
#
|
93
93
|
# Return the batch progress percentage.
|
94
94
|
#
|
95
|
+
# A `min_total` can be specified to linearize the calculation, while jobs get added at
|
96
|
+
# the start of the batch.
|
97
|
+
#
|
98
|
+
# Similarly a `smoothing` parameter can be specified to add a constant to the total
|
99
|
+
# and linearize the calculation, which becomes: `done / (total + smoothing)`
|
100
|
+
#
|
101
|
+
# @param [Integer] min_total The minimum for the total number of jobs
|
102
|
+
# @param [Integer] smoothing An additive smoothing for the total number of jobs
|
103
|
+
#
|
95
104
|
# @return [Float] The progress percentage.
|
96
105
|
#
|
97
|
-
def percent
|
98
|
-
|
106
|
+
def percent(min_total: 0, smoothing: 0)
|
107
|
+
# Get the total value to use
|
108
|
+
actual_total = [min_total, total + smoothing].max
|
109
|
+
|
110
|
+
# Abort if we cannot divide
|
111
|
+
return 0 if actual_total.zero?
|
99
112
|
|
100
|
-
|
113
|
+
# Calculate progress
|
114
|
+
(done.to_f / actual_total) * 100
|
101
115
|
end
|
102
116
|
|
103
117
|
#
|
@@ -10,7 +10,20 @@ module Cloudtasker
|
|
10
10
|
JOBS_NAMESPACE = 'jobs'
|
11
11
|
STATES_NAMESPACE = 'states'
|
12
12
|
|
13
|
-
# List of statuses
|
13
|
+
# List of sub-job statuses taken into account when evaluating
|
14
|
+
# if the batch is complete.
|
15
|
+
#
|
16
|
+
# Batch jobs go through the following states:
|
17
|
+
# - scheduled: the parent batch has enqueued a worker for the child job
|
18
|
+
# - processing: the child job is running
|
19
|
+
# - completed: the child job has completed successfully
|
20
|
+
# - errored: the child job has encountered an error and must retry
|
21
|
+
# - dead: the child job has exceeded its max number of retries
|
22
|
+
#
|
23
|
+
# The 'dead' status is considered to be a completion status as it
|
24
|
+
# means that the job will never succeed. There is no point in blocking
|
25
|
+
# the batch forever so we proceed forward eventually.
|
26
|
+
#
|
14
27
|
COMPLETION_STATUSES = %w[completed dead].freeze
|
15
28
|
|
16
29
|
# These callbacks do not need to raise errors on their own
|
@@ -237,20 +250,14 @@ module Cloudtasker
|
|
237
250
|
end
|
238
251
|
|
239
252
|
#
|
240
|
-
# Save the
|
253
|
+
# Save serialized version of the worker.
|
254
|
+
#
|
255
|
+
# This is required to be able to invoke callback methods in the
|
256
|
+
# context of the worker (= instantiated worker) when child workers
|
257
|
+
# complete (success or failure).
|
241
258
|
#
|
242
259
|
def save
|
243
|
-
# Save serialized version of the worker. This is required to
|
244
|
-
# be able to invoke callback methods in the context of
|
245
|
-
# the worker (= instantiated worker) when child workers
|
246
|
-
# complete (success or failure).
|
247
260
|
redis.write(batch_gid, worker.to_h)
|
248
|
-
|
249
|
-
# Stop there if no jobs to save
|
250
|
-
return if jobs.empty?
|
251
|
-
|
252
|
-
# Save list of child workers
|
253
|
-
redis.hset(batch_state_gid, jobs.map { |e| [e.job_id, 'scheduled'] }.to_h)
|
254
261
|
end
|
255
262
|
|
256
263
|
#
|
@@ -263,7 +270,7 @@ module Cloudtasker
|
|
263
270
|
migrate_batch_state_to_redis_hash
|
264
271
|
|
265
272
|
# Update the batch state batch_id entry with the new status
|
266
|
-
redis.hset(batch_state_gid, batch_id, status)
|
273
|
+
redis.hset(batch_state_gid, batch_id, status)
|
267
274
|
end
|
268
275
|
|
269
276
|
#
|
@@ -369,6 +376,9 @@ module Cloudtasker
|
|
369
376
|
#
|
370
377
|
# Calculate the progress of the batch.
|
371
378
|
#
|
379
|
+
# @param [Integer] depth The depth of calculation. Zero (default) means only immediate
|
380
|
+
# children will be taken into account.
|
381
|
+
#
|
372
382
|
# @return [Cloudtasker::Batch::BatchProgress] The batch progress.
|
373
383
|
#
|
374
384
|
def progress(depth: 0)
|
@@ -391,16 +401,25 @@ module Cloudtasker
|
|
391
401
|
#
|
392
402
|
# Save the batch and enqueue all child workers attached to it.
|
393
403
|
#
|
394
|
-
# @return [Array<Cloudtasker::CloudTask>] The Google Task responses
|
395
|
-
#
|
396
404
|
def setup
|
397
405
|
return true if jobs.empty?
|
398
406
|
|
399
407
|
# Save batch
|
400
408
|
save
|
401
409
|
|
402
|
-
#
|
403
|
-
jobs.
|
410
|
+
# Schedule all child workers
|
411
|
+
jobs.each do |j|
|
412
|
+
# Schedule the job
|
413
|
+
j.schedule
|
414
|
+
|
415
|
+
# Initialize the batch state unless the job has already started (and taken
|
416
|
+
# hold of its own status)
|
417
|
+
# The batch state is initialized only after the job is scheduled to avoid
|
418
|
+
# having never-ending batches - which could occur if a batch was crashing
|
419
|
+
# while enqueuing children due to a OOM error and since 'scheduled' is a
|
420
|
+
# blocking status.
|
421
|
+
redis.hsetnx(batch_state_gid, j.job_id, 'scheduled')
|
422
|
+
end
|
404
423
|
end
|
405
424
|
|
406
425
|
#
|
data/lib/cloudtasker/config.rb
CHANGED
@@ -5,9 +5,9 @@ require 'logger'
|
|
5
5
|
module Cloudtasker
|
6
6
|
# Holds cloudtasker configuration. See Cloudtasker#configure
|
7
7
|
class Config
|
8
|
-
attr_accessor :redis, :store_payloads_in_redis
|
8
|
+
attr_accessor :redis, :store_payloads_in_redis, :gcp_queue_prefix
|
9
9
|
attr_writer :secret, :gcp_location_id, :gcp_project_id,
|
10
|
-
:
|
10
|
+
:processor_path, :logger, :mode, :max_retries,
|
11
11
|
:dispatch_deadline, :on_error, :on_dead
|
12
12
|
|
13
13
|
# Max Cloud Task size in bytes
|
@@ -70,11 +70,6 @@ module Cloudtasker
|
|
70
70
|
Missing host for processing.
|
71
71
|
Please specify a processor hostname in form of `https://some-public-dns.example.com`'
|
72
72
|
DOC
|
73
|
-
QUEUE_PREFIX_MISSING_ERROR = <<~DOC
|
74
|
-
Missing GCP queue prefix.
|
75
|
-
Please specify a queue prefix in the form of `my-app`.
|
76
|
-
You can create a default queue using the Google SDK via `gcloud tasks queues create my-app-default`
|
77
|
-
DOC
|
78
73
|
PROJECT_ID_MISSING_ERROR = <<~DOC
|
79
74
|
Missing GCP project ID.
|
80
75
|
Please specify a project ID in the cloudtasker configurator.
|
@@ -189,15 +184,6 @@ module Cloudtasker
|
|
189
184
|
@processor_path || DEFAULT_PROCESSOR_PATH
|
190
185
|
end
|
191
186
|
|
192
|
-
#
|
193
|
-
# Return the prefix used for queues.
|
194
|
-
#
|
195
|
-
# @return [String] The prefix of the processing queues.
|
196
|
-
#
|
197
|
-
def gcp_queue_prefix
|
198
|
-
@gcp_queue_prefix || raise(StandardError, QUEUE_PREFIX_MISSING_ERROR)
|
199
|
-
end
|
200
|
-
|
201
187
|
#
|
202
188
|
# Return the GCP project ID.
|
203
189
|
#
|
@@ -43,7 +43,7 @@ module Cloudtasker
|
|
43
43
|
# to use Schedule Set instead.
|
44
44
|
redis.search(key('*')).map do |gid|
|
45
45
|
schedule_id = gid.sub(key(''), '')
|
46
|
-
redis.sadd(key, schedule_id)
|
46
|
+
redis.sadd(key, [schedule_id])
|
47
47
|
find(schedule_id)
|
48
48
|
end
|
49
49
|
end
|
@@ -108,7 +108,7 @@ module Cloudtasker
|
|
108
108
|
|
109
109
|
# Delete task and stored schedule
|
110
110
|
CloudTask.delete(schedule.task_id) if schedule.task_id
|
111
|
-
redis.srem(key, schedule.id)
|
111
|
+
redis.srem(key, [schedule.id])
|
112
112
|
redis.del(schedule.gid)
|
113
113
|
end
|
114
114
|
end
|
@@ -278,7 +278,7 @@ module Cloudtasker
|
|
278
278
|
|
279
279
|
# Save schedule
|
280
280
|
config_was_changed = config_changed?
|
281
|
-
redis.sadd(self.class.key, id)
|
281
|
+
redis.sadd(self.class.key, [id])
|
282
282
|
redis.write(gid, to_h)
|
283
283
|
|
284
284
|
# Stop there if backend does not need update
|
data/lib/cloudtasker/version.rb
CHANGED
data/lib/cloudtasker.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'active_support/core_ext/
|
3
|
+
require 'active_support/core_ext/object/blank'
|
4
4
|
require 'active_support/core_ext/object/try'
|
5
|
+
require 'active_support/core_ext/string/inflections'
|
5
6
|
|
6
7
|
require 'cloudtasker/version'
|
7
8
|
require 'cloudtasker/config'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloudtasker
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.13.
|
4
|
+
version: 0.13.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Arnaud Lachaume
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2023-06-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -257,7 +257,7 @@ extensions: []
|
|
257
257
|
extra_rdoc_files: []
|
258
258
|
files:
|
259
259
|
- ".github/workflows/lint_rubocop.yml"
|
260
|
-
- ".github/workflows/test_ruby_2.
|
260
|
+
- ".github/workflows/test_ruby_2.6.yml"
|
261
261
|
- ".github/workflows/test_ruby_2.7.yml"
|
262
262
|
- ".github/workflows/test_ruby_3.x.yml"
|
263
263
|
- ".gitignore"
|
@@ -372,7 +372,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
372
372
|
- !ruby/object:Gem::Version
|
373
373
|
version: '0'
|
374
374
|
requirements: []
|
375
|
-
rubygems_version: 3.
|
375
|
+
rubygems_version: 3.4.8
|
376
376
|
signing_key:
|
377
377
|
specification_version: 4
|
378
378
|
summary: Background jobs for Ruby using Google Cloud Tasks (beta)
|
@@ -1,44 +0,0 @@
|
|
1
|
-
name: Ruby 2.5/2.6
|
2
|
-
|
3
|
-
on: [push, pull_request]
|
4
|
-
|
5
|
-
jobs:
|
6
|
-
build:
|
7
|
-
runs-on: ubuntu-latest
|
8
|
-
strategy:
|
9
|
-
matrix:
|
10
|
-
ruby:
|
11
|
-
- '2.5.9'
|
12
|
-
- '2.6.7'
|
13
|
-
appraisal:
|
14
|
-
- 'google-cloud-tasks-1.0'
|
15
|
-
- 'google-cloud-tasks-1.1'
|
16
|
-
- 'google-cloud-tasks-1.2'
|
17
|
-
- 'google-cloud-tasks-1.3'
|
18
|
-
- 'google-cloud-tasks-1.4'
|
19
|
-
- 'google-cloud-tasks-1.5'
|
20
|
-
- 'google-cloud-tasks-2.0'
|
21
|
-
- 'google-cloud-tasks-2.1'
|
22
|
-
- 'rails-5.2'
|
23
|
-
- 'rails-6.0'
|
24
|
-
- 'rails-6.1'
|
25
|
-
- 'semantic_logger-3.4'
|
26
|
-
- 'semantic_logger-4.6'
|
27
|
-
- 'semantic_logger-4.7.0'
|
28
|
-
- 'semantic_logger-4.7.2'
|
29
|
-
steps:
|
30
|
-
- uses: actions/checkout@v2
|
31
|
-
- uses: zhulik/redis-action@1.1.0
|
32
|
-
- name: Set up Ruby
|
33
|
-
uses: ruby/setup-ruby@v1
|
34
|
-
with:
|
35
|
-
ruby-version: ${{ matrix.ruby }}
|
36
|
-
bundler-cache: true
|
37
|
-
- name: Build and test with Rake
|
38
|
-
env:
|
39
|
-
APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
|
40
|
-
run: |
|
41
|
-
gem install bundler
|
42
|
-
bundle install --jobs 4 --retry 3
|
43
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
|
44
|
-
bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
|