cloudtasker 0.13.0 → 0.13.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (36) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/lint_rubocop.yml +2 -7
  3. data/.github/workflows/test_ruby_2.6.yml +37 -0
  4. data/.github/workflows/test_ruby_2.7.yml +21 -27
  5. data/.github/workflows/test_ruby_3.x.yml +21 -26
  6. data/.rubocop.yml +4 -1
  7. data/Appraisals +32 -32
  8. data/CHANGELOG.md +11 -2
  9. data/README.md +10 -11
  10. data/gemfiles/google_cloud_tasks_1.0.gemfile +1 -1
  11. data/gemfiles/google_cloud_tasks_1.1.gemfile +1 -1
  12. data/gemfiles/google_cloud_tasks_1.2.gemfile +1 -1
  13. data/gemfiles/google_cloud_tasks_1.3.gemfile +1 -1
  14. data/gemfiles/google_cloud_tasks_1.4.gemfile +1 -1
  15. data/gemfiles/google_cloud_tasks_1.5.gemfile +1 -1
  16. data/gemfiles/google_cloud_tasks_2.0.gemfile +1 -1
  17. data/gemfiles/google_cloud_tasks_2.1.gemfile +1 -1
  18. data/gemfiles/rails_5.2.gemfile +1 -1
  19. data/gemfiles/rails_6.0.gemfile +1 -1
  20. data/gemfiles/rails_6.1.gemfile +1 -1
  21. data/gemfiles/rails_7.0.gemfile +1 -1
  22. data/gemfiles/semantic_logger_3.4.gemfile +1 -1
  23. data/gemfiles/semantic_logger_4.6.gemfile +1 -1
  24. data/gemfiles/semantic_logger_4.7.0.gemfile +1 -1
  25. data/gemfiles/semantic_logger_4.7.2.gemfile +1 -1
  26. data/lib/cloudtasker/backend/google_cloud_task_v1.rb +24 -6
  27. data/lib/cloudtasker/backend/google_cloud_task_v2.rb +25 -6
  28. data/lib/cloudtasker/backend/redis_task.rb +4 -4
  29. data/lib/cloudtasker/batch/batch_progress.rb +17 -3
  30. data/lib/cloudtasker/batch/job.rb +36 -17
  31. data/lib/cloudtasker/config.rb +2 -16
  32. data/lib/cloudtasker/cron/schedule.rb +3 -3
  33. data/lib/cloudtasker/version.rb +1 -1
  34. data/lib/cloudtasker.rb +2 -1
  35. metadata +4 -4
  36. data/.github/workflows/test_ruby_2.5_2.6.yml +0 -44
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 59faa788a8c1238974bac7f81169c7df4fc460e0b58d181918c7c9b087e6ea29
4
- data.tar.gz: 106d98dc7571bd65f52774946b5e995f47e7c45be9b54dca41644946cd1c61d1
3
+ metadata.gz: acd6db925df54f7f8b10459c94e76fbd4f4316b706bf7918afaac9857161335d
4
+ data.tar.gz: 690a9de1bc62150eb9d6e76dc8689c28a7811996dd036edb3351dce88ee2e082
5
5
  SHA512:
6
- metadata.gz: 5a22f2df22f4b84a330bd613689f63b456cbcc6b87318d1501029e0794cedeb0951e73a7c974e93957c4333a674bdb47a6270995eb81bbe281f5ec5e56009f50
7
- data.tar.gz: 84120f64e690ebcd06c93890c14278d1797620b1289b05f10a9d6846bff965febc06261aa560b3179315b64d26a1b193d19a65651d6b0b2f6ff46d54e779356f
6
+ metadata.gz: fc66650e5d56d77fe026acbe01dd74eaf44526d2218849731cfc8dae65b8bb932016940370b2e478904f45d23197b222c400f788ee9b368a4493746f1a14cb38
7
+ data.tar.gz: 4385ec200c8d5a2b6a1e7fdf9b700a294184894ab4d1c97f83d31a0c7adf636cb02694f1b85394f030e54fff66a0a78de1b01a4ebbb8dd1bf041c708ca64137e
@@ -8,13 +8,8 @@ jobs:
8
8
  steps:
9
9
  - uses: actions/checkout@v2
10
10
  - uses: zhulik/redis-action@1.1.0
11
- - name: Set up Ruby
12
- uses: ruby/setup-ruby@v1
11
+ - uses: ruby/setup-ruby@v1
13
12
  with:
14
13
  ruby-version: '3.0.1'
15
14
  bundler-cache: true
16
- - name: Lint codebase
17
- run: |
18
- gem install bundler
19
- bundle install --jobs 4 --retry 3
20
- bundle exec rubocop
15
+ - run: bundle exec rubocop
@@ -0,0 +1,37 @@
1
+ name: Ruby 2.6
2
+
3
+ on: [push, pull_request]
4
+
5
+ jobs:
6
+ build:
7
+ runs-on: ubuntu-latest
8
+ strategy:
9
+ matrix:
10
+ ruby:
11
+ - '2.6'
12
+ appraisal:
13
+ - 'google_cloud_tasks_1.0'
14
+ - 'google_cloud_tasks_1.1'
15
+ - 'google_cloud_tasks_1.2'
16
+ - 'google_cloud_tasks_1.3'
17
+ - 'google_cloud_tasks_1.4'
18
+ - 'google_cloud_tasks_1.5'
19
+ - 'google_cloud_tasks_2.0'
20
+ - 'google_cloud_tasks_2.1'
21
+ - 'rails_5.2'
22
+ - 'rails_6.0'
23
+ - 'rails_6.1'
24
+ - 'semantic_logger_3.4'
25
+ - 'semantic_logger_4.6'
26
+ - 'semantic_logger_4.7.0'
27
+ - 'semantic_logger_4.7.2'
28
+ env:
29
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
30
+ steps:
31
+ - uses: actions/checkout@v2
32
+ - uses: zhulik/redis-action@1.1.0
33
+ - uses: ruby/setup-ruby@v1
34
+ with:
35
+ ruby-version: ${{ matrix.ruby }}
36
+ bundler-cache: true
37
+ - run: bundle exec rspec
@@ -8,37 +8,31 @@ jobs:
8
8
  strategy:
9
9
  matrix:
10
10
  ruby:
11
- - '2.7.5'
11
+ - '2.7'
12
12
  appraisal:
13
- - 'google-cloud-tasks-1.0'
14
- - 'google-cloud-tasks-1.1'
15
- - 'google-cloud-tasks-1.2'
16
- - 'google-cloud-tasks-1.3'
17
- - 'google-cloud-tasks-1.4'
18
- - 'google-cloud-tasks-1.5'
19
- - 'google-cloud-tasks-2.0'
20
- - 'google-cloud-tasks-2.1'
21
- - 'rails-5.2'
22
- - 'rails-6.0'
23
- - 'rails-6.1'
24
- - 'rails-7.0'
25
- - 'semantic_logger-3.4'
26
- - 'semantic_logger-4.6'
27
- - 'semantic_logger-4.7.0'
28
- - 'semantic_logger-4.7.2'
13
+ - 'google_cloud_tasks_1.0'
14
+ - 'google_cloud_tasks_1.1'
15
+ - 'google_cloud_tasks_1.2'
16
+ - 'google_cloud_tasks_1.3'
17
+ - 'google_cloud_tasks_1.4'
18
+ - 'google_cloud_tasks_1.5'
19
+ - 'google_cloud_tasks_2.0'
20
+ - 'google_cloud_tasks_2.1'
21
+ - 'rails_5.2'
22
+ - 'rails_6.0'
23
+ - 'rails_6.1'
24
+ - 'rails_7.0'
25
+ - 'semantic_logger_3.4'
26
+ - 'semantic_logger_4.6'
27
+ - 'semantic_logger_4.7.0'
28
+ - 'semantic_logger_4.7.2'
29
+ env:
30
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
29
31
  steps:
30
32
  - uses: actions/checkout@v2
31
33
  - uses: zhulik/redis-action@1.1.0
32
- - name: Set up Ruby
33
- uses: ruby/setup-ruby@v1
34
+ - uses: ruby/setup-ruby@v1
34
35
  with:
35
36
  ruby-version: ${{ matrix.ruby }}
36
37
  bundler-cache: true
37
- - name: Build and test with Rake
38
- env:
39
- APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
40
- run: |
41
- gem install bundler
42
- bundle install --jobs 4 --retry 3
43
- bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
44
- bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
38
+ - run: bundle exec rspec
@@ -8,36 +8,31 @@ jobs:
8
8
  strategy:
9
9
  matrix:
10
10
  ruby:
11
- - '3.0.1'
12
- # - '3.1.0'
11
+ - '3.0'
12
+ - '3.1'
13
+ - '3.2'
13
14
  appraisal:
14
- - 'google-cloud-tasks-1.0'
15
- - 'google-cloud-tasks-1.1'
16
- - 'google-cloud-tasks-1.2'
17
- - 'google-cloud-tasks-1.3'
18
- - 'google-cloud-tasks-1.4'
19
- - 'google-cloud-tasks-1.5'
20
- - 'google-cloud-tasks-2.0'
21
- - 'google-cloud-tasks-2.1'
22
- - 'rails-6.1'
23
- - 'rails-7.0'
24
- - 'semantic_logger-3.4'
25
- - 'semantic_logger-4.6'
26
- - 'semantic_logger-4.7.0'
27
- - 'semantic_logger-4.7.2'
15
+ - 'google_cloud_tasks_1.0'
16
+ - 'google_cloud_tasks_1.1'
17
+ - 'google_cloud_tasks_1.2'
18
+ - 'google_cloud_tasks_1.3'
19
+ - 'google_cloud_tasks_1.4'
20
+ - 'google_cloud_tasks_1.5'
21
+ - 'google_cloud_tasks_2.0'
22
+ - 'google_cloud_tasks_2.1'
23
+ - 'rails_6.1'
24
+ - 'rails_7.0'
25
+ - 'semantic_logger_3.4'
26
+ - 'semantic_logger_4.6'
27
+ - 'semantic_logger_4.7.0'
28
+ - 'semantic_logger_4.7.2'
29
+ env:
30
+ BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
28
31
  steps:
29
32
  - uses: actions/checkout@v2
30
33
  - uses: zhulik/redis-action@1.1.0
31
- - name: Set up Ruby
32
- uses: ruby/setup-ruby@v1
34
+ - uses: ruby/setup-ruby@v1
33
35
  with:
34
36
  ruby-version: ${{ matrix.ruby }}
35
37
  bundler-cache: true
36
- - name: Build and test with Rake
37
- env:
38
- APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
39
- run: |
40
- gem install bundler
41
- bundle install --jobs 4 --retry 3
42
- bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
43
- bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
38
+ - run: bundle exec rspec
data/.rubocop.yml CHANGED
@@ -59,4 +59,7 @@ RSpec/MessageSpies:
59
59
  RSpec/MultipleExpectations:
60
60
  Exclude:
61
61
  - 'examples/**/*'
62
- - 'spec/integration/**/*'
62
+ - 'spec/integration/**/*'
63
+
64
+ RSpec/AnyInstance:
65
+ Enabled: false
data/Appraisals CHANGED
@@ -1,85 +1,85 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- appraise 'google-cloud-tasks-1.0' do
4
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
3
+ appraise 'google_cloud_tasks_1.0' do
4
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
5
5
  gem 'google-cloud-tasks', '~> 1.0.0'
6
6
  end
7
7
 
8
- appraise 'google-cloud-tasks-1.1' do
9
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
8
+ appraise 'google_cloud_tasks_1.1' do
9
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
10
10
  gem 'google-cloud-tasks', '~> 1.1.0'
11
11
  end
12
12
 
13
- appraise 'google-cloud-tasks-1.2' do
14
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
13
+ appraise 'google_cloud_tasks_1.2' do
14
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
15
15
  gem 'google-cloud-tasks', '~> 1.2.0'
16
16
  end
17
17
 
18
- appraise 'google-cloud-tasks-1.3' do
19
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
18
+ appraise 'google_cloud_tasks_1.3' do
19
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
20
20
  gem 'google-cloud-tasks', '~> 1.3.0'
21
21
  end
22
22
 
23
- appraise 'google-cloud-tasks-1.4' do
24
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
23
+ appraise 'google_cloud_tasks_1.4' do
24
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
25
25
  gem 'google-cloud-tasks', '~> 1.4.0'
26
26
  end
27
27
 
28
- appraise 'google-cloud-tasks-1.5' do
29
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
28
+ appraise 'google_cloud_tasks_1.5' do
29
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
30
30
  gem 'google-cloud-tasks', '~> 1.5.0'
31
31
  end
32
32
 
33
- appraise 'google-cloud-tasks-2.0' do
34
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
33
+ appraise 'google_cloud_tasks_2.0' do
34
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
35
35
  gem 'google-cloud-tasks', '~> 2.0.0'
36
36
  end
37
37
 
38
- appraise 'google-cloud-tasks-2.1' do
39
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
38
+ appraise 'google_cloud_tasks_2.1' do
39
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
40
40
  gem 'google-cloud-tasks', '~> 2.1.0'
41
41
  end
42
42
 
43
43
  if RUBY_VERSION < '3'
44
- appraise 'rails-5.2' do
45
- gem 'rails', '5.2'
44
+ appraise 'rails_5.2' do
45
+ gem 'rails', '~> 5.2.0'
46
46
  gem 'rspec-rails'
47
47
  end
48
48
 
49
- appraise 'rails-6.0' do
50
- gem 'rails', '6.0'
49
+ appraise 'rails_6.0' do
50
+ gem 'rails', '~> 6.0.0'
51
51
  gem 'rspec-rails'
52
52
  end
53
53
  end
54
54
 
55
- appraise 'rails-6.1' do
56
- gem 'rails', '6.1'
55
+ appraise 'rails_6.1' do
56
+ gem 'rails', '~> 6.1.0'
57
57
  gem 'rspec-rails'
58
58
  end
59
59
 
60
60
  if RUBY_VERSION >= '2.7'
61
- appraise 'rails-7.0' do
62
- gem 'rails', '7.0'
61
+ appraise 'rails_7.0' do
62
+ gem 'rails', '~> 7.0.0'
63
63
  gem 'rspec-rails'
64
64
  end
65
65
  end
66
66
 
67
- appraise 'semantic_logger-3.4' do
68
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
67
+ appraise 'semantic_logger_3.4' do
68
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
69
69
  gem 'semantic_logger', '3.4.1'
70
70
  end
71
71
 
72
- appraise 'semantic_logger-4.6' do
73
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
72
+ appraise 'semantic_logger_4.6' do
73
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
74
74
  gem 'semantic_logger', '4.6.1'
75
75
  end
76
76
 
77
- appraise 'semantic_logger-4.7.0' do
78
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
77
+ appraise 'semantic_logger_4.7.0' do
78
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
79
79
  gem 'semantic_logger', '4.7.0'
80
80
  end
81
81
 
82
- appraise 'semantic_logger-4.7.2' do
83
- gem 'activesupport', '6.1' # ruby 2.5 / 2.6 compatibility
82
+ appraise 'semantic_logger_4.7.2' do
83
+ gem 'activesupport', '~> 6.1.0' # ruby 2.6 compatibility
84
84
  gem 'semantic_logger', '4.7.2'
85
85
  end
data/CHANGELOG.md CHANGED
@@ -1,8 +1,17 @@
1
1
  # Changelog
2
2
 
3
- ## [v0.13.rc2](https://github.com/keypup-io/cloudtasker/tree/v0.13.rc2) (2022-01-25)
3
+ ## [v0.13.1](https://github.com/keypup-io/cloudtasker/tree/v0.13.1) (2023-06-19)
4
4
 
5
- [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.12.2...v0.13.rc2)
5
+ [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.13.0...v0.13.1)
6
+
7
+ **Fixed bugs:**
8
+ - Batch jobs: Gracefully handle errors (e.g. Redis connection error) while setting up child jobs. Previously this could lead to ghost child jobs preventing the parent batch from finishing (child job is registered but is never enqueued)
9
+ - Protobuf: fix formatting of protobuf payload by removig `nil` values and formatting the `dispatch_deadline` as a proper `Google::Protobuf::Duration` (instead of `integer`). Fixes [#94](https://github.com/keypup-io/cloudtasker/issues/94)
10
+
11
+
12
+ ## [v0.13.0](https://github.com/keypup-io/cloudtasker/tree/v0.13.0) (2022-03-11)
13
+
14
+ [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.12.2...v0.13.0)
6
15
 
7
16
  **Improvements:**
8
17
  - Dependencies: add support for google-cloud-tasks v2 and while keeping backward compatibility with v1. For existing projects, you may need to run `bundle update google-cloud-tasks` after upgrading cloudtasker to update google-cloud-tasks to the latest version.
data/README.md CHANGED
@@ -12,8 +12,6 @@ Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOB
12
12
 
13
13
  A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
14
14
 
15
- **Maturity**: This gem is production-ready. We at Keypup have already processed millions of jobs using Cloudtasker and all related extensions (cron, batch and unique jobs). We are planning to release the official `v1.0.0` somewhere in 2022, in case we've missed any edge-case bug.
16
-
17
15
  ## Summary
18
16
 
19
17
  1. [Installation](#installation)
@@ -51,9 +49,6 @@ Add this line to your application's Gemfile:
51
49
 
52
50
  ```ruby
53
51
  gem 'cloudtasker'
54
-
55
- # For Ruby 3 support, please use v0.13.rc1
56
- gem 'cloudtasker, '~> 0.13.rc1'
57
52
  ```
58
53
 
59
54
  And then execute:
@@ -235,15 +230,16 @@ Open a Rails console and enqueue some jobs
235
230
 
236
231
  The Google Cloud library authenticates via the Google Cloud SDK by default. If you do not have it setup then we recommend you [install it](https://cloud.google.com/sdk/docs/quickstarts).
237
232
 
238
- Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-bigquery/AUTHENTICATION.md).
233
+ Other options are available such as using a service account. You can see all authentication options in the [Google Cloud Authentication guide](https://github.com/googleapis/google-cloud-ruby/blob/main/AUTHENTICATION.md).
239
234
 
240
235
  In order to function properly Cloudtasker requires the authenticated account to have the following IAM permissions:
241
236
  - `cloudtasks.tasks.get`
242
237
  - `cloudtasks.tasks.create`
243
238
  - `cloudtasks.tasks.delete`
244
239
 
245
- To get started quickly you can add the `roles/cloudtasks.queueAdmin` role to your account via the [IAM Console](https://console.cloud.google.com/iam-admin/iam). This is not required if your account is a project admin account.
240
+ To get started quickly you can add the `roles/cloudtasks.admin` role to your account via the [IAM Console](https://console.cloud.google.com/iam-admin/iam). This is not required if your account is a project admin account.
246
241
 
242
+ The GCP project ID and region values are not loaded automatically by the Google Cloud library, and must be explicitly defined in the initializer when using Google Cloud Tasks.
247
243
 
248
244
  ### Cloudtasker initializer
249
245
 
@@ -263,16 +259,19 @@ Cloudtasker.configure do |config|
263
259
  # config.secret = 'some-long-token'
264
260
 
265
261
  #
266
- # Specify the details of your Google Cloud Task location.
267
- #
268
- # This not required in development using the Cloudtasker local server.
269
- #
262
+ # Specify the details of your Google Cloud Task location.
263
+ #
264
+ # This is required when the mode of operation is set to :production
265
+ #
270
266
  config.gcp_location_id = 'us-central1' # defaults to 'us-east1'
271
267
  config.gcp_project_id = 'my-gcp-project'
272
268
 
273
269
  #
274
270
  # Specify the namespace for your Cloud Task queues.
275
271
  #
272
+ # Specifying a namespace is optional but strongly recommended to keep
273
+ # queues organised, especially in a micro-service environment.
274
+ #
276
275
  # The gem assumes that a least a default queue named 'my-app-default'
277
276
  # exists in Cloud Tasks. You can create this default queue using the
278
277
  # gcloud SDK or via the `rake cloudtasker:setup_queue` task if you use Rails.
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.0.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.1.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.2.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.3.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.4.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 1.5.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 2.0.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "google-cloud-tasks", "~> 2.1.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "5.2"
5
+ gem "rails", "~> 5.2.0"
6
6
  gem "rspec-rails"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "6.0"
5
+ gem "rails", "~> 6.0.0"
6
6
  gem "rspec-rails"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "6.1"
5
+ gem "rails", "~> 6.1.0"
6
6
  gem "rspec-rails"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "rails", "7.0"
5
+ gem "rails", "~> 7.0.0"
6
6
  gem "rspec-rails"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "semantic_logger", "3.4.1"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "semantic_logger", "4.6.1"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "semantic_logger", "4.7.0"
7
7
 
8
8
  gemspec path: "../"
@@ -2,7 +2,7 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activesupport", "6.1"
5
+ gem "activesupport", "~> 6.1.0"
6
6
  gem "semantic_logger", "4.7.2"
7
7
 
8
8
  gemspec path: "../"
@@ -70,7 +70,7 @@ module Cloudtasker
70
70
  client.queue_path(
71
71
  config.gcp_project_id,
72
72
  config.gcp_location_id,
73
- [config.gcp_queue_prefix, queue_name].join('-')
73
+ [config.gcp_queue_prefix, queue_name].map(&:presence).compact.join('-')
74
74
  )
75
75
  end
76
76
 
@@ -82,13 +82,27 @@ module Cloudtasker
82
82
  #
83
83
  # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
84
84
  #
85
- def self.format_schedule_time(schedule_time)
85
+ def self.format_protobuf_time(schedule_time)
86
86
  return nil unless schedule_time
87
87
 
88
88
  # Generate protobuf timestamp
89
89
  Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
90
90
  end
91
91
 
92
+ #
93
+ # Return a protobuf duration.
94
+ #
95
+ # @param [Integer, nil] duration A duration in seconds.
96
+ #
97
+ # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
98
+ #
99
+ def self.format_protobuf_duration(duration)
100
+ return nil unless duration
101
+
102
+ # Generate protobuf timestamp
103
+ Google::Protobuf::Duration.new.tap { |e| e.seconds = duration.to_i }
104
+ end
105
+
92
106
  #
93
107
  # Format the job payload sent to Cloud Tasks.
94
108
  #
@@ -99,8 +113,11 @@ module Cloudtasker
99
113
  def self.format_task_payload(payload)
100
114
  payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
101
115
 
102
- # Format schedule time to Google Protobuf timestamp
103
- payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
116
+ # Format schedule time to Google::Protobuf::Timestamp
117
+ payload[:schedule_time] = format_protobuf_time(payload[:schedule_time])
118
+
119
+ # Format dispatch_deadline to Google::Protobuf::Duration
120
+ payload[:dispatch_deadline] = format_protobuf_duration(payload[:dispatch_deadline])
104
121
 
105
122
  # Encode job content to support UTF-8. Google Cloud Task
106
123
  # expect content to be ASCII-8BIT compatible (binary)
@@ -109,7 +126,7 @@ module Cloudtasker
109
126
  payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
110
127
  payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
111
128
 
112
- payload
129
+ payload.compact
113
130
  end
114
131
 
115
132
  #
@@ -186,7 +203,7 @@ module Cloudtasker
186
203
  .match(%r{/queues/([^/]+)})
187
204
  &.captures
188
205
  &.first
189
- &.sub("#{self.class.config.gcp_queue_prefix}-", '')
206
+ &.sub(/^#{self.class.config.gcp_queue_prefix}-/, '')
190
207
  end
191
208
 
192
209
  #
@@ -199,6 +216,7 @@ module Cloudtasker
199
216
  id: gcp_task.name,
200
217
  http_request: gcp_task.to_h[:http_request],
201
218
  schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
219
+ dispatch_deadline: gcp_task.to_h.dig(:dispatch_deadline, :seconds).to_i,
202
220
  retries: gcp_task.to_h[:response_count],
203
221
  queue: relative_queue
204
222
  }
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'google/cloud/tasks'
4
+ require 'google/protobuf/duration_pb'
4
5
  require 'google/protobuf/timestamp_pb'
5
6
  require 'retriable'
6
7
 
@@ -71,7 +72,7 @@ module Cloudtasker
71
72
  client.queue_path(
72
73
  project: config.gcp_project_id,
73
74
  location: config.gcp_location_id,
74
- queue: [config.gcp_queue_prefix, queue_name].join('-')
75
+ queue: [config.gcp_queue_prefix, queue_name].map(&:presence).compact.join('-')
75
76
  )
76
77
  end
77
78
 
@@ -83,13 +84,27 @@ module Cloudtasker
83
84
  #
84
85
  # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
85
86
  #
86
- def self.format_schedule_time(schedule_time)
87
+ def self.format_protobuf_time(schedule_time)
87
88
  return nil unless schedule_time
88
89
 
89
90
  # Generate protobuf timestamp
90
91
  Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
91
92
  end
92
93
 
94
+ #
95
+ # Return a protobuf duration.
96
+ #
97
+ # @param [Integer, nil] duration A duration in seconds.
98
+ #
99
+ # @return [Google::Protobuf::Timestamp, nil] The protobuff timestamp
100
+ #
101
+ def self.format_protobuf_duration(duration)
102
+ return nil unless duration
103
+
104
+ # Generate protobuf timestamp
105
+ Google::Protobuf::Duration.new.tap { |e| e.seconds = duration.to_i }
106
+ end
107
+
93
108
  #
94
109
  # Format the job payload sent to Cloud Tasks.
95
110
  #
@@ -100,8 +115,11 @@ module Cloudtasker
100
115
  def self.format_task_payload(payload)
101
116
  payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
102
117
 
103
- # Format schedule time to Google Protobuf timestamp
104
- payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
118
+ # Format schedule time to Google::Protobuf::Timestamp
119
+ payload[:schedule_time] = format_protobuf_time(payload[:schedule_time])
120
+
121
+ # Format dispatch_deadline to Google::Protobuf::Duration
122
+ payload[:dispatch_deadline] = format_protobuf_duration(payload[:dispatch_deadline])
105
123
 
106
124
  # Encode job content to support UTF-8.
107
125
  # Google Cloud Task expect content to be ASCII-8BIT compatible (binary)
@@ -110,7 +128,7 @@ module Cloudtasker
110
128
  payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
111
129
  payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
112
130
 
113
- payload
131
+ payload.compact
114
132
  end
115
133
 
116
134
  #
@@ -188,7 +206,7 @@ module Cloudtasker
188
206
  .match(%r{/queues/([^/]+)})
189
207
  &.captures
190
208
  &.first
191
- &.sub("#{self.class.config.gcp_queue_prefix}-", '')
209
+ &.sub(/^#{self.class.config.gcp_queue_prefix}-/, '')
192
210
  end
193
211
 
194
212
  #
@@ -201,6 +219,7 @@ module Cloudtasker
201
219
  id: gcp_task.name,
202
220
  http_request: gcp_task.to_h[:http_request],
203
221
  schedule_time: gcp_task.to_h.dig(:schedule_time, :seconds).to_i,
222
+ dispatch_deadline: gcp_task.to_h.dig(:dispatch_deadline, :seconds).to_i,
204
223
  retries: gcp_task.to_h[:response_count],
205
224
  queue: relative_queue
206
225
  }
@@ -45,7 +45,7 @@ module Cloudtasker
45
45
  # to use Task Set instead.
46
46
  redis.search(key('*')).map do |gid|
47
47
  task_id = gid.sub(key(''), '')
48
- redis.sadd(key, task_id)
48
+ redis.sadd(key, [task_id])
49
49
  find(task_id)
50
50
  end
51
51
  end
@@ -88,7 +88,7 @@ module Cloudtasker
88
88
 
89
89
  # Save job
90
90
  redis.write(key(id), payload)
91
- redis.sadd(key, id)
91
+ redis.sadd(key, [id])
92
92
  new(**payload.merge(id: id))
93
93
  end
94
94
 
@@ -112,7 +112,7 @@ module Cloudtasker
112
112
  # @param [String] id The task id.
113
113
  #
114
114
  def self.delete(id)
115
- redis.srem(key, id)
115
+ redis.srem(key, [id])
116
116
  redis.del(key(id))
117
117
  end
118
118
 
@@ -186,7 +186,7 @@ module Cloudtasker
186
186
  queue: queue,
187
187
  dispatch_deadline: dispatch_deadline
188
188
  )
189
- redis.sadd(self.class.key, id)
189
+ redis.sadd(self.class.key, [id])
190
190
  end
191
191
 
192
192
  #
@@ -92,12 +92,26 @@ module Cloudtasker
92
92
  #
93
93
  # Return the batch progress percentage.
94
94
  #
95
+ # A `min_total` can be specified to linearize the calculation, while jobs get added at
96
+ # the start of the batch.
97
+ #
98
+ # Similarly a `smoothing` parameter can be specified to add a constant to the total
99
+ # and linearize the calculation, which becomes: `done / (total + smoothing)`
100
+ #
101
+ # @param [Integer] min_total The minimum for the total number of jobs
102
+ # @param [Integer] smoothing An additive smoothing for the total number of jobs
103
+ #
95
104
  # @return [Float] The progress percentage.
96
105
  #
97
- def percent
98
- return 0 if total.zero?
106
+ def percent(min_total: 0, smoothing: 0)
107
+ # Get the total value to use
108
+ actual_total = [min_total, total + smoothing].max
109
+
110
+ # Abort if we cannot divide
111
+ return 0 if actual_total.zero?
99
112
 
100
- (done.to_f / total) * 100
113
+ # Calculate progress
114
+ (done.to_f / actual_total) * 100
101
115
  end
102
116
 
103
117
  #
@@ -10,7 +10,20 @@ module Cloudtasker
10
10
  JOBS_NAMESPACE = 'jobs'
11
11
  STATES_NAMESPACE = 'states'
12
12
 
13
- # List of statuses triggering a completion callback
13
+ # List of sub-job statuses taken into account when evaluating
14
+ # if the batch is complete.
15
+ #
16
+ # Batch jobs go through the following states:
17
+ # - scheduled: the parent batch has enqueued a worker for the child job
18
+ # - processing: the child job is running
19
+ # - completed: the child job has completed successfully
20
+ # - errored: the child job has encountered an error and must retry
21
+ # - dead: the child job has exceeded its max number of retries
22
+ #
23
+ # The 'dead' status is considered to be a completion status as it
24
+ # means that the job will never succeed. There is no point in blocking
25
+ # the batch forever so we proceed forward eventually.
26
+ #
14
27
  COMPLETION_STATUSES = %w[completed dead].freeze
15
28
 
16
29
  # These callbacks do not need to raise errors on their own
@@ -237,20 +250,14 @@ module Cloudtasker
237
250
  end
238
251
 
239
252
  #
240
- # Save the batch.
253
+ # Save serialized version of the worker.
254
+ #
255
+ # This is required to be able to invoke callback methods in the
256
+ # context of the worker (= instantiated worker) when child workers
257
+ # complete (success or failure).
241
258
  #
242
259
  def save
243
- # Save serialized version of the worker. This is required to
244
- # be able to invoke callback methods in the context of
245
- # the worker (= instantiated worker) when child workers
246
- # complete (success or failure).
247
260
  redis.write(batch_gid, worker.to_h)
248
-
249
- # Stop there if no jobs to save
250
- return if jobs.empty?
251
-
252
- # Save list of child workers
253
- redis.hset(batch_state_gid, jobs.map { |e| [e.job_id, 'scheduled'] }.to_h)
254
261
  end
255
262
 
256
263
  #
@@ -263,7 +270,7 @@ module Cloudtasker
263
270
  migrate_batch_state_to_redis_hash
264
271
 
265
272
  # Update the batch state batch_id entry with the new status
266
- redis.hset(batch_state_gid, batch_id, status) if redis.hexists(batch_state_gid, batch_id)
273
+ redis.hset(batch_state_gid, batch_id, status)
267
274
  end
268
275
 
269
276
  #
@@ -369,6 +376,9 @@ module Cloudtasker
369
376
  #
370
377
  # Calculate the progress of the batch.
371
378
  #
379
+ # @param [Integer] depth The depth of calculation. Zero (default) means only immediate
380
+ # children will be taken into account.
381
+ #
372
382
  # @return [Cloudtasker::Batch::BatchProgress] The batch progress.
373
383
  #
374
384
  def progress(depth: 0)
@@ -391,16 +401,25 @@ module Cloudtasker
391
401
  #
392
402
  # Save the batch and enqueue all child workers attached to it.
393
403
  #
394
- # @return [Array<Cloudtasker::CloudTask>] The Google Task responses
395
- #
396
404
  def setup
397
405
  return true if jobs.empty?
398
406
 
399
407
  # Save batch
400
408
  save
401
409
 
402
- # Enqueue all child workers
403
- jobs.map(&:schedule)
410
+ # Schedule all child workers
411
+ jobs.each do |j|
412
+ # Schedule the job
413
+ j.schedule
414
+
415
+ # Initialize the batch state unless the job has already started (and taken
416
+ # hold of its own status)
417
+ # The batch state is initialized only after the job is scheduled to avoid
418
+ # having never-ending batches - which could occur if a batch was crashing
419
+ # while enqueuing children due to a OOM error and since 'scheduled' is a
420
+ # blocking status.
421
+ redis.hsetnx(batch_state_gid, j.job_id, 'scheduled')
422
+ end
404
423
  end
405
424
 
406
425
  #
@@ -5,9 +5,9 @@ require 'logger'
5
5
  module Cloudtasker
6
6
  # Holds cloudtasker configuration. See Cloudtasker#configure
7
7
  class Config
8
- attr_accessor :redis, :store_payloads_in_redis
8
+ attr_accessor :redis, :store_payloads_in_redis, :gcp_queue_prefix
9
9
  attr_writer :secret, :gcp_location_id, :gcp_project_id,
10
- :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries,
10
+ :processor_path, :logger, :mode, :max_retries,
11
11
  :dispatch_deadline, :on_error, :on_dead
12
12
 
13
13
  # Max Cloud Task size in bytes
@@ -70,11 +70,6 @@ module Cloudtasker
70
70
  Missing host for processing.
71
71
  Please specify a processor hostname in form of `https://some-public-dns.example.com`'
72
72
  DOC
73
- QUEUE_PREFIX_MISSING_ERROR = <<~DOC
74
- Missing GCP queue prefix.
75
- Please specify a queue prefix in the form of `my-app`.
76
- You can create a default queue using the Google SDK via `gcloud tasks queues create my-app-default`
77
- DOC
78
73
  PROJECT_ID_MISSING_ERROR = <<~DOC
79
74
  Missing GCP project ID.
80
75
  Please specify a project ID in the cloudtasker configurator.
@@ -189,15 +184,6 @@ module Cloudtasker
189
184
  @processor_path || DEFAULT_PROCESSOR_PATH
190
185
  end
191
186
 
192
- #
193
- # Return the prefix used for queues.
194
- #
195
- # @return [String] The prefix of the processing queues.
196
- #
197
- def gcp_queue_prefix
198
- @gcp_queue_prefix || raise(StandardError, QUEUE_PREFIX_MISSING_ERROR)
199
- end
200
-
201
187
  #
202
188
  # Return the GCP project ID.
203
189
  #
@@ -43,7 +43,7 @@ module Cloudtasker
43
43
  # to use Schedule Set instead.
44
44
  redis.search(key('*')).map do |gid|
45
45
  schedule_id = gid.sub(key(''), '')
46
- redis.sadd(key, schedule_id)
46
+ redis.sadd(key, [schedule_id])
47
47
  find(schedule_id)
48
48
  end
49
49
  end
@@ -108,7 +108,7 @@ module Cloudtasker
108
108
 
109
109
  # Delete task and stored schedule
110
110
  CloudTask.delete(schedule.task_id) if schedule.task_id
111
- redis.srem(key, schedule.id)
111
+ redis.srem(key, [schedule.id])
112
112
  redis.del(schedule.gid)
113
113
  end
114
114
  end
@@ -278,7 +278,7 @@ module Cloudtasker
278
278
 
279
279
  # Save schedule
280
280
  config_was_changed = config_changed?
281
- redis.sadd(self.class.key, id)
281
+ redis.sadd(self.class.key, [id])
282
282
  redis.write(gid, to_h)
283
283
 
284
284
  # Stop there if backend does not need update
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Cloudtasker
4
- VERSION = '0.13.0'
4
+ VERSION = '0.13.1'
5
5
  end
data/lib/cloudtasker.rb CHANGED
@@ -1,7 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'active_support/core_ext/string/inflections'
3
+ require 'active_support/core_ext/object/blank'
4
4
  require 'active_support/core_ext/object/try'
5
+ require 'active_support/core_ext/string/inflections'
5
6
 
6
7
  require 'cloudtasker/version'
7
8
  require 'cloudtasker/config'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cloudtasker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.13.0
4
+ version: 0.13.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Arnaud Lachaume
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2022-03-11 00:00:00.000000000 Z
11
+ date: 2023-06-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -257,7 +257,7 @@ extensions: []
257
257
  extra_rdoc_files: []
258
258
  files:
259
259
  - ".github/workflows/lint_rubocop.yml"
260
- - ".github/workflows/test_ruby_2.5_2.6.yml"
260
+ - ".github/workflows/test_ruby_2.6.yml"
261
261
  - ".github/workflows/test_ruby_2.7.yml"
262
262
  - ".github/workflows/test_ruby_3.x.yml"
263
263
  - ".gitignore"
@@ -372,7 +372,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
372
372
  - !ruby/object:Gem::Version
373
373
  version: '0'
374
374
  requirements: []
375
- rubygems_version: 3.1.6
375
+ rubygems_version: 3.4.8
376
376
  signing_key:
377
377
  specification_version: 4
378
378
  summary: Background jobs for Ruby using Google Cloud Tasks (beta)
@@ -1,44 +0,0 @@
1
- name: Ruby 2.5/2.6
2
-
3
- on: [push, pull_request]
4
-
5
- jobs:
6
- build:
7
- runs-on: ubuntu-latest
8
- strategy:
9
- matrix:
10
- ruby:
11
- - '2.5.9'
12
- - '2.6.7'
13
- appraisal:
14
- - 'google-cloud-tasks-1.0'
15
- - 'google-cloud-tasks-1.1'
16
- - 'google-cloud-tasks-1.2'
17
- - 'google-cloud-tasks-1.3'
18
- - 'google-cloud-tasks-1.4'
19
- - 'google-cloud-tasks-1.5'
20
- - 'google-cloud-tasks-2.0'
21
- - 'google-cloud-tasks-2.1'
22
- - 'rails-5.2'
23
- - 'rails-6.0'
24
- - 'rails-6.1'
25
- - 'semantic_logger-3.4'
26
- - 'semantic_logger-4.6'
27
- - 'semantic_logger-4.7.0'
28
- - 'semantic_logger-4.7.2'
29
- steps:
30
- - uses: actions/checkout@v2
31
- - uses: zhulik/redis-action@1.1.0
32
- - name: Set up Ruby
33
- uses: ruby/setup-ruby@v1
34
- with:
35
- ruby-version: ${{ matrix.ruby }}
36
- bundler-cache: true
37
- - name: Build and test with Rake
38
- env:
39
- APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
40
- run: |
41
- gem install bundler
42
- bundle install --jobs 4 --retry 3
43
- bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
44
- bundle exec appraisal ${APPRAISAL_CONTEXT} rspec