cloudtasker 0.10.rc6 → 0.11.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 796e69d0470947fe0af6ea530cfd6c5b69e95b6a1be42b307e84b27dd9246198
-  data.tar.gz: 5c51dd25b3033546a7115d83d638123e80a9ee26fb74845b88a02247edb39461
+  metadata.gz: 1a0759638a4af47fcc26467b93039b3c6355908db5cafd0a20972b7e48649b81
+  data.tar.gz: a0f76c953bc0f64276f5b10f90300bcf7415dbfe305fac6697e4bd9fb83b53c0
 SHA512:
-  metadata.gz: 134e6d344e75a9500850b135ea215c2b64d31cf366892acc0aab1b37c47e758dfa7e2bd00015ec6d8bb47556aba002bb578f2365ea34196d3343b24cee0f2156
-  data.tar.gz: 73bd3cbd91fa1938b97df2cd7f5125a3f90205fb31b5d12efe5b64c4c0aa03e74f6f83cd8c9267b95b5553be93395a45db4f365bbb49b5623a97bf982bf9d2d7
+  metadata.gz: ef16621727a56793623e1c582bac1f6e4971b8ff1af3ee36a3d37b22d239f7bb3504a35b5f1ffc9f250343b2ed5ae888e75b8d583b49487f34dc69eb45cd9d13
+  data.tar.gz: aea0fa6eb2873f6b99df05c613d89363a3c6103fe814123d25a7508ede1fbf92cf653d10e7d57863d7ec6065d1c96e5c5f3ecf371a75ac925a588693849c320e

data/.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, 0.9-stable ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, 0.9-stable ]
 
 jobs:
   build:
@@ -21,6 +21,10 @@ jobs:
           - 'google-cloud-tasks-1.3'
           - 'rails-5.2'
           - 'rails-6.0'
+          - 'semantic_logger-3.4'
+          - 'semantic_logger-4.6'
+          - 'semantic_logger-4.7.0'
+          - 'semantic_logger-4.7.2'
     steps:
       - name: Setup System
         run: sudo apt-get install libsqlite3-dev
@@ -38,4 +42,4 @@ jobs:
           bundle install --jobs 4 --retry 3
           bundle exec rubocop
           bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
-          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.rubocop.yml

@@ -40,4 +40,7 @@ Style/Documentation:
     - 'spec/**/*'
 
 Metrics/ParameterLists:
-  CountKeywordArgs: false
+  CountKeywordArgs: false
+
+RSpec/MessageSpies:
+  Enabled: false

data/Appraisals

@@ -23,3 +23,19 @@ end
 appraise 'rails-6.0' do
   gem 'rails', '6.0'
 end
+
+appraise 'semantic_logger-3.4' do
+  gem 'semantic_logger', '3.4.1'
+end
+
+appraise 'semantic_logger-4.6' do
+  gem 'semantic_logger', '4.6.1'
+end
+
+appraise 'semantic_logger-4.7.0' do
+  gem 'semantic_logger', '4.7.0'
+end
+
+appraise 'semantic_logger-4.7.2' do
+  gem 'semantic_logger', '4.7.2'
+end

data/CHANGELOG.md

@@ -1,5 +1,46 @@
 # Changelog
 
+## [v0.10.1](https://github.com/keypup-io/cloudtasker/tree/v0.10.1) (2020-10-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.10.1)
+
+**Fixed bugs:**
+- Local server: delete dead task from local server queue
+- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
+- Worker: fix configuration of `max_retries` at worker level
+
+## [v0.10.0](https://github.com/keypup-io/cloudtasker/tree/v0.10.0) (2020-09-02)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.10.0)
+
+**Improvements:**
+- Logging: Add worker name in log messages
+- Logging: Add job duration in log messages
+- Logging: Add Cloud Cloud Task ID in log messages
+- Unique Job: Support TTL for lock keys. This feature prevents queues from being dead-locked when a critical crash occurs while processing a unique job.
+- Worker: support payload storage in Redis instead of sending the payload to Google Cloud Tasks. This is useful when job arguments are expected to exceed 100kb, which is the limit set by Google Cloud Tasks
+
+**Fixed bugs:**
+- Local processing error: improve error handling and retries around network interruptions
+- Redis client: prevent deadlocks in high concurrency scenario by slowing down poll time and enforcing lock expiration
+- Redis client: use connecion pool with Redis to prevent race conditions
+- Google API: improve error handling on job creation
+- Google API: use the `X-CloudTasks-TaskRetryCount` instead of `X-CloudTasks-TaskExecutionCount` to detect how many retries Google Cloud Tasks has performed. Using `X-CloudTasks-TaskRetryCount` is theoretically less accurate than using `X-CloudTasks-TaskExecutionCount` because it includes the number of "app unreachable" retries but `X-CloudTasks-TaskExecutionCount` is currently bugged and remains at zero all the time. See [this issue](https://github.com/keypup-io/cloudtasker/issues/6)
+
+## [v0.9.4](https://github.com/keypup-io/cloudtasker/tree/v0.9.4) (2020-10-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.9.4)
+
+**Fixed bugs:**
+- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
+
+## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
+
+**Fixed bugs:**
+- Google Cloud Tasks: lock version to `~> 1.0` (Google recently released a v2 which changes its bindings completely). An [issue](https://github.com/keypup-io/cloudtasker/issues/11) has been raised to upgrade Cloudtasker to `google-cloud-tasks` `v2`.
+
 ## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
@@ -71,7 +112,3 @@ For Sinatra applications please update your Cloudtasker controller according to
 ## [v0.1.0](https://github.com/keypup-io/cloudtasker/tree/v0.1.0) (2019-11-17)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/c137feb1ceaaaa4e2fecac0d1f0b4c73151ae002...v0.1.0)
-
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*

data/README.md

@@ -6,11 +6,11 @@ Background jobs for Ruby using Google Cloud Tasks.
 
 Cloudtasker provides an easy to manage interface to Google Cloud Tasks for background job processing. Workers can be defined programmatically using the Cloudtasker DSL and enqueued for processing using a simple to use API.
 
-Cloudtasker is particularly suited for serverless applications only responding to HTTP requests and where running a dedicated job processing is not an option (e.g. deploy via [Cloud Run](https://cloud.google.com/run)). All jobs enqueued in Cloud Tasks via Cloudtasker eventually get processed by your application via HTTP requests.
+Cloudtasker is particularly suited for serverless applications only responding to HTTP requests and where running a dedicated job processing server is not an option (e.g. deploy via [Cloud Run](https://cloud.google.com/run)). All jobs enqueued in Cloud Tasks via Cloudtasker eventually get processed by your application via HTTP requests.
 
 Cloudtasker also provides optional modules for running [cron jobs](docs/CRON_JOBS.md), [batch jobs](docs/BATCH_JOBS.md) and [unique jobs](docs/UNIQUE_JOBS.md).
 
-A local processing server is also available in development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
+A local processing server is also available for development. This local server processes jobs in lieu of Cloud Tasks and allows you to work offline.
 
 ## Summary
 
@@ -34,7 +34,11 @@ A local processing server is also available in development. This local server pr
     1. [HTTP Error codes](#http-error-codes)
     2. [Error callbacks](#error-callbacks)
     3. [Max retries](#max-retries)
-10. [Best practices building workers](#best-practices-building-workers)
+10. [Testing](#testing)
+    1. [Test helper setup](#test-helper-setup)
+    2. [In-memory queues](#in-memory-queues)
+    3. [Unit tests](#unit-tests)
+11. [Best practices building workers](#best-practices-building-workers)
 
 ## Installation
 
@@ -48,7 +52,7 @@ And then execute:
 
     $ bundle
 
-Or install it yourself as:
+Or install it yourself with:
 
     $ gem install cloudtasker
 
@@ -71,7 +75,7 @@ Cloudtasker.configure do |config|
   # Adapt the server port to be the one used by your Rails web process
   #
   config.processor_host = 'http://localhost:3000'
-  
+
   #
   # If you do not have any Rails secret_key_base defined, uncomment the following
   # This secret is used to authenticate jobs sent to the processing endpoint
@@ -154,14 +158,14 @@ The gem can be configured through an initializer. See below all the available co
 Cloudtasker.configure do |config|
   #
   # If you do not have any Rails secret_key_base defined, uncomment the following.
-  # This secret is used to authenticate jobs sent to the processing endpoint 
+  # This secret is used to authenticate jobs sent to the processing endpoint
   # of your application.
   #
   # Default with Rails: Rails.application.credentials.secret_key_base
   #
   # config.secret = 'some-long-token'
 
-  # 
+  #
   # Specify the details of your Google Cloud Task location.
   #
   # This not required in development using the Cloudtasker local server.
@@ -185,21 +189,21 @@ Cloudtasker.configure do |config|
   #
   # Specific queues can be created in Cloud Tasks using the gcloud SDK or
   # via the `rake cloudtasker:setup_queue name=<queue_name>` task.
-  # 
+  #
   config.gcp_queue_prefix = 'my-app'
 
-  # 
+  #
   # Specify the publicly accessible host for your application
   #
   # > E.g. in development, using the cloudtasker local server
   # config.processor_host = 'http://localhost:3000'
-  # 
+  #
   # > E.g. in development, using `config.mode = :production` and ngrok
   # config.processor_host = 'https://111111.ngrok.io'
   #
   config.processor_host = 'https://app.mydomain.com'
 
-  # 
+  #
   # Specify the mode of operation:
   # - :development => jobs will be pushed to Redis and picked up by the Cloudtasker local server
   # - :production => jobs will be pushed to Google Cloud Tasks. Requires a publicly accessible domain.
@@ -208,20 +212,20 @@ Cloudtasker.configure do |config|
   #
   # config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
 
-  # 
+  #
   # Specify the logger to use
-  # 
+  #
   # Default with Rails: Rails.logger
   # Default without Rails: Logger.new(STDOUT)
-  # 
+  #
   # config.logger = MyLogger.new(STDOUT)
 
-  # 
+  #
   # Specify how many retries are allowed on jobs. This number of retries excludes any
-  # connectivity error that would be due to the application being down or unreachable.
-  # 
+  # connectivity error due to the application being down or unreachable.
+  #
   # Default: 25
-  # 
+  #
   # config.max_retries = 10
 
   #
@@ -246,7 +250,7 @@ Cloudtasker.configure do |config|
   # You can set this configuration parameter to a KB value if you want to store jobs
   # args in redis only if the JSONified arguments payload exceeds that threshold.
   #
-  # Supported since: v0.10.rc1
+  # Supported since: v0.10.0
   #
   # Default: false
   #
@@ -258,7 +262,7 @@ Cloudtasker.configure do |config|
 end
 ```
 
-If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues). 
+If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
 
 Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
 ```bash
@@ -289,7 +293,7 @@ MyWorker.schedule(args: [arg1, arg2], time_at: Time.parse('2025-01-01 00:50:00Z'
 MyWorker.schedule(args: [arg1, arg2], time_in: 5 * 60, queue: 'critical')
 ```
 
-Cloudtasker also provides a helper for re-enqueuing jobs. Re-enqueued jobs keep the same worker id. Some middlewares may rely on this to track the fact that that a job didn't actually complete (e.g. Cloustasker batch). This is optional and you can always fallback to using exception management (raise an error) to retry/re-enqueue jobs.
+Cloudtasker also provides a helper for re-enqueuing jobs. Re-enqueued jobs keep the same job id. Some middlewares may rely on this to track the fact that that a job didn't actually complete (e.g. Cloustasker batch). This is optional and you can always fallback to using exception management (raise an error) to retry/re-enqueue jobs.
 
 E.g.
 ```ruby
@@ -360,7 +364,7 @@ CriticalWorker.schedule(args: [1], queue: :important)
 Cloudtasker comes with three optional features:
 - Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
 - Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
-- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments. 
+- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
 
 ## Working locally
 
@@ -377,9 +381,9 @@ You can configure your application to use the Cloudtasker local server using the
 
 Cloudtasker.configure do |config|
   # ... other options
-  
+
   # Push jobs to redis and let the Cloudtasker local server collect them
-  # This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set 
+  # This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
   # to a non-development environment
   config.mode = :development
 end
@@ -423,7 +427,7 @@ Cloudtasker.configure do |config|
 
   # Use your ngrok domain as the processor host
   config.processor_host = 'https://your-tunnel-id.ngrok.io'
-  
+
   # Force Cloudtasker to use Google Cloud Tasks in development
   config.mode = :production
 end
@@ -474,7 +478,7 @@ The way contextual information is displayed depends on the logger itself. For ex
 
 Contextual information can be customised globally and locally using a log context_processor. By default the `Cloudtasker::WorkerLogger` is configured the following way:
 ```ruby
-Cloudtasker::WorkerLogger.log_context_processor = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta) }
+Cloudtasker::WorkerLogger.log_context_processor = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue, :task_id) }
 ```
 
 You can decide to add a global identifier for your worker logs using the following:
@@ -482,7 +486,7 @@ You can decide to add a global identifier for your worker logs using the followi
 # config/initializers/cloudtasker.rb
 
 Cloudtasker::WorkerLogger.log_context_processor = lambda { |worker|
-  worker.to_h.slice(:worker, :job_id, :job_meta).merge(app: 'my-app')
+  worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue, :task_id).merge(app: 'my-app')
 }
 ```
 
@@ -504,7 +508,7 @@ end
 See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
 
 ### Searching logs: Job ID vs Task ID
-**Note**: `task_id` field is available in logs starting with `0.10.rc6`
+**Note**: `task_id` field is available in logs starting with `0.10.0`
 
 Job instances are assigned two different different IDs for tracking and logging purpose: `job_id` and `task_id`. These IDs are found in each log entry to facilitate search.
 
@@ -520,7 +524,7 @@ The Google Cloud Task UI (GCP console) lists all the tasks pending/retrying and
 
 ## Error Handling
 
-Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of Cloud Task retries Cloud Task will depend on the configuration of your queue in Cloud Tasks.
+Jobs failures will return an HTTP error to Cloud Task and trigger a retry at a later time. The number of Cloud Task retries depends on the configuration of your queue in Cloud Tasks.
 
 ### HTTP Error codes
 
@@ -528,6 +532,7 @@ Jobs failing will automatically return the following HTTP error code to Cloud Ta
 
 | Code | Description |
 |------|-------------|
+| 204 | The job was processed successfully |
 | 205 | The job is dead and has been removed from the queue |
 | 404 | The job has specified an incorrect worker class.  |
 | 422 | An error happened during the execution of the worker (`perform` method) |
@@ -566,26 +571,24 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
 
 Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
 
-**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `0.10.rc3` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
+**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `v0.10.0` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
 
 E.g. Set max number of retries globally via the cloudtasker initializer.
 ```ruby
 # config/initializers/cloudtasker.rb
 
 Cloudtasker.configure do |config|
-  # 
+  #
   # Specify how many retries are allowed on jobs. This number of retries excludes any
   # connectivity error that would be due to the application being down or unreachable.
-  # 
+  #
   # Default: 25
-  # 
+  #
   config.max_retries = 10
 end
 ```
 
 E.g. Set max number of retries to 3 on a given worker
-
-E.g.
 ```ruby
 # app/workers/some_error_worker.rb
 
@@ -595,12 +598,126 @@ class SomeErrorWorker
   # This will override the global setting
   cloudtasker_options max_retries: 3
 
-  def perform()
+  def perform
     raise(ArgumentError)
   end
 end
 ```
 
+E.g. Evaluate the number of max retries at runtime (Supported since: v0.10.1)
+```ruby
+# app/workers/some_error_worker.rb
+
+class SomeErrorWorker
+  include Cloudtasker::Worker
+
+  # Return the number of max retries based on
+  # worker arguments.
+  #
+  # If this method returns nil then max_retries
+  # will delegate to the class `max_retries` setting or Cloudtasker
+  # `max_retries` configuration otion.
+  def max_retries(arg1, arg2)
+    arg1 == 'foo' ? 13 : nil
+  end
+
+  def perform(arg1, arg2)
+    raise(ArgumentError)
+  end
+end
+```
+
+## Testing
+Cloudtasker provides several options to test your workers.
+
+### Test helper setup
+Require `cloudtasker/testing` in your `rails_helper.rb` (Rspec Rails) or `spec_helper.rb` (Rspec) or test unit helper file then enable one of the three modes:
+
+```ruby
+require 'cloudtasker/testing'
+
+# Mode 1 (default): Push jobs to Google Cloud Tasks (env != development) or Redis (env == development)
+Cloudtasker::Testing.enable!
+
+# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
+# Cloudtasker::Worker.drain_all (process all jobs) or MyWorker.drain (process jobs for specific worker)
+Cloudtasker::Testing.fake!
+
+# Mode 3: Push jobs to an in-memory queue. Jobs will be processed immediately.
+Cloudtasker::Testing.inline!
+```
+
+You can query the current testing mode with:
+```ruby
+Cloudtasker::Testing.enabled?
+Cloudtasker::Testing.fake?
+Cloudtasker::Testing.inline?
+```
+
+Each testing mode accepts a block argument to temporarily switch to it:
+```ruby
+# Enable fake mode for all tests
+Cloudtasker::Testing.fake!
+
+# Enable inline! mode temporarily for a given test
+Cloudtasker.inline! do
+   MyWorker.perform_async(1,2)
+end
+```
+
+Note that extension middlewares - e.g. unique job, batch job etc. - run in test mode. You can disable middlewares in your tests by adding the following to your test helper:
+```ruby
+# Remove all middlewares
+Cloudtasker.configure do |c|
+  c.client_middleware.clear
+  c.server_middleware.clear
+end
+
+# Remove all unique job middlewares
+Cloudtasker.configure do |c|
+  c.client_middleware.remove(Cloudtasker::UniqueJob::Middleware::Client)
+  c.server_middleware.remove(Cloudtasker::UniqueJob::Middleware::Server)
+end
+```
+
+### In-memory queues
+The `fake!` or `inline!` modes use in-memory queues, which can be queried and controlled using the following methods:
+
+```ruby
+# Perform all jobs in queue
+Cloudtasker::Worker.drain_all
+
+# Remove all jobs in queue
+Cloudtasker::Worker.clear_all
+
+# Perform all jobs in queue for a specific worker type
+MyWorker.drain
+
+# Return the list of jobs in queue for a specific worker type
+MyWorker.jobs
+```
+
+### Unit tests
+Below are examples of rspec tests. It is assumed that `Cloudtasker::Testing.fake!` has been set in the test helper.
+
+**Example 1**: Testing that a job is scheduled
+```ruby
+describe 'worker scheduling'
+  subject(:enqueue_job) { MyWorker.perform_async(1,2) }
+
+  it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
+end
+```
+
+**Example 2**: Testing job execution logic
+```ruby
+describe 'worker calls api'
+  subject { Cloudtasker::Testing.inline! { MyApiWorker.perform_async(1,2) } }
+
+  before { expect(MyApi).to receive(:fetch).and_return([]) }
+  it { is_expected.to be_truthy }
+end
+```
 
 ## Best practices building workers
 
@@ -676,7 +793,7 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
 Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 #### Option 1: Use Cloudtasker optional support for payload storage in Redis
-**Supported since**: `0.10.rc1`
+**Supported since**: `0.10.0`
 
 Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
 

data/app/controllers/cloudtasker/worker_controller.rb

@@ -19,21 +19,32 @@ module Cloudtasker
       # Process payload
       WorkerHandler.execute_from_payload!(payload)
       head :no_content
-    rescue DeadWorkerError
+    rescue DeadWorkerError, MissingWorkerArgumentsError => e
       # 205: job will NOT be retried
+      log_error(e)
       head :reset_content
-    rescue InvalidWorkerError
+    rescue InvalidWorkerError => e
       # 404: Job will be retried
+      log_error(e)
       head :not_found
     rescue StandardError => e
       # 404: Job will be retried
-      Cloudtasker.logger.error(e)
-      Cloudtasker.logger.error(e.backtrace.join("\n"))
+      log_error(e)
       head :unprocessable_entity
     end
 
     private
 
+    #
+    # Log an error via cloudtasker logger.
+    #
+    # @param [Exception] e The error to log
+    #
+    def log_error(error)
+      Cloudtasker.logger.error(error)
+      Cloudtasker.logger.error(error.backtrace.join("\n"))
+    end
+
     #
     # Parse the request body and return the actual job
     # payload.

data/cloudtasker.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
   spec.add_dependency 'connection_pool'
   spec.add_dependency 'fugit'
-  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'google-cloud-tasks', '~> 1.0'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
   spec.add_dependency 'retriable'
@@ -43,6 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'
   spec.add_development_dependency 'rubocop-rspec', '1.37.0'
+  spec.add_development_dependency 'semantic_logger'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/gemfiles/semantic_logger_3.4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "3.4.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.6.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.6.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.0"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/missing_worker_arguments_error'
 require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'

data/lib/cloudtasker/backend/memory_task.rb

@@ -8,6 +8,15 @@ module Cloudtasker
       attr_accessor :job_retries
       attr_reader :id, :http_request, :schedule_time, :queue
 
+      #
+      # Return true if we are in test inline execution mode.
+      #
+      # @return [Boolean] True if inline mode enabled.
+      #
+      def self.inline_mode?
+        defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
+      end
+
       #
       # Return the task queue. A worker class name
       #
@@ -57,7 +66,7 @@ module Cloudtasker
         queue << task
 
         # Execute task immediately if in testing and inline mode enabled
-        task.execute if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
+        task.execute if inline_mode?
 
         task
       end
@@ -157,8 +166,12 @@ module Cloudtasker
         # Delete task
         self.class.delete(id)
         resp
-      rescue StandardError
+      rescue DeadWorkerError => e
+        self.class.delete(id)
+        raise(e) if self.class.inline_mode?
+      rescue StandardError => e
         self.job_retries += 1
+        raise(e) if self.class.inline_mode?
       end
 
       #

data/lib/cloudtasker/missing_worker_arguments_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class MissingWorkerArgumentsError < StandardError
+  end
+end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.10.rc6'
+  VERSION = '0.11.rc1'
 end

data/lib/cloudtasker/worker.rb

@@ -296,6 +296,20 @@ module Cloudtasker
       other.is_a?(self.class) && other.job_id == job_id
     end
 
+    #
+    # Return the max number of retries allowed for this job.
+    #
+    # The order of precedence for retry lookup is:
+    # - Worker `max_retries` method
+    # - Class `max_retries` option
+    # - Cloudtasker `max_retries` config option
+    #
+    # @return [Integer] The number of retries
+    #
+    def job_max_retries
+      @job_max_retries ||= (try(:max_retries, *job_args) || self.class.max_retries)
+    end
+
     #
     # Return true if the job has excceeded its maximum number
     # of retries
@@ -303,7 +317,7 @@ module Cloudtasker
     # @return [Boolean] True if the job is dead
     #
     def job_dead?
-      job_retries >= Cloudtasker.config.max_retries
+      job_retries >= job_max_retries
     end
 
     #
@@ -333,6 +347,13 @@ module Cloudtasker
 
       Cloudtasker.config.server_middleware.invoke(self) do
         begin
+          # Abort if arguments are missing. This may happen with redis arguments storage
+          # if Cloud Tasks times out on a job but the job still succeeds
+          if job_args.empty? && [0, -1].exclude?(method(:perform).arity)
+            raise(MissingWorkerArgumentsError, 'worker arguments are missing')
+          end
+
+          # Perform the job
           perform(*job_args)
         rescue StandardError => e
           try(:on_error, e)

data/lib/cloudtasker/worker_handler.rb

@@ -85,7 +85,7 @@ module Cloudtasker
       redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
 
       resp
-    rescue DeadWorkerError => e
+    rescue DeadWorkerError, MissingWorkerArgumentsError => e
       # Delete stored args payload if job is dead
       redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
       raise(e)

data/lib/cloudtasker/worker_logger.rb

@@ -142,7 +142,7 @@ module Cloudtasker
     #
     def log_message(level, msg, &block)
       # Merge log-specific context into worker-specific context
-      payload_block = -> { log_block.call.merge(block&.call || {}) }
+      payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.10.rc6
+  version: 0.11.rc1
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -56,16 +56,16 @@ dependencies:
   name: google-cloud-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jwt
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +206,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.37.0
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -324,6 +338,11 @@ files:
 - gemfiles/rails_5.2.gemfile.lock
 - gemfiles/rails_6.0.gemfile
 - gemfiles/rails_6.0.gemfile.lock
+- gemfiles/semantic_logger_3.4.gemfile
+- gemfiles/semantic_logger_4.6.gemfile
+- gemfiles/semantic_logger_4.7.0.gemfile
+- gemfiles/semantic_logger_4.7.2.gemfile
+- gemfiles/semantic_logger_4.7.gemfile
 - lib/cloudtasker.rb
 - lib/cloudtasker/authentication_error.rb
 - lib/cloudtasker/authenticator.rb
@@ -351,6 +370,7 @@ files:
 - lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
+- lib/cloudtasker/missing_worker_arguments_error.rb
 - lib/cloudtasker/redis_client.rb
 - lib/cloudtasker/testing.rb
 - lib/cloudtasker/unique_job.rb