cloudtasker 0.10.rc7 → 0.11.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7c5440326c0fa695e469a341d58b976bd4027179440a607ce34c15c9dd4d697e
-  data.tar.gz: 671c2bcf90c199c68136a14efa62410d17e3a850dad1e6b5bf6a02b083169d75
+  metadata.gz: c5d91132bd3363bf3d7c08cd207b385f502eae3056be4734f73f60ef220f94c5
+  data.tar.gz: fe935947d1ece340202d32eb54927336840ca236510a5044c2b1996c2161080d
 SHA512:
-  metadata.gz: f6adad797eedb6ff0d589ea2dfa6b109709f629ad82cf64e9755e7da235b4a4fb902d4c48794981dcd527a6933196686a72fce46b2572912fd84b1a3396bcd33
-  data.tar.gz: 78fa73bafda04c16b46bed2bd8930e5e5ef3badf046f2e665b41d156e9644c20468788a0ffa173fcb6585bdb24a1bdcff599be11f41a5420c1302747e209bc52
+  metadata.gz: ed918a89e19dca6068f6c4d7b2aa0e55f838829f19741b1509edcbf3e03117e102454791efd808c5521da30e862d8cfc1d5969444cfb6ee250589e5b7dd281de
+  data.tar.gz: d4fbb9afebcf84efb8bbe3f11b126aefc93a14c62d9d09b1a2f27186defb9c30ee0d28c13688b6bcc38411a3cbe8a736010fbea1a75e7dfa6334870e0522f2ff

data/.github/workflows/test.yml

@@ -2,9 +2,9 @@ name: Test
 
 on:
   push:
-    branches: [ master ]
+    branches: [ master, 0.9-stable ]
   pull_request:
-    branches: [ master ]
+    branches: [ master, 0.9-stable ]
 
 jobs:
   build:
@@ -21,6 +21,10 @@ jobs:
           - 'google-cloud-tasks-1.3'
           - 'rails-5.2'
           - 'rails-6.0'
+          - 'semantic_logger-3.4'
+          - 'semantic_logger-4.6'
+          - 'semantic_logger-4.7.0'
+          - 'semantic_logger-4.7.2'
     steps:
       - name: Setup System
         run: sudo apt-get install libsqlite3-dev
@@ -38,4 +42,4 @@ jobs:
           bundle install --jobs 4 --retry 3
           bundle exec rubocop
           bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
-          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.rubocop.yml

@@ -20,6 +20,10 @@ Metrics/LineLength:
 Metrics/MethodLength:
   Max: 20
 
+RSpec/DescribeClass:
+  Exclude:
+    - 'spec/integration/**/*_spec.rb'
+
 RSpec/ExpectInHook:
   Enabled: false
 

data/Appraisals

@@ -23,3 +23,19 @@ end
 appraise 'rails-6.0' do
   gem 'rails', '6.0'
 end
+
+appraise 'semantic_logger-3.4' do
+  gem 'semantic_logger', '3.4.1'
+end
+
+appraise 'semantic_logger-4.6' do
+  gem 'semantic_logger', '4.6.1'
+end
+
+appraise 'semantic_logger-4.7.0' do
+  gem 'semantic_logger', '4.7.0'
+end
+
+appraise 'semantic_logger-4.7.2' do
+  gem 'semantic_logger', '4.7.2'
+end

data/CHANGELOG.md

@@ -1,5 +1,54 @@
 # Changelog
 
+## Latest RC: [v0.11.rc2](https://github.com/keypup-io/cloudtasker/tree/v0.11.rc2) (2020-11-23)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.11.rc2)
+
+**Improvements:**
+- Worker: drop job (return 205 response) when worker arguments are not available (e.g. arguments were stored in Redis and the latter was flushed)
+- Rails: add ActiveJob adapter (thanks @vovimayhem)
+
+## [v0.10.1](https://github.com/keypup-io/cloudtasker/tree/v0.10.1) (2020-10-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.10.0...v0.10.1)
+
+**Fixed bugs:**
+- Local server: delete dead task from local server queue
+- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
+- Worker: fix configuration of `max_retries` at worker level
+
+## [v0.10.0](https://github.com/keypup-io/cloudtasker/tree/v0.10.0) (2020-09-02)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.10.0)
+
+**Improvements:**
+- Logging: Add worker name in log messages
+- Logging: Add job duration in log messages
+- Logging: Add Cloud Cloud Task ID in log messages
+- Unique Job: Support TTL for lock keys. This feature prevents queues from being dead-locked when a critical crash occurs while processing a unique job.
+- Worker: support payload storage in Redis instead of sending the payload to Google Cloud Tasks. This is useful when job arguments are expected to exceed 100kb, which is the limit set by Google Cloud Tasks
+
+**Fixed bugs:**
+- Local processing error: improve error handling and retries around network interruptions
+- Redis client: prevent deadlocks in high concurrency scenario by slowing down poll time and enforcing lock expiration
+- Redis client: use connecion pool with Redis to prevent race conditions
+- Google API: improve error handling on job creation
+- Google API: use the `X-CloudTasks-TaskRetryCount` instead of `X-CloudTasks-TaskExecutionCount` to detect how many retries Google Cloud Tasks has performed. Using `X-CloudTasks-TaskRetryCount` is theoretically less accurate than using `X-CloudTasks-TaskExecutionCount` because it includes the number of "app unreachable" retries but `X-CloudTasks-TaskExecutionCount` is currently bugged and remains at zero all the time. See [this issue](https://github.com/keypup-io/cloudtasker/issues/6)
+
+## [v0.9.4](https://github.com/keypup-io/cloudtasker/tree/v0.9.4) (2020-10-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.3...v0.9.4)
+
+**Fixed bugs:**
+- Logging: fix log processing with `semantic_logger` `v4.7.2`. Accept any args on block passed to the logger.
+
+## [v0.9.3](https://github.com/keypup-io/cloudtasker/tree/v0.9.3) (2020-06-25)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.2...v0.9.3)
+
+**Fixed bugs:**
+- Google Cloud Tasks: lock version to `~> 1.0` (Google recently released a v2 which changes its bindings completely). An [issue](https://github.com/keypup-io/cloudtasker/issues/11) has been raised to upgrade Cloudtasker to `google-cloud-tasks` `v2`.
+
 ## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
@@ -71,7 +120,3 @@ For Sinatra applications please update your Cloudtasker controller according to
 ## [v0.1.0](https://github.com/keypup-io/cloudtasker/tree/v0.1.0) (2019-11-17)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/c137feb1ceaaaa4e2fecac0d1f0b4c73151ae002...v0.1.0)
-
-
-
-\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)*

data/README.md

@@ -16,6 +16,7 @@ A local processing server is also available for development. This local server p
 
 1. [Installation](#installation)
 2. [Get started with Rails](#get-started-with-rails)
+2. [Get started with Rails & ActiveJob](#get-started-with-rails--activejob)
 3. [Configuring Cloudtasker](#configuring-cloudtasker)
     1. [Cloud Tasks authentication & permissions](#cloud-tasks-authentication--permissions)
     2. [Cloudtasker initializer](#cloudtasker-initializer)
@@ -75,7 +76,7 @@ Cloudtasker.configure do |config|
   # Adapt the server port to be the one used by your Rails web process
   #
   config.processor_host = 'http://localhost:3000'
-  
+
   #
   # If you do not have any Rails secret_key_base defined, uncomment the following
   # This secret is used to authenticate jobs sent to the processing endpoint
@@ -132,6 +133,95 @@ That's it! Your job was picked up by the Cloudtasker local server and sent for p
 
 Now jump to the next section to configure your app to use Google Cloud Tasks as a backend.
 
+## Get started with Rails & ActiveJob
+**Note**: ActiveJob is supported since `0.11.rc2`  
+**Note**: Cloudtasker extensions (cron, batch and unique jobs) are not available when using cloudtasker via ActiveJob.
+
+Cloudtasker is pre-integrated with ActiveJob. Follow the steps below to get started.
+
+Install redis on your machine (this is required by the Cloudtasker local processing server)
+```bash
+# E.g. using brew
+brew install redis
+```
+
+Add the following initializer
+```ruby
+# config/initializers/cloudtasker.rb
+
+Cloudtasker.configure do |config|
+  #
+  # Adapt the server port to be the one used by your Rails web process
+  #
+  config.processor_host = 'http://localhost:3000'
+
+  #
+  # If you do not have any Rails secret_key_base defined, uncomment the following
+  # This secret is used to authenticate jobs sent to the processing endpoint
+  # of your application.
+  #
+  # config.secret = 'some-long-token'
+end
+```
+
+Configure ActiveJob to use Cloudtasker. You can also configure ActiveJob per environment via the config/environments/:env.rb files
+```ruby
+# config/application.rb
+
+require_relative 'boot'
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+
+module Dummy
+  class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 6.0
+
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration can go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded after loading
+    # the framework and any gems in your application.
+
+    # Use cloudtasker as the ActiveJob backend:
+    config.active_job.queue_adapter = :cloudtasker
+  end
+end
+
+```
+
+Define your first job:
+```ruby
+# app/jobs/example_job.rb
+
+class ExampleJob < ApplicationJob
+  queue_as :default
+
+  def perform(some_arg)
+    logger.info("Job run with #{some_arg}. This is working!")
+  end
+end
+```
+
+Launch Rails and the local Cloudtasker processing server (or add `cloudtasker` to your foreman config as a `worker` process)
+```bash
+# In one terminal
+> rails s -p 3000
+
+# In another terminal
+> cloudtasker
+```
+
+Open a Rails console and enqueue some jobs
+```ruby
+  # Process job as soon as possible
+  ExampleJob.perform_later('foo')
+
+  # Process job in 60 seconds
+  ExampleJob.set(wait: 60).perform_later('foo')
+```
+
+
 ## Configuring Cloudtasker
 
 ### Cloud Tasks authentication & permissions
@@ -158,14 +248,14 @@ The gem can be configured through an initializer. See below all the available co
 Cloudtasker.configure do |config|
   #
   # If you do not have any Rails secret_key_base defined, uncomment the following.
-  # This secret is used to authenticate jobs sent to the processing endpoint 
+  # This secret is used to authenticate jobs sent to the processing endpoint
   # of your application.
   #
   # Default with Rails: Rails.application.credentials.secret_key_base
   #
   # config.secret = 'some-long-token'
 
-  # 
+  #
   # Specify the details of your Google Cloud Task location.
   #
   # This not required in development using the Cloudtasker local server.
@@ -189,21 +279,21 @@ Cloudtasker.configure do |config|
   #
   # Specific queues can be created in Cloud Tasks using the gcloud SDK or
   # via the `rake cloudtasker:setup_queue name=<queue_name>` task.
-  # 
+  #
   config.gcp_queue_prefix = 'my-app'
 
-  # 
+  #
   # Specify the publicly accessible host for your application
   #
   # > E.g. in development, using the cloudtasker local server
   # config.processor_host = 'http://localhost:3000'
-  # 
+  #
   # > E.g. in development, using `config.mode = :production` and ngrok
   # config.processor_host = 'https://111111.ngrok.io'
   #
   config.processor_host = 'https://app.mydomain.com'
 
-  # 
+  #
   # Specify the mode of operation:
   # - :development => jobs will be pushed to Redis and picked up by the Cloudtasker local server
   # - :production => jobs will be pushed to Google Cloud Tasks. Requires a publicly accessible domain.
@@ -212,20 +302,20 @@ Cloudtasker.configure do |config|
   #
   # config.mode = Rails.env.production? || Rails.env.my_other_env? ? :production : :development
 
-  # 
+  #
   # Specify the logger to use
-  # 
+  #
   # Default with Rails: Rails.logger
   # Default without Rails: Logger.new(STDOUT)
-  # 
+  #
   # config.logger = MyLogger.new(STDOUT)
 
-  # 
+  #
   # Specify how many retries are allowed on jobs. This number of retries excludes any
   # connectivity error due to the application being down or unreachable.
-  # 
+  #
   # Default: 25
-  # 
+  #
   # config.max_retries = 10
 
   #
@@ -250,7 +340,7 @@ Cloudtasker.configure do |config|
   # You can set this configuration parameter to a KB value if you want to store jobs
   # args in redis only if the JSONified arguments payload exceeds that threshold.
   #
-  # Supported since: v0.10.rc1
+  # Supported since: v0.10.0
   #
   # Default: false
   #
@@ -262,7 +352,7 @@ Cloudtasker.configure do |config|
 end
 ```
 
-If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues). 
+If the default queue `<gcp_queue_prefix>-default` does not exist in Cloud Tasks you should [create it using the gcloud sdk](https://cloud.google.com/tasks/docs/creating-queues).
 
 Alternatively with Rails you can simply run the following rake task if you have queue admin permissions (`cloudtasks.queues.get` and `cloudtasks.queues.create`).
 ```bash
@@ -361,10 +451,12 @@ CriticalWorker.schedule(args: [1], queue: :important)
 ```
 
 ## Extensions
+**Note**: Extensions are not available when using cloudtasker via ActiveJob.
+
 Cloudtasker comes with three optional features:
 - Cron Jobs [[docs](docs/CRON_JOBS.md)]: Run jobs at fixed intervals.
 - Batch Jobs [[docs](docs/BATCH_JOBS.md)]: Run jobs in jobs and track completion of the overall batch.
-- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments. 
+- Unique Jobs [[docs](docs/UNIQUE_JOBS.md)]: Ensure uniqueness of jobs based on job arguments.
 
 ## Working locally
 
@@ -381,9 +473,9 @@ You can configure your application to use the Cloudtasker local server using the
 
 Cloudtasker.configure do |config|
   # ... other options
-  
+
   # Push jobs to redis and let the Cloudtasker local server collect them
-  # This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set 
+  # This is the default mode unless CLOUDTASKER_ENV or RAILS_ENV or RACK_ENV is set
   # to a non-development environment
   config.mode = :development
 end
@@ -427,7 +519,7 @@ Cloudtasker.configure do |config|
 
   # Use your ngrok domain as the processor host
   config.processor_host = 'https://your-tunnel-id.ngrok.io'
-  
+
   # Force Cloudtasker to use Google Cloud Tasks in development
   config.mode = :production
 end
@@ -508,7 +600,7 @@ end
 See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
 
 ### Searching logs: Job ID vs Task ID
-**Note**: `task_id` field is available in logs starting with `0.10.rc6`
+**Note**: `task_id` field is available in logs starting with `0.10.0`
 
 Job instances are assigned two different different IDs for tracking and logging purpose: `job_id` and `task_id`. These IDs are found in each log entry to facilitate search.
 
@@ -571,26 +663,24 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
 
 Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
 
-**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `v0.10.rc3` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
+**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `v0.10.0` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
 
 E.g. Set max number of retries globally via the cloudtasker initializer.
 ```ruby
 # config/initializers/cloudtasker.rb
 
 Cloudtasker.configure do |config|
-  # 
+  #
   # Specify how many retries are allowed on jobs. This number of retries excludes any
   # connectivity error that would be due to the application being down or unreachable.
-  # 
+  #
   # Default: 25
-  # 
+  #
   config.max_retries = 10
 end
 ```
 
 E.g. Set max number of retries to 3 on a given worker
-
-E.g.
 ```ruby
 # app/workers/some_error_worker.rb
 
@@ -600,7 +690,30 @@ class SomeErrorWorker
   # This will override the global setting
   cloudtasker_options max_retries: 3
 
-  def perform()
+  def perform
+    raise(ArgumentError)
+  end
+end
+```
+
+E.g. Evaluate the number of max retries at runtime (Supported since: v0.10.1)
+```ruby
+# app/workers/some_error_worker.rb
+
+class SomeErrorWorker
+  include Cloudtasker::Worker
+
+  # Return the number of max retries based on
+  # worker arguments.
+  #
+  # If this method returns nil then max_retries
+  # will delegate to the class `max_retries` setting or Cloudtasker
+  # `max_retries` configuration otion.
+  def max_retries(arg1, arg2)
+    arg1 == 'foo' ? 13 : nil
+  end
+
+  def perform(arg1, arg2)
     raise(ArgumentError)
   end
 end
@@ -618,7 +731,7 @@ require 'cloudtasker/testing'
 # Mode 1 (default): Push jobs to Google Cloud Tasks (env != development) or Redis (env == development)
 Cloudtasker::Testing.enable!
 
-# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call 
+# Mode 2: Push jobs to an in-memory queue. Jobs will not be processed until you call
 # Cloudtasker::Worker.drain_all (process all jobs) or MyWorker.drain (process jobs for specific worker)
 Cloudtasker::Testing.fake!
 
@@ -682,9 +795,9 @@ Below are examples of rspec tests. It is assumed that `Cloudtasker::Testing.fake
 **Example 1**: Testing that a job is scheduled
 ```ruby
 describe 'worker scheduling'
-  subject(:enqueue_job) { MyWorker.perform_async(1,2) } 
-  
-  it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) } 
+  subject(:enqueue_job) { MyWorker.perform_async(1,2) }
+
+  it { expect { enqueue_job }.to change(MyWorker.jobs, :size).by(1) }
 end
 ```
 
@@ -692,7 +805,7 @@ end
 ```ruby
 describe 'worker calls api'
   subject { Cloudtasker::Testing.inline! { MyApiWorker.perform_async(1,2) } }
-  
+
   before { expect(MyApi).to receive(:fetch).and_return([]) }
   it { is_expected.to be_truthy }
 end
@@ -772,7 +885,7 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
 Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 #### Option 1: Use Cloudtasker optional support for payload storage in Redis
-**Supported since**: `0.10.rc1`
+**Supported since**: `0.10.0`
 
 Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
 

data/app/controllers/cloudtasker/worker_controller.rb

@@ -19,21 +19,32 @@ module Cloudtasker
       # Process payload
       WorkerHandler.execute_from_payload!(payload)
       head :no_content
-    rescue DeadWorkerError
+    rescue DeadWorkerError, MissingWorkerArgumentsError => e
       # 205: job will NOT be retried
+      log_error(e)
       head :reset_content
-    rescue InvalidWorkerError
+    rescue InvalidWorkerError => e
       # 404: Job will be retried
+      log_error(e)
       head :not_found
     rescue StandardError => e
       # 404: Job will be retried
-      Cloudtasker.logger.error(e)
-      Cloudtasker.logger.error(e.backtrace.join("\n"))
+      log_error(e)
       head :unprocessable_entity
     end
 
     private
 
+    #
+    # Log an error via cloudtasker logger.
+    #
+    # @param [Exception] e The error to log
+    #
+    def log_error(error)
+      Cloudtasker.logger.error(error)
+      Cloudtasker.logger.error(error.backtrace.join("\n"))
+    end
+
     #
     # Parse the request body and return the actual job
     # payload.

data/cloudtasker.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
   spec.add_dependency 'connection_pool'
   spec.add_dependency 'fugit'
-  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'google-cloud-tasks', '~> 1.0'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
   spec.add_dependency 'retriable'
@@ -41,8 +41,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'github_changelog_generator'
   spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec-json_expectations', '~> 2.2'
   spec.add_development_dependency 'rubocop', '0.76.0'
   spec.add_development_dependency 'rubocop-rspec', '1.37.0'
+  spec.add_development_dependency 'semantic_logger'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/gemfiles/semantic_logger_3.4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "3.4.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.6.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.6.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.0"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/lib/active_job/queue_adapters/cloudtasker_adapter.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# ActiveJob docs: http://guides.rubyonrails.org/active_job_basics.html
+# Example adapters ref: https://github.com/rails/rails/tree/master/activejob/lib/active_job/queue_adapters
+
+module ActiveJob
+  module QueueAdapters
+    # == Cloudtasker adapter for Active Job
+    #
+    # To use Cloudtasker set the queue_adapter config to +:cloudtasker+.
+    #
+    #   Rails.application.config.active_job.queue_adapter = :cloudtasker
+    class CloudtaskerAdapter
+      SERIALIZATION_FILTERED_KEYS = [
+        'executions', # Given by the worker at processing
+        'provider_job_id', # Also given by the worker at processing
+        'priority' # Not used
+      ].freeze
+
+      # Enqueues the given ActiveJob instance for execution
+      #
+      # @param job [ActiveJob::Base] The ActiveJob instance
+      #
+      # @return [Cloudtasker::CloudTask] The Google Task response
+      #
+      def enqueue(job)
+        build_worker(job).schedule
+      end
+
+      # Enqueues the given ActiveJob instance for execution at a given time
+      #
+      # @param job [ActiveJob::Base] The ActiveJob instance
+      # @param precise_timestamp [Integer] The timestamp at which the job must be executed
+      #
+      # @return [Cloudtasker::CloudTask] The Google Task response
+      #
+      def enqueue_at(job, precise_timestamp)
+        build_worker(job).schedule(time_at: Time.at(precise_timestamp))
+      end
+
+      private
+
+      def build_worker(job)
+        job_serialization = job.serialize.except(*SERIALIZATION_FILTERED_KEYS)
+
+        JobWrapper.new(
+          job_id: job_serialization.delete('job_id'),
+          job_queue: job_serialization.delete('queue_name'),
+          job_args: [job_serialization]
+        )
+      end
+
+      # == Job Wrapper for the Cloudtasker adapter
+      #
+      # Executes jobs scheduled by the Cloudtasker ActiveJob adapter
+      class JobWrapper #:nodoc:
+        include Cloudtasker::Worker
+
+        # Executes the given serialized ActiveJob call.
+        # - See https://api.rubyonrails.org/classes/ActiveJob/Core.html#method-i-serialize
+        #
+        # @param [Hash] job_serialization The serialized ActiveJob call
+        #
+        # @return [any] The execution of the ActiveJob call
+        #
+        def perform(job_serialization, *_extra_options)
+          job_executions = job_retries < 1 ? 0 : (job_retries + 1)
+
+          job_serialization.merge!(
+            'job_id' => job_id,
+            'queue_name' => job_queue,
+            'provider_job_id' => task_id,
+            'executions' => job_executions,
+            'priority' => nil
+          )
+
+          Base.execute job_serialization
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/missing_worker_arguments_error'
 require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'

data/lib/cloudtasker/backend/memory_task.rb

@@ -166,6 +166,9 @@ module Cloudtasker
         # Delete task
         self.class.delete(id)
         resp
+      rescue DeadWorkerError => e
+        self.class.delete(id)
+        raise(e) if self.class.inline_mode?
       rescue StandardError => e
         self.job_retries += 1
         raise(e) if self.class.inline_mode?

data/lib/cloudtasker/cli.rb

@@ -64,6 +64,7 @@ module Cloudtasker
       return false unless File.exist?('./config/environment.rb')
 
       require 'rails'
+      require 'cloudtasker/engine'
       require File.expand_path('./config/environment.rb')
     end
 

data/lib/cloudtasker/engine.rb

@@ -5,12 +5,18 @@ module Cloudtasker
   class Engine < ::Rails::Engine
     isolate_namespace Cloudtasker
 
+    # Setup cloudtasker processing route
     initializer 'cloudtasker', before: :load_config_initializers do
       Rails.application.routes.append do
         mount Cloudtasker::Engine, at: '/cloudtasker'
       end
     end
 
+    # Setup active job adapter
+    initializer 'cloudtasker.active_job', after: :load_config_initializers do
+      require 'active_job/queue_adapters/cloudtasker_adapter' if defined?(::ActiveJob::Railtie)
+    end
+
     config.generators do |g|
       g.test_framework :rspec, fixture: false
       g.assets false

data/lib/cloudtasker/missing_worker_arguments_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class MissingWorkerArgumentsError < StandardError
+  end
+end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.10.rc7'
+  VERSION = '0.11.rc2'
 end

data/lib/cloudtasker/worker.rb

@@ -296,6 +296,20 @@ module Cloudtasker
       other.is_a?(self.class) && other.job_id == job_id
     end
 
+    #
+    # Return the max number of retries allowed for this job.
+    #
+    # The order of precedence for retry lookup is:
+    # - Worker `max_retries` method
+    # - Class `max_retries` option
+    # - Cloudtasker `max_retries` config option
+    #
+    # @return [Integer] The number of retries
+    #
+    def job_max_retries
+      @job_max_retries ||= (try(:max_retries, *job_args) || self.class.max_retries)
+    end
+
     #
     # Return true if the job has excceeded its maximum number
     # of retries
@@ -303,7 +317,7 @@ module Cloudtasker
     # @return [Boolean] True if the job is dead
     #
     def job_dead?
-      job_retries >= Cloudtasker.config.max_retries
+      job_retries >= job_max_retries
     end
 
     #
@@ -333,6 +347,13 @@ module Cloudtasker
 
       Cloudtasker.config.server_middleware.invoke(self) do
         begin
+          # Abort if arguments are missing. This may happen with redis arguments storage
+          # if Cloud Tasks times out on a job but the job still succeeds
+          if job_args.empty? && [0, -1].exclude?(method(:perform).arity)
+            raise(MissingWorkerArgumentsError, 'worker arguments are missing')
+          end
+
+          # Perform the job
           perform(*job_args)
         rescue StandardError => e
           try(:on_error, e)

data/lib/cloudtasker/worker_handler.rb

@@ -85,7 +85,7 @@ module Cloudtasker
       redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
 
       resp
-    rescue DeadWorkerError => e
+    rescue DeadWorkerError, MissingWorkerArgumentsError => e
       # Delete stored args payload if job is dead
       redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
       raise(e)

data/lib/cloudtasker/worker_logger.rb

@@ -142,7 +142,7 @@ module Cloudtasker
     #
     def log_message(level, msg, &block)
       # Merge log-specific context into worker-specific context
-      payload_block = -> { log_block.call.merge(block&.call || {}) }
+      payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.10.rc7
+  version: 0.11.rc2
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -56,16 +56,16 @@ dependencies:
   name: google-cloud-tasks
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: jwt
   requirement: !ruby/object:Gem::Requirement
@@ -178,6 +178,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-json_expectations
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -206,6 +220,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.37.0
+- !ruby/object:Gem::Dependency
+  name: semantic_logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -324,6 +352,12 @@ files:
 - gemfiles/rails_5.2.gemfile.lock
 - gemfiles/rails_6.0.gemfile
 - gemfiles/rails_6.0.gemfile.lock
+- gemfiles/semantic_logger_3.4.gemfile
+- gemfiles/semantic_logger_4.6.gemfile
+- gemfiles/semantic_logger_4.7.0.gemfile
+- gemfiles/semantic_logger_4.7.2.gemfile
+- gemfiles/semantic_logger_4.7.gemfile
+- lib/active_job/queue_adapters/cloudtasker_adapter.rb
 - lib/cloudtasker.rb
 - lib/cloudtasker/authentication_error.rb
 - lib/cloudtasker/authenticator.rb
@@ -351,6 +385,7 @@ files:
 - lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
+- lib/cloudtasker/missing_worker_arguments_error.rb
 - lib/cloudtasker/redis_client.rb
 - lib/cloudtasker/testing.rb
 - lib/cloudtasker/unique_job.rb