cloudtasker 0.10.rc1 → 0.10.rc6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d4cba7de3e429d612adf6c9c2f4424b6ef73db39d4db93b70804800300011e1b
-  data.tar.gz: 3775cdf3f16430cf8decd49dfc28be9e26f0ef6a63d45224bdc5ed11b13a86fc
+  metadata.gz: 796e69d0470947fe0af6ea530cfd6c5b69e95b6a1be42b307e84b27dd9246198
+  data.tar.gz: 5c51dd25b3033546a7115d83d638123e80a9ee26fb74845b88a02247edb39461
 SHA512:
-  metadata.gz: 5e2e15dc54fad72e3508763855a99804b591126968ccbcaccd8211d51b8b1e28bf6d2907c746f2b14c53c3c065ce1eb06871f30b39419df1f3d7b8a4e1b1fded
-  data.tar.gz: a2808491a7251b5212587351deb84a99f688a62f7a54c3f5b9c8ebe3a3b6a1ca6adda6ffe898424069223cef8260f1a49c8a086010c0e59dedc5e50ed371e830
+  metadata.gz: 134e6d344e75a9500850b135ea215c2b64d31cf366892acc0aab1b37c47e758dfa7e2bd00015ec6d8bb47556aba002bb578f2365ea34196d3343b24cee0f2156
+  data.tar.gz: 73bd3cbd91fa1938b97df2cd7f5125a3f90205fb31b5d12efe5b64c4c0aa03e74f6f83cd8c9267b95b5553be93395a45db4f365bbb49b5623a97bf982bf9d2d7

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - '2.5.x'
+          - '2.6.x'
+        appraisal:
+          - 'google-cloud-tasks-1.0'
+          - 'google-cloud-tasks-1.1'
+          - 'google-cloud-tasks-1.2'
+          - 'google-cloud-tasks-1.3'
+          - 'rails-5.2'
+          - 'rails-6.0'
+    steps:
+      - name: Setup System
+        run: sudo apt-get install libsqlite3-dev
+      - uses: actions/checkout@v2
+      - uses: zhulik/redis-action@1.1.0
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        env:
+            APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rubocop
+          bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.rubocop.yml

@@ -2,12 +2,15 @@ require: rubocop-rspec
 
 AllCops:
   Exclude:
-    - 'gemfiles/vendor/**/*'
+    - 'gemfiles/**/*'
     - 'vendor/**/*'
 
 Metrics/ClassLength:
   Max: 150
 
+Metrics/ModuleLength:
+  Max: 150
+
 Metrics/AbcSize:
   Max: 20
 
@@ -34,4 +37,7 @@ Metrics/BlockLength:
 Style/Documentation:
   Exclude:
     - 'examples/**/*'
-    - 'spec/**/*'
+    - 'spec/**/*'
+
+Metrics/ParameterLists:
+  CountKeywordArgs: false

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/keypup-io/cloudtasker.svg?branch=master)](https://travis-ci.org/keypup-io/cloudtasker) [![Gem Version](https://badge.fury.io/rb/cloudtasker.svg)](https://badge.fury.io/rb/cloudtasker)
+![Build Status](https://github.com/keypup-io/cloudtasker/workflows/Test/badge.svg) [![Gem Version](https://badge.fury.io/rb/cloudtasker.svg)](https://badge.fury.io/rb/cloudtasker)
 
 # Cloudtasker
 
@@ -246,6 +246,8 @@ Cloudtasker.configure do |config|
   # You can set this configuration parameter to a KB value if you want to store jobs
   # args in redis only if the JSONified arguments payload exceeds that threshold.
   #
+  # Supported since: v0.10.rc1
+  #
   # Default: false
   #
   # Store all job payloads in Redis:
@@ -465,7 +467,7 @@ end
 
 Will generate the following log with context `{:worker=> ..., :job_id=> ..., :job_meta=> ...}`
 ```log
-[Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Job run with foo. This is working!: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}}
+[Cloudtasker][d76040a1-367e-4e3b-854e-e05a74d5f773] Job run with foo. This is working!: {:worker=>"DummyWorker", :job_id=>"d76040a1-367e-4e3b-854e-e05a74d5f773", :job_meta=>{}, :task_id => "4e755d3f-6de0-426c-b4ac-51edd445c045"}
 ```
 
 The way contextual information is displayed depends on the logger itself. For example with [semantic_logger](http://rocketjob.github.io/semantic_logger) contextual information might not appear in the log message but show up as payload data on the log entry itself (e.g. using the fluentd adapter).
@@ -501,9 +503,24 @@ end
 
 See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more information on attributes available to be logged in your `log_context_processor` proc.
 
+### Searching logs: Job ID vs Task ID
+**Note**: `task_id` field is available in logs starting with `0.10.rc6`
+
+Job instances are assigned two different different IDs for tracking and logging purpose: `job_id` and `task_id`. These IDs are found in each log entry to facilitate search.
+
+| Field | Definition |
+|------|-------------|
+| `job_id` | This ID is generated by Cloudtasker. It identifies the job along its entire lifecyle. It is persistent across retries and reschedules. |
+| `task_id` | This ID is generated by Google Cloud Tasks. It identifies a job instance on the Google Cloud Task side. It is persistent across retries but NOT across reschedules. |
+
+The Google Cloud Task UI (GCP console) lists all the tasks pending/retrying and their associated task id (also called "Task name"). From there you can:
+1. Use a task ID to lookup the logs of a specific job instance in Stackdriver Logging (or any other logging solution).
+2. From (1) you can retrieve the `job_id` attribute of the job.
+3. From (2) you can use the `job_id` to lookup the job logs along its entire lifecycle.
+
 ## Error Handling
 
-Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of retries Cloud Task will do depends on the configuration of your queue in Cloud Tasks.
+Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of Cloud Task retries Cloud Task will depend on the configuration of your queue in Cloud Tasks.
 
 ### HTTP Error codes
 
@@ -549,6 +566,8 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
 
 Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
 
+**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `0.10.rc3` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
+
 E.g. Set max number of retries globally via the cloudtasker initializer.
 ```ruby
 # config/initializers/cloudtasker.rb
@@ -583,7 +602,6 @@ end
 ```
 
 
-
 ## Best practices building workers
 
 Below are recommendations and notes about creating workers.
@@ -658,6 +676,8 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
 Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 #### Option 1: Use Cloudtasker optional support for payload storage in Redis
+**Supported since**: `0.10.rc1`
+
 Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
 
 To enable it simply put the following in your Cloudtasker initializer:

data/app/controllers/cloudtasker/worker_controller.rb

@@ -51,19 +51,28 @@ module Cloudtasker
         end
 
         # Return content parsed as JSON and add job retries count
-        JSON.parse(content).merge(job_retries: job_retries)
+        JSON.parse(content).merge(job_retries: job_retries, task_id: task_id)
       end
     end
 
     #
     # Extract the number of times this task failed at runtime.
     #
-    # @return [Integer] The number of failures
+    # @return [Integer] The number of failures.
     #
     def job_retries
       request.headers[Cloudtasker::Config::RETRY_HEADER].to_i
     end
 
+    #
+    # Return the Google Cloud Task ID from headers.
+    #
+    # @return [String] The task ID.
+    #
+    def task_id
+      request.headers[Cloudtasker::Config::TASK_ID_HEADER]
+    end
+
     #
     # Authenticate incoming requests using a bearer token
     #

data/cloudtasker.gemspec

@@ -15,8 +15,6 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/keypup-io/cloudtasker'
   spec.license       = 'MIT'
 
-  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
-
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/keypup-io/cloudtasker'
   spec.metadata['changelog_uri'] = 'https://github.com/keypup-io/cloudtasker/master/tree/CHANGELOG.md'
@@ -31,10 +29,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'connection_pool'
   spec.add_dependency 'fugit'
   spec.add_dependency 'google-cloud-tasks'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
+  spec.add_dependency 'retriable'
 
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'

data/docs/UNIQUE_JOBS.md

@@ -81,6 +81,68 @@ Below is the list of available conflict strategies can be specified through the
 | `raise` | All locks | A `Cloudtasker::UniqueJob::LockError` will be raised when a conflict occurs |
 | `reschedule` | `while_executing` | The job will be rescheduled 5 seconds later when a conflict occurs |
 
+## Lock Time To Live (TTL) & deadlocks
+**Note**: Lock TTL has been introduced in `v0.10.rc6`
+
+To make jobs unique Cloudtasker sets a lock key - a hash of class name + job arguments - in Redis. Unique crash situations may lead to lock keys not being cleaned up when jobs complete - e.g. Redis crash with rollback from last known state on disk. Situations like these may lead to having a unique job deadlock: jobs with the same class and arguments would stop being processed because they're unable to acquire a lock that will never be cleaned up.
+
+In order to prevent deadlocks Cloudtasker configures lock keys to automatically expire in Redis after `job schedule time + lock_ttl (default: 10 minutes)`. This forced expiration ensures that deadlocks eventually get cleaned up shortly after the expected run time of a job.
+
+The `lock_ttl (default: 10 minutes)` duration represent the expected max duration of the job. The default 10 minutes value was chosen because it's twice the default request timeout value in Cloud Run. This usually leaves enough room for queue lag (5 minutes) + job processing (5 minutes).
+
+Queue lag is certainly the most unpredictable factor here. Job processing time is less of a factor. Jobs running for more than 5 minutes should be split into sub-jobs to limit invocation time over HTTP anyway. Cloudtasker [batch jobs](BATCH_JOBS.md) can help split big jobs into sub-jobs in an atomic way.
+
+The default lock key expiration of `job schedule time + 10 minutes` may look aggressive but it is a better choice than having real-time jobs stuck for X hours after a crash recovery.
+
+We **strongly recommend** adapting the `lock_ttl` option either globally or for each worker based on expected queue lag and job duration.
+
+**Example 1**: Global configuration
+```ruby
+# config/initializers/cloudtasker.rb
+
+# General Cloudtasker configuration
+Cloudtasker.configure do |config|
+  # ...
+end
+
+# Unique job extension configuration
+Cloudtasker::UniqueJob.configure do |config|
+  config.lock_ttl = 3 * 60 # 3 minutes
+end
+```
+
+**Example 2**: Worker-level - fast
+```ruby
+# app/workers/realtime_worker_on_fast_queue.rb
+
+class RealtimeWorkerOnFastQueue
+  include Cloudtasker::Worker
+
+  # Ensure lock is removed 30 seconds after schedule time
+  cloudtasker_options lock: :until_executing, lock_ttl: 30
+
+  def perform(arg1, arg2)
+    # ...
+  end
+end
+```
+
+**Example 3**: Worker-level - slow
+```ruby
+# app/workers/non_critical_worker_on_slow_queue.rb
+
+class NonCriticalWorkerOnSlowQueue
+  include Cloudtasker::Worker
+
+  # Ensure lock is removed 24 hours after schedule time
+  cloudtasker_options lock: :until_executing, lock_ttl: 3600 * 24
+
+  def perform(arg1, arg2)
+    # ...
+  end
+end
+```
+
 ## Configuring unique arguments
 
 By default Cloudtasker considers all job arguments to evaluate the uniqueness of a job. This behaviour is configurable per worker by defining a `unique_args` method on the worker itself returning the list of args defining uniqueness.

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.0'
+gem "google-cloud-tasks", "1.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.1.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.1'
+gem "google-cloud-tasks", "1.1"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.2'
+gem "google-cloud-tasks", "1.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.3.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.3'
+gem "google-cloud-tasks", "1.3"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '5.2'
+gem "rails", "5.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '6.0'
+gem "rails", "6.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'google/cloud/tasks'
+require 'retriable'
+
 module Cloudtasker
   module Backend
     # Manage tasks pushed to GCP Cloud Task
@@ -113,9 +116,10 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The retrieved task.
       #
       def self.find(id)
-        resp = client.get_task(id)
+        resp = with_gax_retries { client.get_task(id) }
         resp ? new(resp) : nil
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::RetryError, Google::Gax::NotFoundError, GRPC::NotFound
+        # The ID does not exist
         nil
       end
 
@@ -133,10 +137,8 @@ module Cloudtasker
         relative_queue = payload.delete(:queue)
 
         # Create task
-        resp = client.create_task(queue_path(relative_queue), payload)
+        resp = with_gax_retries { client.create_task(queue_path(relative_queue), payload) }
         resp ? new(resp) : nil
-      rescue Google::Gax::RetryError
-        nil
       end
 
       #
@@ -145,11 +147,21 @@ module Cloudtasker
       # @param [String] id The id of the task.
       #
       def self.delete(id)
-        client.delete_task(id)
-      rescue Google::Gax::RetryError, GRPC::NotFound, Google::Gax::PermissionDeniedError
+        with_gax_retries { client.delete_task(id) }
+      rescue Google::Gax::RetryError, Google::Gax::NotFoundError, GRPC::NotFound, Google::Gax::PermissionDeniedError
+        # The ID does not exist
         nil
       end
 
+      #
+      # Helper method encapsulating the retry strategy for GAX calls
+      #
+      def self.with_gax_retries
+        Retriable.retriable(on: [Google::Gax::UnavailableError], tries: 3) do
+          yield
+        end
+      end
+
       #
       # Build a new instance of the class.
       #

data/lib/cloudtasker/backend/memory_task.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'cloudtasker/redis_client'
-
 module Cloudtasker
   module Backend
     # Manage local tasks pushed to memory.
@@ -153,7 +151,8 @@ module Cloudtasker
       #
       def execute
         # Execute worker
-        resp = WorkerHandler.with_worker_handling(payload, &:execute)
+        worker_payload = payload.merge(job_retries: job_retries, task_id: id)
+        resp = WorkerHandler.with_worker_handling(worker_payload, &:execute)
 
         # Delete task
         self.class.delete(id)

data/lib/cloudtasker/backend/redis_task.rb

@@ -247,8 +247,9 @@ module Cloudtasker
           uri = URI(http_request[:url])
           req = Net::HTTP::Post.new(uri.path, http_request[:headers])
 
-          # Add retries header
-          req['X-CloudTasks-TaskExecutionCount'] = retries
+          # Add task headers
+          req[Cloudtasker::Config::TASK_ID_HEADER] = id
+          req[Cloudtasker::Config::RETRY_HEADER] = retries
 
           # Set job payload
           req.body = http_request[:body]

data/lib/cloudtasker/batch/middleware/server.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.for(worker).execute { yield }
         end
       end

data/lib/cloudtasker/config.rb

@@ -13,7 +13,20 @@ module Cloudtasker
     MAX_TASK_SIZE = 100 * 1024 # 100 KB
 
     # Retry header in Cloud Task responses
-    RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
+    #
+    # TODO: use 'X-CloudTasks-TaskExecutionCount' instead of 'X-CloudTasks-TaskRetryCount'
+    #   'X-CloudTasks-TaskExecutionCount' is currently bugged and remains at 0 even on retries.
+    #
+    # See bug: https://issuetracker.google.com/issues/154532072
+    #
+    # Definitions:
+    #   X-CloudTasks-TaskRetryCount: total number of retries (including 504 "instance unreachable")
+    #   X-CloudTasks-TaskExecutionCount: number of non-503 retries (= actual number of job failures)
+    #
+    RETRY_HEADER = 'X-CloudTasks-TaskRetryCount'
+
+    # Cloud Task ID header
+    TASK_ID_HEADER = 'X-CloudTasks-TaskName'
 
     # Content-Transfer-Encoding header in Cloud Task responses
     ENCODING_HEADER = 'Content-Transfer-Encoding'
@@ -33,7 +46,15 @@ module Cloudtasker
     DEFAULT_QUEUE_CONCURRENCY = 10
     DEFAULT_QUEUE_RETRIES = -1 # unlimited
 
-    # The number of times jobs will be attempted before declaring them dead
+    # The number of times jobs will be attempted before declaring them dead.
+    #
+    # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
+    # it means that jobs will be declared dead after 20h of consecutive failing.
+    #
+    # Note that this configuration parameter is internal to Cloudtasker and does not
+    # affect the Cloud Task queue configuration. The number of retries configured
+    # on the Cloud Task queue should be higher than the number below to also cover
+    # failures due to the instance being unreachable.
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
     PROCESSOR_HOST_MISSING = <<~DOC

data/lib/cloudtasker/cron/job.rb

@@ -4,15 +4,10 @@ require 'fugit'
 
 module Cloudtasker
   module Cron
-    # TODO: handle deletion of cron jobs
-    #
     # Manage cron jobs
     class Job
       attr_reader :worker
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'job'
-
       #
       # Build a new instance of the class
       #

data/lib/cloudtasker/cron/middleware/server.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.new(worker).execute { yield }
         end
       end

data/lib/cloudtasker/cron/schedule.rb

@@ -9,9 +9,6 @@ module Cloudtasker
     class Schedule
       attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'schedule'
-
       #
       # Return the redis client.
       #

data/lib/cloudtasker/local_server.rb

@@ -12,6 +12,9 @@ module Cloudtasker
     # Default number of threads to allocate to process a specific queue
     QUEUE_CONCURRENCY = 1
 
+    # Job Polling. How frequently to poll jobs in redis.
+    JOB_POLLING_FREQUENCY = 0.5 # seconds
+
     #
     # Stop the local server.
     #
@@ -46,7 +49,7 @@ module Cloudtasker
       @start ||= Thread.new do
         until @done
           queues.each { |(n, c)| process_jobs(n, c) }
-          sleep 1
+          sleep JOB_POLLING_FREQUENCY
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
       end
@@ -82,7 +85,7 @@ module Cloudtasker
       # Deliver task
       begin
         Thread.current['task'].deliver
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::EBADF, Errno::ECONNREFUSED => e
         raise(e) unless Thread.current['attempts'] < 3
 
         # Retry on connection error, in case the web server is not

data/lib/cloudtasker/redis_client.rb

@@ -1,15 +1,28 @@
 # frozen_string_literal: true
 
 require 'redis'
+require 'connection_pool'
 
 module Cloudtasker
   # A wrapper with helper methods for redis
   class RedisClient
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
+    LOCK_DURATION = 2 # seconds
+    LOCK_WAIT_DURATION = 0.03 # seconds
+
+    # Default pool size used for Redis
+    DEFAULT_POOL_SIZE = ENV.fetch('RAILS_MAX_THREADS') { 25 }
+    DEFAULT_POOL_TIMEOUT = 5
 
     def self.client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= begin
+        pool_size = Cloudtasker.config.redis&.dig(:pool_size) || DEFAULT_POOL_SIZE
+        pool_timeout = Cloudtasker.config.redis&.dig(:pool_timeout) || DEFAULT_POOL_TIMEOUT
+        ConnectionPool.new(size: pool_size, timeout: pool_timeout) do
+          Redis.new(Cloudtasker.config.redis || {})
+        end
+      end
     end
 
     #
@@ -29,7 +42,7 @@ module Cloudtasker
     # @return [Hash, Array] The content of the cache key, parsed as JSON.
     #
     def fetch(key)
-      return nil unless (val = client.get(key.to_s))
+      return nil unless (val = get(key.to_s))
 
       JSON.parse(val, symbolize_names: true)
     rescue JSON::ParserError
@@ -45,12 +58,15 @@ module Cloudtasker
     # @return [String] Redis response code.
     #
     def write(key, content)
-      client.set(key.to_s, content.to_json)
+      set(key.to_s, content.to_json)
     end
 
     #
     # Acquire a lock on a cache entry.
     #
+    # Locks are enforced to be short-lived (2s).
+    # The yielded block should limit its logic to short operations (e.g. redis get/set).
+    #
     # @example
     #   redis = RedisClient.new
     #   redis.with_lock('foo')
@@ -65,12 +81,14 @@ module Cloudtasker
 
       # Wait to acquire lock
       lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
-      true until client.setnx(lock_key, true)
+      client.with do |conn|
+        sleep(LOCK_WAIT_DURATION) until conn.set(lock_key, true, nx: true, ex: LOCK_DURATION)
+      end
 
       # yield content
       yield
     ensure
-      client.del(lock_key)
+      del(lock_key)
     end
 
     #
@@ -99,10 +117,12 @@ module Cloudtasker
       list = []
 
       # Scan and capture matching keys
-      while cursor != 0
-        scan = client.scan(cursor || 0, match: pattern)
-        list += scan[1]
-        cursor = scan[0].to_i
+      client.with do |conn|
+        while cursor != 0
+          scan = conn.scan(cursor || 0, match: pattern)
+          list += scan[1]
+          cursor = scan[0].to_i
+        end
       end
 
       list
@@ -118,8 +138,8 @@ module Cloudtasker
     # @return [Any] The method return value
     #
     def method_missing(name, *args, &block)
-      if client.respond_to?(name)
-        client.send(name, *args, &block)
+      if Redis.method_defined?(name)
+        client.with { |c| c.send(name, *args, &block) }
       else
         super
       end
@@ -134,7 +154,7 @@ module Cloudtasker
     # @return [Boolean] Return true if the class respond to this method.
     #
     def respond_to_missing?(name, include_private = false)
-      client.respond_to?(name) || super
+      Redis.method_defined?(name) || super
     end
   end
 end

data/lib/cloudtasker/unique_job.rb

@@ -3,3 +3,30 @@
 require_relative 'unique_job/middleware'
 
 Cloudtasker::UniqueJob::Middleware.configure
+
+module Cloudtasker
+  # UniqueJob configurator
+  module UniqueJob
+    # The maximum duration a lock can remain in place
+    # after schedule time.
+    DEFAULT_LOCK_TTL = 10 * 60 # 10 minutes
+
+    class << self
+      attr_writer :lock_ttl
+
+      # Configure the middleware
+      def configure
+        yield(self)
+      end
+
+      #
+      # Return the max TTL for locks
+      #
+      # @return [Integer] The lock TTL.
+      #
+      def lock_ttl
+        @lock_ttl || DEFAULT_LOCK_TTL
+      end
+    end
+  end
+end

data/lib/cloudtasker/unique_job/job.rb

@@ -5,21 +5,19 @@ module Cloudtasker
     # Wrapper class for Cloudtasker::Worker delegating to lock
     # and conflict strategies
     class Job
-      attr_reader :worker
+      attr_reader :worker, :call_opts
 
       # The default lock strategy to use. Defaults to "no lock".
       DEFAULT_LOCK = UniqueJob::Lock::NoOp
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'job'
-
       #
       # Build a new instance of the class.
       #
       # @param [Cloudtasker::Worker] worker The worker at hand
       #
-      def initialize(worker)
+      def initialize(worker, **kwargs)
         @worker = worker
+        @call_opts = kwargs
       end
 
       #
@@ -31,6 +29,43 @@ module Cloudtasker
         worker.class.cloudtasker_options_hash
       end
 
+      #
+      # Return the Time To Live (TTL) that should be set in Redis for
+      # the lock key. Having a TTL on lock keys ensures that jobs
+      # do not end up stuck due to a dead lock situation.
+      #
+      # The TTL is calculated using schedule time + expected
+      # max job duration.
+      #
+      # The expected max job duration is set to 10 minutes by default.
+      # This value was chosen because it's twice the default request timeout
+      # value in Cloud Run. This leaves enough room for queue lag (5 minutes)
+      # + job processing (5 minutes).
+      #
+      # Queue lag is certainly the most unpredictable factor here.
+      # Job processing time is less of a factor. Jobs running for more than 5 minutes
+      # should be split into sub-jobs to limit invocation time over HTTP. Cloudtasker batch
+      # jobs can help achieve that if you need to make one big job split into sub-jobs "atomic".
+      #
+      # The default lock key expiration of "time_at + 10 minutes" may look aggressive but it
+      # is still a better choice than potentially having real-time jobs stuck for X hours.
+      #
+      # The expected max job duration can be configured via the `lock_ttl`
+      # option on the job itself.
+      #
+      # @return [Integer] The TTL in seconds
+      #
+      def lock_ttl
+        now = Time.now.to_i
+
+        # Get scheduled at and lock duration
+        scheduled_at = [call_opts[:time_at].to_i, now].compact.max
+        lock_duration = (options[:lock_ttl] || Cloudtasker::UniqueJob.lock_ttl).to_i
+
+        # Return TTL
+        scheduled_at + lock_duration - now
+      end
+
       #
       # Return the instantiated lock.
       #
@@ -121,7 +156,7 @@ module Cloudtasker
           raise(LockError, locked_id) if locked_id && locked_id != id
 
           # Take job lock if the lock is currently free
-          redis.set(unique_gid, id) unless locked_id
+          redis.set(unique_gid, id, ex: lock_ttl) unless locked_id
         end
       end
 

data/lib/cloudtasker/unique_job/middleware/client.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Client middleware, invoked when jobs are scheduled
       class Client
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.new(worker).lock_instance.schedule { yield }
         end
       end

data/lib/cloudtasker/unique_job/middleware/server.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.new(worker).lock_instance.execute { yield }
         end
       end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.10.rc1'
+  VERSION = '0.10.rc6'
 end

data/lib/cloudtasker/worker.rb

@@ -7,7 +7,8 @@ module Cloudtasker
     def self.included(base)
       base.extend(ClassMethods)
       base.attr_writer :job_queue
-      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
+      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries,
+                         :perform_started_at, :perform_ended_at, :task_id
     end
 
     #
@@ -46,7 +47,7 @@ module Cloudtasker
       return nil unless worker_klass.include?(self)
 
       # Return instantiated worker
-      worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries))
+      worker_klass.new(payload.slice(:job_queue, :job_args, :job_id, :job_meta, :job_retries, :task_id))
     rescue NameError
       nil
     end
@@ -139,12 +140,13 @@ module Cloudtasker
     # @param [Array<any>] job_args The list of perform args.
     # @param [String] job_id A unique ID identifying this job.
     #
-    def initialize(job_queue: nil, job_args: nil, job_id: nil, job_meta: {}, job_retries: 0)
+    def initialize(job_queue: nil, job_args: nil, job_id: nil, job_meta: {}, job_retries: 0, task_id: nil)
       @job_args = job_args || []
       @job_id = job_id || SecureRandom.uuid
       @job_meta = MetaStore.new(job_meta)
       @job_retries = job_retries || 0
       @job_queue = job_queue
+      @task_id = task_id
     end
 
     #
@@ -181,35 +183,51 @@ module Cloudtasker
     #
     def execute
       logger.info('Starting job...')
-      resp = Cloudtasker.config.server_middleware.invoke(self) do
-        begin
-          perform(*job_args)
-        rescue StandardError => e
-          try(:on_error, e)
-          return raise(e) unless job_dead?
 
-          # Flag job as dead
-          logger.info('Job dead')
-          try(:on_dead, e)
-          raise(DeadWorkerError, e)
-        end
-      end
-      logger.info('Job done')
+      # Perform job logic
+      resp = execute_middleware_chain
+
+      # Log job completion and return result
+      logger.info("Job done after #{job_duration}s") { { duration: job_duration } }
       resp
+    rescue DeadWorkerError => e
+      logger.info("Job dead after #{job_duration}s and #{job_retries} retries") { { duration: job_duration } }
+      raise(e)
+    rescue StandardError => e
+      logger.info("Job failed after #{job_duration}s") { { duration: job_duration } }
+      raise(e)
+    end
+
+    #
+    # Return a unix timestamp specifying when to run the task.
+    #
+    # @param [Integer, nil] interval The time to wait.
+    # @param [Integer, nil] time_at The time at which the job should run.
+    #
+    # @return [Integer, nil] The Unix timestamp.
+    #
+    def schedule_time(interval: nil, time_at: nil)
+      return nil unless interval || time_at
+
+      # Generate the complete Unix timestamp
+      (time_at || Time.now).to_i + interval.to_i
     end
 
     #
     # Enqueue a worker, with or without delay.
     #
     # @param [Integer] interval The delay in seconds.
-    #
     # @param [Time, Integer] interval The time at which the job should run
     #
     # @return [Cloudtasker::CloudTask] The Google Task response
     #
-    def schedule(interval: nil, time_at: nil)
-      Cloudtasker.config.client_middleware.invoke(self) do
-        WorkerHandler.new(self).schedule(interval: interval, time_at: time_at)
+    def schedule(**args)
+      # Evaluate when to schedule the job
+      time_at = schedule_time(args)
+
+      # Schedule job through client middlewares
+      Cloudtasker.config.client_middleware.invoke(self, time_at: time_at) do
+        WorkerHandler.new(self).schedule(time_at: time_at)
       end
     end
 
@@ -251,7 +269,8 @@ module Cloudtasker
         job_args: job_args,
         job_meta: job_meta.to_h,
         job_retries: job_retries,
-        job_queue: job_queue
+        job_queue: job_queue,
+        task_id: task_id
       }
     end
 
@@ -286,5 +305,46 @@ module Cloudtasker
     def job_dead?
       job_retries >= Cloudtasker.config.max_retries
     end
+
+    #
+    # Return the time taken (in seconds) to perform the job. This duration
+    # includes the middlewares and the actual perform method.
+    #
+    # @return [Float] The time taken in seconds as a floating point number.
+    #
+    def job_duration
+      return 0.0 unless perform_ended_at && perform_started_at
+
+      (perform_ended_at - perform_started_at).ceil(3)
+    end
+
+    #=============================
+    # Private
+    #=============================
+    private
+
+    #
+    # Execute the worker perform method through the middleware chain.
+    #
+    # @return [Any] The result of the perform method.
+    #
+    def execute_middleware_chain
+      self.perform_started_at = Time.now
+
+      Cloudtasker.config.server_middleware.invoke(self) do
+        begin
+          perform(*job_args)
+        rescue StandardError => e
+          try(:on_error, e)
+          return raise(e) unless job_dead?
+
+          # Flag job as dead
+          try(:on_dead, e)
+          raise(DeadWorkerError, e)
+        end
+      end
+    ensure
+      self.perform_ended_at = Time.now
+    end
   end
 end

data/lib/cloudtasker/worker_handler.rb

@@ -56,11 +56,6 @@ module Cloudtasker
       with_worker_handling(input_payload, &:execute)
     end
 
-    # TODO: do not delete redis payload if job has been re-enqueued
-    # worker.job_reenqueued
-    #
-    # Idea: change with_worker_handling to with_worker_handling and build the worker
-    # inside the with_worker_handling block.
     #
     # Local middleware used to retrieve the job arg payload from cache
     # if a arg payload reference is present.
@@ -210,35 +205,17 @@ module Cloudtasker
       }.merge(worker_args_payload)
     end
 
-    #
-    # Return a protobuf timestamp specifying how to wait
-    # before running a task.
-    #
-    # @param [Integer, nil] interval The time to wait.
-    # @param [Integer, nil] time_at The time at which the job should run.
-    #
-    # @return [Integer, nil] The Unix timestamp.
-    #
-    def schedule_time(interval: nil, time_at: nil)
-      return nil unless interval || time_at
-
-      # Generate the complete Unix timestamp
-      (time_at || Time.now).to_i + interval.to_i
-    end
-
     #
     # Schedule the task on GCP Cloud Task.
     #
-    # @param [Integer, nil] interval How to wait before running the task.
+    # @param [Integer, nil] time_at A unix timestamp specifying when to run the job.
     #   Leave to `nil` to run now.
     #
     # @return [Cloudtasker::CloudTask] The Google Task response
     #
-    def schedule(interval: nil, time_at: nil)
+    def schedule(time_at: nil)
       # Generate task payload
-      task = task_payload.merge(
-        schedule_time: schedule_time(interval: interval, time_at: time_at)
-      ).compact
+      task = task_payload.merge(schedule_time: time_at).compact
 
       # Create and return remote task
       CloudTask.create(task)

data/lib/cloudtasker/worker_logger.rb

@@ -11,7 +11,7 @@ module Cloudtasker
     end
 
     # Only log the job meta information by default (exclude arguments)
-    DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue) }
+    DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue, :task_id) }
 
     #
     # Build a new instance of the class.
@@ -59,7 +59,7 @@ module Cloudtasker
     # @return [String] The formatted log message
     #
     def formatted_message(msg)
-      "[Cloudtasker][#{worker.job_id}] #{msg}"
+      "[Cloudtasker][#{worker.class}][#{worker.job_id}] #{msg}"
     end
 
     #
@@ -141,7 +141,8 @@ module Cloudtasker
     # @param [Proc] &block Optional context block.
     #
     def log_message(level, msg, &block)
-      payload_block = block || log_block
+      # Merge log-specific context into worker-specific context
+      payload_block = -> { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.10.rc1
+  version: 0.10.rc6
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-09 00:00:00.000000000 Z
+date: 2020-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: connection_pool
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: fugit
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: retriable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: appraisal
   requirement: !ruby/object:Gem::Requirement
@@ -256,10 +284,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -368,8 +396,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.0.0
 signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)

data/.travis.yml

@@ -1,16 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.5
-services:
-  - redis-server
-before_install: gem install bundler -v 2.0.2
-before_script: bundle exec rubocop
-gemfile:
-  - gemfiles/google_cloud_tasks_1.0.gemfile
-  - gemfiles/google_cloud_tasks_1.1.gemfile
-  - gemfiles/google_cloud_tasks_1.2.gemfile
-  - gemfiles/google_cloud_tasks_1.3.gemfile
-  - gemfiles/rails_5.2.gemfile
-  - gemfiles/rails_6.0.gemfile