cloudtasker 0.10.rc5 → 0.10.1

data/app/controllers/cloudtasker/worker_controller.rb

@@ -51,19 +51,28 @@ module Cloudtasker
         end
 
         # Return content parsed as JSON and add job retries count
-        JSON.parse(content).merge(job_retries: job_retries)
+        JSON.parse(content).merge(job_retries: job_retries, task_id: task_id)
       end
     end
 
     #
     # Extract the number of times this task failed at runtime.
     #
-    # @return [Integer] The number of failures
+    # @return [Integer] The number of failures.
     #
     def job_retries
       request.headers[Cloudtasker::Config::RETRY_HEADER].to_i
     end
 
+    #
+    # Return the Google Cloud Task ID from headers.
+    #
+    # @return [String] The task ID.
+    #
+    def task_id
+      request.headers[Cloudtasker::Config::TASK_ID_HEADER]
+    end
+
     #
     # Authenticate incoming requests using a bearer token
     #

data/cloudtasker.gemspec

@@ -15,8 +15,6 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/keypup-io/cloudtasker'
   spec.license       = 'MIT'
 
-  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
-
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/keypup-io/cloudtasker'
   spec.metadata['changelog_uri'] = 'https://github.com/keypup-io/cloudtasker/master/tree/CHANGELOG.md'
@@ -33,9 +31,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'activesupport'
   spec.add_dependency 'connection_pool'
   spec.add_dependency 'fugit'
-  spec.add_dependency 'google-cloud-tasks'
+  spec.add_dependency 'google-cloud-tasks', '~> 1.0'
   spec.add_dependency 'jwt'
   spec.add_dependency 'redis'
+  spec.add_dependency 'retriable'
 
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'
@@ -44,6 +43,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'
   spec.add_development_dependency 'rubocop-rspec', '1.37.0'
+  spec.add_development_dependency 'semantic_logger'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/docs/UNIQUE_JOBS.md

@@ -81,6 +81,68 @@ Below is the list of available conflict strategies can be specified through the
 | `raise` | All locks | A `Cloudtasker::UniqueJob::LockError` will be raised when a conflict occurs |
 | `reschedule` | `while_executing` | The job will be rescheduled 5 seconds later when a conflict occurs |
 
+## Lock Time To Live (TTL) & deadlocks
+**Note**: Lock TTL has been introduced in `v0.10.rc6`
+
+To make jobs unique Cloudtasker sets a lock key - a hash of class name + job arguments - in Redis. Unique crash situations may lead to lock keys not being cleaned up when jobs complete - e.g. Redis crash with rollback from last known state on disk. Situations like these may lead to having a unique job deadlock: jobs with the same class and arguments would stop being processed because they're unable to acquire a lock that will never be cleaned up.
+
+In order to prevent deadlocks Cloudtasker configures lock keys to automatically expire in Redis after `job schedule time + lock_ttl (default: 10 minutes)`. This forced expiration ensures that deadlocks eventually get cleaned up shortly after the expected run time of a job.
+
+The `lock_ttl (default: 10 minutes)` duration represent the expected max duration of the job. The default 10 minutes value was chosen because it's twice the default request timeout value in Cloud Run. This usually leaves enough room for queue lag (5 minutes) + job processing (5 minutes).
+
+Queue lag is certainly the most unpredictable factor here. Job processing time is less of a factor. Jobs running for more than 5 minutes should be split into sub-jobs to limit invocation time over HTTP anyway. Cloudtasker [batch jobs](BATCH_JOBS.md) can help split big jobs into sub-jobs in an atomic way.
+
+The default lock key expiration of `job schedule time + 10 minutes` may look aggressive but it is a better choice than having real-time jobs stuck for X hours after a crash recovery.
+
+We **strongly recommend** adapting the `lock_ttl` option either globally or for each worker based on expected queue lag and job duration.
+
+**Example 1**: Global configuration
+```ruby
+# config/initializers/cloudtasker.rb
+
+# General Cloudtasker configuration
+Cloudtasker.configure do |config|
+  # ...
+end
+
+# Unique job extension configuration
+Cloudtasker::UniqueJob.configure do |config|
+  config.lock_ttl = 3 * 60 # 3 minutes
+end
+```
+
+**Example 2**: Worker-level - fast
+```ruby
+# app/workers/realtime_worker_on_fast_queue.rb
+
+class RealtimeWorkerOnFastQueue
+  include Cloudtasker::Worker
+
+  # Ensure lock is removed 30 seconds after schedule time
+  cloudtasker_options lock: :until_executing, lock_ttl: 30
+
+  def perform(arg1, arg2)
+    # ...
+  end
+end
+```
+
+**Example 3**: Worker-level - slow
+```ruby
+# app/workers/non_critical_worker_on_slow_queue.rb
+
+class NonCriticalWorkerOnSlowQueue
+  include Cloudtasker::Worker
+
+  # Ensure lock is removed 24 hours after schedule time
+  cloudtasker_options lock: :until_executing, lock_ttl: 3600 * 24
+
+  def perform(arg1, arg2)
+    # ...
+  end
+end
+```
+
 ## Configuring unique arguments
 
 By default Cloudtasker considers all job arguments to evaluate the uniqueness of a job. This behaviour is configurable per worker by defining a `unique_args` method on the worker itself returning the list of args defining uniqueness.

data/gemfiles/semantic_logger_3.4.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "3.4.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.6.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.6.1"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.0.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.0"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.2.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/gemfiles/semantic_logger_4.7.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "semantic_logger", "4.7.2"
+
+gemspec path: "../"

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require 'google/cloud/tasks'
+require 'retriable'
+
 module Cloudtasker
   module Backend
     # Manage tasks pushed to GCP Cloud Task
@@ -113,9 +116,10 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The retrieved task.
       #
       def self.find(id)
-        resp = client.get_task(id)
+        resp = with_gax_retries { client.get_task(id) }
         resp ? new(resp) : nil
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::RetryError, Google::Gax::NotFoundError, GRPC::NotFound
+        # The ID does not exist
         nil
       end
 
@@ -133,10 +137,8 @@ module Cloudtasker
         relative_queue = payload.delete(:queue)
 
         # Create task
-        resp = client.create_task(queue_path(relative_queue), payload)
+        resp = with_gax_retries { client.create_task(queue_path(relative_queue), payload) }
         resp ? new(resp) : nil
-      rescue Google::Gax::RetryError
-        nil
       end
 
       #
@@ -145,11 +147,21 @@ module Cloudtasker
       # @param [String] id The id of the task.
       #
       def self.delete(id)
-        client.delete_task(id)
-      rescue Google::Gax::NotFoundError, Google::Gax::RetryError, GRPC::NotFound, Google::Gax::PermissionDeniedError
+        with_gax_retries { client.delete_task(id) }
+      rescue Google::Gax::RetryError, Google::Gax::NotFoundError, GRPC::NotFound, Google::Gax::PermissionDeniedError
+        # The ID does not exist
         nil
       end
 
+      #
+      # Helper method encapsulating the retry strategy for GAX calls
+      #
+      def self.with_gax_retries
+        Retriable.retriable(on: [Google::Gax::UnavailableError], tries: 3) do
+          yield
+        end
+      end
+
       #
       # Build a new instance of the class.
       #

data/lib/cloudtasker/backend/memory_task.rb

@@ -1,7 +1,5 @@
 # frozen_string_literal: true
 
-require 'cloudtasker/redis_client'
-
 module Cloudtasker
   module Backend
     # Manage local tasks pushed to memory.
@@ -10,6 +8,15 @@ module Cloudtasker
       attr_accessor :job_retries
       attr_reader :id, :http_request, :schedule_time, :queue
 
+      #
+      # Return true if we are in test inline execution mode.
+      #
+      # @return [Boolean] True if inline mode enabled.
+      #
+      def self.inline_mode?
+        defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
+      end
+
       #
       # Return the task queue. A worker class name
       #
@@ -59,7 +66,7 @@ module Cloudtasker
         queue << task
 
         # Execute task immediately if in testing and inline mode enabled
-        task.execute if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.inline?
+        task.execute if inline_mode?
 
         task
       end
@@ -153,13 +160,18 @@ module Cloudtasker
       #
       def execute
         # Execute worker
-        resp = WorkerHandler.with_worker_handling(payload, &:execute)
+        worker_payload = payload.merge(job_retries: job_retries, task_id: id)
+        resp = WorkerHandler.with_worker_handling(worker_payload, &:execute)
 
         # Delete task
         self.class.delete(id)
         resp
-      rescue StandardError
+      rescue DeadWorkerError => e
+        self.class.delete(id)
+        raise(e) if self.class.inline_mode?
+      rescue StandardError => e
         self.job_retries += 1
+        raise(e) if self.class.inline_mode?
       end
 
       #

data/lib/cloudtasker/backend/redis_task.rb

@@ -247,7 +247,8 @@ module Cloudtasker
           uri = URI(http_request[:url])
           req = Net::HTTP::Post.new(uri.path, http_request[:headers])
 
-          # Add retries header
+          # Add task headers
+          req[Cloudtasker::Config::TASK_ID_HEADER] = id
           req[Cloudtasker::Config::RETRY_HEADER] = retries
 
           # Set job payload

data/lib/cloudtasker/batch/middleware/server.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.for(worker).execute { yield }
         end
       end

data/lib/cloudtasker/config.rb

@@ -25,6 +25,9 @@ module Cloudtasker
     #
     RETRY_HEADER = 'X-CloudTasks-TaskRetryCount'
 
+    # Cloud Task ID header
+    TASK_ID_HEADER = 'X-CloudTasks-TaskName'
+
     # Content-Transfer-Encoding header in Cloud Task responses
     ENCODING_HEADER = 'Content-Transfer-Encoding'
 

data/lib/cloudtasker/cron/job.rb

@@ -4,15 +4,10 @@ require 'fugit'
 
 module Cloudtasker
   module Cron
-    # TODO: handle deletion of cron jobs
-    #
     # Manage cron jobs
     class Job
       attr_reader :worker
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'job'
-
       #
       # Build a new instance of the class
       #

data/lib/cloudtasker/cron/middleware/server.rb

@@ -5,7 +5,7 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker)
+        def call(worker, **_kwargs)
           Job.new(worker).execute { yield }
         end
       end

data/lib/cloudtasker/cron/schedule.rb

@@ -9,9 +9,6 @@ module Cloudtasker
     class Schedule
       attr_accessor :id, :cron, :worker, :task_id, :job_id, :queue, :args
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'schedule'
-
       #
       # Return the redis client.
       #

data/lib/cloudtasker/unique_job.rb

@@ -3,3 +3,30 @@
 require_relative 'unique_job/middleware'
 
 Cloudtasker::UniqueJob::Middleware.configure
+
+module Cloudtasker
+  # UniqueJob configurator
+  module UniqueJob
+    # The maximum duration a lock can remain in place
+    # after schedule time.
+    DEFAULT_LOCK_TTL = 10 * 60 # 10 minutes
+
+    class << self
+      attr_writer :lock_ttl
+
+      # Configure the middleware
+      def configure
+        yield(self)
+      end
+
+      #
+      # Return the max TTL for locks
+      #
+      # @return [Integer] The lock TTL.
+      #
+      def lock_ttl
+        @lock_ttl || DEFAULT_LOCK_TTL
+      end
+    end
+  end
+end

data/lib/cloudtasker/unique_job/job.rb

@@ -5,21 +5,19 @@ module Cloudtasker
     # Wrapper class for Cloudtasker::Worker delegating to lock
     # and conflict strategies
     class Job
-      attr_reader :worker
+      attr_reader :worker, :call_opts
 
       # The default lock strategy to use. Defaults to "no lock".
       DEFAULT_LOCK = UniqueJob::Lock::NoOp
 
-      # Key Namespace used for object saved under this class
-      SUB_NAMESPACE = 'job'
-
       #
       # Build a new instance of the class.
       #
       # @param [Cloudtasker::Worker] worker The worker at hand
       #
-      def initialize(worker)
+      def initialize(worker, **kwargs)
         @worker = worker
+        @call_opts = kwargs
       end
 
       #
@@ -31,6 +29,43 @@ module Cloudtasker
         worker.class.cloudtasker_options_hash
       end
 
+      #
+      # Return the Time To Live (TTL) that should be set in Redis for
+      # the lock key. Having a TTL on lock keys ensures that jobs
+      # do not end up stuck due to a dead lock situation.
+      #
+      # The TTL is calculated using schedule time + expected
+      # max job duration.
+      #
+      # The expected max job duration is set to 10 minutes by default.
+      # This value was chosen because it's twice the default request timeout
+      # value in Cloud Run. This leaves enough room for queue lag (5 minutes)
+      # + job processing (5 minutes).
+      #
+      # Queue lag is certainly the most unpredictable factor here.
+      # Job processing time is less of a factor. Jobs running for more than 5 minutes
+      # should be split into sub-jobs to limit invocation time over HTTP. Cloudtasker batch
+      # jobs can help achieve that if you need to make one big job split into sub-jobs "atomic".
+      #
+      # The default lock key expiration of "time_at + 10 minutes" may look aggressive but it
+      # is still a better choice than potentially having real-time jobs stuck for X hours.
+      #
+      # The expected max job duration can be configured via the `lock_ttl`
+      # option on the job itself.
+      #
+      # @return [Integer] The TTL in seconds
+      #
+      def lock_ttl
+        now = Time.now.to_i
+
+        # Get scheduled at and lock duration
+        scheduled_at = [call_opts[:time_at].to_i, now].compact.max
+        lock_duration = (options[:lock_ttl] || Cloudtasker::UniqueJob.lock_ttl).to_i
+
+        # Return TTL
+        scheduled_at + lock_duration - now
+      end
+
       #
       # Return the instantiated lock.
       #
@@ -121,7 +156,7 @@ module Cloudtasker
           raise(LockError, locked_id) if locked_id && locked_id != id
 
           # Take job lock if the lock is currently free
-          redis.set(unique_gid, id) unless locked_id
+          redis.set(unique_gid, id, ex: lock_ttl) unless locked_id
         end
       end