cloudtasker 0.12.rc7 → 0.12.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 96524dc4a6825a3760a462277f7b0a710d779521e759bce44374ea044dbd649d
-  data.tar.gz: 908f1497b7ad316549f4f347c363b183181f0e0a17a92bda3f50ac9c34e2247c
+  metadata.gz: 6ac7405c882f6cc73a1d18d5f0ffd4aaf1032d2b22973b03ac0c05b579321d47
+  data.tar.gz: ab4e12890ba7f1cf2f8cfde952a532f8563998352ba3c0ba348f8253c0f562b4
 SHA512:
-  metadata.gz: e18579d27321e09f1e997ad3fdd3e3e0d1d0c344cd5fc553b64f665937fbe510cd11957381eee1fee5c100e2f14d8390434f26c60307fec4ecf4a681b15885d8
-  data.tar.gz: 3b688838f1a518f2f5c7b914a590f035d8a828c2c2b67838173a1712b8765e2ce34cddb8ddf1214aaefc26ce8ec43b7cc1dbe4eb450ccb24feb9e06bcd19bac7
+  metadata.gz: 99a740769048aa11f4a7b2a3710ed98973c44c7d9fa5e059b37027f408d10e5c731cd01e28bbf7e0e2cc4df49290384fdbc073d4a7869d511af661d8c5363e1b
+  data.tar.gz: 4214ca35db358f7f027cad6fe0e36353034789c3969726484c2af50cc019a34ce03840718609aa6941ab5a8d1d6b312f174d884a9c88a2e112c4219f6e1a26d8

data/.rubocop.yml

@@ -12,7 +12,7 @@ Metrics/ModuleLength:
   Max: 150
 
 Metrics/AbcSize:
-  Max: 20
+  Max: 25
   Exclude:
     - 'spec/support/*'
 

data/CHANGELOG.md

@@ -1,11 +1,12 @@
 # Changelog
 
-## Latest RC [v0.12.rc7](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc7) (2021-03-31)
+## Latest RC [v0.12.rc8](https://github.com/keypup-io/cloudtasker/tree/v0.12.rc8) (2021-04-06)
 
-[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc7)
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.11.0...v0.12.rc8)
 
 **Improvements:**
 - ActiveJob: do not double log errors (ActiveJob has its own error logging)
+- Configuration: allow configuration of Cloud Tasks `dispatch deadline` at global and worker level
 - Cron jobs: Use Redis Sets instead of key pattern matching for resource listing
 - Error logging: Use worker logger so as to include context (job args etc.)
 - Error logging: Do not log exception and stack trace separately, combine them instead.

data/README.md

@@ -37,6 +37,7 @@ A local processing server is also available for development. This local server p
     1. [HTTP Error codes](#http-error-codes)
     2. [Error callbacks](#error-callbacks)
     3. [Max retries](#max-retries)
+    4. [Dispatch deadline](#dispatch-deadline)
 10. [Testing](#testing)
     1. [Test helper setup](#test-helper-setup)
     2. [In-memory queues](#in-memory-queues)
@@ -351,6 +352,23 @@ Cloudtasker.configure do |config|
   #
   # Store all job payloads in Redis exceeding 50 KB:
   # config.store_payloads_in_redis = 50
+
+  #
+  # Specify the dispatch deadline for jobs in Cloud Tasks, in seconds.
+  # Jobs taking longer will be retried by Cloud Tasks, even if they eventually
+  # complete on the server side.
+  #
+  # Note that this option is applied when jobs are enqueued job. Changing this value
+  # will not impact already enqueued jobs.
+  #
+  # This option can also be configured on a per worker basis via
+  # the cloudtasker_options directive.
+  #
+  # Supported since: v0.12.rc8
+  #
+  # Default: 600 seconds (10 minutes)
+  #
+  # config.dispatch_deadline = 600
 end
 ```
 
@@ -721,6 +739,48 @@ class SomeErrorWorker
 end
 ```
 
+### Dispatch deadline
+**Supported since**: `0.12.rc8`  
+
+By default Cloud Tasks will automatically timeout your jobs after 10 minutes, independently of your server HTTP timeout configuration.
+
+You can modify the dispatch deadline for jobs at a global level or on a per job basis.
+
+E.g. Set the default dispatch deadline to 20 minutes.
+```ruby
+# config/initializers/cloudtasker.rb
+
+Cloudtasker.configure do |config|
+  #
+  # Specify the dispatch deadline for jobs in Cloud Tasks, in seconds. 
+  # Jobs taking longer will be retried by Cloud Tasks, even if they eventually
+  # complete on the server side.
+  #
+  # Note that this option is applied when jobs are enqueued job. Changing this value
+  # will not impact already enqueued jobs.
+  #
+  # Default: 600 (10 minutes)
+  #
+  config.dispatch_deadline = 20 * 60 # 20 minutes
+end
+```
+
+E.g. Set a dispatch deadline of 5 minutes on a specific worker
+```ruby
+# app/workers/some_error_worker.rb
+
+class SomeFasterWorker
+  include Cloudtasker::Worker
+
+  # This will override the global setting
+  cloudtasker_options dispatch_deadline: 5 * 60
+
+  def perform
+    # ... do things ...
+  end
+end
+```
+
 ## Testing
 Cloudtasker provides several options to test your workers.
 

data/lib/cloudtasker/backend/memory_task.rb

@@ -113,7 +113,7 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0, **_xargs)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)

data/lib/cloudtasker/backend/redis_task.rb

@@ -7,7 +7,7 @@ module Cloudtasker
   module Backend
     # Manage local tasks pushed to Redis
     class RedisTask
-      attr_reader :id, :http_request, :schedule_time, :retries, :queue
+      attr_reader :id, :http_request, :schedule_time, :retries, :queue, :dispatch_deadline
 
       RETRY_INTERVAL = 20 # seconds
 
@@ -123,13 +123,15 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       # @param [Integer] retries The number of times the job failed.
+      # @param [Integer] dispatch_deadline The dispatch_deadline in seconds.
       #
-      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
+      def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil, dispatch_deadline: nil)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @retries = retries || 0
-        @queue = queue || Cloudtasker::Config::DEFAULT_JOB_QUEUE
+        @queue = queue || Config::DEFAULT_JOB_QUEUE
+        @dispatch_deadline = dispatch_deadline || Config::DEFAULT_DISPATCH_DEADLINE
       end
 
       #
@@ -152,7 +154,8 @@ module Cloudtasker
           http_request: http_request,
           schedule_time: schedule_time.to_i,
           retries: retries,
-          queue: queue
+          queue: queue,
+          dispatch_deadline: dispatch_deadline
         }
       end
 
@@ -176,7 +179,8 @@ module Cloudtasker
           retries: is_error ? retries + 1 : retries,
           http_request: http_request,
           schedule_time: (Time.now + interval).to_i,
-          queue: queue
+          queue: queue,
+          dispatch_deadline: dispatch_deadline
         )
         redis.sadd(self.class.key, id)
       end
@@ -207,6 +211,13 @@ module Cloudtasker
         end
 
         resp
+      rescue Net::ReadTimeout
+        retry_later(RETRY_INTERVAL)
+        Cloudtasker.logger.info(
+          format_log_message(
+            "Task deadline exceeded (#{dispatch_deadline}s) - Retry in #{RETRY_INTERVAL} seconds..."
+          )
+        )
       end
 
       #
@@ -242,7 +253,7 @@ module Cloudtasker
         @http_client ||=
           begin
             uri = URI(http_request[:url])
-            Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = 60 * 10 }
+            Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = dispatch_deadline }
           end
       end
 

data/lib/cloudtasker/cloud_task.rb

@@ -3,7 +3,7 @@
 module Cloudtasker
   # An interface class to manage tasks on the backend (Cloud Task or Redis)
   class CloudTask
-    attr_accessor :id, :http_request, :schedule_time, :retries, :queue
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue, :dispatch_deadline
 
     #
     # The backend to use for cloud tasks.
@@ -73,12 +73,13 @@ module Cloudtasker
     # @param [Integer] retries The number of times the job failed.
     # @param [String] queue The queue the task is in.
     #
-    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil)
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil, dispatch_deadline: nil)
       @id = id
       @http_request = http_request
       @schedule_time = schedule_time
       @retries = retries || 0
       @queue = queue
+      @dispatch_deadline = dispatch_deadline
     end
 
     #

data/lib/cloudtasker/config.rb

@@ -7,7 +7,7 @@ module Cloudtasker
   class Config
     attr_accessor :redis, :store_payloads_in_redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
-                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
+                :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries, :dispatch_deadline
 
     # Max Cloud Task size in bytes
     MAX_TASK_SIZE = 100 * 1024 # 100 KB
@@ -46,6 +46,11 @@ module Cloudtasker
     DEFAULT_QUEUE_CONCURRENCY = 10
     DEFAULT_QUEUE_RETRIES = -1 # unlimited
 
+    # Job timeout configuration for Cloud Tasks
+    DEFAULT_DISPATCH_DEADLINE = 10 * 60 # 10 minutes
+    MIN_DISPATCH_DEADLINE = 15 # seconds
+    MAX_DISPATCH_DEADLINE = 30 * 60 # 30 minutes
+
     # The number of times jobs will be attempted before declaring them dead.
     #
     # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
@@ -207,6 +212,16 @@ module Cloudtasker
       @gcp_location_id || DEFAULT_LOCATION_ID
     end
 
+    #
+    # Return the Dispatch deadline duration. Cloud Tasks will timeout the job after
+    # this duration is elapsed.
+    #
+    # @return [Integer] The value in seconds.
+    #
+    def dispatch_deadline
+      @dispatch_deadline || DEFAULT_DISPATCH_DEADLINE
+    end
+
     #
     # Return the secret to use to sign the verification tokens
     # attached to tasks.

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.12.rc7'
+  VERSION = '0.12.rc8'
 end

data/lib/cloudtasker/worker.rb

@@ -167,6 +167,22 @@ module Cloudtasker
       (@job_queue ||= self.class.cloudtasker_options_hash[:queue] || Config::DEFAULT_JOB_QUEUE).to_s
     end
 
+    #
+    # Return the Dispatch deadline duration. Cloud Tasks will timeout the job after
+    # this duration is elapsed.
+    #
+    # @return [Integer] The value in seconds.
+    #
+    def dispatch_deadline
+      @dispatch_deadline ||= [
+        [
+          Config::MIN_DISPATCH_DEADLINE,
+          (self.class.cloudtasker_options_hash[:dispatch_deadline] || Cloudtasker.config.dispatch_deadline).to_i
+        ].max,
+        Config::MAX_DISPATCH_DEADLINE
+      ].min
+    end
+
     #
     # Return the Cloudtasker logger instance.
     #

data/lib/cloudtasker/worker_handler.rb

@@ -165,6 +165,7 @@ module Cloudtasker
           },
           body: worker_payload.to_json
         },
+        dispatch_deadline: worker.dispatch_deadline.to_i,
         queue: worker.job_queue
       }
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.12.rc7
+  version: 0.12.rc8
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-03-31 00:00:00.000000000 Z
+date: 2021-04-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport