cloudtasker 0.10.rc2 → 0.10.rc3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9de65884c24b1902d30af0f9b140caf36749d40e4613a13fa88940593e9cd12
-  data.tar.gz: 9033cd6fca6bab0dc5a1387c6e8a8f82a12460e86165072d987282e5bb0e2b6d
+  metadata.gz: 4b2692b6e4fe2fbd98cf70fb2fe7a2027992d83be63188108113d523cf410160
+  data.tar.gz: 892910f2b59d84e2f516478926fefb63bd913bb4db4507133cde614eca933482
 SHA512:
-  metadata.gz: edfcbf887b455566b52718e06a41dd5061de05e4e93a7a2e2df42d148d863853852545374f27d8b41e4b445866bc0f76fc8ad3866abe182dea7728a302664a1c
-  data.tar.gz: 631b9929e8b4365e64d47eca6354765773ee79780b60c857a41bca41fd8d7c55f946c621876167deb9067f446e0fbdc54ad2673358f59d0143368203ea8195b9
+  metadata.gz: 3f7a4bc94036af569901cde01fe7596acc9eea3019d9216db033de2833eb471668f231c6dbf00ac231247f599291fdd0bec0ed4eb700cf06521b01ea2a753739
+  data.tar.gz: f37e39c7ce6fff9a755a457694f5b46aca1c64b9a46b8b1fa74e10208e19c6c6beffc7e5c4e50a74040d1cca8032088478be2168f80f17e2be4d1845d4d69306

data/.rubocop.yml

@@ -8,6 +8,9 @@ AllCops:
 Metrics/ClassLength:
   Max: 150
 
+Metrics/ModuleLength:
+  Max: 150
+
 Metrics/AbcSize:
   Max: 20
 
@@ -34,7 +37,4 @@ Metrics/BlockLength:
 Style/Documentation:
   Exclude:
     - 'examples/**/*'
-    - 'spec/**/*'
-
-RSpec/RepeatedExampleGroupBody:
-  Enabled: false
+    - 'spec/**/*'

data/README.md

@@ -246,6 +246,8 @@ Cloudtasker.configure do |config|
   # You can set this configuration parameter to a KB value if you want to store jobs
   # args in redis only if the JSONified arguments payload exceeds that threshold.
   #
+  # Supported since: v0.10.rc1
+  #
   # Default: false
   #
   # Store all job payloads in Redis:
@@ -503,7 +505,7 @@ See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more informat
 
 ## Error Handling
 
-Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of retries Cloud Task will do depends on the configuration of your queue in Cloud Tasks.
+Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of Cloud Task retries Cloud Task will depend on the configuration of your queue in Cloud Tasks.
 
 ### HTTP Error codes
 
@@ -549,6 +551,8 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
 
 Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
 
+**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `0.10.rc3` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
+
 E.g. Set max number of retries globally via the cloudtasker initializer.
 ```ruby
 # config/initializers/cloudtasker.rb
@@ -583,7 +587,6 @@ end
 ```
 
 
-
 ## Best practices building workers
 
 Below are recommendations and notes about creating workers.
@@ -658,6 +661,8 @@ Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into acco
 Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
 
 #### Option 1: Use Cloudtasker optional support for payload storage in Redis
+**Supported since**: `0.10.rc1`
+
 Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
 
 To enable it simply put the following in your Cloudtasker initializer:

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.0'
+gem "google-cloud-tasks", "1.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.1.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.1'
+gem "google-cloud-tasks", "1.1"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.2'
+gem "google-cloud-tasks", "1.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.3.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.3'
+gem "google-cloud-tasks", "1.3"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '5.2'
+gem "rails", "5.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '6.0'
+gem "rails", "6.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/lib/cloudtasker/backend/redis_task.rb

@@ -248,7 +248,7 @@ module Cloudtasker
           req = Net::HTTP::Post.new(uri.path, http_request[:headers])
 
           # Add retries header
-          req['X-CloudTasks-TaskExecutionCount'] = retries
+          req[Cloudtasker::Config::RETRY_HEADER] = retries
 
           # Set job payload
           req.body = http_request[:body]

data/lib/cloudtasker/config.rb

@@ -13,7 +13,17 @@ module Cloudtasker
     MAX_TASK_SIZE = 100 * 1024 # 100 KB
 
     # Retry header in Cloud Task responses
-    RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
+    #
+    # TODO: use 'X-CloudTasks-TaskExecutionCount' instead of 'X-CloudTasks-TaskRetryCount'
+    #   'X-CloudTasks-TaskExecutionCount' is currently bugged and remains at 0 even on retries.
+    #
+    # See bug: https://issuetracker.google.com/issues/154532072
+    #
+    # Definitions:
+    #   X-CloudTasks-TaskRetryCount: total number of retries (including 504 "instance unreachable")
+    #   X-CloudTasks-TaskExecutionCount: number of non-503 retries (= actual number of job failures)
+    #
+    RETRY_HEADER = 'X-CloudTasks-TaskRetryCount'
 
     # Content-Transfer-Encoding header in Cloud Task responses
     ENCODING_HEADER = 'Content-Transfer-Encoding'
@@ -33,7 +43,15 @@ module Cloudtasker
     DEFAULT_QUEUE_CONCURRENCY = 10
     DEFAULT_QUEUE_RETRIES = -1 # unlimited
 
-    # The number of times jobs will be attempted before declaring them dead
+    # The number of times jobs will be attempted before declaring them dead.
+    #
+    # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
+    # it means that jobs will be declared dead after 20h of consecutive failing.
+    #
+    # Note that this configuration parameter is internal to Cloudtasker and does not
+    # affect the Cloud Task queue configuration. The number of retries configured
+    # on the Cloud Task queue should be higher than the number below to also cover
+    # failures due to the instance being unreachable.
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
     PROCESSOR_HOST_MISSING = <<~DOC

data/lib/cloudtasker/local_server.rb

@@ -12,6 +12,9 @@ module Cloudtasker
     # Default number of threads to allocate to process a specific queue
     QUEUE_CONCURRENCY = 1
 
+    # Job Polling. How frequently to poll jobs in redis.
+    JOB_POLLING_FREQUENCY = 0.5 # seconds
+
     #
     # Stop the local server.
     #
@@ -46,7 +49,7 @@ module Cloudtasker
       @start ||= Thread.new do
         until @done
           queues.each { |(n, c)| process_jobs(n, c) }
-          sleep 1
+          sleep JOB_POLLING_FREQUENCY
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
       end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.10.rc2'
+  VERSION = '0.10.rc3'
 end

data/lib/cloudtasker/worker.rb

@@ -7,7 +7,8 @@ module Cloudtasker
     def self.included(base)
       base.extend(ClassMethods)
       base.attr_writer :job_queue
-      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
+      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries,
+                         :perform_started_at, :perform_ended_at
     end
 
     #
@@ -181,21 +182,19 @@ module Cloudtasker
     #
     def execute
       logger.info('Starting job...')
-      resp = Cloudtasker.config.server_middleware.invoke(self) do
-        begin
-          perform(*job_args)
-        rescue StandardError => e
-          try(:on_error, e)
-          return raise(e) unless job_dead?
 
-          # Flag job as dead
-          logger.info('Job dead')
-          try(:on_dead, e)
-          raise(DeadWorkerError, e)
-        end
-      end
-      logger.info('Job done')
+      # Perform job logic
+      resp = execute_middleware_chain
+
+      # Log job completion and return result
+      logger.info("Job done after #{job_duration}s") { { duration: job_duration } }
       resp
+    rescue DeadWorkerError => e
+      logger.info("Job dead after #{job_duration}s and #{job_retries} retries") { { duration: job_duration } }
+      raise(e)
+    rescue StandardError => e
+      logger.info("Job failed after #{job_duration}s") { { duration: job_duration } }
+      raise(e)
     end
 
     #
@@ -286,5 +285,46 @@ module Cloudtasker
     def job_dead?
       job_retries >= Cloudtasker.config.max_retries
     end
+
+    #
+    # Return the time taken (in seconds) to perform the job. This duration
+    # includes the middlewares and the actual perform method.
+    #
+    # @return [Float] The time taken in seconds as a floating point number.
+    #
+    def job_duration
+      return 0.0 unless perform_ended_at && perform_started_at
+
+      (perform_ended_at - perform_started_at).ceil(3)
+    end
+
+    #=============================
+    # Private
+    #=============================
+    private
+
+    #
+    # Execute the worker perform method through the middleware chain.
+    #
+    # @return [Any] The result of the perform method.
+    #
+    def execute_middleware_chain
+      self.perform_started_at = Time.now
+
+      Cloudtasker.config.server_middleware.invoke(self) do
+        begin
+          perform(*job_args)
+        rescue StandardError => e
+          try(:on_error, e)
+          return raise(e) unless job_dead?
+
+          # Flag job as dead
+          try(:on_dead, e)
+          raise(DeadWorkerError, e)
+        end
+      end
+    ensure
+      self.perform_ended_at = Time.now
+    end
   end
 end

data/lib/cloudtasker/worker_logger.rb

@@ -59,7 +59,7 @@ module Cloudtasker
     # @return [String] The formatted log message
     #
     def formatted_message(msg)
-      "[Cloudtasker][#{worker.job_id}] #{msg}"
+      "[Cloudtasker][#{worker.class}][#{worker.job_id}] #{msg}"
     end
 
     #
@@ -141,7 +141,8 @@ module Cloudtasker
     # @param [Proc] &block Optional context block.
     #
     def log_message(level, msg, &block)
-      payload_block = block || log_block
+      # Merge log-specific context into worker-specific context
+      payload_block = -> { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.10.rc2
+  version: 0.10.rc3
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2020-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport