cloudtasker 0.8.1 → 0.10.rc4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3459c464d92cbd6e9c920a0f67a96ba143bf5746af3e8dde4fe13cade0a75ffb
-  data.tar.gz: 42313e898816af4d2eb670622569e1545e13eb54f2de1acaee2f4cddb923e101
+  metadata.gz: 228f7fc8ef7ca52a1a9827b233ae1ebfea77e2a6d6683d68caa3fffef697ffb4
+  data.tar.gz: b58fee4392d02847c462684744a5b90ff413564a02e3e74b51514b7f8aee4107
 SHA512:
-  metadata.gz: 7ce1fcf26ec35f3e21a8f932f5f530c54a136193458944535097e84186c23fdf61306059fbe7449af38c3747ba48b0aad380cf706c3d4d27ffa70eae36c2f042
-  data.tar.gz: ad8c3a5529ef16e699a5db45594b19ee72eaaa3ff22315e463fc68e7548b961ce399ce7a44af784270cf7cac02b057cc5f31bb41188eabbb6e29224fbc0e87ef
+  metadata.gz: 88d7bfd9d4bbdb38ca48cf5e3af1fd9f901a3f89b84bce7d41609baa22d7ded5ce0c3a657d8aaaa9ad3b9274ad389825bdd7e17272f65ef80a68c6cb96538d25
+  data.tar.gz: b0a1f08dd892a9c709ecb24fb9a38c6507489179ce06779e49c9641d1ed849b02af6c3717476d85e7cf852879d0a5c8429123bc843a4ce60933f8319d0241e5a

data/.github/workflows/test.yml

@@ -0,0 +1,41 @@
+name: Test
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby:
+          - '2.5.x'
+          - '2.6.x'
+        appraisal:
+          - 'google-cloud-tasks-1.0'
+          - 'google-cloud-tasks-1.1'
+          - 'google-cloud-tasks-1.2'
+          - 'google-cloud-tasks-1.3'
+          - 'rails-5.2'
+          - 'rails-6.0'
+    steps:
+      - name: Setup System
+        run: sudo apt-get install libsqlite3-dev
+      - uses: actions/checkout@v2
+      - uses: zhulik/redis-action@1.1.0
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+      - name: Build and test with Rake
+        env:
+            APPRAISAL_CONTEXT: ${{ matrix.appraisal }}
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+          bundle exec rubocop
+          bundle exec appraisal ${APPRAISAL_CONTEXT} bundle
+          bundle exec appraisal ${APPRAISAL_CONTEXT} rspec

data/.rubocop.yml

@@ -2,12 +2,15 @@ require: rubocop-rspec
 
 AllCops:
   Exclude:
-    - 'gemfiles/vendor/**/*'
+    - 'gemfiles/**/*'
     - 'vendor/**/*'
 
 Metrics/ClassLength:
   Max: 150
 
+Metrics/ModuleLength:
+  Max: 150
+
 Metrics/AbcSize:
   Max: 20
 

data/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## [v0.9.2](https://github.com/keypup-io/cloudtasker/tree/v0.9.2) (2020-03-04)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.1...v0.9.2)
+
+**Fixed bugs:**
+- Cloud Task: ignore "not found" errors when trying to delete an already deleted task.
+
+## [v0.9.1](https://github.com/keypup-io/cloudtasker/tree/v0.9.1) (2020-02-11)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.9.0...v0.9.1)
+
+**Fixed bugs:**
+- Cloud Task: raise `Cloudtasker::MaxTaskSizeExceededError` if job payload exceeds 100 KB. This is mainly to have production parity in development when running the local processing server.
+
+## [v0.9.0](https://github.com/keypup-io/cloudtasker/tree/v0.9.0) (2020-01-23)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.2...v0.9.0)
+
+**Fixed bugs:**
+- Cloud Task: Base64 encode task body to support UTF-8 characters (e.g. emojis).
+- Redis: Restrict to one connection (class level) to avoid too many DNS lookups
+
+**Migration**
+For Sinatra applications please update your Cloudtasker controller according to [this diff](https://github.com/keypup-io/cloudtasker/commit/311fa8f9beec91fbae012164a25b2ee6e261a2e4#diff-c2a0ea6c6e6c31c749d2e1acdc574f0f).
+
+## [v0.8.2](https://github.com/keypup-io/cloudtasker/tree/v0.8.2) (2019-12-05)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.1...v0.8.2)
+
+**Fixed bugs:**
+- Config: do not add processor host to `Rails.application.config.hosts` if originally empty.
+
+## [v0.8.1](https://github.com/keypup-io/cloudtasker/tree/v0.8.1) (2019-12-03)
+
+[Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.8.0...v0.8.1)
+
+**Fixed bugs:**
+- Local dev server: ensure job queue name is kept when taks is retried
+- Rails/Controller: bypass Rails munge logic to preserve nil values inside job arguments.
+
 ## [v0.8.0](https://github.com/keypup-io/cloudtasker/tree/v0.8.0) (2019-11-27)
 
 [Full Changelog](https://github.com/keypup-io/cloudtasker/compare/v0.7.0...v0.8.0)

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/keypup-io/cloudtasker.svg?branch=master)](https://travis-ci.org/keypup-io/cloudtasker) [![Gem Version](https://badge.fury.io/rb/cloudtasker.svg)](https://badge.fury.io/rb/cloudtasker)
+![Build Status](https://github.com/keypup-io/cloudtasker/workflows/Test/badge.svg) [![Gem Version](https://badge.fury.io/rb/cloudtasker.svg)](https://badge.fury.io/rb/cloudtasker)
 
 # Cloudtasker
 
@@ -224,7 +224,7 @@ Cloudtasker.configure do |config|
   # 
   # config.max_retries = 10
 
-  # 
+  #
   # Specify the redis connection hash.
   #
   # This is ONLY required in development for the Cloudtasker local server and in
@@ -235,6 +235,26 @@ Cloudtasker.configure do |config|
   # Default: redis-rb connects to redis://127.0.0.1:6379/0
   #
   # config.redis = { url: 'redis://localhost:6379/5' }
+
+  #
+  # Set to true to store job arguments in Redis instead of sending arguments as part
+  # of the job payload to Google Cloud Tasks.
+  #
+  # This is useful if you expect to process jobs with payloads exceeding 100KB, which
+  # is the limit enforced by Google Cloud Tasks.
+  #
+  # You can set this configuration parameter to a KB value if you want to store jobs
+  # args in redis only if the JSONified arguments payload exceeds that threshold.
+  #
+  # Supported since: v0.10.rc1
+  #
+  # Default: false
+  #
+  # Store all job payloads in Redis:
+  # config.store_payloads_in_redis = true
+  #
+  # Store all job payloads in Redis exceeding 50 KB:
+  # config.store_payloads_in_redis = 50
 end
 ```
 
@@ -485,7 +505,7 @@ See the [Cloudtasker::Worker class](lib/cloudtasker/worker.rb) for more informat
 
 ## Error Handling
 
-Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of retries Cloud Task will do depends on the configuration of your queue in Cloud Tasks.
+Jobs failing will automatically return an HTTP error to Cloud Task and trigger a retry at a later time. The number of Cloud Task retries Cloud Task will depend on the configuration of your queue in Cloud Tasks.
 
 ### HTTP Error codes
 
@@ -531,6 +551,8 @@ By default jobs are retried 25 times - using an exponential backoff - before bei
 
 Note that the number of retries set on your Cloud Task queue should be many times higher than the number of retries configured in Cloudtasker because Cloud Task also includes failures to connect to your application. Ideally set the number of retries to `unlimited` in Cloud Tasks.
 
+**Note**: The `X-CloudTasks-TaskExecutionCount` header sent by Google Cloud Tasks and providing the number of retries outside of `HTTP 503` (instance not reachable) is currently bugged and remains at `0` all the time. Starting with `0.10.rc3` Cloudtasker uses the `X-CloudTasks-TaskRetryCount` header to detect the number of retries. This header includes `HTTP 503` errors which means that if your application is down at some point, jobs will fail and these failures will be counted toward the maximum number of retries. A [bug report](https://issuetracker.google.com/issues/154532072) has been raised with GCP to address this issue. Once fixed we will revert to using `X-CloudTasks-TaskExecutionCount` to avoid counting `HTTP 503` as job failures.
+
 E.g. Set max number of retries globally via the cloudtasker initializer.
 ```ruby
 # config/initializers/cloudtasker.rb
@@ -565,7 +587,6 @@ end
 ```
 
 
-
 ## Best practices building workers
 
 Below are recommendations and notes about creating workers.
@@ -635,7 +656,32 @@ If you enqueue this worker by omitting the second argument `MyWorker.perform_asy
 - The `time_at` argument will be ignored by the `unique-job` extension, meaning that job uniqueness will be only based on the `user_id` argument.
 
 ### Handling big job payloads
-Keep in mind that jobs are pushed to Google Cloud Tasks via API and then delivered to your application via API as well. Therefore any excessive job payload will slow down the enqueuing of jobs and create additional processing when receiving the job.
+Google Cloud Tasks enforces a limit of 100 KB for job payloads. Taking into accounts Cloudtasker authentication headers and meta information this leave ~85 KB of free space for JSONified job arguments.
+
+Any excessive job payload (> 100 KB) will raise a `Cloudtasker::MaxTaskSizeExceededError`, both in production and development mode.
+
+#### Option 1: Use Cloudtasker optional support for payload storage in Redis
+**Supported since**: `0.10.rc1`
+
+Cloudtasker provides optional support for storing argument payloads in Redis instead of sending them to Google Cloud Tasks.
+
+To enable it simply put the following in your Cloudtasker initializer:
+```ruby
+# config/initializers/cloudtasker.rb
+
+Cloudtasker.configure do |config|
+  # Enable Redis support. Specify your redis connection
+  config.redis = { url: 'redis://localhost:6379/5' }
+
+  # Store all job payloads in Redis:
+  config.store_payloads_in_redis = true
+
+  # OR: store all job payloads in Redis exceeding 50 KB:
+  # config.store_payloads_in_redis = 50
+end
+```
+
+#### Option 2: Do it yourself solution
 
 If you feel that a job payload is going to get big, prefer to store the payload using a datastore (e.g. Redis) and pass a reference to the job to retrieve the payload inside your job `perform` method.
 

data/app/controllers/cloudtasker/worker_controller.rb

@@ -16,9 +16,6 @@ module Cloudtasker
     # Run a worker from a Cloud Task payload
     #
     def run
-      # Build payload
-      payload = JSON.parse(request.body.read).merge(job_retries: job_retries)
-
       # Process payload
       WorkerHandler.execute_from_payload!(payload)
       head :no_content
@@ -37,6 +34,27 @@ module Cloudtasker
 
     private
 
+    #
+    # Parse the request body and return the actual job
+    # payload.
+    #
+    # @return [Hash] The job payload
+    #
+    def payload
+      @payload ||= begin
+        # Get raw body
+        content = request.body.read
+
+        # Decode content if the body is Base64 encoded
+        if request.headers[Cloudtasker::Config::ENCODING_HEADER].to_s.downcase == 'base64'
+          content = Base64.decode64(content)
+        end
+
+        # Return content parsed as JSON and add job retries count
+        JSON.parse(content).merge(job_retries: job_retries)
+      end
+    end
+
     #
     # Extract the number of times this task failed at runtime.
     #

data/cloudtasker.gemspec

@@ -39,10 +39,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'github_changelog_generator'
-  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rake', '>= 12.3.3'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'rubocop', '0.76.0'
-  spec.add_development_dependency 'rubocop-rspec'
+  spec.add_development_dependency 'rubocop-rspec', '1.37.0'
   spec.add_development_dependency 'timecop'
   spec.add_development_dependency 'webmock'
 

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.0'
+gem "google-cloud-tasks", "1.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.1.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.1'
+gem "google-cloud-tasks", "1.1"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.2'
+gem "google-cloud-tasks", "1.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.3.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'google-cloud-tasks', '1.3'
+gem "google-cloud-tasks", "1.3"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '5.2'
+gem "rails", "5.2"
 
-gemspec path: '../'
+gemspec path: "../"

data/gemfiles/rails_6.0.gemfile

@@ -1,9 +1,7 @@
-# frozen_string_literal: true
-
 # This file was generated by Appraisal
 
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
-gem 'rails', '6.0'
+gem "rails", "6.0"
 
-gemspec path: '../'
+gemspec path: "../"

data/lib/cloudtasker.rb

@@ -8,6 +8,7 @@ require 'cloudtasker/config'
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
 require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/max_task_size_exceeded_error'
 
 require 'cloudtasker/middleware/chain'
 require 'cloudtasker/authenticator'

data/lib/cloudtasker/backend/google_cloud_task.rb

@@ -82,6 +82,29 @@ module Cloudtasker
         Google::Protobuf::Timestamp.new.tap { |e| e.seconds = schedule_time.to_i }
       end
 
+      #
+      # Format the job payload sent to Cloud Tasks.
+      #
+      # @param [Hash] hash The worker payload.
+      #
+      # @return [Hash] The Cloud Task payloadd.
+      #
+      def self.format_task_payload(payload)
+        payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
+
+        # Format schedule time to Google Protobuf timestamp
+        payload[:schedule_time] = format_schedule_time(payload[:schedule_time])
+
+        # Encode job content to support UTF-8. Google Cloud Task
+        # expect content to be ASCII-8BIT compatible (binary)
+        payload[:http_request][:headers] ||= {}
+        payload[:http_request][:headers][Cloudtasker::Config::CONTENT_TYPE_HEADER] = 'text/json'
+        payload[:http_request][:headers][Cloudtasker::Config::ENCODING_HEADER] = 'Base64'
+        payload[:http_request][:body] = Base64.encode64(payload[:http_request][:body])
+
+        payload
+      end
+
       #
       # Find a task by id.
       #
@@ -104,10 +127,7 @@ module Cloudtasker
       # @return [Cloudtasker::Backend::GoogleCloudTask, nil] The created task.
       #
       def self.create(payload)
-        # Format payload
-        payload = payload.merge(
-          schedule_time: format_schedule_time(payload[:schedule_time])
-        ).compact
+        payload = format_task_payload(payload)
 
         # Extract relative queue name
         relative_queue = payload.delete(:queue)
@@ -126,7 +146,7 @@ module Cloudtasker
       #
       def self.delete(id)
         client.delete_task(id)
-      rescue Google::Gax::RetryError
+      rescue Google::Gax::NotFoundError, Google::Gax::RetryError, GRPC::NotFound, Google::Gax::PermissionDeniedError
         nil
       end
 

data/lib/cloudtasker/backend/memory_task.rb

@@ -7,6 +7,7 @@ module Cloudtasker
     # Manage local tasks pushed to memory.
     # Used for testing.
     class MemoryTask
+      attr_accessor :job_retries
       attr_reader :id, :http_request, :schedule_time, :queue
 
       #
@@ -18,17 +19,6 @@ module Cloudtasker
         @queue ||= []
       end
 
-      #
-      # Return the workers currently in the queue.
-      #
-      # @param [String] worker_class_name Filter jobs on worker class name.
-      #
-      # @return [Array<Cloudtasker::Worker] The list of workers
-      #
-      def self.jobs(worker_class_name = nil)
-        all(worker_class_name).map(&:worker)
-      end
-
       #
       # Run all Tasks in the queue. Optionally filter which tasks to run based
       # on the worker class name.
@@ -116,11 +106,12 @@ module Cloudtasker
       # @param [Hash] http_request The HTTP request content.
       # @param [Integer] schedule_time When to run the task (Unix timestamp)
       #
-      def initialize(id:, http_request:, schedule_time: nil, queue: nil)
+      def initialize(id:, http_request:, schedule_time: nil, queue: nil, job_retries: 0)
         @id = id
         @http_request = http_request
         @schedule_time = Time.at(schedule_time || 0)
         @queue = queue
+        @job_retries = job_retries || 0
       end
 
       #
@@ -155,26 +146,20 @@ module Cloudtasker
         }
       end
 
-      #
-      # Return the worker attached to this task.
-      #
-      # @return [Cloudtasker::Worker] The task worker.
-      #
-      def worker
-        @worker ||= Worker.from_hash(payload)
-      end
-
       #
       # Execute the task.
       #
       # @return [Any] The return value of the worker perform method.
       #
       def execute
-        resp = worker.execute
+        # Execute worker
+        resp = WorkerHandler.with_worker_handling(payload, &:execute)
+
+        # Delete task
         self.class.delete(id)
         resp
       rescue StandardError
-        worker.job_retries += 1
+        self.job_retries += 1
       end
 
       #

data/lib/cloudtasker/backend/redis_task.rb

@@ -248,7 +248,7 @@ module Cloudtasker
           req = Net::HTTP::Post.new(uri.path, http_request[:headers])
 
           # Add retries header
-          req['X-CloudTasks-TaskExecutionCount'] = retries
+          req[Cloudtasker::Config::RETRY_HEADER] = retries
 
           # Set job payload
           req.body = http_request[:body]

data/lib/cloudtasker/cloud_task.rb

@@ -48,6 +48,8 @@ module Cloudtasker
     # @return [Cloudtasker::CloudTask] The created task.
     #
     def self.create(payload)
+      raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
+
       resp = backend.create(payload)&.to_h
       resp ? new(resp) : nil
     end

data/lib/cloudtasker/config.rb

@@ -5,12 +5,34 @@ require 'logger'
 module Cloudtasker
   # Holds cloudtasker configuration. See Cloudtasker#configure
   class Config
-    attr_accessor :redis
+    attr_accessor :redis, :store_payloads_in_redis
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :gcp_queue_prefix, :processor_path, :logger, :mode, :max_retries
 
+    # Max Cloud Task size in bytes
+    MAX_TASK_SIZE = 100 * 1024 # 100 KB
+
     # Retry header in Cloud Task responses
-    RETRY_HEADER = 'X-CloudTasks-TaskExecutionCount'
+    #
+    # TODO: use 'X-CloudTasks-TaskExecutionCount' instead of 'X-CloudTasks-TaskRetryCount'
+    #   'X-CloudTasks-TaskExecutionCount' is currently bugged and remains at 0 even on retries.
+    #
+    # See bug: https://issuetracker.google.com/issues/154532072
+    #
+    # Definitions:
+    #   X-CloudTasks-TaskRetryCount: total number of retries (including 504 "instance unreachable")
+    #   X-CloudTasks-TaskExecutionCount: number of non-503 retries (= actual number of job failures)
+    #
+    RETRY_HEADER = 'X-CloudTasks-TaskRetryCount'
+
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # Authorization header
+    AUTHORIZATION_HEADER = 'Authorization'
 
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
@@ -21,7 +43,15 @@ module Cloudtasker
     DEFAULT_QUEUE_CONCURRENCY = 10
     DEFAULT_QUEUE_RETRIES = -1 # unlimited
 
-    # The number of times jobs will be attempted before declaring them dead
+    # The number of times jobs will be attempted before declaring them dead.
+    #
+    # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
+    # it means that jobs will be declared dead after 20h of consecutive failing.
+    #
+    # Note that this configuration parameter is internal to Cloudtasker and does not
+    # affect the Cloud Task queue configuration. The number of retries configured
+    # on the Cloud Task queue should be higher than the number below to also cover
+    # failures due to the instance being unreachable.
     DEFAULT_MAX_RETRY_ATTEMPTS = 25
 
     PROCESSOR_HOST_MISSING = <<~DOC
@@ -42,6 +72,21 @@ module Cloudtasker
       Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
     DOC
 
+    #
+    # Return the threshold above which job arguments must be stored
+    # in Redis instead of being sent to the backend as part of the job
+    # payload.
+    #
+    # Return nil if redis payload storage is disabled.
+    #
+    # @return [Integer, nil] The threshold above which payloads will be stored in Redis.
+    #
+    def redis_payload_storage_threshold
+      return nil unless store_payloads_in_redis
+
+      store_payloads_in_redis.respond_to?(:to_i) ? store_payloads_in_redis.to_i : 0
+    end
+
     #
     # The number of times jobs will be retried. This number of
     # retries does not include failures due to the application being unreachable.
@@ -102,7 +147,10 @@ module Cloudtasker
       @processor_host = val
 
       # Check if Rails supports host filtering
-      return unless val && defined?(Rails) && Rails.application.config.respond_to?(:hosts)
+      return unless val &&
+                    defined?(Rails) &&
+                    Rails.application.config.respond_to?(:hosts) &&
+                    Rails.application.config.hosts&.any?
 
       # Add processor host to the list of authorized hosts
       Rails.application.config.hosts << val.gsub(%r{https?://}, '')

data/lib/cloudtasker/local_server.rb

@@ -12,6 +12,9 @@ module Cloudtasker
     # Default number of threads to allocate to process a specific queue
     QUEUE_CONCURRENCY = 1
 
+    # Job Polling. How frequently to poll jobs in redis.
+    JOB_POLLING_FREQUENCY = 0.5 # seconds
+
     #
     # Stop the local server.
     #
@@ -46,7 +49,7 @@ module Cloudtasker
       @start ||= Thread.new do
         until @done
           queues.each { |(n, c)| process_jobs(n, c) }
-          sleep 1
+          sleep JOB_POLLING_FREQUENCY
         end
         Cloudtasker.logger.info('[Cloudtasker/Server] Local server exiting...')
       end
@@ -82,7 +85,7 @@ module Cloudtasker
       # Deliver task
       begin
         Thread.current['task'].deliver
-      rescue Errno::ECONNREFUSED => e
+      rescue Errno::EBADF, Errno::ECONNREFUSED => e
         raise(e) unless Thread.current['attempts'] < 3
 
         # Retry on connection error, in case the web server is not

data/lib/cloudtasker/max_task_size_exceeded_error.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Handle Cloud Task size quota
+  # See: https://cloud.google.com/appengine/quotas#Task_Queue
+  #
+  class MaxTaskSizeExceededError < StandardError
+    MSG = 'The size of Cloud Tasks must not exceed 100KB'
+
+    def initialize(msg = MSG)
+      super
+    end
+  end
+end

data/lib/cloudtasker/redis_client.rb

@@ -7,6 +7,12 @@ module Cloudtasker
   class RedisClient
     # Suffix added to cache keys when locking them
     LOCK_KEY_PREFIX = 'cloudtasker/lock'
+    LOCK_DURATION = 2 # seconds
+    LOCK_WAIT_DURATION = 0.03 # seconds
+
+    def self.client
+      @client ||= Redis.new(Cloudtasker.config.redis || {})
+    end
 
     #
     # Return the underlying redis client.
@@ -14,7 +20,7 @@ module Cloudtasker
     # @return [Redis] The redis client.
     #
     def client
-      @client ||= Redis.new(Cloudtasker.config.redis || {})
+      @client ||= self.class.client
     end
 
     #
@@ -47,6 +53,9 @@ module Cloudtasker
     #
     # Acquire a lock on a cache entry.
     #
+    # Locks are enforced to be short-lived (2s).
+    # The yielded block should limit its logic to short operations (e.g. redis get/set).
+    #
     # @example
     #   redis = RedisClient.new
     #   redis.with_lock('foo')
@@ -61,7 +70,7 @@ module Cloudtasker
 
       # Wait to acquire lock
       lock_key = [LOCK_KEY_PREFIX, cache_key].join('/')
-      true until client.setnx(lock_key, true)
+      sleep(LOCK_WAIT_DURATION) until client.set(lock_key, true, nx: true, ex: LOCK_DURATION)
 
       # yield content
       yield

data/lib/cloudtasker/testing.rb

@@ -114,10 +114,10 @@ module Cloudtasker
       #
       # Return all jobs related to this worker class.
       #
-      # @return [Array<Cloudtasker::Worker] The list of workers
+      # @return [Array<Cloudtasker::Backend::MemoryTask>] The list of tasks
       #
       def jobs
-        Backend::MemoryTask.jobs(to_s)
+        Backend::MemoryTask.all(to_s)
       end
 
       #

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.8.1'
+  VERSION = '0.10.rc4'
 end

data/lib/cloudtasker/worker.rb

@@ -7,7 +7,8 @@ module Cloudtasker
     def self.included(base)
       base.extend(ClassMethods)
       base.attr_writer :job_queue
-      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries
+      base.attr_accessor :job_args, :job_id, :job_meta, :job_reenqueued, :job_retries,
+                         :perform_started_at, :perform_ended_at
     end
 
     #
@@ -181,21 +182,19 @@ module Cloudtasker
     #
     def execute
       logger.info('Starting job...')
-      resp = Cloudtasker.config.server_middleware.invoke(self) do
-        begin
-          perform(*job_args)
-        rescue StandardError => e
-          try(:on_error, e)
-          return raise(e) unless job_dead?
 
-          # Flag job as dead
-          logger.info('Job dead')
-          try(:on_dead, e)
-          raise(DeadWorkerError, e)
-        end
-      end
-      logger.info('Job done')
+      # Perform job logic
+      resp = execute_middleware_chain
+
+      # Log job completion and return result
+      logger.info("Job done after #{job_duration}s") { { duration: job_duration } }
       resp
+    rescue DeadWorkerError => e
+      logger.info("Job dead after #{job_duration}s and #{job_retries} retries") { { duration: job_duration } }
+      raise(e)
+    rescue StandardError => e
+      logger.info("Job failed after #{job_duration}s") { { duration: job_duration } }
+      raise(e)
     end
 
     #
@@ -286,5 +285,46 @@ module Cloudtasker
     def job_dead?
       job_retries >= Cloudtasker.config.max_retries
     end
+
+    #
+    # Return the time taken (in seconds) to perform the job. This duration
+    # includes the middlewares and the actual perform method.
+    #
+    # @return [Float] The time taken in seconds as a floating point number.
+    #
+    def job_duration
+      return 0.0 unless perform_ended_at && perform_started_at
+
+      (perform_ended_at - perform_started_at).ceil(3)
+    end
+
+    #=============================
+    # Private
+    #=============================
+    private
+
+    #
+    # Execute the worker perform method through the middleware chain.
+    #
+    # @return [Any] The result of the perform method.
+    #
+    def execute_middleware_chain
+      self.perform_started_at = Time.now
+
+      Cloudtasker.config.server_middleware.invoke(self) do
+        begin
+          perform(*job_args)
+        rescue StandardError => e
+          try(:on_error, e)
+          return raise(e) unless job_dead?
+
+          # Flag job as dead
+          try(:on_dead, e)
+          raise(DeadWorkerError, e)
+        end
+      end
+    ensure
+      self.perform_ended_at = Time.now
+    end
   end
 end

data/lib/cloudtasker/worker_handler.rb

@@ -10,16 +10,113 @@ module Cloudtasker
     # Alrogith used to sign the verification token
     JWT_ALG = 'HS256'
 
+    # Sub-namespace to use for redis keys when storing
+    # payloads in Redis
+    REDIS_PAYLOAD_NAMESPACE = 'payload'
+
+    # Arg payload cache keys get expired instead of deleted
+    # in case jobs are re-processed due to connection interruption
+    # (job is successful but Cloud Task considers it as failed due
+    # to network interruption)
+    ARGS_PAYLOAD_CLEANUP_TTL = 3600 # 1 hour
+
+    #
+    # Return a namespaced key
+    #
+    # @param [String, Symbol] val The key to namespace
+    #
+    # @return [String] The namespaced key.
+    #
+    def self.key(val)
+      return nil if val.nil?
+
+      [to_s.underscore, val.to_s].join('/')
+    end
+
+    #
+    # Return the cloudtasker redis client
+    #
+    # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+    #
+    def self.redis
+      @redis ||= begin
+        require 'cloudtasker/redis_client'
+        RedisClient.new
+      end
+    end
+
     #
     # Execute a task worker from a task payload
     #
-    # @param [Hash] payload The Cloud Task payload.
+    # @param [Hash] input_payload The Cloud Task payload.
     #
     # @return [Any] The return value of the worker perform method.
     #
-    def self.execute_from_payload!(payload)
+    def self.execute_from_payload!(input_payload)
+      with_worker_handling(input_payload, &:execute)
+    end
+
+    # TODO: do not delete redis payload if job has been re-enqueued
+    # worker.job_reenqueued
+    #
+    # Idea: change with_worker_handling to with_worker_handling and build the worker
+    # inside the with_worker_handling block.
+    #
+    # Local middleware used to retrieve the job arg payload from cache
+    # if a arg payload reference is present.
+    #
+    # @param [Hash] payload The full job payload
+    #
+    # @yield [Hash] The actual payload to use to process the job.
+    #
+    # @return [Any] The block result
+    #
+    def self.with_worker_handling(input_payload)
+      # Extract payload information
+      extracted_payload = extract_payload(input_payload)
+      payload = extracted_payload[:payload]
+      args_payload_key = extracted_payload[:args_payload_key]
+
+      # Build worker
       worker = Cloudtasker::Worker.from_hash(payload) || raise(InvalidWorkerError)
-      worker.execute
+
+      # Yied worker
+      resp = yield(worker)
+
+      # Schedule args payload deletion after job has been successfully processed
+      # Note: we expire the key instead of deleting it immediately in case the job
+      # succeeds but is considered as failed by Cloud Task due to network interruption.
+      # In such case the job is likely to be re-processed soon after.
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key && !worker.job_reenqueued
+
+      resp
+    rescue DeadWorkerError => e
+      # Delete stored args payload if job is dead
+      redis.expire(args_payload_key, ARGS_PAYLOAD_CLEANUP_TTL) if args_payload_key
+      raise(e)
+    end
+
+    #
+    # Return the argument payload key (if present) along with the actual worker payload.
+    #
+    # If the payload was stored in Redis then retrieve it.
+    #
+    # @return [Hash] Hash
+    #
+    def self.extract_payload(input_payload)
+      # Get references
+      payload = JSON.parse(input_payload.to_json, symbolize_names: true)
+      args_payload_id = payload.delete(:job_args_payload_id)
+      args_payload_key = args_payload_id ? key([REDIS_PAYLOAD_NAMESPACE, args_payload_id].join('/')) : nil
+
+      # Retrieve the actual worker args payload
+      args_payload = args_payload_key ? redis.fetch(args_payload_key) : payload[:job_args]
+
+      # Return the payload
+      {
+        args_payload_key: args_payload_key,
+        payload: payload.merge(job_args: args_payload)
+      }
     end
 
     #
@@ -42,8 +139,8 @@ module Cloudtasker
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
-            'Content-Type' => 'application/json',
-            'Authorization' => "Bearer #{Authenticator.verification_token}"
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
           },
           body: worker_payload.to_json
         },
@@ -51,6 +148,47 @@ module Cloudtasker
       }
     end
 
+    #
+    # Return true if the worker args must be stored in Redis.
+    #
+    # @return [Boolean] True if the payload must be stored in redis.
+    #
+    def store_payload_in_redis?
+      Cloudtasker.config.redis_payload_storage_threshold &&
+        worker.job_args.to_json.bytesize > (Cloudtasker.config.redis_payload_storage_threshold * 1024)
+    end
+
+    #
+    # Return the payload to use for job arguments. This payload
+    # is merged inside the #worker_payload.
+    #
+    # If the argument payload must be stored in Redis then returns:
+    # `{ job_args_payload_id: <worker_id> }`
+    #
+    # If the argument payload must be natively handled by the backend
+    # then returns:
+    # `{ job_args: [...] }`
+    #
+    # @return [Hash] The worker args payload.
+    #
+    def worker_args_payload
+      @worker_args_payload ||= begin
+        if store_payload_in_redis?
+          # Store payload in Redis
+          self.class.redis.write(
+            self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
+            worker.job_args
+          )
+
+          # Return reference to args payload
+          { job_args_payload_id: worker.job_id }
+        else
+          # Return regular job args payload
+          { job_args: worker.job_args }
+        end
+      end
+    end
+
     #
     # Return the task payload that Google Task will eventually
     # send to the job processor.
@@ -68,9 +206,8 @@ module Cloudtasker
         worker: worker.job_class_name,
         job_queue: worker.job_queue,
         job_id: worker.job_id,
-        job_args: worker.job_args,
         job_meta: worker.job_meta.to_h
-      }
+      }.merge(worker_args_payload)
     end
 
     #

data/lib/cloudtasker/worker_logger.rb

@@ -59,7 +59,7 @@ module Cloudtasker
     # @return [String] The formatted log message
     #
     def formatted_message(msg)
-      "[Cloudtasker][#{worker.job_id}] #{msg}"
+      "[Cloudtasker][#{worker.class}][#{worker.job_id}] #{msg}"
     end
 
     #
@@ -141,7 +141,8 @@ module Cloudtasker
     # @param [Proc] &block Optional context block.
     #
     def log_message(level, msg, &block)
-      payload_block = block || log_block
+      # Merge log-specific context into worker-specific context
+      payload_block = -> { log_block.call.merge(block&.call || {}) }
 
       # ActiveSupport::Logger does not support passing a payload through a block on top
       # of a message.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.10.rc4
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-03 00:00:00.000000000 Z
+date: 2020-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -126,16 +126,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -168,16 +168,16 @@ dependencies:
   name: rubocop-rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.37.0
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
@@ -256,10 +256,10 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - Appraisals
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -320,6 +320,7 @@ files:
 - lib/cloudtasker/engine.rb
 - lib/cloudtasker/invalid_worker_error.rb
 - lib/cloudtasker/local_server.rb
+- lib/cloudtasker/max_task_size_exceeded_error.rb
 - lib/cloudtasker/meta_store.rb
 - lib/cloudtasker/middleware/chain.rb
 - lib/cloudtasker/redis_client.rb
@@ -363,12 +364,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.9
+rubygems_version: 3.0.0
 signing_key: 
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)

data/.travis.yml

@@ -1,16 +0,0 @@
----
-language: ruby
-cache: bundler
-rvm:
-  - 2.5.5
-services:
-  - redis-server
-before_install: gem install bundler -v 2.0.2
-before_script: bundle exec rubocop
-gemfile:
-  - gemfiles/google_cloud_tasks_1.0.gemfile
-  - gemfiles/google_cloud_tasks_1.1.gemfile
-  - gemfiles/google_cloud_tasks_1.2.gemfile
-  - gemfiles/google_cloud_tasks_1.3.gemfile
-  - gemfiles/rails_5.2.gemfile
-  - gemfiles/rails_6.0.gemfile