cloudtasker 0.13.2 → 0.14.rc1

data/docs/STORABLE_JOBS.md

@@ -0,0 +1,68 @@
+# Cloudtasker Storable Jobs
+
+**Supported since**: `v0.14.rc1`  
+**Note**: this extension requires redis
+
+The Cloudtasker storage extension allows you to park jobs in a specific garage lane and enqueue (pull) them when specific conditions have been met.
+
+This extension is useful when you need to prepare some jobs (e.g. you are retrieving data from an API and must process some of it asynchronously) but only process them when some programmatic conditions have been met (e.g. a series of preliminary preparation jobs have run successfully). Using parked jobs is a leaner (and cheaper) approach than using guard logic in the `perform` method to re-enqueue a job until a set of conditions is satisfied. The latter tends to generate a lot of jobs/logs pollution.
+
+## Configuration
+
+You can enable storable jobs by adding the following to your cloudtasker initializer:
+```ruby
+# The storable extension is optional and must be explicitly required
+require 'cloudtasker/storable'
+
+Cloudtasker.configure do |config|
+  # Specify your redis url.
+  # Defaults to `redis://localhost:6379/0` if unspecified
+  config.redis = { url: 'redis://some-host:6379/0' }
+end
+```
+
+Then you can make workers storable by including the `Cloudtasker::Storable::Worker` concern into your workers:
+```ruby
+class MyWorker
+  include Cloudtasker::Worker
+  include Cloudtasker::Storable::Worker
+
+  def perform(...)
+    # Do stuff
+  end
+end
+```
+
+## Parking jobs
+You can park jobs to a specific garage lane using the `push_to_store(store_name, *worker_args)` class method:
+```ruby
+MyWorker.push_to_store('some-customer-reference:some-task-group', job_arg1, job_arg2)
+```
+
+## Pulling jobs
+You can pull and enqueue jobs using the `pull_all_from_store(store_name)` class method:
+```ruby
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group')
+```
+
+If you need to enqueue jobs with specific options or using any special means, you can call `pull_all_from_store(store_name)` with a block. When a block is passed the method yield each worker's set of arguments.
+```ruby
+# Delay the enqueuing of parked jobs by 30 seconds
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  MyWorker.perform_in(30, *args)
+end
+
+# Enqueue parked jobs on a specific queue, with a 10s delay
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  MyWorker.schedule(args: args, time_in: 10, queue: 'critical')
+end
+
+# Enqueue parked jobs as part of a job's current batch (the logic below assumes
+# we are inside a job's `perform` method)
+MyWorker.pull_all_from_store('some-customer-reference:some-task-group') do |args|
+  batch.add(MyWorker, *args)
+
+  # Or with a specific queue
+  # batch.add_to_queue('critical', SubWorker, *args)
+end
+```

data/exe/cloudtasker

@@ -9,8 +9,11 @@ options = {}
 OptionParser.new do |opts|
   opts.banner = 'Usage: cloudtasker [options]'
 
-  opts.on('-q QUEUE', '--queue=QUEUE', 'Queue to process and number of threads. ' \
-    "Examples: '-q critical' | '-q critical,2' | '-q critical,3 -q defaults,2'") do |o|
+  opts.on(
+    '-q QUEUE', '--queue=QUEUE',
+    'Queue to process and number of threads. ' \
+    "Examples: '-q critical' | '-q critical,2' | '-q critical,3 -q defaults,2'"
+  ) do |o|
     options[:queues] ||= []
     options[:queues] << o.split(',')
   end

data/gemfiles/google_cloud_tasks_1.0.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.0.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.1.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.1.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.2.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.2.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.3.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.3.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.4.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.4.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_1.5.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 1.5.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_2.0.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 2.0.0"
 
 gemspec path: "../"

data/gemfiles/google_cloud_tasks_2.1.gemfile

@@ -2,7 +2,16 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "google-cloud-tasks", "~> 2.1.0"
 
 gemspec path: "../"

data/gemfiles/rails_5.2.gemfile

@@ -2,6 +2,16 @@
 
 source "https://rubygems.org"
 
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "rails", "~> 5.2.0"
 gem "rspec-rails"
 

data/gemfiles/rails_6.0.gemfile

@@ -2,6 +2,16 @@
 
 source "https://rubygems.org"
 
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "rails", "~> 6.0.0"
 gem "rspec-rails"
 

data/gemfiles/rails_6.1.gemfile

@@ -2,6 +2,16 @@
 
 source "https://rubygems.org"
 
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "rails", "~> 6.1.0"
 gem "rspec-rails"
 

data/gemfiles/rails_7.0.gemfile

@@ -2,6 +2,16 @@
 
 source "https://rubygems.org"
 
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
+gem "semantic_logger"
+gem "timecop"
+gem "webmock"
 gem "rails", "~> 7.0.0"
 gem "rspec-rails"
 

data/gemfiles/semantic_logger_3.4.gemfile

@@ -2,7 +2,15 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
 gem "semantic_logger", "3.4.1"
+gem "timecop"
+gem "webmock"
 
 gemspec path: "../"

data/gemfiles/semantic_logger_4.6.gemfile

@@ -2,7 +2,15 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
 gem "semantic_logger", "4.6.1"
+gem "timecop"
+gem "webmock"
 
 gemspec path: "../"

data/gemfiles/semantic_logger_4.7.0.gemfile

@@ -2,7 +2,15 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
 gem "semantic_logger", "4.7.0"
+gem "timecop"
+gem "webmock"
 
 gemspec path: "../"

data/gemfiles/semantic_logger_4.7.2.gemfile

@@ -2,7 +2,15 @@
 
 source "https://rubygems.org"
 
-gem "activesupport", "~> 6.1.0"
+gem "appraisal", github: "thoughtbot/appraisal"
+gem "bundler", "~> 2.0"
+gem "rake", ">= 12.3.3"
+gem "rspec", "~> 3.0"
+gem "rspec-json_expectations", "~> 2.2"
+gem "rubocop", "~> 1.64.1"
+gem "rubocop-rspec", "~> 3.0.1"
 gem "semantic_logger", "4.7.2"
+gem "timecop"
+gem "webmock"
 
 gemspec path: "../"

data/lib/active_job/queue_adapters/cloudtasker_adapter.rb

@@ -38,6 +38,13 @@ module ActiveJob
         build_worker(job).schedule(time_at: Time.at(precise_timestamp))
       end
 
+      # Determines if enqueuing will check and wait for an associated transaction completes before enqueuing
+      #
+      # @return [Boolean] True always as this is the default from QueueAdapters::AbstractAdapter
+      def enqueue_after_transaction_commit?
+        true
+      end
+
       private
 
       def build_worker(job)
@@ -53,7 +60,7 @@ module ActiveJob
       # == Job Wrapper for the Cloudtasker adapter
       #
       # Executes jobs scheduled by the Cloudtasker ActiveJob adapter
-      class JobWrapper #:nodoc:
+      class JobWrapper # :nodoc:
         include Cloudtasker::Worker
 
         # Executes the given serialized ActiveJob call.

data/lib/cloudtasker/authenticator.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'openssl'
+
 module Cloudtasker
   # Manage token generation and verification
   module Authenticator
@@ -27,6 +29,15 @@ module Cloudtasker
       JWT.encode({ iat: Time.now.to_i }, config.secret, JWT_ALG)
     end
 
+    #
+    # The Authorization header content
+    #
+    # @return [String] The Bearer authorization header
+    #
+    def bearer_token
+      "Bearer #{verification_token}"
+    end
+
     #
     # Verify a bearer token (jwt token)
     #
@@ -51,5 +62,29 @@ module Cloudtasker
     def verify!(bearer_token)
       verify(bearer_token) || raise(AuthenticationError)
     end
+
+    #
+    # Generate a signature for a payload
+    #
+    # @param [String] payload The JSON payload
+    #
+    # @return [String] The HMAC signature
+    #
+    def sign_payload(payload)
+      OpenSSL::HMAC.hexdigest('sha256', config.secret, payload)
+    end
+
+    #
+    # Verify that a signature matches the payload and raise a `Cloudtasker::AuthenticationError`
+    # if the signature is invalid.
+    #
+    # @param [String] signature The tested signature
+    # @param [String] payload The JSON payload
+    #
+    # @return [Boolean] Return true if the signature is valid
+    #
+    def verify_signature!(signature, payload)
+      ActiveSupport::SecurityUtils.secure_compare(signature, sign_payload(payload)) || raise(AuthenticationError)
+    end
   end
 end

data/lib/cloudtasker/backend/google_cloud_task_v1.rb

@@ -177,10 +177,8 @@ module Cloudtasker
       #
       # Helper method encapsulating the retry strategy for GAX calls
       #
-      def self.with_gax_retries
-        Retriable.retriable(on: [Google::Gax::UnavailableError], tries: 3) do
-          yield
-        end
+      def self.with_gax_retries(&block)
+        Retriable.retriable(on: [Google::Gax::UnavailableError], tries: 3, &block)
       end
 
       #

data/lib/cloudtasker/backend/google_cloud_task_v2.rb

@@ -110,7 +110,7 @@ module Cloudtasker
       #
       # @param [Hash] hash The worker payload.
       #
-      # @return [Hash] The Cloud Task payloadd.
+      # @return [Hash] The Cloud Task payload.
       #
       def self.format_task_payload(payload)
         payload = JSON.parse(payload.to_json, symbolize_names: true) # deep dup
@@ -180,10 +180,8 @@ module Cloudtasker
       #
       # Helper method encapsulating the retry strategy for Google API calls
       #
-      def self.with_gapi_retries
-        Retriable.retriable(on: [Google::Cloud::UnavailableError], tries: 3) do
-          yield
-        end
+      def self.with_gapi_retries(&block)
+        Retriable.retriable(on: [Google::Cloud::UnavailableError], tries: 3, &block)
       end
 
       #

data/lib/cloudtasker/backend/memory_task.rb

@@ -131,7 +131,7 @@ module Cloudtasker
       # @return [Hash] The task payload.
       #
       def payload
-        @payload ||= JSON.parse(http_request.dig(:body), symbolize_names: true)
+        @payload ||= JSON.parse(http_request[:body], symbolize_names: true)
       end
 
       #

data/lib/cloudtasker/backend/redis_task.rb

@@ -12,9 +12,9 @@ module Cloudtasker
       RETRY_INTERVAL = 20 # seconds
 
       #
-      # Return the cloudtasker redis client
+      # Return the Cloudtasker redis client
       #
-      # @return [Cloudtasker::RedisClient] The cloudtasker redis client..
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
       #
       def self.redis
         @redis ||= RedisClient.new
@@ -52,7 +52,7 @@ module Cloudtasker
       end
 
       #
-      # Reeturn all tasks ready to process.
+      # Return all tasks ready to process.
       #
       # @param [String] queue The queue to retrieve items from.
       #
@@ -215,6 +215,9 @@ module Cloudtasker
         end
 
         resp
+      rescue Errno::ECONNREFUSED
+        retry_later(RETRY_INTERVAL)
+        Cloudtasker.logger.info(format_log_message("Processor not ready - Retry in #{RETRY_INTERVAL} seconds..."))
       rescue Net::ReadTimeout
         retry_later(RETRY_INTERVAL)
         Cloudtasker.logger.info(
@@ -257,7 +260,10 @@ module Cloudtasker
         @http_client ||=
           begin
             uri = URI(http_request[:url])
-            Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = dispatch_deadline }
+            http = Net::HTTP.new(uri.host, uri.port).tap { |e| e.read_timeout = dispatch_deadline }
+            http.use_ssl = true if uri.instance_of?(URI::HTTPS)
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE unless Cloudtasker.config.local_server_ssl_verify
+            http
           end
       end
 

data/lib/cloudtasker/batch/batch_progress.rb

@@ -6,15 +6,28 @@ module Cloudtasker
   module Batch
     # Capture the progress of a batch
     class BatchProgress
-      attr_reader :batch_state
+      attr_reader :batches
 
       #
       # Build a new instance of the class.
       #
-      # @param [Hash] batch_state The batch state
+      # @param [Array<Cloudtasker::Batch::Job>] batches The batches to consider
       #
-      def initialize(batch_state = {})
-        @batch_state = batch_state
+      def initialize(batches = [])
+        @batches = batches
+      end
+
+      # Count the number of items in a given status
+
+      #
+      # Count the number of items in a given status
+      #
+      # @param [String] status The status to count
+      #
+      # @return [Integer] The number of jobs in the status
+      #
+      def count(status = 'all')
+        batches.sum { |e| e.batch_state_count(status) }
       end
 
       #
@@ -122,16 +135,7 @@ module Cloudtasker
       # @return [Cloudtasker::Batch::BatchProgress] The sum of the two batch progresses.
       #
       def +(other)
-        self.class.new(batch_state.to_h.merge(other.batch_state.to_h))
-      end
-
-      private
-
-      # Count the number of items in a given status
-      def count(status = nil)
-        return batch_state.to_h.keys.size unless status
-
-        batch_state.to_h.values.count { |e| e == status }
+        self.class.new(batches + other.batches)
       end
     end
   end