cloudtasker 0.13.1 → 0.14.rc1

data/lib/cloudtasker/unique_job/lock/until_executing.rb

@@ -10,11 +10,11 @@ module Cloudtasker
         # Acquire a lock for the job and trigger a conflict
         # if the lock could not be acquired.
         #
-        def schedule
+        def schedule(&block)
           job.lock!
           yield
         rescue LockError
-          conflict_instance.on_schedule { yield }
+          conflict_instance.on_schedule(&block)
         end
 
         #

data/lib/cloudtasker/unique_job/lock/while_executing.rb

@@ -10,11 +10,11 @@ module Cloudtasker
         # Acquire a lock for the job and trigger a conflict
         # if the lock could not be acquired.
         #
-        def execute
+        def execute(&block)
           job.lock!
           yield
         rescue LockError
-          conflict_instance.on_execute { yield }
+          conflict_instance.on_execute(&block)
         ensure
           # Unlock the job on any error to avoid deadlocks.
           job.unlock!

data/lib/cloudtasker/unique_job/middleware/client.rb

@@ -6,8 +6,8 @@ module Cloudtasker
       # TODO: kwargs to job otherwise it won't get the time_at
       # Client middleware, invoked when jobs are scheduled
       class Client
-        def call(worker, _opts = {})
-          Job.new(worker).lock_instance.schedule { yield }
+        def call(worker, _opts = {}, &block)
+          Job.new(worker).lock_instance.schedule(&block)
         end
       end
     end

data/lib/cloudtasker/unique_job/middleware/server.rb

@@ -5,8 +5,8 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker, **_kwargs)
-          Job.new(worker).lock_instance.execute { yield }
+        def call(worker, **_kwargs, &block)
+          Job.new(worker).lock_instance.execute(&block)
         end
       end
     end

data/lib/cloudtasker/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Cloudtasker
-  VERSION = '0.13.1'
+  VERSION = '0.14.rc1'
 end

data/lib/cloudtasker/worker.rb

@@ -54,6 +54,15 @@ module Cloudtasker
 
     # Module class methods
     module ClassMethods
+      #
+      # Return the Cloudtasker redis client
+      #
+      # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+      #
+      def redis
+        @redis ||= RedisClient.new
+      end
+
       #
       # Set the worker runtime options.
       #
@@ -63,7 +72,7 @@ module Cloudtasker
       #
       def cloudtasker_options(opts = {})
         opt_list = opts&.map { |k, v| [k.to_sym, v] } || [] # symbolize
-        @cloudtasker_options_hash = Hash[opt_list]
+        @cloudtasker_options_hash = opt_list.to_h
       end
 
       #
@@ -75,6 +84,17 @@ module Cloudtasker
         @cloudtasker_options_hash || {}
       end
 
+      #
+      # Return a namespaced cache key.
+      #
+      # @param [Any, Array<Any>, nil] val The key to namespace
+      #
+      # @return [String] The namespaced key(s).
+      #
+      def cache_key(val = nil)
+        [to_s.underscore, val].flatten.compact.map(&:to_s).join('/')
+      end
+
       #
       # Enqueue worker in the backgroundf.
       #
@@ -174,13 +194,13 @@ module Cloudtasker
     # @return [Integer] The value in seconds.
     #
     def dispatch_deadline
-      @dispatch_deadline ||= [
-        [
-          Config::MIN_DISPATCH_DEADLINE,
-          (self.class.cloudtasker_options_hash[:dispatch_deadline] || Cloudtasker.config.dispatch_deadline).to_i
-        ].max,
-        Config::MAX_DISPATCH_DEADLINE
-      ].min
+      @dispatch_deadline ||= begin
+        configured_deadline = (
+          self.class.cloudtasker_options_hash[:dispatch_deadline] ||
+          Cloudtasker.config.dispatch_deadline
+        ).to_i
+        configured_deadline.clamp(Config::MIN_DISPATCH_DEADLINE, Config::MAX_DISPATCH_DEADLINE)
+      end
     end
 
     #
@@ -204,13 +224,16 @@ module Cloudtasker
       resp = execute_middleware_chain
 
       # Log job completion and return result
-      logger.info("Job done after #{job_duration}s") { { duration: job_duration } }
+      logger.info("Job done after #{job_duration}s") { { duration: job_duration * 1000 } }
       resp
     rescue DeadWorkerError => e
-      logger.info("Job dead after #{job_duration}s and #{job_retries} retries") { { duration: job_duration } }
+      logger.info("Job dead after #{job_duration}s and #{job_retries} retries") { { duration: job_duration * 1000 } }
+      raise(e)
+    rescue RetryWorkerError => e
+      logger.info("Job done after #{job_duration}s (retry requested)") { { duration: job_duration * 1000 } }
       raise(e)
     rescue StandardError => e
-      logger.info("Job failed after #{job_duration}s") { { duration: job_duration } }
+      logger.info("Job failed after #{job_duration}s") { { duration: job_duration * 1000 } }
       raise(e)
     end
 
@@ -235,7 +258,7 @@ module Cloudtasker
     # @param [Integer] interval The delay in seconds.
     # @param [Time, Integer] interval The time at which the job should run
     #
-    # @return [Cloudtasker::CloudTask] The Google Task response
+    # @return [Cloudtasker::CloudTask, nil] The Google Task response or nil if the job was not scheduled
     #
     def schedule(**args)
       # Evaluate when to schedule the job
@@ -323,7 +346,7 @@ module Cloudtasker
     # @return [Integer] The number of retries
     #
     def job_max_retries
-      @job_max_retries ||= (try(:max_retries, *job_args) || self.class.max_retries)
+      @job_max_retries ||= try(:max_retries, *job_args) || self.class.max_retries
     end
 
     #
@@ -373,7 +396,7 @@ module Cloudtasker
     def job_duration
       return 0.0 unless perform_ended_at && perform_started_at
 
-      (perform_ended_at - perform_started_at).ceil(3)
+      @job_duration ||= (perform_ended_at - perform_started_at).ceil(3)
     end
 
     #
@@ -422,7 +445,7 @@ module Cloudtasker
           # Perform the job
           perform(*job_args)
         rescue StandardError => e
-          run_callback(:on_error, e)
+          run_callback(:on_error, e) unless e.is_a?(RetryWorkerError)
           return raise(e) unless job_must_die?
 
           # Flag job as dead

data/lib/cloudtasker/worker_handler.rb

@@ -53,6 +53,9 @@ module Cloudtasker
       # ActiveJob::QueueAdapters::CloudtaskerAdapter::JobWrapper might not be loaded
       return if worker.class.to_s =~ /^ActiveJob::/
 
+      # Do not log error when a retry was specifically requested
+      return if error.is_a?(RetryWorkerError)
+
       # Choose logger to use based on context
       # Worker will be nil on InvalidWorkerError - in that case we use generic logging
       logger = worker&.logger || Cloudtasker.logger
@@ -148,16 +151,21 @@ module Cloudtasker
     # @return [Hash] The task body
     #
     def task_payload
+      # Generate content
+      worker_payload_json = worker_payload.to_json
+
+      # Build payload
       {
         http_request: {
           http_method: 'POST',
           url: Cloudtasker.config.processor_url,
           headers: {
             Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
-            Cloudtasker::Config::AUTHORIZATION_HEADER => "Bearer #{Authenticator.verification_token}"
-          },
-          body: worker_payload.to_json
-        },
+            Cloudtasker::Config::CT_SIGNATURE_HEADER => Authenticator.sign_payload(worker_payload_json)
+          }.compact,
+          oidc_token: Cloudtasker.config.oidc,
+          body: worker_payload_json
+        }.compact,
         dispatch_deadline: worker.dispatch_deadline.to_i,
         queue: worker.job_queue
       }
@@ -187,21 +195,19 @@ module Cloudtasker
     # @return [Hash] The worker args payload.
     #
     def worker_args_payload
-      @worker_args_payload ||= begin
-        if store_payload_in_redis?
-          # Store payload in Redis
-          self.class.redis.write(
-            self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
-            worker.job_args
-          )
-
-          # Return reference to args payload
-          { job_args_payload_id: worker.job_id }
-        else
-          # Return regular job args payload
-          { job_args: worker.job_args }
-        end
-      end
+      @worker_args_payload ||= if store_payload_in_redis?
+                                 # Store payload in Redis
+                                 self.class.redis.write(
+                                   self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
+                                   worker.job_args
+                                 )
+
+                                 # Return reference to args payload
+                                 { job_args_payload_id: worker.job_id }
+                               else
+                                 # Return regular job args payload
+                                 { job_args: worker.job_args }
+                               end
     end
 
     #

data/lib/cloudtasker/worker_logger.rb

@@ -8,6 +8,54 @@ module Cloudtasker
 
     class << self
       attr_accessor :log_context_processor
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      #
+      # Truncate an array or hash payload.
+      #
+      # This can be used to log arguments on jobs while still keeping logs to a
+      # reasonable size.
+      #
+      # @param [Hash,Array] payload The payload to truncate
+      # @param [Integer] string_limit The maximum size for strings. Set to -1 to disable.
+      # @param [Integer] array_limit The maximum length for arrays. Set to -1 to disable.
+      # @param [Hash] max_depth The maximum recursive depth. Set to -1 to disable.
+      #
+      # @return [Hash,Array] The truncated payload
+      #
+      def truncate(payload, **kwargs)
+        depth = kwargs[:depth].to_i
+        max_depth = kwargs[:max_depth] || 3
+        string_limit = kwargs[:string_limit] || 64
+        array_limit = kwargs[:array_limit] || 10
+
+        case payload
+        when Array
+          if max_depth > -1 && depth > max_depth
+            ["...#{payload.size} items..."]
+          elsif array_limit > -1
+            payload.take(array_limit).map { |e| truncate(e, **kwargs, depth: depth + 1) } +
+              (payload.size > array_limit ? ["...#{payload.size - array_limit} items..."] : [])
+          else
+            payload.map { |e| truncate(e, **kwargs, depth: depth + 1) }
+          end
+        when Hash
+          if max_depth > -1 && depth > max_depth
+            '{hash}'
+          else
+            payload.transform_values { |e| truncate(e, **kwargs, depth: depth + 1) }
+          end
+        when String
+          if string_limit > -1 && payload.size > string_limit
+            payload.truncate(string_limit)
+          else
+            payload
+          end
+        else
+          payload
+        end
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
     end
 
     # Only log the job meta information by default (exclude arguments)

data/lib/cloudtasker.rb

@@ -3,12 +3,15 @@
 require 'active_support/core_ext/object/blank'
 require 'active_support/core_ext/object/try'
 require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/string/filters'
+require 'active_support/security_utils'
 
 require 'cloudtasker/version'
 require 'cloudtasker/config'
 
 require 'cloudtasker/authentication_error'
 require 'cloudtasker/dead_worker_error'
+require 'cloudtasker/retry_worker_error'
 require 'cloudtasker/invalid_worker_error'
 require 'cloudtasker/missing_worker_arguments_error'
 require 'cloudtasker/max_task_size_exceeded_error'
@@ -51,4 +54,4 @@ module Cloudtasker
   end
 end
 
-require 'cloudtasker/engine' if defined?(::Rails::Engine)
+require 'cloudtasker/engine' if defined?(Rails::Engine)

data/lib/tasks/setup_queue.rake

@@ -7,14 +7,14 @@ ENV['GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS'] ||= 'true'
 
 namespace :cloudtasker do
   desc 'Setup a Cloud Task queue. (default options: ' \
-    "name=#{Cloudtasker::Config::DEFAULT_JOB_QUEUE}, " \
-    "concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
-    "retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
+       "name=#{Cloudtasker::Config::DEFAULT_JOB_QUEUE}, " \
+       "concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
+       "retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
   task setup_queue: :environment do
     puts Cloudtasker::CloudTask.setup_production_queue(
-      name: ENV['name'],
-      concurrency: ENV['concurrency'],
-      retries: ENV['retries']
+      name: ENV.fetch('name', nil),
+      concurrency: ENV.fetch('concurrency', nil),
+      retries: ENV.fetch('retries', nil)
     )
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloudtasker
 version: !ruby/object:Gem::Version
-  version: 0.13.1
+  version: 0.14.rc1
 platform: ruby
 authors:
 - Arnaud Lachaume
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-06-19 00:00:00.000000000 Z
+date: 2024-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -108,146 +108,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 12.3.3
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: rspec-json_expectations
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.2'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.76.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 0.76.0
-- !ruby/object:Gem::Dependency
-  name: rubocop-rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.37.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
-      - !ruby/object:Gem::Version
-        version: 1.37.0
-- !ruby/object:Gem::Dependency
-  name: semantic_logger
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: timecop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Background jobs for Ruby using Google Cloud Tasks (beta)
 email:
 - arnaud.lachaume@keypup.io
@@ -257,7 +117,6 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/lint_rubocop.yml"
-- ".github/workflows/test_ruby_2.6.yml"
 - ".github/workflows/test_ruby_2.7.yml"
 - ".github/workflows/test_ruby_3.x.yml"
 - ".gitignore"
@@ -278,6 +137,7 @@ files:
 - config/routes.rb
 - docs/BATCH_JOBS.md
 - docs/CRON_JOBS.md
+- docs/STORABLE_JOBS.md
 - docs/UNIQUE_JOBS.md
 - exe/cloudtasker
 - gemfiles/.bundle/config
@@ -328,6 +188,9 @@ files:
 - lib/cloudtasker/middleware/chain.rb
 - lib/cloudtasker/missing_worker_arguments_error.rb
 - lib/cloudtasker/redis_client.rb
+- lib/cloudtasker/retry_worker_error.rb
+- lib/cloudtasker/storable.rb
+- lib/cloudtasker/storable/worker.rb
 - lib/cloudtasker/testing.rb
 - lib/cloudtasker/unique_job.rb
 - lib/cloudtasker/unique_job/conflict_strategy/base_strategy.rb
@@ -357,6 +220,7 @@ metadata:
   homepage_uri: https://github.com/keypup-io/cloudtasker
   source_code_uri: https://github.com/keypup-io/cloudtasker
   changelog_uri: https://github.com/keypup-io/cloudtasker/master/tree/CHANGELOG.md
+  rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -365,14 +229,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.8
+rubygems_version: 3.5.4
 signing_key:
 specification_version: 4
 summary: Background jobs for Ruby using Google Cloud Tasks (beta)

data/.github/workflows/test_ruby_2.6.yml

@@ -1,37 +0,0 @@
-name: Ruby 2.6
-
-on: [push, pull_request]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        ruby:
-          - '2.6'
-        appraisal:
-          - 'google_cloud_tasks_1.0'
-          - 'google_cloud_tasks_1.1'
-          - 'google_cloud_tasks_1.2'
-          - 'google_cloud_tasks_1.3'
-          - 'google_cloud_tasks_1.4'
-          - 'google_cloud_tasks_1.5'
-          - 'google_cloud_tasks_2.0'
-          - 'google_cloud_tasks_2.1'
-          - 'rails_5.2'
-          - 'rails_6.0'
-          - 'rails_6.1'
-          - 'semantic_logger_3.4'
-          - 'semantic_logger_4.6'
-          - 'semantic_logger_4.7.0'
-          - 'semantic_logger_4.7.2'
-    env:
-      BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.appraisal }}.gemfile
-    steps:
-      - uses: actions/checkout@v2
-      - uses: zhulik/redis-action@1.1.0
-      - uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
-      - run: bundle exec rspec