cloudtasker 0.13.1 → 0.14.rc1

data/lib/cloudtasker/batch/job.rb

@@ -24,6 +24,7 @@ module Cloudtasker
       # means that the job will never succeed. There is no point in blocking
       # the batch forever so we proceed forward eventually.
       #
+      BATCH_STATUSES = %w[scheduled processing completed errored dead all].freeze
       COMPLETION_STATUSES = %w[completed dead].freeze
 
       # These callbacks do not need to raise errors on their own
@@ -183,12 +184,40 @@ module Cloudtasker
       end
 
       #
-      # The list of jobs in the batch
+      # Return the key under which the batch progress is stored
+      # for a specific state.
+      #
+      # @return [String] The batch progress state namespaced id.
+      #
+      def batch_state_count_gid(state)
+        "#{batch_state_gid}/state_count/#{state}"
+      end
+
+      #
+      # Return the number of jobs in a given state
+      #
+      # @return [String] The batch progress state namespaced id.
+      #
+      def batch_state_count(state)
+        redis.get(batch_state_count_gid(state)).to_i
+      end
+
+      #
+      # The list of jobs to be enqueued in the batch
       #
       # @return [Array<Cloudtasker::Worker>] The jobs to enqueue at the end of the batch.
       #
-      def jobs
-        @jobs ||= []
+      def pending_jobs
+        @pending_jobs ||= []
+      end
+
+      #
+      # The list of jobs that have been enqueued as part of the batch
+      #
+      # @return [Array<Cloudtasker::Worker>] The jobs enqueued as part of the batch.
+      #
+      def enqueued_jobs
+        @enqueued_jobs ||= []
       end
 
       #
@@ -208,7 +237,7 @@ module Cloudtasker
       # @param [Class] worker_klass The worker class.
       # @param [Array<any>] *args The worker arguments.
       #
-      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      # @return [Array<Cloudtasker::Worker>] The updated list of pending jobs.
       #
       def add(worker_klass, *args)
         add_to_queue(worker.job_queue, worker_klass, *args)
@@ -221,10 +250,10 @@ module Cloudtasker
       # @param [Class] worker_klass The worker class.
       # @param [Array<any>] *args The worker arguments.
       #
-      # @return [Array<Cloudtasker::Worker>] The updated list of jobs.
+      # @return [Array<Cloudtasker::Worker>] The updated list of pending jobs.
       #
       def add_to_queue(queue, worker_klass, *args)
-        jobs << worker_klass.new(
+        pending_jobs << worker_klass.new(
           job_args: args,
           job_meta: { key(:parent_id) => batch_id },
           job_queue: queue
@@ -249,6 +278,28 @@ module Cloudtasker
         end
       end
 
+      #
+      # This method initializes the batch job counters if not set already
+      #
+      def migrate_progress_stats_to_redis_counters
+        # Abort if counters have already been set. The 'all' counter acts as a feature flag.
+        return if redis.exists?(batch_state_count_gid('all'))
+
+        # Get all job states
+        values = batch_state.values
+
+        # Count by value
+        redis.multi do |m|
+          # Per status
+          values.tally.each do |k, v|
+            m.set(batch_state_count_gid(k), v)
+          end
+
+          # All counter
+          m.set(batch_state_count_gid('all'), values.size)
+        end
+      end
+
       #
       # Save serialized version of the worker.
       #
@@ -269,8 +320,17 @@ module Cloudtasker
       def update_state(batch_id, status)
         migrate_batch_state_to_redis_hash
 
+        # Get current status
+        current_status = redis.hget(batch_state_gid, batch_id)
+        return if current_status == status.to_s
+
         # Update the batch state batch_id entry with the new status
-        redis.hset(batch_state_gid, batch_id, status)
+        # and update counters
+        redis.multi do |m|
+          m.hset(batch_state_gid, batch_id, status)
+          m.decr(batch_state_count_gid(current_status))
+          m.incr(batch_state_count_gid(status))
+        end
       end
 
       #
@@ -295,7 +355,14 @@ module Cloudtasker
       # @return [any] The callback return value
       #
       def run_worker_callback(callback, *args)
-        worker.try(callback, *args)
+        worker.try(callback, *args).tap do
+          # Enqueue pending jobs if batch was expanded in callback
+          # A completed batch cannot receive additional jobs
+          schedule_pending_jobs if callback.to_sym != :on_batch_complete
+
+          # Schedule pending jobs on parent if batch was expanded
+          parent_batch&.schedule_pending_jobs
+        end
       rescue StandardError => e
         # There is no point in retrying jobs due to failure callbacks failing
         # Only completion callbacks will trigger a re-run of the job because
@@ -369,8 +436,11 @@ module Cloudtasker
         redis.hkeys(batch_state_gid).each { |id| self.class.find(id)&.cleanup }
 
         # Delete batch redis entries
-        redis.del(batch_gid)
-        redis.del(batch_state_gid)
+        redis.multi do |m|
+          m.del(batch_gid)
+          m.del(batch_state_gid)
+          BATCH_STATUSES.each { |e| m.del(batch_state_count_gid(e)) }
+        end
       end
 
       #
@@ -384,33 +454,31 @@ module Cloudtasker
       def progress(depth: 0)
         depth = depth.to_i
 
-        # Capture batch state
-        state = batch_state
+        # Initialize counters from batch state. This is only applicable to running batches
+        # that started before the counter-based progress was implemented/released.
+        migrate_progress_stats_to_redis_counters
 
         # Return immediately if we do not need to go down the tree
-        return BatchProgress.new(state) if depth <= 0
+        return BatchProgress.new([self]) if depth <= 0
 
         # Sum batch progress of current batch and sub-batches up to the specified
         # depth
-        state.to_h.reduce(BatchProgress.new(state)) do |memo, (child_id, child_status)|
-          memo + (self.class.find(child_id)&.progress(depth: depth - 1) ||
-            BatchProgress.new(child_id => child_status))
+        batch_state.to_h.reduce(BatchProgress.new([self])) do |memo, (child_id, _)|
+          memo + (self.class.find(child_id)&.progress(depth: depth - 1) || BatchProgress.new)
         end
       end
 
       #
-      # Save the batch and enqueue all child workers attached to it.
+      # Schedule the child workers that were added to the batch
       #
-      def setup
-        return true if jobs.empty?
+      def schedule_pending_jobs
+        ret_list = []
 
-        # Save batch
-        save
-
-        # Schedule all child workers
-        jobs.each do |j|
+        while (j = pending_jobs.shift)
           # Schedule the job
-          j.schedule
+          # Skip batch registration if the job was not actually scheduled
+          # E.g. the job was evicted due to uniqueness requirements
+          next unless j.schedule
 
           # Initialize the batch state unless the job has already started (and taken
           # hold of its own status)
@@ -418,15 +486,39 @@ module Cloudtasker
           # having never-ending batches - which could occur if a batch was crashing
           # while enqueuing children due to a OOM error and since 'scheduled' is a
           # blocking status.
-          redis.hsetnx(batch_state_gid, j.job_id, 'scheduled')
+          redis.multi do |m|
+            m.hsetnx(batch_state_gid, j.job_id, 'scheduled')
+            m.incr(batch_state_count_gid('scheduled'))
+            m.incr(batch_state_count_gid('all'))
+          end
+
+          # Flag job as enqueued
+          ret_list << j
+          enqueued_jobs << j
         end
+
+        # Return the list of jobs just enqueued
+        ret_list
+      end
+
+      #
+      # Save the batch and enqueue all child workers attached to it.
+      #
+      def setup
+        return true if pending_jobs.empty?
+
+        # Save batch
+        save
+
+        # Schedule all child workers
+        schedule_pending_jobs
       end
 
       #
       # Post-perform logic. The parent batch is notified if the job is complete.
       #
       def complete(status = :completed)
-        return true if reenqueued? || jobs.any?
+        return true if reenqueued?
 
         # Notify the parent batch that a child is complete
         on_complete(status) if complete?
@@ -445,11 +537,12 @@ module Cloudtasker
         # Perform job
         yield
 
-        # Save batch if child jobs added
-        setup if jobs.any?
+        # Setup batch
+        # Only applicable if the batch has pending_jobs
+        setup
 
-        # Save parent batch if batch expanded
-        parent_batch&.setup if parent_batch&.jobs&.any?
+        # Save parent batch if batch was expanded
+        parent_batch&.schedule_pending_jobs
 
         # Complete batch
         complete(:completed)

data/lib/cloudtasker/batch/middleware/server.rb

@@ -5,8 +5,8 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker, **_kwargs)
-          Job.for(worker).execute { yield }
+        def call(worker, **_kwargs, &block)
+          Job.for(worker).execute(&block)
         end
       end
     end

data/lib/cloudtasker/cli.rb

@@ -58,7 +58,7 @@ module Cloudtasker
     #
     def boot_system
       # Sync logs
-      STDOUT.sync = true
+      $stdout.sync = true
 
       # Check for Rails
       return false unless File.exist?('./config/environment.rb')
@@ -103,7 +103,7 @@ module Cloudtasker
     def run_server(read_pipe, opts = {})
       local_server.start(opts)
 
-      while (readable_io = IO.select([read_pipe]))
+      while (readable_io = read_pipe.wait_readable)
         signal = readable_io.first[0].gets.strip
         handle_signal(signal)
       end
@@ -124,11 +124,9 @@ module Cloudtasker
       # USR1 and USR2 don't work on the JVM
       sigs << 'USR2' unless jruby?
       sigs.each do |sig|
-        begin
-          trap(sig) { write_pipe.puts(sig) }
-        rescue ArgumentError
-          puts "Signal #{sig} not supported"
-        end
+        trap(sig) { write_pipe.puts(sig) }
+      rescue ArgumentError
+        puts "Signal #{sig} not supported"
       end
     end
 

data/lib/cloudtasker/cloud_task.rb

@@ -19,17 +19,15 @@ module Cloudtasker
       # Re-evaluate backend every time if testing mode enabled
       @backend = nil if defined?(Cloudtasker::Testing)
 
-      @backend ||= begin
-        if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.in_memory?
-          require 'cloudtasker/backend/memory_task'
-          Backend::MemoryTask
-        elsif Cloudtasker.config.mode.to_sym == :development
-          require 'cloudtasker/backend/redis_task'
-          Backend::RedisTask
-        else
-          gct_backend
-        end
-      end
+      @backend ||= if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.in_memory?
+                     require 'cloudtasker/backend/memory_task'
+                     Backend::MemoryTask
+                   elsif Cloudtasker.config.mode.to_sym == :development
+                     require 'cloudtasker/backend/redis_task'
+                     Backend::RedisTask
+                   else
+                     gct_backend
+                   end
     end
 
     #
@@ -42,15 +40,13 @@ module Cloudtasker
     # ] The google cloud task backend.
     #
     def self.gct_backend
-      @gct_backend ||= begin
-        if !defined?(Google::Cloud::Tasks::VERSION) || Google::Cloud::Tasks::VERSION < '2'
-          require 'cloudtasker/backend/google_cloud_task_v1'
-          Backend::GoogleCloudTaskV1
-        else
-          require 'cloudtasker/backend/google_cloud_task_v2'
-          Backend::GoogleCloudTaskV2
-        end
-      end
+      @gct_backend ||= if !defined?(Google::Cloud::Tasks::VERSION) || Google::Cloud::Tasks::VERSION < '2'
+                         require 'cloudtasker/backend/google_cloud_task_v1'
+                         Backend::GoogleCloudTaskV1
+                       else
+                         require 'cloudtasker/backend/google_cloud_task_v2'
+                         Backend::GoogleCloudTaskV2
+                       end
     end
 
     #

data/lib/cloudtasker/config.rb

@@ -8,23 +8,18 @@ module Cloudtasker
     attr_accessor :redis, :store_payloads_in_redis, :gcp_queue_prefix
     attr_writer :secret, :gcp_location_id, :gcp_project_id,
                 :processor_path, :logger, :mode, :max_retries,
-                :dispatch_deadline, :on_error, :on_dead
+                :dispatch_deadline, :on_error, :on_dead, :oidc, :local_server_ssl_verify
 
     # Max Cloud Task size in bytes
     MAX_TASK_SIZE = 100 * 1024 # 100 KB
 
     # Retry header in Cloud Task responses
     #
-    # TODO: use 'X-CloudTasks-TaskExecutionCount' instead of 'X-CloudTasks-TaskRetryCount'
-    #   'X-CloudTasks-TaskExecutionCount' is currently bugged and remains at 0 even on retries.
-    #
-    # See bug: https://issuetracker.google.com/issues/154532072
-    #
     # Definitions:
     #   X-CloudTasks-TaskRetryCount: total number of retries (including 504 "instance unreachable")
     #   X-CloudTasks-TaskExecutionCount: number of non-503 retries (= actual number of job failures)
     #
-    RETRY_HEADER = 'X-CloudTasks-TaskRetryCount'
+    RETRY_HEADER = 'X-Cloudtasks-Taskexecutioncount'
 
     # Cloud Task ID header
     TASK_ID_HEADER = 'X-CloudTasks-TaskName'
@@ -35,12 +30,18 @@ module Cloudtasker
     # Content Type
     CONTENT_TYPE_HEADER = 'Content-Type'
 
-    # Authorization header
-    AUTHORIZATION_HEADER = 'Authorization'
+    # OIDC Authorization header
+    OIDC_AUTHORIZATION_HEADER = 'Authorization'
+
+    # Custom authentication header that does not conflict with
+    # OIDC authorization header
+    CT_AUTHORIZATION_HEADER = 'X-Cloudtasker-Authorization'
+    CT_SIGNATURE_HEADER = 'X-Cloudtasker-Signature'
 
     # Default values
     DEFAULT_LOCATION_ID = 'us-east1'
     DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
+    DEFAULT_LOCAL_SERVER_SSL_VERIFY_MODE = true
 
     # Default queue values
     DEFAULT_JOB_QUEUE = 'default'
@@ -55,6 +56,10 @@ module Cloudtasker
     # Default on_error Proc
     DEFAULT_ON_ERROR = ->(error, worker) {}
 
+    # Cache key prefix used to store workers in cache and retrieve
+    # them later.
+    WORKER_STORE_PREFIX = 'worker_store'
+
     # The number of times jobs will be attempted before declaring them dead.
     #
     # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
@@ -78,6 +83,10 @@ module Cloudtasker
       Missing cloudtasker secret.
       Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
     DOC
+    OIDC_EMAIL_MISSING_ERROR = <<~DOC
+      Missing OpenID Connect (OIDC) service account email.
+      You specified an OIDC configuration hash but the :service_account_email property is missing.
+    DOC
 
     #
     # Return the threshold above which job arguments must be stored
@@ -131,7 +140,7 @@ module Cloudtasker
     # @return [Logger, any] The cloudtasker logger.
     #
     def logger
-      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new(STDOUT)
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stdout)
     end
 
     #
@@ -244,6 +253,21 @@ module Cloudtasker
       @on_dead || DEFAULT_ON_ERROR
     end
 
+    #
+    # Return the Open ID Connect configuration to use for tasks.
+    #
+    # @return [Hash] The OIDC configuration
+    #
+    def oidc
+      return unless @oidc
+      raise(StandardError, OIDC_EMAIL_MISSING_ERROR) unless @oidc[:service_account_email]
+
+      {
+        service_account_email: @oidc[:service_account_email],
+        audience: @oidc[:audience] || processor_host
+      }
+    end
+
     #
     # Return the chain of client middlewares.
     #
@@ -265,5 +289,14 @@ module Cloudtasker
       yield @server_middleware if block_given?
       @server_middleware
     end
+
+    #
+    # Return the ssl verify mode for the Cloudtasker local server.
+    #
+    # @return [Boolean] The ssl verify mode for the Cloudtasker local server.
+    #
+    def local_server_ssl_verify
+      @local_server_ssl_verify.nil? ? DEFAULT_LOCAL_SERVER_SSL_VERIFY_MODE : @local_server_ssl_verify
+    end
   end
 end

data/lib/cloudtasker/cron/middleware/server.rb

@@ -5,8 +5,8 @@ module Cloudtasker
     module Middleware
       # Server middleware, invoked when jobs are executed
       class Server
-        def call(worker, **_kwargs)
-          Job.new(worker).execute { yield }
+        def call(worker, **_kwargs, &block)
+          Job.new(worker).execute(&block)
         end
       end
     end

data/lib/cloudtasker/cron/schedule.rb

@@ -159,6 +159,9 @@ module Cloudtasker
       #
       def valid?
         id && cron_schedule && worker
+      rescue ArgumentError
+        # Rescue invalid cron expressions
+        false
       end
 
       #
@@ -225,7 +228,7 @@ module Cloudtasker
       # @return [Fugit::Cron] The cron schedule.
       #
       def cron_schedule
-        @cron_schedule ||= Fugit::Cron.parse(cron)
+        @cron_schedule ||= Fugit::Cron.do_parse(cron)
       end
 
       #
@@ -255,7 +258,7 @@ module Cloudtasker
       #
       def assign_attributes(opts)
         opts
-          .select { |k, _| instance_variables.include?("@#{k}".to_sym) }
+          .select { |k, _| instance_variables.include?(:"@#{k}") }
           .each { |k, v| instance_variable_set("@#{k}", v) }
       end
 

data/lib/cloudtasker/middleware/chain.rb

@@ -159,7 +159,7 @@ module Cloudtasker
       def insert_after(oldklass, newklass, *args)
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
-        i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
+        i = entries.index { |entry| entry.klass == oldklass } || (entries.count - 1)
         entries.insert(i + 1, new_entry)
       end
 

data/lib/cloudtasker/redis_client.rb

@@ -12,7 +12,7 @@ module Cloudtasker
     LOCK_WAIT_DURATION = 0.03 # seconds
 
     # Default pool size used for Redis
-    DEFAULT_POOL_SIZE = ENV.fetch('RAILS_MAX_THREADS') { 25 }
+    DEFAULT_POOL_SIZE = ENV.fetch('RAILS_MAX_THREADS', 25)
     DEFAULT_POOL_TIMEOUT = 5
 
     def self.client
@@ -132,7 +132,6 @@ module Cloudtasker
       list
     end
 
-    # rubocop:disable Style/MissingRespondToMissing
     if RUBY_VERSION < '3'
       #
       # Delegate all methods to the redis client.
@@ -172,8 +171,6 @@ module Cloudtasker
         end
       end
     end
-    # rubocop:enable Style/MissingRespondToMissing
-
     #
     # Check if the class respond to a certain method.
     #

data/lib/cloudtasker/retry_worker_error.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  class RetryWorkerError < StandardError
+  end
+end

data/lib/cloudtasker/storable/worker.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  module Storable
+    # Add ability to store and pull workers in Redis under a specific namespace
+    module Worker
+      # Add class method to including class
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Module class methods
+      module ClassMethods
+        #
+        # Return the namespaced store key used to store jobs that
+        # have been parked and should be manually popped later.
+        #
+        # @param [String] namespace The user-provided store namespace
+        #
+        # @return [String] The full store cache key
+        #
+        def store_cache_key(namespace)
+          cache_key([Config::WORKER_STORE_PREFIX, namespace])
+        end
+
+        #
+        # Push the worker to a namespaced store.
+        #
+        # @param [String] namespace The store namespace
+        # @param [Array<any>] *args List of worker arguments
+        #
+        # @return [String] The number of elements added to the store
+        #
+        def push_to_store(namespace, *args)
+          redis.rpush(store_cache_key(namespace), [args.to_json])
+        end
+
+        #
+        # Push many workers to a namespaced store at once.
+        #
+        # @param [String] namespace The store namespace
+        # @param [Array<Array<any>>] args_list A list of arguments for each worker
+        #
+        # @return [String] The number of elements added to the store
+        #
+        def push_many_to_store(namespace, args_list)
+          redis.rpush(store_cache_key(namespace), args_list.map(&:to_json))
+        end
+
+        #
+        # Pull the jobs from the namespaced store and enqueue them.
+        #
+        # @param [String] namespace The store namespace.
+        # @param [Integer] page_size The number of items to pull on each page. Defaults to 1000.
+        #
+        def pull_all_from_store(namespace, page_size: 1000)
+          items = nil
+
+          while items.nil? || items.present?
+            # Pull items
+            items = redis.lpop(store_cache_key(namespace), page_size).to_a
+
+            # For each item, execute block or enqueue it
+            items.each do |args_json|
+              worker_args = JSON.parse(args_json)
+
+              if block_given?
+                yield(worker_args)
+              else
+                perform_async(*worker_args)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloudtasker/storable.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require_relative 'storable/worker'

data/lib/cloudtasker/unique_job/conflict_strategy/base_strategy.rb

@@ -20,8 +20,10 @@ module Cloudtasker
         # Handling logic to perform when a conflict occurs while
         # scheduling a job.
         #
+        # We return nil to flag the job as not scheduled
+        #
         def on_schedule
-          true
+          nil
         end
 
         #
@@ -29,7 +31,7 @@ module Cloudtasker
         # executing a job.
         #
         def on_execute
-          true
+          nil
         end
       end
     end

data/lib/cloudtasker/unique_job/lock/until_executed.rb

@@ -10,22 +10,22 @@ module Cloudtasker
         # Acquire a lock for the job and trigger a conflict
         # if the lock could not be acquired.
         #
-        def schedule
+        def schedule(&block)
           job.lock!
           yield
         rescue LockError
-          conflict_instance.on_schedule { yield }
+          conflict_instance.on_schedule(&block)
         end
 
         #
         # Acquire a lock for the job and trigger a conflict
         # if the lock could not be acquired.
         #
-        def execute
+        def execute(&block)
           job.lock!
           yield
         rescue LockError
-          conflict_instance.on_execute { yield }
+          conflict_instance.on_execute(&block)
         ensure
           # Unlock the job on any error to avoid deadlocks.
           job.unlock!