cloudtasker-tonix 0.1.0

data/lib/cloudtasker/worker_handler.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require 'google/cloud/tasks'
+
+module Cloudtasker
+  # Build, serialize and schedule tasks on the processing backend.
+  class WorkerHandler
+    attr_reader :worker
+
+    # Alrogith used to sign the verification token
+    JWT_ALG = 'HS256'
+
+    # Sub-namespace to use for redis keys when storing
+    # payloads in Redis
+    REDIS_PAYLOAD_NAMESPACE = 'payload'
+
+    #
+    # Return a namespaced key
+    #
+    # @param [String, Symbol] val The key to namespace
+    #
+    # @return [String] The namespaced key.
+    #
+    def self.key(val)
+      return nil if val.nil?
+
+      [to_s.underscore, val.to_s].join('/')
+    end
+
+    #
+    # Return the cloudtasker redis client
+    #
+    # @return [Cloudtasker::RedisClient] The cloudtasker redis client.
+    #
+    def self.redis
+      @redis ||= begin
+        require 'cloudtasker/redis_client'
+        RedisClient.new
+      end
+    end
+
+    #
+    # Log error on execution failure.
+    #
+    # @param [Cloudtasker::Worker, nil] worker The worker.
+    # @param [Exception] error The error to log.
+    #
+    # @void
+    #
+    def self.log_execution_error(worker, error)
+      # ActiveJob has its own error logging. No need to double log the error.
+      # Note: we use string matching instead of class matching as
+      # ActiveJob::QueueAdapters::CloudtaskerAdapter::JobWrapper might not be loaded
+      return if worker.class.to_s =~ /^ActiveJob::/
+
+      # Do not log error when a retry was specifically requested
+      return if error.is_a?(RetryWorkerError)
+
+      # Choose logger to use based on context
+      # Worker will be nil on InvalidWorkerError - in that case we use generic logging
+      logger = worker&.logger || Cloudtasker.logger
+
+      # Log error
+      logger.error(error)
+    end
+
+    #
+    # Execute a task worker from a task payload
+    #
+    # @param [Hash] input_payload The Cloud Task payload.
+    #
+    # @return [Any] The return value of the worker perform method.
+    #
+    def self.execute_from_payload!(input_payload)
+      with_worker_handling(input_payload, &:execute)
+    end
+
+    #
+    # Local middleware used to retrieve the job arg payload from cache
+    # if a arg payload reference is present.
+    #
+    # @param [Hash] payload The full job payload
+    #
+    # @yield [Hash] The actual payload to use to process the job.
+    #
+    # @return [Any] The block result
+    #
+    def self.with_worker_handling(input_payload)
+      # Extract payload information
+      extracted_payload = extract_payload(input_payload)
+      payload = extracted_payload[:payload]
+      args_payload_key = extracted_payload[:args_payload_key]
+
+      # Build worker
+      worker = Cloudtasker::Worker.from_hash(payload) || raise(InvalidWorkerError)
+
+      # Yied worker
+      resp = yield(worker)
+
+      # Delete stored args payload if job has completed
+      redis.del(args_payload_key) if args_payload_key && !worker.job_reenqueued
+
+      resp
+    rescue DeadWorkerError => e
+      # Delete stored args payload if job is dead
+      redis.del(args_payload_key) if args_payload_key
+      log_execution_error(worker, e)
+      Cloudtasker.config.on_dead.call(e, worker)
+      raise(e)
+    rescue StandardError => e
+      log_execution_error(worker, e)
+      Cloudtasker.config.on_error.call(e, worker)
+      raise(e)
+    end
+
+    #
+    # Return the argument payload key (if present) along with the actual worker payload.
+    #
+    # If the payload was stored in Redis then retrieve it.
+    #
+    # @return [Hash] Hash
+    #
+    def self.extract_payload(input_payload)
+      # Get references
+      payload = JSON.parse(input_payload.to_json, symbolize_names: true)
+      args_payload_id = payload.delete(:job_args_payload_id)
+      args_payload_key = args_payload_id ? key([REDIS_PAYLOAD_NAMESPACE, args_payload_id].join('/')) : nil
+
+      # Retrieve the actual worker args payload
+      args_payload = args_payload_key ? redis.fetch(args_payload_key) : payload[:job_args]
+
+      # Return the payload
+      {
+        args_payload_key: args_payload_key,
+        payload: payload.merge(job_args: args_payload)
+      }
+    end
+
+    #
+    # Prepare a new cloud task.
+    #
+    # @param [Cloudtasker::Worker] worker The worker instance.
+    #
+    def initialize(worker)
+      @worker = worker
+    end
+
+    #
+    # Return the full task configuration sent to Cloud Task
+    #
+    # @return [Hash] The task body
+    #
+    def task_payload
+      # Generate content
+      worker_payload_json = worker_payload.to_json
+
+      # Build payload
+      {
+        http_request: {
+          http_method: 'POST',
+          url: Cloudtasker.config.processor_url,
+          headers: {
+            Cloudtasker::Config::CONTENT_TYPE_HEADER => 'application/json',
+            Cloudtasker::Config::CT_SIGNATURE_HEADER => Authenticator.sign_payload(worker_payload_json)
+          }.compact,
+          oidc_token: Cloudtasker.config.oidc,
+          body: worker_payload_json
+        }.compact,
+        dispatch_deadline: worker.dispatch_deadline.to_i,
+        queue: worker.job_queue
+      }
+    end
+
+    #
+    # Return true if the worker args must be stored in Redis.
+    #
+    # @return [Boolean] True if the payload must be stored in redis.
+    #
+    def store_payload_in_redis?
+      Cloudtasker.config.redis_payload_storage_threshold &&
+        worker.job_args.to_json.bytesize > (Cloudtasker.config.redis_payload_storage_threshold * 1024)
+    end
+
+    #
+    # Return the payload to use for job arguments. This payload
+    # is merged inside the #worker_payload.
+    #
+    # If the argument payload must be stored in Redis then returns:
+    # `{ job_args_payload_id: <worker_id> }`
+    #
+    # If the argument payload must be natively handled by the backend
+    # then returns:
+    # `{ job_args: [...] }`
+    #
+    # @return [Hash] The worker args payload.
+    #
+    def worker_args_payload
+      @worker_args_payload ||= if store_payload_in_redis?
+                                 # Store payload in Redis
+                                 self.class.redis.write(
+                                   self.class.key([REDIS_PAYLOAD_NAMESPACE, worker.job_id].join('/')),
+                                   worker.job_args
+                                 )
+
+                                 # Return reference to args payload
+                                 { job_args_payload_id: worker.job_id }
+                               else
+                                 # Return regular job args payload
+                                 { job_args: worker.job_args }
+                               end
+    end
+
+    #
+    # Return the task payload that Google Task will eventually
+    # send to the job processor.
+    #
+    # The payload includes the worker name and the arguments to
+    # pass to the worker.
+    #
+    # The worker arguments should use primitive types as much
+    # as possible as all arguments will be serialized to JSON.
+    #
+    # @return [Hash] The job payload
+    #
+    def worker_payload
+      @worker_payload ||= {
+        worker: worker.job_class_name,
+        job_queue: worker.job_queue,
+        job_id: worker.job_id,
+        job_meta: worker.job_meta.to_h
+      }.merge(worker_args_payload)
+    end
+
+    #
+    # Schedule the task on GCP Cloud Task.
+    #
+    # @param [Integer, nil] time_at A unix timestamp specifying when to run the job.
+    #   Leave to `nil` to run now.
+    #
+    # @return [Cloudtasker::CloudTask] The Google Task response
+    #
+    def schedule(time_at: nil)
+      # Generate task payload
+      task = task_payload.merge(schedule_time: time_at).compact
+
+      # Create and return remote task
+      CloudTask.create(task)
+    end
+  end
+end

data/lib/cloudtasker/worker_logger.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # Add contextual information to logs generated
+  # by workers
+  class WorkerLogger
+    attr_accessor :worker
+
+    class << self
+      attr_accessor :log_context_processor
+
+      # rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+      #
+      # Truncate an array or hash payload.
+      #
+      # This can be used to log arguments on jobs while still keeping logs to a
+      # reasonable size.
+      #
+      # @param [Hash,Array] payload The payload to truncate
+      # @param [Integer] string_limit The maximum size for strings. Set to -1 to disable.
+      # @param [Integer] array_limit The maximum length for arrays. Set to -1 to disable.
+      # @param [Hash] max_depth The maximum recursive depth. Set to -1 to disable.
+      #
+      # @return [Hash,Array] The truncated payload
+      #
+      def truncate(payload, **kwargs)
+        depth = kwargs[:depth].to_i
+        max_depth = kwargs[:max_depth] || 3
+        string_limit = kwargs[:string_limit] || 64
+        array_limit = kwargs[:array_limit] || 10
+
+        case payload
+        when Array
+          if max_depth > -1 && depth > max_depth
+            ["...#{payload.size} items..."]
+          elsif array_limit > -1
+            payload.take(array_limit).map { |e| truncate(e, **kwargs, depth: depth + 1) } +
+              (payload.size > array_limit ? ["...#{payload.size - array_limit} items..."] : [])
+          else
+            payload.map { |e| truncate(e, **kwargs, depth: depth + 1) }
+          end
+        when Hash
+          if max_depth > -1 && depth > max_depth
+            '{hash}'
+          else
+            payload.transform_values { |e| truncate(e, **kwargs, depth: depth + 1) }
+          end
+        when String
+          if string_limit > -1 && payload.size > string_limit
+            payload.truncate(string_limit)
+          else
+            payload
+          end
+        else
+          payload
+        end
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    end
+
+    # Only log the job meta information by default (exclude arguments)
+    DEFAULT_CONTEXT_PROCESSOR = ->(worker) { worker.to_h.slice(:worker, :job_id, :job_meta, :job_queue, :task_id) }
+
+    #
+    # Build a new instance of the class.
+    #
+    # @param [Cloudtasker::Worker] worker The worker.
+    #
+    def initialize(worker)
+      @worker = worker
+    end
+
+    #
+    # Return the Proc responsible for formatting the log payload.
+    #
+    # @return [Proc] The context processor.
+    #
+    def context_processor
+      @context_processor ||= worker.class.cloudtasker_options_hash[:log_context_processor] ||
+                             self.class.log_context_processor ||
+                             DEFAULT_CONTEXT_PROCESSOR
+    end
+
+    #
+    # The block to pass to log messages.
+    #
+    # @return [Proc] The log block.
+    #
+    def log_block
+      @log_block ||= proc { context_processor.call(worker) }
+    end
+
+    #
+    # Return the Cloudtasker logger.
+    #
+    # @return [Logger, any] The cloudtasker logger.
+    #
+    def logger
+      Cloudtasker.logger
+    end
+
+    #
+    # Format the log message as string.
+    #
+    # @param [Object] msg The log message or object.
+    #
+    # @return [String] The formatted message
+    #
+    def formatted_message_as_string(msg)
+      # Format message
+      msg_content = if msg.is_a?(Exception)
+                      [msg.inspect, msg.backtrace].flatten(1).join("\n")
+                    elsif msg.is_a?(String)
+                      msg
+                    else
+                      msg.inspect
+                    end
+
+      "[Cloudtasker][#{worker.class}][#{worker.job_id}] #{msg_content}"
+    end
+
+    #
+    # Format main log message.
+    #
+    # @param [String] msg The message to log.
+    #
+    # @return [String] The formatted log message
+    #
+    def formatted_message(msg)
+      if msg.is_a?(String)
+        formatted_message_as_string(msg)
+      else
+        # Delegate object formatting to logger
+        msg
+      end
+    end
+
+    #
+    # Log an info message.
+    #
+    # @param [String] msg The message to log.
+    # @param [Proc] &block Optional context block.
+    #
+    def info(msg, &block)
+      log_message(:info, msg, &block)
+    end
+
+    #
+    # Log an error message.
+    #
+    # @param [String] msg The message to log.
+    # @param [Proc] &block Optional context block.
+    #
+    def error(msg, &block)
+      log_message(:error, msg, &block)
+    end
+
+    #
+    # Log an fatal message.
+    #
+    # @param [String] msg The message to log.
+    # @param [Proc] &block Optional context block.
+    #
+    def fatal(msg, &block)
+      log_message(:fatal, msg, &block)
+    end
+
+    #
+    # Log an debut message.
+    #
+    # @param [String] msg The message to log.
+    # @param [Proc] &block Optional context block.
+    #
+    def debug(msg, &block)
+      log_message(:debug, msg, &block)
+    end
+
+    #
+    # Delegate all methods to the underlying logger.
+    #
+    # @param [String, Symbol] name The method to delegate.
+    # @param [Array<any>] *args The list of method arguments.
+    # @param [Proc] &block Block passed to the method.
+    #
+    # @return [Any] The method return value
+    #
+    def method_missing(name, ...)
+      if logger.respond_to?(name)
+        logger.send(name, ...)
+      else
+        super
+      end
+    end
+
+    #
+    # Check if the class respond to a certain method.
+    #
+    # @param [String, Symbol] name The name of the method.
+    # @param [Boolean] include_private Whether to check private methods or not. Default to false.
+    #
+    # @return [Boolean] Return true if the class respond to this method.
+    #
+    def respond_to_missing?(name, include_private = false)
+      logger.respond_to?(name) || super
+    end
+
+    private
+
+    #
+    # Log a message for the provided log level.
+    #
+    # @param [String, Symbol] level The log level
+    # @param [String] msg The message to log.
+    # @param [Proc] &block Optional context block.
+    #
+    def log_message(level, msg, &block)
+      # Merge log-specific context into worker-specific context
+      payload_block = ->(*_args) { log_block.call.merge(block&.call || {}) }
+
+      # ActiveSupport::Logger does not support passing a payload through a block on top
+      # of a message.
+      if defined?(ActiveSupport::Logger) && logger.is_a?(ActiveSupport::Logger)
+        # The logger is fairly basic in terms of formatting. All inputs get converted
+        # as regular strings.
+        logger.send(level) { "#{formatted_message_as_string(msg)} -- #{payload_block.call}" }
+      else
+        logger.send(level, formatted_message(msg), &payload_block)
+      end
+    end
+  end
+end

data/lib/cloudtasker/worker_wrapper.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/worker'
+
+module Cloudtasker
+  # A worker class used to schedule jobs without actually
+  # instantiating the worker class. This is useful for middlewares
+  # needing to enqueue jobs in a Rails initializer. Rails 6 complains
+  # about instantiating workers in an iniitializer because of autoloading
+  # in zeitwerk mode.
+  #
+  # Downside of this wrapper: any cloudtasker_options specified on on the
+  # worker_class will be ignored.
+  #
+  # See: https://github.com/rails/rails/issues/36363
+  #
+  class WorkerWrapper
+    include Worker
+
+    attr_accessor :worker_name
+
+    #
+    # Build a new instance of the class.
+    #
+    # @param [String] worker_class The name of the worker class.
+    # @param [Hash] **opts The worker arguments.
+    #
+    def initialize(worker_name:, **opts)
+      @worker_name = worker_name
+      super(**opts)
+    end
+
+    #
+    # Override parent. Return the underlying worker class name.
+    #
+    # @return [String] The worker class.
+    #
+    def job_class_name
+      worker_name
+    end
+
+    #
+    # Return a new instance of the worker with the same args and metadata
+    # but with a different id.
+    #
+    # @return [Cloudtasker::WorkerWrapper] <description>
+    #
+    def new_instance
+      self.class.new(worker_name: worker_name, job_queue: job_queue, job_args: job_args, job_meta: job_meta)
+    end
+  end
+end

data/lib/cloudtasker.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/try'
+require 'active_support/core_ext/string/inflections'
+require 'active_support/core_ext/string/filters'
+require 'active_support/security_utils'
+
+require 'cloudtasker/version'
+require 'cloudtasker/config'
+
+require 'cloudtasker/authentication_error'
+require 'cloudtasker/dead_worker_error'
+require 'cloudtasker/retry_worker_error'
+require 'cloudtasker/invalid_worker_error'
+require 'cloudtasker/missing_worker_arguments_error'
+require 'cloudtasker/max_task_size_exceeded_error'
+
+require 'cloudtasker/middleware/chain'
+require 'cloudtasker/authenticator'
+require 'cloudtasker/cloud_task'
+require 'cloudtasker/worker_logger'
+require 'cloudtasker/worker_handler'
+require 'cloudtasker/meta_store'
+require 'cloudtasker/worker'
+
+# Define and manage Cloud Task based workers
+module Cloudtasker
+  attr_writer :config
+
+  #
+  # Cloudtasker configurator.
+  #
+  def self.configure
+    yield(config)
+  end
+
+  #
+  # Return the Cloudtasker configuration.
+  #
+  # @return [Cloudtasker::Config] The Cloudtasker configuration.
+  #
+  def self.config
+    @config ||= Config.new
+  end
+
+  #
+  # Return the Cloudtasker logger.
+  #
+  # @return [Logger] The Cloudtasker logger.
+  #
+  def self.logger
+    config.logger
+  end
+end
+
+require 'cloudtasker/engine' if defined?(Rails::Engine)

data/lib/tasks/setup_queue.rake

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'cloudtasker/config'
+require 'cloudtasker/cloud_task'
+
+ENV['GOOGLE_AUTH_SUPPRESS_CREDENTIALS_WARNINGS'] ||= 'true'
+
+namespace :cloudtasker do
+  desc 'Setup a Cloud Task queue. (default options: ' \
+       "name=#{Cloudtasker::Config::DEFAULT_JOB_QUEUE}, " \
+       "concurrency=#{Cloudtasker::Config::DEFAULT_QUEUE_CONCURRENCY}, " \
+       "retries=#{Cloudtasker::Config::DEFAULT_QUEUE_RETRIES})"
+  task setup_queue: :environment do
+    puts Cloudtasker::CloudTask.setup_production_queue(
+      name: ENV.fetch('name', nil),
+      concurrency: ENV.fetch('concurrency', nil),
+      retries: ENV.fetch('retries', nil)
+    )
+  end
+end