cloudtasker-tonix 0.1.0

data/lib/cloudtasker/cli.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require 'cloudtasker'
+require 'cloudtasker/local_server'
+
+module Cloudtasker
+  # Cloudtasker executable logic
+  module CLI
+    module_function
+
+    #
+    # Return the current environment.
+    #
+    # @return [String] The environment name.
+    #
+    def environment
+      Cloudtasker.config.environment
+    end
+
+    #
+    # Return true if we are running in Rails.
+    #
+    # @return [Boolean] True if rails is loaded.
+    #
+    def rails_app?
+      defined?(::Rails)
+    end
+
+    #
+    # Return true if we are running in JRuby.
+    #
+    # @return [Boolean] True if JRuby is loaded.
+    #
+    def jruby?
+      defined?(::JRUBY_VERSION)
+    end
+
+    #
+    # Return the Cloudtasker logger
+    #
+    # @return [Logger] The Cloudtasker logger.
+    #
+    def logger
+      Cloudtasker.logger
+    end
+
+    #
+    # Return the local Cloudtasker server.
+    #
+    # @return [Cloudtasker::LocalServer] The local Cloudtasker server.
+    #
+    def local_server
+      @local_server ||= LocalServer.new
+    end
+
+    #
+    # Load Rails if defined
+    #
+    def boot_system
+      # Sync logs
+      $stdout.sync = true
+
+      # Check for Rails
+      return false unless File.exist?('./config/environment.rb')
+
+      require 'rails'
+      require 'cloudtasker/engine'
+      require File.expand_path('./config/environment.rb')
+    end
+
+    #
+    # Run the cloudtasker development server.
+    #
+    def run(opts = {})
+      boot_system
+
+      # Print banner
+      environment == 'development' ? print_banner : print_non_dev_warning
+
+      # Print rails info
+      if rails_app?
+        logger.info "[Cloudtasker/Server] Booted Rails #{::Rails.version} application in #{environment} environment"
+      end
+
+      # Get internal read/write pipe
+      self_read, self_write = IO.pipe
+
+      # Setup signals to trap
+      setup_signals(self_write)
+
+      logger.info "[Cloudtasker/Server] Running in #{RUBY_DESCRIPTION}"
+
+      # Wait for signals
+      run_server(self_read, opts)
+    end
+
+    #
+    # Run server and wait for signals.
+    #
+    # @param [IO] read_pipe Where to read signals.
+    # @param [Hash] opts Server options.
+    #
+    def run_server(read_pipe, opts = {})
+      local_server.start(opts)
+
+      while (readable_io = read_pipe.wait_readable)
+        signal = readable_io.first.chomp
+        handle_signal(signal)
+      end
+    rescue Interrupt
+      logger.info 'Shutting down'
+      local_server.stop
+      logger.info 'Stopped'
+    end
+
+    #
+    # Define which signals to trap
+    #
+    # @param [IO] write_pipe Where to write signals.
+    #
+    def setup_signals(write_pipe)
+      # Display signals on log output
+      sigs = %w[INT TERM TTIN TSTP]
+      # USR1 and USR2 don't work on the JVM
+      sigs << 'USR2' unless jruby?
+      sigs.each do |sig|
+        trap(sig) { write_pipe.puts(sig) }
+      rescue ArgumentError
+        puts "Signal #{sig} not supported"
+      end
+    end
+
+    #
+    # Handle process signals
+    #
+    # @param [String] sig The signal.
+    #
+    def handle_signal(sig)
+      raise(Interrupt) if %w[INT TERM].include?(sig)
+    end
+
+    #
+    # Return the server banner
+    #
+    # @return [String] The server banner
+    #
+    def banner
+      <<~'TEXT'
+           ___ _                 _ _            _
+          / __\ | ___  _   _  __| | |_ __ _ ___| | _____ _ __
+         / /  | |/ _ \| | | |/ _` | __/ _` / __| |/ / _ \ '__|
+        / /___| | (_) | |_| | (_| | || (_| \__ \   <  __/ |
+        \____/|_|\___/ \__,_|\__,_|\__\__,_|___/_|\_\___|_|
+
+      TEXT
+    end
+
+    #
+    # Display a warning message when run in non-dev env.
+    #
+    # @return [<Type>] <description>
+    #
+    def print_non_dev_warning
+      puts "\e[31m"
+      puts non_dev_warning_message
+      puts "\e[0m"
+    end
+
+    #
+    # Return the message to display when users attempt to run
+    # the local development server in non-dev environments.
+    #
+    # @return [String] The warning message.
+    #
+    def non_dev_warning_message
+      <<~'TEXT'
+        ============================================ /!\ ====================================================
+        Your are running the Cloudtasker local development server in a NON-DEVELOPMENT environment.
+        This is not recommended as the the development server is not designed for production-like load.
+        If you need a job processing server to run yourself please use Sidekiq instead (https://sidekiq.org)
+        ============================================ /!\ ====================================================
+      TEXT
+    end
+
+    #
+    # Print the server banner
+    #
+    def print_banner
+      puts "\e[96m"
+      puts banner
+      puts "\e[0m"
+    end
+  end
+end

data/lib/cloudtasker/cloud_task.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+module Cloudtasker
+  # An interface class to manage tasks on the backend (Cloud Task or Redis)
+  class CloudTask
+    attr_accessor :id, :http_request, :schedule_time, :retries, :queue, :dispatch_deadline
+
+    #
+    # The backend to use for cloud tasks.
+    #
+    # @return [
+    #   Backend::MemoryTask,
+    #   Cloudtasker::Backend::GoogleCloudTaskV1,
+    #   Cloudtasker::Backend::GoogleCloudTaskV2,
+    #   Cloudtasker::Backend::RedisTask
+    # ] The cloud task backend.
+    #
+    def self.backend
+      # Re-evaluate backend every time if testing mode enabled
+      @backend = nil if defined?(Cloudtasker::Testing)
+
+      @backend ||= if defined?(Cloudtasker::Testing) && Cloudtasker::Testing.in_memory?
+                     require 'cloudtasker/backend/memory_task'
+                     Backend::MemoryTask
+                   elsif Cloudtasker.config.mode.to_sym == :development
+                     require 'cloudtasker/backend/redis_task'
+                     Backend::RedisTask
+                   else
+                     gct_backend
+                   end
+    end
+
+    #
+    # Return the GoogleCloudTaskV* backend to use based on the version
+    # of the currently installed google-cloud-tasks gem
+    #
+    # @return [
+    #   Cloudtasker::Backend::GoogleCloudTaskV1,
+    #   Cloudtasker::Backend::GoogleCloudTaskV2
+    # ] The google cloud task backend.
+    #
+    def self.gct_backend
+      @gct_backend ||= if !defined?(Google::Cloud::Tasks::VERSION) || Google::Cloud::Tasks::VERSION < '2'
+                         require 'cloudtasker/backend/google_cloud_task_v1'
+                         Backend::GoogleCloudTaskV1
+                       else
+                         require 'cloudtasker/backend/google_cloud_task_v2'
+                         Backend::GoogleCloudTaskV2
+                       end
+    end
+
+    #
+    # Create the google cloud task queue based on provided parameters if it does not exist already.
+    #
+    # @param [String] :name The queue name
+    # @param [Integer] :concurrency The queue concurrency
+    # @param [Integer] :retries The number of retries for the queue
+    #
+    # @return [Google::Cloud::Tasks::V2::Queue, Google::Cloud::Tasks::V2beta3::Queue] The queue
+    #
+    def self.setup_production_queue(**kwargs)
+      gct_backend.setup_queue(**kwargs)
+    end
+
+    #
+    # Find a cloud task by id.
+    #
+    # @param [String] id The id of the task.
+    #
+    # @return [Cloudtasker::Cloudtask] The task.
+    #
+    def self.find(id)
+      payload = backend.find(id)&.to_h
+      payload ? new(**payload) : nil
+    end
+
+    #
+    # Create a new cloud task.
+    #
+    # @param [Hash] payload Thee task payload
+    #
+    # @return [Cloudtasker::CloudTask] The created task.
+    #
+    def self.create(payload)
+      raise MaxTaskSizeExceededError if payload.to_json.bytesize > Config::MAX_TASK_SIZE
+
+      resp = backend.create(payload)&.to_h
+      resp ? new(**resp) : nil
+    end
+
+    #
+    # Delete a cloud task by id.
+    #
+    # @param [String] id The task id.
+    #
+    def self.delete(id)
+      backend.delete(id)
+    end
+
+    #
+    # Build a new instance of the class using a backend response
+    # payload.
+    #
+    # @param [String] id The task id.
+    # @param [Hash] http_request The content of the http request.
+    # @param [Integer] schedule_time When to run the job (Unix timestamp)
+    # @param [Integer] retries The number of times the job failed.
+    # @param [String] queue The queue the task is in.
+    #
+    def initialize(id:, http_request:, schedule_time: nil, retries: 0, queue: nil, dispatch_deadline: nil)
+      @id = id
+      @http_request = http_request
+      @schedule_time = schedule_time
+      @retries = retries || 0
+      @queue = queue
+      @dispatch_deadline = dispatch_deadline
+    end
+
+    #
+    # Equality operator.
+    #
+    # @param [Any] other The object to compare.
+    #
+    # @return [Boolean] True if the object is equal.
+    #
+    def ==(other)
+      other.is_a?(self.class) && other.id == id
+    end
+  end
+end

data/lib/cloudtasker/config.rb

@@ -0,0 +1,319 @@
+# frozen_string_literal: true
+
+require 'logger'
+
+module Cloudtasker
+  # Holds cloudtasker configuration. See Cloudtasker#configure
+  class Config
+    attr_accessor :redis, :store_payloads_in_redis, :gcp_queue_prefix
+    attr_writer :secret, :gcp_location_id, :gcp_project_id,
+                :processor_path, :logger, :mode, :max_retries,
+                :dispatch_deadline, :on_error, :on_dead, :oidc, :local_server_ssl_verify,
+                :base64_encode_body
+
+    # Max Cloud Task size in bytes
+    MAX_TASK_SIZE = 100 * 1024 # 100 KB
+
+    # Retry header in Cloud Task responses
+    #
+    # Definitions:
+    #   X-CloudTasks-TaskRetryCount: total number of retries (including 504 "instance unreachable")
+    #   X-CloudTasks-TaskExecutionCount: number of non-503 retries (= actual number of job failures)
+    #
+    RETRY_HEADER = 'X-Cloudtasks-Taskexecutioncount'
+
+    # Cloud Task ID header
+    TASK_ID_HEADER = 'X-CloudTasks-TaskName'
+
+    # Content-Transfer-Encoding header in Cloud Task responses
+    ENCODING_HEADER = 'Content-Transfer-Encoding'
+
+    # Content Type
+    CONTENT_TYPE_HEADER = 'Content-Type'
+
+    # OIDC Authorization header
+    OIDC_AUTHORIZATION_HEADER = 'Authorization'
+
+    # Custom authentication header that does not conflict with
+    # OIDC authorization header
+    CT_AUTHORIZATION_HEADER = 'X-Cloudtasker-Authorization'
+    CT_SIGNATURE_HEADER = 'X-Cloudtasker-Signature'
+
+    # Default values
+    DEFAULT_LOCATION_ID = 'us-east1'
+    DEFAULT_PROCESSOR_PATH = '/cloudtasker/run'
+    DEFAULT_LOCAL_SERVER_SSL_VERIFY_MODE = true
+
+    # Default queue values
+    DEFAULT_JOB_QUEUE = 'default'
+    DEFAULT_QUEUE_CONCURRENCY = 10
+    DEFAULT_QUEUE_RETRIES = -1 # unlimited
+
+    # Job timeout configuration for Cloud Tasks
+    DEFAULT_DISPATCH_DEADLINE = 10 * 60 # 10 minutes
+    MIN_DISPATCH_DEADLINE = 15 # seconds
+    MAX_DISPATCH_DEADLINE = 30 * 60 # 30 minutes
+
+    # Default on_error Proc
+    DEFAULT_ON_ERROR = ->(error, worker) {}
+
+    # Default base64 encoding flag
+    DEFAULT_BASE64_ENCODE_BODY = true
+
+    # Cache key prefix used to store workers in cache and retrieve
+    # them later.
+    WORKER_STORE_PREFIX = 'worker_store'
+
+    # The number of times jobs will be attempted before declaring them dead.
+    #
+    # With the default retry configuration (maxDoublings = 16 and minBackoff = 0.100s)
+    # it means that jobs will be declared dead after 20h of consecutive failing.
+    #
+    # Note that this configuration parameter is internal to Cloudtasker and does not
+    # affect the Cloud Task queue configuration. The number of retries configured
+    # on the Cloud Task queue should be higher than the number below to also cover
+    # failures due to the instance being unreachable.
+    DEFAULT_MAX_RETRY_ATTEMPTS = 25
+
+    # How long to wait between retries in local server mode
+    LOCAL_SERVER_RETRY_DELAY = (ENV['CLOUDTASKER_LOCAL_RETRY_DELAY'] || 20).to_i # seconds
+
+    PROCESSOR_HOST_MISSING = <<~DOC
+      Missing host for processing.
+      Please specify a processor hostname in form of `https://some-public-dns.example.com`'
+    DOC
+    PROJECT_ID_MISSING_ERROR = <<~DOC
+      Missing GCP project ID.
+      Please specify a project ID in the cloudtasker configurator.
+    DOC
+    SECRET_MISSING_ERROR = <<~DOC
+      Missing cloudtasker secret.
+      Please specify a secret in the cloudtasker initializer or add Rails secret_key_base in your credentials
+    DOC
+    OIDC_EMAIL_MISSING_ERROR = <<~DOC
+      Missing OpenID Connect (OIDC) service account email.
+      You specified an OIDC configuration hash but the :service_account_email property is missing.
+    DOC
+
+    #
+    # Return the threshold above which job arguments must be stored
+    # in Redis instead of being sent to the backend as part of the job
+    # payload.
+    #
+    # Return nil if redis payload storage is disabled.
+    #
+    # @return [Integer, nil] The threshold above which payloads will be stored in Redis.
+    #
+    def redis_payload_storage_threshold
+      return nil unless store_payloads_in_redis
+
+      store_payloads_in_redis.respond_to?(:to_i) ? store_payloads_in_redis.to_i : 0
+    end
+
+    #
+    # The number of times jobs will be retried. This number of
+    # retries does not include failures due to the application being unreachable.
+    #
+    #
+    # @return [Integer] The number of retries
+    #
+    def max_retries
+      @max_retries ||= DEFAULT_MAX_RETRY_ATTEMPTS
+    end
+
+    #
+    # The operating mode.
+    #   - :production => process tasks via GCP Cloud Task.
+    #   - :development => process tasks locally via Redis.
+    #
+    # @return [<Type>] <description>
+    #
+    def mode
+      @mode ||= environment == 'development' ? :development : :production
+    end
+
+    #
+    # Return the current environment.
+    #
+    # @return [String] The environment name.
+    #
+    def environment
+      ENV['CLOUDTASKER_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+
+    #
+    # Return the Cloudtasker logger.
+    #
+    # @return [Logger, any] The cloudtasker logger.
+    #
+    def logger
+      @logger ||= defined?(Rails) ? Rails.logger : ::Logger.new($stdout)
+    end
+
+    #
+    # Return the full URL of the processor. Worker payloads will be sent
+    # to this URL.
+    #
+    # @return [String] The processor URL.
+    #
+    def processor_url
+      File.join(processor_host, processor_path)
+    end
+
+    #
+    # Set the processor host. In the context of Rails the host will
+    # also be added to the list of authorized Rails hosts.
+    #
+    # @param [String] val The processor host to set.
+    #
+    def processor_host=(val)
+      @processor_host = val
+
+      # Check if Rails supports host filtering
+      return unless val &&
+                    defined?(Rails) &&
+                    Rails.application.config.respond_to?(:hosts) &&
+                    Rails.application.config.hosts&.any?
+
+      # Add processor host to the list of authorized hosts
+      Rails.application.config.hosts << val.gsub(%r{https?://}, '')
+    end
+
+    #
+    # The hostname of the application processing the workers. The hostname must
+    # be reachable from Cloud Task.
+    #
+    # @return [String] The processor host.
+    #
+    def processor_host
+      @processor_host || raise(StandardError, PROCESSOR_HOST_MISSING)
+    end
+
+    #
+    # The path on the host when worker payloads will be sent.
+    # Default to `/cloudtasker/run`
+    #
+    #
+    # @return [String] The processor path
+    #
+    def processor_path
+      @processor_path || DEFAULT_PROCESSOR_PATH
+    end
+
+    #
+    # Return the GCP project ID.
+    #
+    # @return [String] The ID of the project for which tasks will be processed.
+    #
+    def gcp_project_id
+      @gcp_project_id || raise(StandardError, PROJECT_ID_MISSING_ERROR)
+    end
+
+    #
+    # Return the GCP location ID. Default to 'us-east1'
+    #
+    # @return [String] The location ID where tasks will be processed.
+    #
+    def gcp_location_id
+      @gcp_location_id || DEFAULT_LOCATION_ID
+    end
+
+    #
+    # Return the Dispatch deadline duration. Cloud Tasks will timeout the job after
+    # this duration is elapsed.
+    #
+    # @return [Integer] The value in seconds.
+    #
+    def dispatch_deadline
+      @dispatch_deadline || DEFAULT_DISPATCH_DEADLINE
+    end
+
+    #
+    # Return the secret to use to sign the verification tokens
+    # attached to tasks.
+    #
+    # @return [String] The cloudtasker secret
+    #
+    def secret
+      @secret ||= (
+        defined?(Rails) && Rails.application.credentials&.dig(:secret_key_base)
+      ) || raise(StandardError, SECRET_MISSING_ERROR)
+    end
+
+    #
+    # Return a Proc invoked whenever a worker runtime error is raised.
+    # See Cloudtasker::WorkerHandler.with_worker_handling
+    #
+    # @return [Proc] A Proc handler
+    #
+    def on_error
+      @on_error || DEFAULT_ON_ERROR
+    end
+
+    #
+    # Return a Proc invoked whenever a worker DeadWorkerError is raised.
+    # See Cloudtasker::WorkerHandler.with_worker_handling
+    #
+    # @return [Proc] A Proc handler
+    #
+    def on_dead
+      @on_dead || DEFAULT_ON_ERROR
+    end
+
+    #
+    # Return the Open ID Connect configuration to use for tasks.
+    #
+    # @return [Hash] The OIDC configuration
+    #
+    def oidc
+      return unless @oidc
+      raise(StandardError, OIDC_EMAIL_MISSING_ERROR) unless @oidc[:service_account_email]
+
+      {
+        service_account_email: @oidc[:service_account_email],
+        audience: @oidc[:audience] || processor_host
+      }
+    end
+
+    #
+    # Return the chain of client middlewares.
+    #
+    # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
+    #
+    def client_middleware
+      @client_middleware ||= Middleware::Chain.new
+      yield @client_middleware if block_given?
+      @client_middleware
+    end
+
+    #
+    # Return the chain of server middlewares.
+    #
+    # @return [Cloudtasker::Middleware::Chain] The chain of middlewares.
+    #
+    def server_middleware
+      @server_middleware ||= Middleware::Chain.new
+      yield @server_middleware if block_given?
+      @server_middleware
+    end
+
+    #
+    # Return the ssl verify mode for the Cloudtasker local server.
+    #
+    # @return [Boolean] The ssl verify mode for the Cloudtasker local server.
+    #
+    def local_server_ssl_verify
+      @local_server_ssl_verify.nil? ? DEFAULT_LOCAL_SERVER_SSL_VERIFY_MODE : @local_server_ssl_verify
+    end
+
+    #
+    # Return whether to base64 encode the task body when sending to Cloud Tasks.
+    # Encoding is enabled by default to support UTF-8 content.
+    #
+    # @return [Boolean] Whether to base64 encode the body.
+    #
+    def base64_encode_body
+      @base64_encode_body.nil? ? DEFAULT_BASE64_ENCODE_BODY : @base64_encode_body
+    end
+  end
+end