activejob-temporal 0.1.0 → 0.3.0

data/lib/activejob/temporal/configuration.rb

@@ -38,6 +38,8 @@ module ActiveJob
     TARGET_HOST_LABEL_PATTERN = /\A[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?\z/
     TARGET_PORT_PATTERN = /\A[1-9]\d{0,4}\z/
     NAMESPACE_PATTERN = /\A[A-Za-z0-9](?:[A-Za-z0-9_-]*[A-Za-z0-9])?\z/
+    ENV_BOOLEAN_TRUE_VALUES = %w[1 true yes on].freeze
+    ENV_BOOLEAN_FALSE_VALUES = %w[0 false no off].freeze
 
     # Central registry of all configuration attributes.
     #
@@ -134,6 +136,29 @@ module ActiveJob
         description: "Signal used by workers to reload TLS certificates manually"
       },
 
+      api_key: {
+        default: nil,
+        env_var: "ACTIVEJOB_TEMPORAL_API_KEY",
+        type: :string,
+        description: "Optional API key sent as the Authorization bearer token"
+      },
+
+      api_key_file: {
+        default: nil,
+        env_var: "ACTIVEJOB_TEMPORAL_API_KEY_FILE",
+        type: :string,
+        description: "Optional file read for the API key when the client is built, for example a " \
+                     "projected Kubernetes ServiceAccount token. An explicit api_key takes precedence"
+      },
+
+      api_key_watch: {
+        default: false,
+        env_var: "ACTIVEJOB_TEMPORAL_API_KEY_WATCH",
+        type: :boolean,
+        description: "Watch api_key_file and reload worker clients when it changes (kubelet rewrites " \
+                     "projected tokens in place long before they expire)"
+      },
+
       priority_task_queues: {
         default: -> { {} },
         type: :hash,
@@ -311,6 +336,13 @@ module ActiveJob
         description: "Base64-encoded 32-byte AES-256-GCM payload encryption key or key metadata"
       },
 
+      allow_legacy_encrypted_payloads: {
+        default: false,
+        env_var: "ACTIVEJOB_TEMPORAL_ALLOW_LEGACY_ENCRYPTED_PAYLOADS",
+        type: :boolean,
+        description: "Accept version 1 encrypted payloads, which are not bound to a namespace/workflow context"
+      },
+
       encryption_old_keys: {
         default: -> { [] },
         type: :array,
@@ -330,6 +362,34 @@ module ActiveJob
         description: "Optional workflow history event threshold for continuing ActiveJob workflows as new"
       },
 
+      dependency_wait_timeout: {
+        default: -> { 1.day },
+        env_var: "ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_TIMEOUT_SECONDS",
+        type: :duration,
+        description: "Maximum time a workflow waits for dependencies before timing out"
+      },
+
+      dependency_wait_initial_interval: {
+        default: -> { 10.seconds },
+        env_var: "ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_INITIAL_INTERVAL_SECONDS",
+        type: :duration,
+        description: "Initial durable sleep interval between dependency checks"
+      },
+
+      dependency_wait_max_interval: {
+        default: -> { 1.minute },
+        env_var: "ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_MAX_INTERVAL_SECONDS",
+        type: :duration,
+        description: "Maximum durable sleep interval between dependency checks"
+      },
+
+      dependency_wait_backoff: {
+        default: 2.0,
+        env_var: "ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_BACKOFF",
+        type: :float,
+        description: "Backoff coefficient for dependency polling intervals"
+      },
+
       local_activity_helpers: {
         default: -> { [] },
         type: :array,
@@ -348,6 +408,29 @@ module ActiveJob
         env_var: "ACTIVEJOB_TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASKS",
         type: :integer,
         description: "Maximum concurrent workflow tasks per worker"
+      },
+
+      worker_activities: {
+        default: -> { [] },
+        type: :array,
+        description: "Additional activity classes (or class names) the worker registers, for example " \
+                     "activities invoked by workflows owned by another service"
+      },
+
+      worker_activejob_workloads: {
+        default: true,
+        env_var: "ACTIVEJOB_TEMPORAL_WORKER_ACTIVEJOB_WORKLOADS",
+        type: :boolean,
+        description: "Register the built-in ActiveJob workflows and activities on the worker. Disable " \
+                     "for workers that only host worker_activities"
+      },
+
+      graceful_shutdown_period: {
+        default: 0.0,
+        env_var: "ACTIVEJOB_TEMPORAL_GRACEFUL_SHUTDOWN_PERIOD",
+        type: :float,
+        description: "Seconds a shutting-down worker lets running activities finish before their " \
+                     "tasks are cancelled (0 cancels immediately, matching the SDK default)"
       }
     }.freeze
 
@@ -357,13 +440,18 @@ module ActiveJob
     # Use {ActiveJob::Temporal.configure} to set values with automatic validation.
     #
     # @note Thread Safety
-    #   The global configuration object is synchronized by {Configurable} while
-    #   configure blocks mutate it. Complete configuration during application boot.
+    #   The global configuration object is synchronized by {Configurable}.
+    #   Configure blocks mutate a candidate copy that replaces the active
+    #   configuration only after validation succeeds.
     #
     # @note Environment Variable Defaults
     #   ACTIVEJOB_TEMPORAL_TARGET, ACTIVEJOB_TEMPORAL_NAMESPACE,
     #   ACTIVEJOB_TEMPORAL_TASK_QUEUE_PREFIX, ACTIVEJOB_TEMPORAL_TASK_QUEUE,
     #   ACTIVEJOB_TEMPORAL_MAX_PAYLOAD_SIZE_KB,
+    #   ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_TIMEOUT_SECONDS,
+    #   ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_INITIAL_INTERVAL_SECONDS,
+    #   ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_MAX_INTERVAL_SECONDS,
+    #   ACTIVEJOB_TEMPORAL_DEPENDENCY_WAIT_BACKOFF,
     #   ACTIVEJOB_TEMPORAL_MAX_CONCURRENT_ACTIVITIES,
     #   ACTIVEJOB_TEMPORAL_MAX_CONCURRENT_WORKFLOW_TASKS,
     #   can provide defaults.
@@ -394,6 +482,14 @@ module ActiveJob
         end
       end
 
+      def initialize_copy(original)
+        super
+        @attributes = original.instance_variable_get(:@attributes).transform_values do |value|
+          duplicate_configuration_value(value)
+        end
+        @in_configure_block = false
+      end
+
       def [](key)
         @attributes[key]
       end
@@ -406,6 +502,26 @@ module ActiveJob
         middleware_chain.add(middleware, ...)
       end
 
+      # Attributes holding secrets, redacted by {#inspect}.
+      REDACTED_ATTRIBUTES = %i[encryption_key encryption_old_keys tls api_key].freeze
+      REDACTED_PLACEHOLDER = "[FILTERED]"
+
+      # Returns the configuration with secret attributes redacted.
+      #
+      # The default `Object#inspect` dumps `@attributes`, which exposes the payload
+      # encryption keys and the inline TLS private key in logs and exception output.
+      #
+      # @return [String] inspect output with encryption keys and TLS settings filtered
+      def inspect
+        pairs = @attributes.map do |attribute, value|
+          value = REDACTED_PLACEHOLDER if REDACTED_ATTRIBUTES.include?(attribute) && value.present?
+
+          "#{attribute}=#{value.inspect}"
+        end
+
+        "#<#{self.class.name} #{pairs.join(', ')}>"
+      end
+
       def validate!
         return if validation_level == :none
 
@@ -415,6 +531,11 @@ module ActiveJob
         handle_validation_errors(validator.errors)
       end
 
+      def finalize_configuration_copy!
+        observability.finalize_configuration_copy! if observability.respond_to?(:finalize_configuration_copy!)
+        self
+      end
+
       def self.format_validation_errors(errors)
         messages = errors.full_messages
         return messages.first if messages.size == 1
@@ -429,6 +550,19 @@ module ActiveJob
 
       private
 
+      def duplicate_configuration_value(value)
+        case value
+        when Hash
+          value.transform_values { |entry| duplicate_configuration_value(entry) }
+        when Array
+          value.map { |entry| duplicate_configuration_value(entry) }
+        when String, Middleware::Chain, Observability::Configuration
+          value.dup
+        else
+          value
+        end
+      end
+
       def build_validator
         validator = ConfigValidator.new
 
@@ -448,24 +582,37 @@ module ActiveJob
       end
 
       def resolve_default_value(metadata)
-        if metadata[:env_var] && ENV[metadata[:env_var]]
-          value = ENV[metadata[:env_var]]
-          return convert_env_value(value, metadata[:type])
+        env_var = metadata[:env_var]
+        if env_var && ENV[env_var]
+          value = ENV[env_var]
+          return convert_env_value(value, metadata[:type], env_var)
         end
 
         default = metadata[:default]
         default.is_a?(Proc) ? default.call : default
       end
 
-      def convert_env_value(value, type)
+      def convert_env_value(value, type, env_var)
         case type
         when :integer then value.to_i
         when :float, :duration then value.to_f
-        when :boolean then value == "true"
+        when :boolean then convert_env_boolean(value, env_var)
         when :symbol then value.to_sym
         else value
         end
       end
+
+      def convert_env_boolean(value, env_var)
+        normalized = value.to_s.strip.downcase
+
+        return true if ENV_BOOLEAN_TRUE_VALUES.include?(normalized)
+        return false if ENV_BOOLEAN_FALSE_VALUES.include?(normalized)
+
+        raise ConfigurationError,
+              "Invalid boolean value for #{env_var}: #{value.inspect}. " \
+              "Accepted true values: #{ENV_BOOLEAN_TRUE_VALUES.join(', ')}. " \
+              "Accepted false values: #{ENV_BOOLEAN_FALSE_VALUES.join(', ')}."
+      end
     end
 
     # Validates ActiveJob::Temporal configuration.
@@ -560,6 +707,13 @@ module ActiveJob
                   allow_nil: false
                 }
 
+      validates :graceful_shutdown_period,
+                numericality: {
+                  greater_than_or_equal_to: 0,
+                  message: :graceful_shutdown_period_negative,
+                  allow_nil: false
+                }
+
       validates :continue_as_new_history_event_threshold,
                 numericality: {
                   greater_than: 0,
@@ -597,7 +751,10 @@ module ActiveJob
       validate :validate_encryption_settings
       validate :validate_payload_storage_settings
       validate :validate_tls_settings
+      validate :validate_api_key_settings
+      validate :validate_worker_registration_settings
       validate :validate_local_activity_helpers
+      validate :validate_dependency_wait_settings
 
       private
 
@@ -645,6 +802,30 @@ module ActiveJob
         validate_duration_attribute(:default_schedule_to_start_timeout, required: false)
         validate_duration_attribute(:default_schedule_to_close_timeout, required: false)
         validate_duration_attribute(:dead_letter_auto_discard_after, required: false)
+        validate_duration_attribute(:dependency_wait_timeout, required: true)
+        validate_duration_attribute(:dependency_wait_initial_interval, required: true)
+        validate_duration_attribute(:dependency_wait_max_interval, required: true)
+      end
+
+      def validate_dependency_wait_settings
+        validate_dependency_wait_backoff
+        validate_dependency_wait_interval_order
+      end
+
+      def validate_dependency_wait_backoff
+        value = dependency_wait_backoff
+        return if value.is_a?(Numeric) && value >= 1.0
+
+        errors.add(:dependency_wait_backoff, "must be greater than or equal to 1")
+      end
+
+      def validate_dependency_wait_interval_order
+        initial_interval = duration_seconds(dependency_wait_initial_interval)
+        max_interval = duration_seconds(dependency_wait_max_interval)
+        return unless initial_interval && max_interval
+        return if max_interval >= initial_interval
+
+        errors.add(:dependency_wait_max_interval, "must be greater than or equal to initial interval")
       end
 
       def validate_workflow_id_generator
@@ -898,6 +1079,30 @@ module ActiveJob
         errors.add(:tls_cert_watch, :requires_paths)
       end
 
+      def validate_api_key_settings
+        validate_tls_file_path(:api_key_file)
+
+        unless [true, false].include?(api_key_watch)
+          errors.add(:api_key_watch, :not_boolean, value: api_key_watch.inspect)
+          return
+        end
+
+        return unless api_key_watch && api_key_file.to_s.strip.empty?
+
+        errors.add(:api_key_watch, :requires_path)
+      end
+
+      def validate_worker_registration_settings
+        unless [true, false].include?(worker_activejob_workloads)
+          errors.add(:worker_activejob_workloads, :not_boolean, value: worker_activejob_workloads.inspect)
+          return
+        end
+
+        return if worker_activejob_workloads || Array(worker_activities).any?
+
+        errors.add(:worker_activejob_workloads, :requires_activities)
+      end
+
       def validate_tls_reload_signal
         unless tls_reload_signal.is_a?(String) && tls_reload_signal.strip.present?
           errors.add(:tls_reload_signal, :blank)
@@ -974,6 +1179,12 @@ module ActiveJob
                    seconds: seconds,
                    attribute: attr_name.to_s.humanize.downcase)
       end
+
+      def duration_seconds(value)
+        return value.to_f if value.is_a?(Numeric) || value.is_a?(ActiveSupport::Duration)
+
+        nil
+      end
     end
     # rubocop:enable Metrics/ClassLength
   end

data/lib/activejob/temporal/configured_job_compatibility.rb

@@ -6,39 +6,121 @@ require_relative "logger"
 module ActiveJob
   module Temporal
     module ConfiguredJobCompatibility
+      SUPPORTED_ACTIVE_JOB_MAJOR_MINOR = [
+        [7, 2],
+        [8, 0],
+        [8, 1]
+      ].freeze
+      FEATURE_REPLACEMENTS = {
+        "chain" => "ActiveJob::Temporal.job",
+        "child_workflows" => "ActiveJob::Temporal.job",
+        "conditional_enqueue" => "JobClass.perform_later_if or an explicit condition before " \
+                                 "JobClass.set(...).perform_later"
+      }.freeze
+      ExtractedConfiguredJob = Struct.new(:job_class, :options, keyword_init: true)
+
       module_function
 
       def payload(value, feature:, normalize_options:)
-        return unless configured_job?(value)
-
-        log_private_api(feature)
-
-        job_class = value.instance_variable_get(:@job_class)
-        return unless active_job_class?(job_class)
+        configured_job = extract(value, feature: feature)
+        return unless configured_job
 
         {
-          job_class: job_class.name,
-          options: normalize_options.call(value.instance_variable_get(:@options) || {})
+          job_class: configured_job.job_class.name,
+          options: normalize_options.call(configured_job.options)
         }
       end
 
+      def job_class(value, feature:)
+        extract(value, feature: feature)&.job_class
+      end
+
       def configured_job?(value)
         defined?(ActiveJob::ConfiguredJob) && value.is_a?(ActiveJob::ConfiguredJob)
       end
 
+      def extract(value, feature:)
+        return unless configured_job?(value)
+
+        validate_active_job_version!(feature)
+
+        job_class = configured_job_instance_variable(value, :@job_class, feature)
+        options = configured_job_instance_variable(value, :@options, feature)
+
+        validate_job_class!(job_class, feature)
+        validate_options!(options, feature)
+
+        log_private_api(feature)
+
+        ExtractedConfiguredJob.new(job_class: job_class, options: options.dup)
+      end
+      private_class_method :extract
+
       def active_job_class?(job_class)
         job_class.is_a?(Class) && job_class < ActiveJob::Base && job_class.name
       end
 
+      def validate_job_class!(job_class, feature)
+        return if active_job_class?(job_class)
+
+        raise ArgumentError, unsupported_internals_message(feature, "@job_class")
+      end
+      private_class_method :validate_job_class!
+
+      def validate_options!(options, feature)
+        return if options.is_a?(Hash)
+
+        raise ArgumentError, unsupported_internals_message(feature, "@options")
+      end
+      private_class_method :validate_options!
+
+      def configured_job_instance_variable(value, name, feature)
+        return value.instance_variable_get(name) if value.instance_variable_defined?(name)
+
+        raise ArgumentError, unsupported_internals_message(feature, name)
+      end
+      private_class_method :configured_job_instance_variable
+
+      def validate_active_job_version!(feature)
+        return if supported_active_job_version?
+
+        raise ArgumentError,
+              "ActiveJob::ConfiguredJob internals are not supported for #{feature} on ActiveJob " \
+              "#{active_job_version}; use #{replacement_for(feature)} instead"
+      end
+      private_class_method :validate_active_job_version!
+
+      def supported_active_job_version?
+        SUPPORTED_ACTIVE_JOB_MAJOR_MINOR.include?(active_job_version.segments.first(2))
+      end
+      private_class_method :supported_active_job_version?
+
+      def active_job_version
+        return ActiveJob.gem_version if ActiveJob.respond_to?(:gem_version)
+
+        Gem::Version.new(ActiveJob::VERSION::STRING)
+      end
+
+      def unsupported_internals_message(feature, name)
+        "ActiveJob::ConfiguredJob internals changed for #{feature}: expected #{name}; " \
+          "use #{replacement_for(feature)} instead"
+      end
+      private_class_method :unsupported_internals_message
+
       def log_private_api(feature)
         ActiveJob::Temporal::Logger.warn(
           "active_job_configured_job_private_api",
           feature: feature,
-          replacement: "ActiveJob::Temporal.job"
+          replacement: replacement_for(feature)
         )
       rescue StandardError
         nil
       end
+
+      def replacement_for(feature)
+        FEATURE_REPLACEMENTS.fetch(feature.to_s, "ActiveJob::Temporal.job")
+      end
+      private_class_method :replacement_for
     end
   end
 end

data/lib/activejob/temporal/connection_worker_pool.rb

@@ -72,12 +72,29 @@ module ActiveJob
           connection = queue.pop
           break unless connection
 
-          @handler.call(connection)
+          begin
+            @handler.call(connection)
+          rescue StandardError => e
+            log_handler_failure(index, e)
+            close_connection(connection)
+          end
         end
       rescue ClosedQueueError
         nil
       end
 
+      def log_handler_failure(index, error)
+        ActiveJob::Temporal::Logger.error(
+          "connection_worker_handler_failed",
+          pool: @name,
+          worker_index: index,
+          error_class: error.class.name,
+          message: error.message.to_s
+        )
+      rescue StandardError
+        nil
+      end
+
       def close_connection(connection)
         connection.close
       rescue IOError, SystemCallError

data/lib/activejob/temporal/dead_letter_queue.rb

@@ -2,11 +2,16 @@
 
 require "temporalio/client"
 
+require_relative "logger"
+require_relative "visibility_query"
+require_relative "workflow_types"
+
 module ActiveJob
   module Temporal
     module DeadLetterQueue
-      WORKFLOW_TYPE = "ActiveJobTemporalDeadLetterWorkflow"
+      WORKFLOW_TYPE = WorkflowTypes::DEAD_LETTER
       DEFAULT_ENTRIES_LIMIT = 100
+      ENTRY_QUERY_CONCURRENCY = 5
 
       module_function
 
@@ -17,22 +22,24 @@ module ActiveJob
       def entries(queue: nil, limit: DEFAULT_ENTRIES_LIMIT, client: ActiveJob::Temporal.client)
         validate_limit!(limit)
 
-        client.list_workflows(entries_query(queue)).each_with_object([]) do |workflow, entries|
-          entry = query_workflow_entry(client, workflow)
-          entries << entry if entry
-          break entries if entries.size >= limit
-        end
+        workflows = client.list_workflows(entries_query(queue)).first(limit)
+        query_workflow_entries(client, workflows)
       end
 
       def retry(job_class, job_id, queue: nil, client: ActiveJob::Temporal.client)
         handle = handle_for(job_class, job_id, client: client)
         entry = handle.query(:entry)
-        return entry.fetch("retry_workflow_id") if retried_entry?(entry)
+        if retried_entry?(entry)
+          workflow_id = entry.fetch("retry_workflow_id")
+          log_retry_requested(entry, workflow_id, queue, duplicate: true)
+          return workflow_id
+        end
 
         ensure_pending_entry!(entry)
 
         workflow_id = retry_workflow_id(entry)
-        start_retry_workflow(client, entry, workflow_id, queue)
+        duplicate = start_retry_workflow(client, entry, workflow_id, queue)
+        log_retry_requested(entry, workflow_id, queue, duplicate: duplicate)
         mark_retried_entry(handle, workflow_id)
         workflow_id
       end
@@ -56,31 +63,72 @@ module ActiveJob
 
       def entries_query(queue)
         query = ["WorkflowType='#{WORKFLOW_TYPE}'", "ExecutionStatus='Running'"]
-        query << "TaskQueue='#{escape_query_value(queue)}'" if queue.to_s.strip.present?
+        query << "TaskQueue=#{VisibilityQuery.quote(queue)}" if queue.to_s.strip.present?
         query.join(" AND ")
       end
       private_class_method :entries_query
 
+      def query_workflow_entries(client, workflows)
+        entries = Array.new(workflows.size)
+        pending = Queue.new
+        workflows.each_with_index { |workflow, index| pending << [workflow, index] }
+
+        Array.new([workflows.size, ENTRY_QUERY_CONCURRENCY].min) do
+          worker = Thread.new do
+            loop do
+              workflow, index = pending.pop(true)
+              entries[index] = query_workflow_entry(client, workflow)
+            rescue ThreadError
+              break
+            end
+          end
+          # Failures are re-raised by Thread#value below, not lost.
+          worker.report_on_exception = false
+          worker
+        end.each(&:value)
+
+        entries
+      end
+      private_class_method :query_workflow_entries
+
       def query_workflow_entry(client, workflow)
         client.workflow_handle(workflow.id, run_id: workflow_run_id(workflow)).query(:entry)
-      rescue Temporalio::Error
-        nil
       end
       private_class_method :query_workflow_entry
 
       def start_retry_workflow(client, entry, workflow_id, queue)
         client.start_workflow(
-          ActiveJob::Temporal::Workflows::AjWorkflow,
+          WorkflowTypes::ACTIVE_JOB,
           retry_payload(entry),
           id: workflow_id,
           task_queue: retry_task_queue(entry, queue),
           id_conflict_policy: Temporalio::WorkflowIDConflictPolicy::FAIL
         )
+        false
       rescue StandardError => e
         raise unless workflow_already_started?(e)
+
+        true
       end
       private_class_method :start_retry_workflow
 
+      def log_retry_requested(entry, workflow_id, queue, duplicate:)
+        Logger.log_event(
+          "dead_letter_retry_requested",
+          {
+            entry_id: entry.fetch("id"),
+            workflow_id: workflow_id,
+            job_class: entry.dig("payload", "job_class"),
+            job_id: entry.dig("payload", "job_id"),
+            task_queue: retry_task_queue(entry, queue),
+            duplicate: duplicate
+          }.compact
+        )
+      rescue StandardError
+        nil
+      end
+      private_class_method :log_retry_requested
+
       def mark_retried_entry(handle, workflow_id)
         handle.signal(:mark_retried, workflow_id)
       rescue StandardError => e
@@ -146,16 +194,14 @@ module ActiveJob
       end
       private_class_method :validate_limit!
 
-      def escape_query_value(value)
-        value.to_s.gsub("'", "''")
-      end
-      private_class_method :escape_query_value
-
       def workflow_already_started?(error)
         (defined?(Temporalio::Error::WorkflowAlreadyStartedError) &&
           error.is_a?(Temporalio::Error::WorkflowAlreadyStartedError)) ||
           (defined?(Temporalio::Client::WorkflowAlreadyStartedError) &&
-            error.is_a?(Temporalio::Client::WorkflowAlreadyStartedError))
+            error.is_a?(Temporalio::Client::WorkflowAlreadyStartedError)) ||
+          (defined?(Temporalio::Error::RPCError::Code::ALREADY_EXISTS) &&
+            error.respond_to?(:code) &&
+            error.code == Temporalio::Error::RPCError::Code::ALREADY_EXISTS)
       end
       private_class_method :workflow_already_started?
     end