activejob-temporal 0.1.0 → 0.2.0

data/lib/activejob/temporal/configured_job_compatibility.rb

@@ -6,39 +6,121 @@ require_relative "logger"
 module ActiveJob
   module Temporal
     module ConfiguredJobCompatibility
+      SUPPORTED_ACTIVE_JOB_MAJOR_MINOR = [
+        [7, 2],
+        [8, 0],
+        [8, 1]
+      ].freeze
+      FEATURE_REPLACEMENTS = {
+        "chain" => "ActiveJob::Temporal.job",
+        "child_workflows" => "ActiveJob::Temporal.job",
+        "conditional_enqueue" => "JobClass.perform_later_if or an explicit condition before " \
+                                 "JobClass.set(...).perform_later"
+      }.freeze
+      ExtractedConfiguredJob = Struct.new(:job_class, :options, keyword_init: true)
+
       module_function
 
       def payload(value, feature:, normalize_options:)
-        return unless configured_job?(value)
-
-        log_private_api(feature)
-
-        job_class = value.instance_variable_get(:@job_class)
-        return unless active_job_class?(job_class)
+        configured_job = extract(value, feature: feature)
+        return unless configured_job
 
         {
-          job_class: job_class.name,
-          options: normalize_options.call(value.instance_variable_get(:@options) || {})
+          job_class: configured_job.job_class.name,
+          options: normalize_options.call(configured_job.options)
         }
       end
 
+      def job_class(value, feature:)
+        extract(value, feature: feature)&.job_class
+      end
+
       def configured_job?(value)
         defined?(ActiveJob::ConfiguredJob) && value.is_a?(ActiveJob::ConfiguredJob)
       end
 
+      def extract(value, feature:)
+        return unless configured_job?(value)
+
+        validate_active_job_version!(feature)
+
+        job_class = configured_job_instance_variable(value, :@job_class, feature)
+        options = configured_job_instance_variable(value, :@options, feature)
+
+        validate_job_class!(job_class, feature)
+        validate_options!(options, feature)
+
+        log_private_api(feature)
+
+        ExtractedConfiguredJob.new(job_class: job_class, options: options.dup)
+      end
+      private_class_method :extract
+
       def active_job_class?(job_class)
         job_class.is_a?(Class) && job_class < ActiveJob::Base && job_class.name
       end
 
+      def validate_job_class!(job_class, feature)
+        return if active_job_class?(job_class)
+
+        raise ArgumentError, unsupported_internals_message(feature, "@job_class")
+      end
+      private_class_method :validate_job_class!
+
+      def validate_options!(options, feature)
+        return if options.is_a?(Hash)
+
+        raise ArgumentError, unsupported_internals_message(feature, "@options")
+      end
+      private_class_method :validate_options!
+
+      def configured_job_instance_variable(value, name, feature)
+        return value.instance_variable_get(name) if value.instance_variable_defined?(name)
+
+        raise ArgumentError, unsupported_internals_message(feature, name)
+      end
+      private_class_method :configured_job_instance_variable
+
+      def validate_active_job_version!(feature)
+        return if supported_active_job_version?
+
+        raise ArgumentError,
+              "ActiveJob::ConfiguredJob internals are not supported for #{feature} on ActiveJob " \
+              "#{active_job_version}; use #{replacement_for(feature)} instead"
+      end
+      private_class_method :validate_active_job_version!
+
+      def supported_active_job_version?
+        SUPPORTED_ACTIVE_JOB_MAJOR_MINOR.include?(active_job_version.segments.first(2))
+      end
+      private_class_method :supported_active_job_version?
+
+      def active_job_version
+        return ActiveJob.gem_version if ActiveJob.respond_to?(:gem_version)
+
+        Gem::Version.new(ActiveJob::VERSION::STRING)
+      end
+
+      def unsupported_internals_message(feature, name)
+        "ActiveJob::ConfiguredJob internals changed for #{feature}: expected #{name}; " \
+          "use #{replacement_for(feature)} instead"
+      end
+      private_class_method :unsupported_internals_message
+
       def log_private_api(feature)
         ActiveJob::Temporal::Logger.warn(
           "active_job_configured_job_private_api",
           feature: feature,
-          replacement: "ActiveJob::Temporal.job"
+          replacement: replacement_for(feature)
         )
       rescue StandardError
         nil
       end
+
+      def replacement_for(feature)
+        FEATURE_REPLACEMENTS.fetch(feature.to_s, "ActiveJob::Temporal.job")
+      end
+      private_class_method :replacement_for
     end
   end
 end

data/lib/activejob/temporal/connection_worker_pool.rb

@@ -72,12 +72,29 @@ module ActiveJob
           connection = queue.pop
           break unless connection
 
-          @handler.call(connection)
+          begin
+            @handler.call(connection)
+          rescue StandardError => e
+            log_handler_failure(index, e)
+            close_connection(connection)
+          end
         end
       rescue ClosedQueueError
         nil
       end
 
+      def log_handler_failure(index, error)
+        ActiveJob::Temporal::Logger.error(
+          "connection_worker_handler_failed",
+          pool: @name,
+          worker_index: index,
+          error_class: error.class.name,
+          message: error.message.to_s
+        )
+      rescue StandardError
+        nil
+      end
+
       def close_connection(connection)
         connection.close
       rescue IOError, SystemCallError

data/lib/activejob/temporal/dead_letter_queue.rb

@@ -2,11 +2,16 @@
 
 require "temporalio/client"
 
+require_relative "logger"
+require_relative "visibility_query"
+require_relative "workflow_types"
+
 module ActiveJob
   module Temporal
     module DeadLetterQueue
-      WORKFLOW_TYPE = "ActiveJobTemporalDeadLetterWorkflow"
+      WORKFLOW_TYPE = WorkflowTypes::DEAD_LETTER
       DEFAULT_ENTRIES_LIMIT = 100
+      ENTRY_QUERY_CONCURRENCY = 5
 
       module_function
 
@@ -17,22 +22,24 @@ module ActiveJob
       def entries(queue: nil, limit: DEFAULT_ENTRIES_LIMIT, client: ActiveJob::Temporal.client)
         validate_limit!(limit)
 
-        client.list_workflows(entries_query(queue)).each_with_object([]) do |workflow, entries|
-          entry = query_workflow_entry(client, workflow)
-          entries << entry if entry
-          break entries if entries.size >= limit
-        end
+        workflows = client.list_workflows(entries_query(queue)).first(limit)
+        query_workflow_entries(client, workflows)
       end
 
       def retry(job_class, job_id, queue: nil, client: ActiveJob::Temporal.client)
         handle = handle_for(job_class, job_id, client: client)
         entry = handle.query(:entry)
-        return entry.fetch("retry_workflow_id") if retried_entry?(entry)
+        if retried_entry?(entry)
+          workflow_id = entry.fetch("retry_workflow_id")
+          log_retry_requested(entry, workflow_id, queue, duplicate: true)
+          return workflow_id
+        end
 
         ensure_pending_entry!(entry)
 
         workflow_id = retry_workflow_id(entry)
-        start_retry_workflow(client, entry, workflow_id, queue)
+        duplicate = start_retry_workflow(client, entry, workflow_id, queue)
+        log_retry_requested(entry, workflow_id, queue, duplicate: duplicate)
         mark_retried_entry(handle, workflow_id)
         workflow_id
       end
@@ -56,31 +63,72 @@ module ActiveJob
 
       def entries_query(queue)
         query = ["WorkflowType='#{WORKFLOW_TYPE}'", "ExecutionStatus='Running'"]
-        query << "TaskQueue='#{escape_query_value(queue)}'" if queue.to_s.strip.present?
+        query << "TaskQueue=#{VisibilityQuery.quote(queue)}" if queue.to_s.strip.present?
         query.join(" AND ")
       end
       private_class_method :entries_query
 
+      def query_workflow_entries(client, workflows)
+        entries = Array.new(workflows.size)
+        pending = Queue.new
+        workflows.each_with_index { |workflow, index| pending << [workflow, index] }
+
+        Array.new([workflows.size, ENTRY_QUERY_CONCURRENCY].min) do
+          worker = Thread.new do
+            loop do
+              workflow, index = pending.pop(true)
+              entries[index] = query_workflow_entry(client, workflow)
+            rescue ThreadError
+              break
+            end
+          end
+          # Failures are re-raised by Thread#value below, not lost.
+          worker.report_on_exception = false
+          worker
+        end.each(&:value)
+
+        entries
+      end
+      private_class_method :query_workflow_entries
+
       def query_workflow_entry(client, workflow)
         client.workflow_handle(workflow.id, run_id: workflow_run_id(workflow)).query(:entry)
-      rescue Temporalio::Error
-        nil
       end
       private_class_method :query_workflow_entry
 
       def start_retry_workflow(client, entry, workflow_id, queue)
         client.start_workflow(
-          ActiveJob::Temporal::Workflows::AjWorkflow,
+          WorkflowTypes::ACTIVE_JOB,
           retry_payload(entry),
           id: workflow_id,
           task_queue: retry_task_queue(entry, queue),
           id_conflict_policy: Temporalio::WorkflowIDConflictPolicy::FAIL
         )
+        false
       rescue StandardError => e
         raise unless workflow_already_started?(e)
+
+        true
       end
       private_class_method :start_retry_workflow
 
+      def log_retry_requested(entry, workflow_id, queue, duplicate:)
+        Logger.log_event(
+          "dead_letter_retry_requested",
+          {
+            entry_id: entry.fetch("id"),
+            workflow_id: workflow_id,
+            job_class: entry.dig("payload", "job_class"),
+            job_id: entry.dig("payload", "job_id"),
+            task_queue: retry_task_queue(entry, queue),
+            duplicate: duplicate
+          }.compact
+        )
+      rescue StandardError
+        nil
+      end
+      private_class_method :log_retry_requested
+
       def mark_retried_entry(handle, workflow_id)
         handle.signal(:mark_retried, workflow_id)
       rescue StandardError => e
@@ -146,16 +194,14 @@ module ActiveJob
       end
       private_class_method :validate_limit!
 
-      def escape_query_value(value)
-        value.to_s.gsub("'", "''")
-      end
-      private_class_method :escape_query_value
-
       def workflow_already_started?(error)
         (defined?(Temporalio::Error::WorkflowAlreadyStartedError) &&
           error.is_a?(Temporalio::Error::WorkflowAlreadyStartedError)) ||
           (defined?(Temporalio::Client::WorkflowAlreadyStartedError) &&
-            error.is_a?(Temporalio::Client::WorkflowAlreadyStartedError))
+            error.is_a?(Temporalio::Client::WorkflowAlreadyStartedError)) ||
+          (defined?(Temporalio::Error::RPCError::Code::ALREADY_EXISTS) &&
+            error.respond_to?(:code) &&
+            error.code == Temporalio::Error::RPCError::Code::ALREADY_EXISTS)
       end
       private_class_method :workflow_already_started?
     end

data/lib/activejob/temporal/dependency_options.rb

@@ -9,8 +9,9 @@ module ActiveJob
       JOB_CLASS_NAME_PATTERN = /\A[A-Z]\w*(?:::[A-Z]\w*)*\z/
       SAFE_ID_PATTERN = /\A[A-Za-z0-9_.:-]+\z/
       FAILURE_POLICIES = %i[fail ignore].freeze
+      WAIT_OPTION_KEYS = %i[timeout initial_interval max_interval backoff].freeze
 
-      attr_reader :temporal_dependencies, :temporal_dependency_failure_policy
+      attr_reader :temporal_dependencies, :temporal_dependency_failure_policy, :temporal_dependency_wait
 
       def self.normalize(depends_on)
         dependencies = depends_on.is_a?(Array) ? depends_on : [depends_on]
@@ -28,6 +29,17 @@ module ActiveJob
         raise ArgumentError, "on_dependency_failure must be :fail or :ignore"
       end
 
+      def self.normalize_wait_options(options)
+        raise ArgumentError, "dependency_wait must be a hash" unless options.is_a?(Hash)
+
+        normalized = options.each_with_object({}) do |(key, value), wait_options|
+          normalized_key = normalize_wait_option_key(key)
+          wait_options[normalized_key] = normalize_wait_option_value(normalized_key, value)
+        end
+        validate_wait_interval_order!(normalized)
+        normalized
+      end
+
       def self.normalize_dependency(dependency)
         return normalize_job_dependency(dependency) if dependency.is_a?(ActiveJob::Base)
         return normalize_hash_dependency(dependency) if dependency.is_a?(Hash)
@@ -55,10 +67,12 @@ module ActiveJob
         normalized = {}
         job_id = hash_value(dependency, :job_id)
         workflow_id = hash_value(dependency, :workflow_id)
+        run_id = hash_value(dependency, :run_id)
         job_class = hash_value(dependency, :job_class)
 
         normalized[:job_id] = normalize_id(job_id, "job_id") if job_id
         normalized[:workflow_id] = normalize_id(workflow_id, "workflow_id") if workflow_id
+        normalized[:run_id] = normalize_id(run_id, "run_id") if run_id
         normalized[:job_class] = normalize_job_class(job_class) if job_class
 
         if normalized[:job_id].nil? && normalized[:workflow_id].nil?
@@ -97,36 +111,132 @@ module ActiveJob
       end
       private_class_method :hash_value
 
-      def set(options = {})
-        enqueue_options = options.dup
-        dependencies_configured = enqueue_options.key?(:depends_on)
-        failure_policy_configured = enqueue_options.key?(:on_dependency_failure)
+      def self.normalize_wait_option_key(key)
+        normalized_key = key.to_sym
+        return normalized_key if WAIT_OPTION_KEYS.include?(normalized_key)
 
-        normalized_dependencies = if dependencies_configured
-                                    DependencyOptions.normalize(enqueue_options.delete(:depends_on))
-                                  end
-        normalized_failure_policy = normalize_dependency_failure_policy(
-          enqueue_options.delete(:on_dependency_failure),
-          failure_policy_configured
-        )
+        raise ArgumentError, "dependency_wait supports: #{WAIT_OPTION_KEYS.join(', ')}"
+      rescue NoMethodError
+        raise ArgumentError, "dependency_wait supports: #{WAIT_OPTION_KEYS.join(', ')}"
+      end
+      private_class_method :normalize_wait_option_key
+
+      def self.normalize_wait_option_value(key, value)
+        return normalize_wait_backoff(value) if key == :backoff
 
-        if failure_policy_configured && !dependencies_configured && normalized_dependencies.nil?
-          raise ArgumentError, "on_dependency_failure requires depends_on"
+        normalize_wait_duration(value, key)
+      end
+      private_class_method :normalize_wait_option_value
+
+      def self.normalize_wait_duration(value, key)
+        unless value.is_a?(Numeric) || value.is_a?(ActiveSupport::Duration)
+          raise ArgumentError, "dependency_wait #{key} must be a duration"
         end
 
+        seconds = value.to_f
+        raise ArgumentError, "dependency_wait #{key} must be positive" unless seconds.positive?
+
+        seconds
+      end
+      private_class_method :normalize_wait_duration
+
+      def self.normalize_wait_backoff(value)
+        backoff = Float(value)
+        raise ArgumentError, "dependency_wait backoff must be greater than or equal to 1" if backoff < 1.0
+
+        backoff
+      rescue ArgumentError, TypeError
+        raise ArgumentError, "dependency_wait backoff must be greater than or equal to 1"
+      end
+      private_class_method :normalize_wait_backoff
+
+      def self.validate_wait_interval_order!(wait_options)
+        initial_interval = wait_options[:initial_interval]
+        max_interval = wait_options[:max_interval]
+        return unless initial_interval && max_interval
+        return if max_interval >= initial_interval
+
+        raise ArgumentError, "dependency_wait max_interval must be greater than or equal to initial_interval"
+      end
+      private_class_method :validate_wait_interval_order!
+
+      def set(options = {})
+        enqueue_options = options.dup
+        dependency_options = normalize_dependency_set_options(enqueue_options)
+        validate_dependency_set_options!(dependency_options)
+
         super(enqueue_options).tap do
-          @temporal_dependencies = normalized_dependencies if dependencies_configured
-          @temporal_dependency_failure_policy = normalized_failure_policy || :fail if dependencies_configured
+          apply_dependency_set_options(dependency_options)
         end
       end
 
       private
 
+      def normalize_dependency_set_options(enqueue_options)
+        dependencies_configured = enqueue_options.key?(:depends_on)
+        failure_policy_configured = enqueue_options.key?(:on_dependency_failure)
+        dependency_wait_configured = enqueue_options.key?(:dependency_wait)
+
+        {
+          dependencies_configured: dependencies_configured,
+          failure_policy_configured: failure_policy_configured,
+          dependency_wait_configured: dependency_wait_configured,
+          dependencies: normalize_dependencies(enqueue_options, dependencies_configured),
+          failure_policy: normalize_dependency_failure_policy(
+            enqueue_options.delete(:on_dependency_failure),
+            failure_policy_configured
+          ),
+          dependency_wait: normalize_dependency_wait(enqueue_options, dependency_wait_configured)
+        }
+      end
+
+      def normalize_dependencies(enqueue_options, configured)
+        return unless configured
+
+        DependencyOptions.normalize(enqueue_options.delete(:depends_on))
+      end
+
+      def normalize_dependency_wait(enqueue_options, configured)
+        return unless configured
+
+        DependencyOptions.normalize_wait_options(enqueue_options.delete(:dependency_wait))
+      end
+
       def normalize_dependency_failure_policy(policy, configured)
         return unless configured
 
         DependencyOptions.normalize_failure_policy(policy)
       end
+
+      def validate_dependency_set_options!(dependency_options)
+        dependencies_configured = dependency_options[:dependencies_configured]
+        validate_dependency_requirement!(
+          dependency_options[:failure_policy_configured],
+          dependencies_configured,
+          "on_dependency_failure requires depends_on"
+        )
+        validate_dependency_requirement!(
+          dependency_options[:dependency_wait_configured],
+          dependencies_configured,
+          "dependency_wait requires depends_on"
+        )
+      end
+
+      def validate_dependency_requirement!(configured, dependencies_configured, message)
+        return if !configured || dependencies_configured
+
+        raise ArgumentError, message
+      end
+
+      def apply_dependency_set_options(dependency_options)
+        if dependency_options[:dependencies_configured]
+          @temporal_dependencies = dependency_options[:dependencies]
+          @temporal_dependency_failure_policy = dependency_options[:failure_policy] || :fail
+        end
+        return unless dependency_options[:dependency_wait_configured]
+
+        @temporal_dependency_wait = dependency_options[:dependency_wait]
+      end
     end
   end
 end

data/lib/activejob/temporal/health_check_server.rb

@@ -6,17 +6,21 @@ require "socket"
 
 require_relative "connection_worker_pool"
 require_relative "bind_policy"
+require_relative "http_request_failure_handling"
 require_relative "http_line_reader"
 
 module ActiveJob
   module Temporal
     class HealthCheckServer
       include HttpLineReader
+      include HttpRequestFailureHandling
 
       DEFAULT_BIND_ADDRESS = "127.0.0.1"
       READ_TIMEOUT_SECONDS = 1
       CONNECTION_WORKERS = 4
       CONNECTION_QUEUE_SIZE = 16
+      REQUEST_FAILURE_EVENT = "health_check_request_failed"
+      REQUEST_FAILURE_FORMAT = :json
 
       attr_reader :port, :bind_address
 
@@ -84,25 +88,24 @@ module ActiveJob
           connection_pool.enqueue(server.accept)
         rescue IOError, Errno::EBADF
           break
+        rescue SystemCallError => e
+          # Transient accept failures (fd exhaustion, aborted connections) must not kill the listener.
+          ActiveJob::Temporal::Logger.error(
+            "health_check_accept_failed", error_class: e.class.name, message: e.message.to_s
+          )
+          sleep 0.05
         end
       ensure
         @mutex.synchronize { @running = false if @server }
       end
 
-      def serve_client(client)
-        handle_client(client)
-      rescue IOError, SystemCallError
-        nil
-      ensure
-        client&.close
-      end
-
       def handle_client(client)
-        request_line = read_line(client)
+        deadline = request_deadline
+        request_line = read_line(client, deadline)
         return unless request_line
 
         method, path = request_line.split.first(2)
-        drain_headers(client)
+        drain_headers(client, deadline)
 
         unless method && path
           write_json(client, 400, { error: "bad_request" })
@@ -128,13 +131,6 @@ module ActiveJob
         payload[:worker_running] ? 200 : 503
       end
 
-      def drain_headers(client)
-        loop do
-          line = read_line(client)
-          break if line.nil? || line == "\r\n" || line == "\n"
-        end
-      end
-
       def write_json(client, status, payload, body: true)
         json = JSON.generate(payload)
         response = "HTTP/1.1 #{status} #{reason_phrase(status)}\r\n"
@@ -151,6 +147,7 @@ module ActiveJob
           400 => "Bad Request",
           404 => "Not Found",
           405 => "Method Not Allowed",
+          500 => "Internal Server Error",
           503 => "Service Unavailable"
         }.fetch(status)
       end

data/lib/activejob/temporal/http_line_reader.rb

@@ -2,11 +2,21 @@
 
 module ActiveJob
   module Temporal
+    # Bounded line reading for the embedded HTTP endpoints: one deadline for the
+    # whole request plus caps on line length and header count, so a slow or
+    # oversized client cannot hold a connection worker.
     module HttpLineReader
+      MAX_LINE_BYTES = 8_192
+      MAX_HEADER_LINES = 100
+
       private
 
-      def read_line(client)
-        deadline = monotonic_time + self.class.const_get(:READ_TIMEOUT_SECONDS)
+      # @return [Float] monotonic deadline covering an entire request
+      def request_deadline
+        monotonic_time + self.class.const_get(:READ_TIMEOUT_SECONDS)
+      end
+
+      def read_line(client, deadline)
         buffer = +""
 
         loop do
@@ -24,10 +34,18 @@ module ActiveJob
           else
             buffer << chunk
             return buffer if chunk == "\n"
+            return if buffer.bytesize >= MAX_LINE_BYTES
           end
         end
       end
 
+      def drain_headers(client, deadline)
+        MAX_HEADER_LINES.times do
+          line = read_line(client, deadline)
+          break if line.nil? || line == "\r\n" || line == "\n"
+        end
+      end
+
       def monotonic_time
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end

data/lib/activejob/temporal/http_request_failure_handling.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module ActiveJob
+  module Temporal
+    module HttpRequestFailureHandling
+      private
+
+      def serve_client(client)
+        handle_client(client)
+      rescue IOError, SystemCallError
+        nil
+      rescue StandardError => e
+        log_request_failure(e)
+        write_failure_response(client)
+      ensure
+        client&.close
+      end
+
+      def write_failure_response(client)
+        case self.class::REQUEST_FAILURE_FORMAT
+        when :json
+          write_json(client, 500, { error: "internal_server_error" })
+        when :text
+          write_text(client, 500, "internal_server_error\n")
+        end
+      rescue IOError, SystemCallError
+        nil
+      end
+
+      def log_request_failure(error)
+        ActiveJob::Temporal::Logger.error(
+          self.class::REQUEST_FAILURE_EVENT,
+          error_class: error.class.name,
+          message: error.message.to_s
+        )
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end