activejob-temporal 0.1.0 → 0.2.0

data/api/job_payload_schema.json

@@ -71,6 +71,15 @@
         "workflow_id_prefix": { "type": "string", "minLength": 1 }
       }
     },
+    "payload_encryption_context": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["namespace", "workflow_id"],
+      "properties": {
+        "namespace": { "type": "string", "minLength": 1 },
+        "workflow_id": { "type": "string", "minLength": 1 }
+      }
+    },
     "external_temporal_options": {
       "type": "object",
       "required": ["task_queue"],
@@ -199,19 +208,35 @@
       "properties": {
         "job_class": { "type": "string", "minLength": 1 },
         "job_id": { "type": "string", "minLength": 1 },
-        "workflow_id": { "type": "string", "minLength": 1 }
+        "workflow_id": { "type": "string", "minLength": 1 },
+        "run_id": { "type": "string", "minLength": 1 }
       }
     },
     "dependencies": {
       "type": "array",
       "minItems": 1,
       "items": { "$ref": "#/definitions/dependency" }
+    },
+    "dependency_wait": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["timeout", "initial_interval", "max_interval", "backoff"],
+      "properties": {
+        "timeout": { "type": "number", "exclusiveMinimum": 0 },
+        "initial_interval": { "type": "number", "exclusiveMinimum": 0 },
+        "max_interval": { "type": "number", "exclusiveMinimum": 0 },
+        "backoff": { "type": "number", "minimum": 1.0 }
+      }
     }
   },
   "oneOf": [
     {
       "additionalProperties": false,
-      "required": ["job_class", "job_id", "queue_name", "arguments"],
+      "required": ["job_class", "job_id", "queue_name"],
+      "anyOf": [
+        { "required": ["active_job"] },
+        { "required": ["arguments"] }
+      ],
       "properties": {
         "job_class": { "type": "string" },
         "job_id": { "type": "string" },
@@ -235,7 +260,12 @@
         "chain": { "$ref": "#/definitions/chain" },
         "dependencies": { "$ref": "#/definitions/dependencies" },
         "dependency_failure_policy": { "type": "string", "enum": ["fail", "ignore"] },
-        "activity_task_queue": { "type": "string", "minLength": 1 }
+        "dependency_wait": { "$ref": "#/definitions/dependency_wait" },
+        "activity_task_queue": { "type": "string", "minLength": 1 },
+        "schedule_id": { "type": "string", "minLength": 1 },
+        "schedule_workflow_id_prefix": { "type": "string", "minLength": 1 },
+        "schedule_execution_job_id": { "type": "string", "minLength": 1 },
+        "payload_encryption_context": { "$ref": "#/definitions/payload_encryption_context" }
       }
     },
     {
@@ -258,7 +288,12 @@
         "chain": { "$ref": "#/definitions/chain" },
         "dependencies": { "$ref": "#/definitions/dependencies" },
         "dependency_failure_policy": { "type": "string", "enum": ["fail", "ignore"] },
-        "activity_task_queue": { "type": "string", "minLength": 1 }
+        "dependency_wait": { "$ref": "#/definitions/dependency_wait" },
+        "activity_task_queue": { "type": "string", "minLength": 1 },
+        "schedule_id": { "type": "string", "minLength": 1 },
+        "schedule_workflow_id_prefix": { "type": "string", "minLength": 1 },
+        "schedule_execution_job_id": { "type": "string", "minLength": 1 },
+        "payload_encryption_context": { "$ref": "#/definitions/payload_encryption_context" }
       }
     },
     {
@@ -285,7 +320,12 @@
         "chain": { "$ref": "#/definitions/chain" },
         "dependencies": { "$ref": "#/definitions/dependencies" },
         "dependency_failure_policy": { "type": "string", "enum": ["fail", "ignore"] },
-        "activity_task_queue": { "type": "string", "minLength": 1 }
+        "dependency_wait": { "$ref": "#/definitions/dependency_wait" },
+        "activity_task_queue": { "type": "string", "minLength": 1 },
+        "schedule_id": { "type": "string", "minLength": 1 },
+        "schedule_workflow_id_prefix": { "type": "string", "minLength": 1 },
+        "schedule_execution_job_id": { "type": "string", "minLength": 1 },
+        "payload_encryption_context": { "$ref": "#/definitions/payload_encryption_context" }
       },
       "dependencies": {
         "payload_serializer": ["payload_serializer_version"],
@@ -311,7 +351,12 @@
         "chain": { "$ref": "#/definitions/chain" },
         "dependencies": { "$ref": "#/definitions/dependencies" },
         "dependency_failure_policy": { "type": "string", "enum": ["fail", "ignore"] },
-        "activity_task_queue": { "type": "string", "minLength": 1 }
+        "dependency_wait": { "$ref": "#/definitions/dependency_wait" },
+        "activity_task_queue": { "type": "string", "minLength": 1 },
+        "schedule_id": { "type": "string", "minLength": 1 },
+        "schedule_workflow_id_prefix": { "type": "string", "minLength": 1 },
+        "schedule_execution_job_id": { "type": "string", "minLength": 1 },
+        "payload_encryption_context": { "$ref": "#/definitions/payload_encryption_context" }
       }
     }
   ]

data/bin/temporal-worker

@@ -1,7 +1,7 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative "../lib/activejob/temporal"
+require_relative "../lib/activejob/temporal/worker_runtime"
 require "optparse"
 
 begin
@@ -263,13 +263,8 @@ certificate_watcher = nil
 if config.tls_cert_watch
   certificate_watcher = ActiveJob::Temporal::CertificateWatcher.new(
     paths: ActiveJob::Temporal::CertificateWatcher.paths_from_config(config),
-    reload_callback: lambda {
-      begin
-        client_reloader.reload(source: "file_watch")
-      rescue StandardError
-        # Reload failures are already logged and should not stop the watcher.
-      end
-    }
+    # Failures are logged by the reloader and retried by the watcher, so let them propagate.
+    reload_callback: -> { client_reloader.reload(source: "file_watch") }
   ).start
 end
 

data/gemfiles/activejob_contract.gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec path: ".."
+
+gem "activejob", ENV.fetch("ACTIVEJOB_VERSION")
+gem "activemodel", ENV.fetch("ACTIVEMODEL_VERSION", ENV.fetch("ACTIVEJOB_VERSION"))

data/gemfiles/temporalio_contract.gemfile

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec path: ".."
+
+gem "temporalio", ENV.fetch("TEMPORALIO_VERSION")

data/lib/activejob/temporal/activities/aj_runner_activity.rb

@@ -76,12 +76,24 @@ module ActiveJob
           end
         end
 
+        class DiscardRequested < StandardError
+          attr_reader :job, :original_error
+
+          def initialize(job, original_error)
+            @job = job
+            @original_error = original_error
+            super(original_error.message)
+            set_backtrace(original_error.backtrace)
+          end
+        end
+
         # Executes the job inside the Temporal activity context.
         #
-        # @param payload [Hash] Job payload with serialized arguments and metadata
+        # @param payload [Hash] Job payload with serialized ActiveJob data and metadata
         # @option payload [String] :job_class Fully-qualified job class name (required)
         # @option payload [String] :job_id Unique job identifier
-        # @option payload [Array] :arguments Serialized job arguments (via ActiveJob::Arguments)
+        # @option payload [Hash] :active_job Full ActiveJob serialized payload
+        # @option payload [Array] :arguments Legacy serialized job arguments
         # @option payload [String] :queue_name Target queue name
         # @option payload [Integer] :executions Current execution count
         # @option payload [Hash] :exception_executions Exception execution counts
@@ -98,7 +110,9 @@ module ActiveJob
         #   execute({
         #     job_class: "MyJob",
         #     job_id: "123",
-        #     arguments: [{ "_aj_serialized" => "ActiveJob::Serializers::ObjectSerializer", "value" => {...} }]
+        #     active_job: {
+        #       "arguments" => [{ "_aj_serialized" => "ActiveJob::Serializers::ObjectSerializer", "value" => {...} }]
+        #     }
         #   })
         #
         # @example Accessing idempotency key in job
@@ -166,8 +180,9 @@ module ActiveJob
           apply_activity_retry_state(job_data, job_class)
           job = deserialize_job(job_data)
           intercept_active_job_retry(job)
+          intercept_active_job_discard(job)
 
-          set_idempotency_key
+          set_idempotency_key(payload)
           perform_job(job)
         end
 
@@ -180,7 +195,7 @@ module ActiveJob
         def apply_schedule_execution_identity(payload)
           return unless payload_value(payload, :schedule_id)
 
-          execution_job_id = payload_value(payload, :schedule_execution_job_id) || activity_workflow_id
+          execution_job_id = payload_value(payload, :schedule_execution_job_id) || activity_workflow_execution_identity
           return unless execution_job_id
 
           payload[:job_id] = execution_job_id
@@ -218,6 +233,18 @@ module ActiveJob
           end
         end
 
+        def intercept_active_job_discard(job)
+          original_rescue_with_handler = job.method(:rescue_with_handler)
+          job.define_singleton_method(:rescue_with_handler) do |exception|
+            handled = original_rescue_with_handler.call(exception)
+            if handled && ActiveJob::Temporal::RetryMapper.discard_exception?(self.class, exception)
+              raise DiscardRequested.new(self, exception)
+            end
+
+            handled
+          end
+        end
+
         def apply_activity_retry_state(job_data, job_class)
           previous_attempts = activity_attempt - 1
           return if previous_attempts <= 0
@@ -256,7 +283,7 @@ module ActiveJob
           side_effects.after_failure("retry_observability") do
             record_retry_observability(retry_payload, observed_error)
           end
-          handle_exception(job_class, error)
+          handle_exception(job_class, error, retry_payload)
         end
 
         def instrument_perform(payload, &)
@@ -313,13 +340,10 @@ module ActiveJob
         end
 
         # @api private
-        def set_idempotency_key
-          workflow_id = if defined?(Temporalio::Activity::Context) && Temporalio::Activity::Context.exist?
-                          Temporalio::Activity::Context.current.info.workflow_id
-                        else
-                          "unknown-workflow"
-                        end
-          idempotency_key = "#{workflow_id}/runner"
+        def set_idempotency_key(payload = nil)
+          identity = payload && payload_value(payload, :schedule_execution_job_id)
+          identity ||= activity_workflow_id || "unknown-workflow"
+          idempotency_key = "#{identity}/runner"
           Thread.current[IDEMPOTENCY_KEY] = idempotency_key
           Fiber[IDEMPOTENCY_KEY] = idempotency_key
         end
@@ -329,10 +353,9 @@ module ActiveJob
           Fiber[IDEMPOTENCY_KEY] = nil
         end
 
+        # The AES-GCM authenticated data must come from the activity context, never from the
+        # payload: a payload-supplied context would let a tampered envelope pick its own AAD.
         def activity_encryption_context(payload = nil)
-          payload_context = payload && (payload[:payload_encryption_context] || payload["payload_encryption_context"])
-          return payload_context if payload_context
-
           return unless defined?(Temporalio::Activity::Context) && Temporalio::Activity::Context.exist?
 
           info = Temporalio::Activity::Context.current.info
@@ -341,7 +364,17 @@ module ActiveJob
                       else
                         ActiveJob::Temporal.config.namespace
                       end
-          { namespace: namespace, workflow_id: activity_workflow_id }
+          { namespace: namespace, workflow_id: encryption_context_workflow_id(payload) }
+        end
+
+        # Scheduled payloads are encrypted once against the schedule's workflow-ID prefix, while
+        # each occurrence runs under a longer workflow ID derived from that prefix.
+        def encryption_context_workflow_id(payload)
+          workflow_id = activity_workflow_id
+          prefix = payload && payload_value(payload, :schedule_workflow_id_prefix)
+          return workflow_id unless prefix && workflow_id.to_s.start_with?(prefix.to_s)
+
+          prefix
         end
 
         def activity_workflow_id
@@ -350,22 +383,45 @@ module ActiveJob
           Temporalio::Activity::Context.current.info.workflow_id
         end
 
+        def activity_workflow_execution_identity
+          return unless defined?(Temporalio::Activity::Context) && Temporalio::Activity::Context.exist?
+
+          info = Temporalio::Activity::Context.current.info
+          [info.workflow_id, activity_workflow_run_id(info)].compact.join(":")
+        end
+
+        def activity_workflow_run_id(info)
+          return info.workflow_run_id if info.respond_to?(:workflow_run_id)
+
+          info.run_id if info.respond_to?(:run_id)
+        end
+
         # Handles exceptions by checking discard_on declarations.
         # @api private
-        def handle_exception(job_class, error)
+        def handle_exception(job_class, error, retry_payload)
           raise retryable_application_error(error) if error.is_a?(RetryRequested)
+          raise non_retryable_application_error(error.original_error) if error.is_a?(DiscardRequested)
 
           raise non_retryable_application_error(error) if job_class.nil? && deserialization_error?(error)
 
-          raise non_retryable_application_error(error) if job_class && retry_attempts_exhausted?(job_class, error)
+          if job_class && retry_attempts_exhausted?(job_class, error)
+            raise error if dead_letter_payload?(retry_payload)
+
+            raise non_retryable_application_error(error)
+          end
 
           raise non_retryable_application_error(error) if job_class && RetryMapper.discard_exception?(job_class, error)
 
           raise error
         end
 
+        def dead_letter_payload?(payload)
+          payload_value(payload, :dead_letter)
+        end
+
         def observed_error_for(error)
           return error.original_error || error if error.is_a?(RetryRequested)
+          return error.original_error || error if error.is_a?(DiscardRequested)
 
           error
         end

data/lib/activejob/temporal/activities/dependency_status_activity.rb

@@ -9,17 +9,18 @@ require_relative "../workflow_id_builder"
 module ActiveJob
   module Temporal
     module Activities
+      DEPENDENCY_WORKFLOW_STATES = {
+        Temporalio::Client::WorkflowExecutionStatus::RUNNING => "running",
+        Temporalio::Client::WorkflowExecutionStatus::COMPLETED => "completed",
+        Temporalio::Client::WorkflowExecutionStatus::FAILED => "failed",
+        Temporalio::Client::WorkflowExecutionStatus::CANCELED => "canceled",
+        Temporalio::Client::WorkflowExecutionStatus::TERMINATED => "terminated",
+        Temporalio::Client::WorkflowExecutionStatus::CONTINUED_AS_NEW => "continued_as_new",
+        Temporalio::Client::WorkflowExecutionStatus::TIMED_OUT => "timed_out"
+      }.freeze
+
       class DependencyStatusActivity < Temporalio::Activity::Definition
         SAFE_QUERY_VALUE_PATTERN = /\A[A-Za-z0-9_.:-]+\z/
-        WORKFLOW_STATES = {
-          Temporalio::Client::WorkflowExecutionStatus::RUNNING => "running",
-          Temporalio::Client::WorkflowExecutionStatus::COMPLETED => "completed",
-          Temporalio::Client::WorkflowExecutionStatus::FAILED => "failed",
-          Temporalio::Client::WorkflowExecutionStatus::CANCELED => "canceled",
-          Temporalio::Client::WorkflowExecutionStatus::TERMINATED => "terminated",
-          Temporalio::Client::WorkflowExecutionStatus::CONTINUED_AS_NEW => "continued_as_new",
-          Temporalio::Client::WorkflowExecutionStatus::TIMED_OUT => "timed_out"
-        }.freeze
 
         def execute(dependencies)
           Array(dependencies).map { |dependency| status_for(normalize_dependency(dependency)) }
@@ -31,7 +32,9 @@ module ActiveJob
           workflow_reference = find_workflow_reference(dependency)
           return status_payload(dependency, "not_found") unless workflow_reference
 
-          status_from_description(dependency, describe_workflow(workflow_reference))
+          description = describe_workflow(workflow_reference)
+          description = follow_continued_run(description, workflow_reference) if workflow_reference[:run_id]
+          status_from_description(dependency, description)
         rescue StandardError => e
           fallback_description = describe_search_attribute_workflow(dependency) if rpc_not_found?(e)
           return status_from_description(dependency, fallback_description) if fallback_description
@@ -43,21 +46,19 @@ module ActiveJob
         def status_from_description(dependency, description)
           status_payload(
             dependency,
-            WORKFLOW_STATES.fetch(description.status, "unknown"),
+            DEPENDENCY_WORKFLOW_STATES.fetch(description.status, "unknown"),
             workflow_id: description.id,
             run_id: description.run_id
           )
         end
 
         def normalize_dependency(dependency)
-          dependency.each_with_object({}) do |(key, value), normalized|
-            normalized[key.to_s] = value
-          end
+          dependency.to_h.transform_keys(&:to_s)
         end
 
         def find_workflow_reference(dependency)
           workflow_id = dependency["workflow_id"]
-          return { workflow_id: workflow_id, run_id: nil } if workflow_id
+          return { workflow_id: workflow_id, run_id: dependency["run_id"] } if workflow_id
 
           search_workflow_reference(dependency) || default_workflow_reference(dependency)
         end
@@ -66,7 +67,7 @@ module ActiveJob
           job_id = dependency["job_id"]
           return unless job_id
 
-          workflow = client.list_workflows(workflow_search_query(dependency)).first
+          workflow = search_ordered_workflow(dependency)
           return unless workflow
 
           {
@@ -79,6 +80,14 @@ module ActiveJob
           raise
         end
 
+        def search_ordered_workflow(dependency)
+          client.list_workflows(workflow_search_query(dependency, ordered: true)).first
+        rescue StandardError => e
+          raise unless rpc_invalid_argument?(e)
+
+          client.list_workflows(workflow_search_query(dependency, ordered: false)).first
+        end
+
         def default_workflow_reference(dependency)
           job_class = dependency["job_class"]
           job_id = dependency["job_id"]
@@ -92,11 +101,12 @@ module ActiveJob
           }
         end
 
-        def workflow_search_query(dependency)
+        def workflow_search_query(dependency, ordered:)
           filters = ["ajJobId='#{safe_query_value(dependency.fetch('job_id'))}'"]
           job_class = dependency["job_class"]
           filters.unshift("ajClass='#{safe_query_value(job_class)}'") if job_class
-          filters.join(" AND ")
+          query = filters.join(" AND ")
+          ordered ? "#{query} ORDER BY StartTime DESC" : query
         end
 
         def safe_query_value(value)
@@ -114,6 +124,13 @@ module ActiveJob
           ).describe
         end
 
+        def follow_continued_run(description, workflow_reference)
+          state = DEPENDENCY_WORKFLOW_STATES.fetch(description.status, "unknown")
+          return description unless state == "continued_as_new"
+
+          describe_workflow(workflow_reference.merge(run_id: nil))
+        end
+
         def describe_search_attribute_workflow(dependency)
           return unless dependency["workflow_id"] && dependency["job_id"]
 
@@ -134,7 +151,7 @@ module ActiveJob
             "job_id" => dependency["job_id"],
             "job_class" => dependency["job_class"],
             "workflow_id" => workflow_id || dependency["workflow_id"],
-            "run_id" => run_id,
+            "run_id" => run_id || dependency["run_id"],
             "state" => state
           }.compact
         end

data/lib/activejob/temporal/adapter.rb

@@ -103,9 +103,10 @@ module ActiveJob
     #
     # @note Transaction Safety
     #   Jobs using the Temporal adapter are opted into ActiveJob's
-    #   `enqueue_after_transaction_commit` setting. This defers workflow starts
-    #   until the current database transaction commits and prevents workflows from
-    #   starting for rolled-back jobs.
+    #   `enqueue_after_transaction_commit` setting unless the job class configured
+    #   it explicitly. This defers workflow starts until the current database
+    #   transaction commits and prevents workflows from starting for rolled-back
+    #   jobs.
     #
     # @example Basic usage
     #   class MyJob < ApplicationJob

data/lib/activejob/temporal/batch_enqueuer.rb

@@ -5,6 +5,9 @@ require_relative "batch_enqueue_result"
 module ActiveJob
   module Temporal
     class BatchEnqueuer
+      MAX_BATCH_SIZE = 10_000
+      QUEUE_STOP = Object.new.freeze
+
       def initialize(enqueue:, validate_job:, validate_scheduled_at:)
         @enqueue_job = enqueue
         @validate_job = validate_job
@@ -12,8 +15,8 @@ module ActiveJob
       end
 
       def enqueue(items, concurrency: 1)
-        entries = validate_entries!(items)
         concurrency = validate_concurrency!(concurrency)
+        entries = validate_entries!(items)
         results = Array.new(entries.length)
 
         enqueue_entries(entries, results, concurrency)
@@ -28,9 +31,14 @@ module ActiveJob
       def validate_entries!(items)
         raise ArgumentError, "batch enqueue jobs must be an Enumerable" unless items.respond_to?(:each)
 
+        validate_batch_size_hint!(items)
+
         errors = []
-        entries = items.each_with_index.map do |item, index|
-          validate_entry(item, index, errors)
+        entries = []
+        items.each.with_index do |item, index|
+          raise_batch_size_error! if index >= MAX_BATCH_SIZE
+
+          entries << validate_entry(item, index, errors)
         end
 
         raise ArgumentError, "batch enqueue jobs cannot be empty" if entries.empty?
@@ -53,6 +61,20 @@ module ActiveJob
         nil
       end
 
+      def validate_batch_size_hint!(items)
+        return unless items.respond_to?(:size)
+
+        size = items.size
+        return if size.nil? || size <= MAX_BATCH_SIZE
+
+        raise_batch_size_error!
+      end
+
+      def raise_batch_size_error!
+        raise ArgumentError,
+              "batch enqueue accepts at most #{MAX_BATCH_SIZE} jobs; split larger inputs into smaller batches"
+      end
+
       def normalize_entry(item)
         return { job: item, scheduled_at: nil } if active_job_instance?(item)
 
@@ -82,19 +104,23 @@ module ActiveJob
       def enqueue_entries(entries, results, concurrency)
         return enqueue_sequentially(entries, results) if concurrency == 1
 
-        entry_queue = Queue.new
-        entries.each { |entry| entry_queue << entry }
         worker_count = [concurrency, entries.length].min
+        entry_queue = SizedQueue.new(worker_count)
 
-        Array.new(worker_count) do
+        workers = Array.new(worker_count) do
           Thread.new do
             loop do
-              enqueue_entry(entry_queue.pop(true), results)
-            rescue ThreadError
-              break
+              entry = entry_queue.pop
+              break if entry.equal?(QUEUE_STOP)
+
+              enqueue_entry(entry, results)
             end
           end
-        end.each(&:value)
+        end
+
+        entries.each { |entry| entry_queue << entry }
+        worker_count.times { entry_queue << QUEUE_STOP }
+        workers.each(&:value)
       end
 
       def enqueue_sequentially(entries, results)
@@ -103,36 +129,19 @@ module ActiveJob
 
       def enqueue_entry(entry, results)
         handle = enqueue_job.call(entry[:job], scheduled_at: entry[:scheduled_at])
-        results[entry[:index]] = success_result(entry, handle)
+        results[entry[:index]] = item_result(entry, status: :success, handle: handle)
       rescue DuplicateEnqueueError => e
-        results[entry[:index]] = duplicate_result(entry, e)
+        results[entry[:index]] = item_result(entry, status: :duplicate, error: e)
       rescue StandardError => e
-        results[entry[:index]] = failed_result(entry, e)
-      end
-
-      def success_result(entry, handle)
-        BatchEnqueueItemResult.new(
-          index: entry[:index],
-          job: entry[:job],
-          status: :success,
-          handle: handle
-        )
-      end
-
-      def duplicate_result(entry, error)
-        BatchEnqueueItemResult.new(
-          index: entry[:index],
-          job: entry[:job],
-          status: :duplicate,
-          error: error
-        )
+        results[entry[:index]] = item_result(entry, status: :failed, error: e)
       end
 
-      def failed_result(entry, error)
+      def item_result(entry, status:, handle: nil, error: nil)
         BatchEnqueueItemResult.new(
           index: entry[:index],
           job: entry[:job],
-          status: :failed,
+          status: status,
+          handle: handle,
           error: error
         )
       end

data/lib/activejob/temporal/bind_policy.rb

@@ -6,7 +6,7 @@ module ActiveJob
   module Temporal
     module BindPolicy
       LOOPBACK_HOSTNAMES = %w[localhost].freeze
-      TRUE_VALUES = %w[1 true yes].freeze
+      TRUE_VALUES = %w[1 true yes on].freeze
 
       module_function