activejob-temporal 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +14 -0
- data/CLAUDE.md +5 -2
- data/README.md +27 -1
- data/activejob-temporal.gemspec +1 -1
- data/lib/activejob/temporal/activities/aj_runner_activity.rb +24 -16
- data/lib/activejob/temporal/adapter.rb +17 -0
- data/lib/activejob/temporal/audit_log.rb +4 -3
- data/lib/activejob/temporal/cancel.rb +63 -28
- data/lib/activejob/temporal/configuration.rb +23 -0
- data/lib/activejob/temporal/dead_letter_queue.rb +1 -8
- data/lib/activejob/temporal/inspect.rb +12 -3
- data/lib/activejob/temporal/job_payload_builder.rb +55 -6
- data/lib/activejob/temporal/job_payload_rate_limits.rb +4 -4
- data/lib/activejob/temporal/locales/en.yml +4 -0
- data/lib/activejob/temporal/observability.rb +5 -4
- data/lib/activejob/temporal/retry_mapper.rb +21 -2
- data/lib/activejob/temporal/schedule.rb +2 -1
- data/lib/activejob/temporal/signal_query.rb +6 -0
- data/lib/activejob/temporal/standalone_activity.rb +84 -0
- data/lib/activejob/temporal/stuck_jobs.rb +118 -0
- data/lib/activejob/temporal/temporal_options.rb +16 -0
- data/lib/activejob/temporal/version.rb +1 -1
- data/lib/activejob/temporal/visibility_query.rb +6 -0
- data/lib/activejob/temporal/workflow_enqueuer.rb +62 -30
- data/lib/activejob/temporal/workflows/aj_workflow.rb +2 -20
- data/lib/activejob/temporal/workflows/workflow_chaining.rb +1 -1
- data/lib/activejob/temporal.rb +32 -0
- metadata +5 -3
|
@@ -141,9 +141,9 @@ module ActiveJob
|
|
|
141
141
|
|
|
142
142
|
info = Temporalio::Activity::Context.current.info
|
|
143
143
|
normalize_payload(
|
|
144
|
-
workflow_id: context_value(info, :workflow_id),
|
|
145
|
-
run_id: context_value(info, :workflow_run_id, :run_id),
|
|
146
|
-
namespace: context_value(info, :workflow_namespace),
|
|
144
|
+
workflow_id: context_value(info, :workflow_id, :activity_id),
|
|
145
|
+
run_id: context_value(info, :workflow_run_id, :activity_run_id, :run_id),
|
|
146
|
+
namespace: context_value(info, :workflow_namespace, :namespace),
|
|
147
147
|
attempt: context_value(info, :attempt)
|
|
148
148
|
)
|
|
149
149
|
rescue StandardError
|
|
@@ -152,7 +152,8 @@ module ActiveJob
|
|
|
152
152
|
|
|
153
153
|
def context_value(object, *methods)
|
|
154
154
|
methods.each do |method_name|
|
|
155
|
-
|
|
155
|
+
value = object.public_send(method_name) if object.respond_to?(method_name)
|
|
156
|
+
return value if value.present?
|
|
156
157
|
end
|
|
157
158
|
|
|
158
159
|
nil
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "active_support/core_ext/string/inflections"
|
|
4
|
+
require "temporalio/retry_policy"
|
|
4
5
|
require_relative "retry_handler_extractor"
|
|
5
6
|
|
|
6
7
|
module ActiveJob
|
|
@@ -54,6 +55,8 @@ module ActiveJob
|
|
|
54
55
|
# @see https://edgeguides.rubyonrails.org/active_job_basics.html#retrying-or-discarding-failed-jobs
|
|
55
56
|
# ActiveJob Retry Guide
|
|
56
57
|
module RetryMapper
|
|
58
|
+
RETRY_POLICY_MAX_ATTEMPT_KEYS = [:maximum_attempts, "maximum_attempts", :max_attempts, "max_attempts"].freeze
|
|
59
|
+
|
|
57
60
|
module_function
|
|
58
61
|
|
|
59
62
|
# Builds a Temporal retry policy hash from a job class's retry configuration.
|
|
@@ -64,6 +67,7 @@ module ActiveJob
|
|
|
64
67
|
#
|
|
65
68
|
# @param job_class [Class] ActiveJob class with retry_on/discard_on declarations
|
|
66
69
|
# @param exception [Exception, nil] Optional exception to match against handlers
|
|
70
|
+
# @param config [Configuration] Configuration for retry defaults
|
|
67
71
|
#
|
|
68
72
|
# @return [Hash] Retry policy with keys:
|
|
69
73
|
# - :initial_interval [Float] Initial retry delay in seconds
|
|
@@ -116,8 +120,7 @@ module ActiveJob
|
|
|
116
120
|
# are evaluated in reverse order (last declared = first matched). If you provide an
|
|
117
121
|
# exception argument, the first matching handler is used. Without an exception, the
|
|
118
122
|
# first handler in the list is used.
|
|
119
|
-
def for(job_class, exception = nil)
|
|
120
|
-
config = ActiveJob::Temporal.config
|
|
123
|
+
def for(job_class, exception = nil, config: ActiveJob::Temporal.config)
|
|
121
124
|
retry_entries = extractor.retry_handlers(job_class)
|
|
122
125
|
retry_entry = select_retry_entry(job_class, exception, retry_entries)
|
|
123
126
|
|
|
@@ -158,6 +161,22 @@ module ActiveJob
|
|
|
158
161
|
extractor.discard_exception?(job_class, exception)
|
|
159
162
|
end
|
|
160
163
|
|
|
164
|
+
# Builds Temporalio::RetryPolicy from hash.
|
|
165
|
+
# @api private
|
|
166
|
+
def temporal_policy(hash)
|
|
167
|
+
# RetryMapper returns maximum_attempts, but Temporalio::RetryPolicy expects max_attempts
|
|
168
|
+
max_attempts_value = RETRY_POLICY_MAX_ATTEMPT_KEYS.filter_map { |key| hash[key] }.first
|
|
169
|
+
retry_policy_options = {
|
|
170
|
+
initial_interval: hash[:initial_interval] || hash["initial_interval"],
|
|
171
|
+
backoff_coefficient: hash[:backoff_coefficient] || hash["backoff_coefficient"],
|
|
172
|
+
max_interval: hash[:max_interval] || hash["max_interval"],
|
|
173
|
+
max_attempts: max_attempts_value,
|
|
174
|
+
non_retryable_error_types: hash[:non_retryable_error_types] || hash["non_retryable_error_types"]
|
|
175
|
+
}.compact
|
|
176
|
+
|
|
177
|
+
Temporalio::RetryPolicy.new(**retry_policy_options)
|
|
178
|
+
end
|
|
179
|
+
|
|
161
180
|
# -- helpers ----------------------------------------------------------------
|
|
162
181
|
|
|
163
182
|
# Returns the handler extractor instance (memoized).
|
|
@@ -100,7 +100,8 @@ module ActiveJob
|
|
|
100
100
|
payload,
|
|
101
101
|
id: workflow_id,
|
|
102
102
|
task_queue: Adapter.resolve_task_queue(job, config: @config),
|
|
103
|
-
search_attributes: search_attributes_for(job)
|
|
103
|
+
search_attributes: search_attributes_for(job),
|
|
104
|
+
priority: Adapter.resolve_priority(job, config: @config) || Temporalio::Priority.default
|
|
104
105
|
)
|
|
105
106
|
end
|
|
106
107
|
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
require "temporalio/error"
|
|
4
4
|
require_relative "job_id_validation"
|
|
5
5
|
require_relative "visibility_query"
|
|
6
|
+
require_relative "standalone_activity"
|
|
6
7
|
require_relative "workflow_id_builder"
|
|
7
8
|
|
|
8
9
|
module ActiveJob
|
|
@@ -64,6 +65,11 @@ module ActiveJob
|
|
|
64
65
|
private
|
|
65
66
|
|
|
66
67
|
def with_running_workflow_handle(job_class, job_id, &)
|
|
68
|
+
if StandaloneActivity.enabled?(job_class)
|
|
69
|
+
raise ConfigurationError,
|
|
70
|
+
"#{job_class.name} standalone activities do not support signals, queries, or updates"
|
|
71
|
+
end
|
|
72
|
+
|
|
67
73
|
client = ActiveJob::Temporal.client
|
|
68
74
|
result = yield_schedule_execution_handle(client, job_id, &)
|
|
69
75
|
return result unless result.equal?(WORKFLOW_NOT_FOUND)
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "temporalio/error"
|
|
4
|
+
require_relative "workflow_id_builder"
|
|
5
|
+
|
|
6
|
+
module ActiveJob
|
|
7
|
+
module Temporal
|
|
8
|
+
# @api private
|
|
9
|
+
module StandaloneActivity
|
|
10
|
+
STATES = {
|
|
11
|
+
RUNNING: :running,
|
|
12
|
+
PAUSED: :running,
|
|
13
|
+
COMPLETED: :completed,
|
|
14
|
+
FAILED: :failed,
|
|
15
|
+
CANCELED: :canceled,
|
|
16
|
+
TERMINATED: :terminated,
|
|
17
|
+
TIMED_OUT: :timed_out
|
|
18
|
+
}.freeze
|
|
19
|
+
|
|
20
|
+
module_function
|
|
21
|
+
|
|
22
|
+
def enabled?(job_class)
|
|
23
|
+
job_class.respond_to?(:temporal_standalone_activity?) && job_class.temporal_standalone_activity?
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def with_handle(client, job_class, job_id)
|
|
27
|
+
return unless client.respond_to?(:activity_handle)
|
|
28
|
+
|
|
29
|
+
activity_id = WorkflowIdBuilder.default_from_job_class(job_class, job_id)
|
|
30
|
+
yield client.activity_handle(activity_id)
|
|
31
|
+
rescue Temporalio::Error::RPCError => e
|
|
32
|
+
return if e.code == Temporalio::Error::RPCError::Code::NOT_FOUND
|
|
33
|
+
return if e.code == Temporalio::Error::RPCError::Code::UNIMPLEMENTED && !enabled?(job_class)
|
|
34
|
+
|
|
35
|
+
raise
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def status(client, job_class, job_id)
|
|
39
|
+
with_handle(client, job_class, job_id) do |handle|
|
|
40
|
+
activity_status(describe_activity(handle))
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def describe_activity(handle)
|
|
45
|
+
if handle.method(:describe).parameters.include?(%i[key include_last_failure])
|
|
46
|
+
handle.describe(include_last_failure: true)
|
|
47
|
+
else
|
|
48
|
+
handle.describe
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def activity_status(description)
|
|
53
|
+
{
|
|
54
|
+
state: state(description.status),
|
|
55
|
+
workflow_id: description.activity_id,
|
|
56
|
+
run_id: description.activity_run_id,
|
|
57
|
+
started_at: description.last_started_time,
|
|
58
|
+
closed_at: description.close_time,
|
|
59
|
+
attempt: description.attempt,
|
|
60
|
+
last_failure: failure_message(description.last_failure)
|
|
61
|
+
}
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def failure_message(failure)
|
|
65
|
+
return unless failure
|
|
66
|
+
|
|
67
|
+
type = failure.type if failure.respond_to?(:type)
|
|
68
|
+
[type, failure.message].compact.reject(&:empty?).join(": ").presence
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private_class_method :describe_activity, :activity_status, :failure_message
|
|
72
|
+
|
|
73
|
+
def state(status)
|
|
74
|
+
return :unknown unless defined?(Temporalio::Client::ActivityExecutionStatus)
|
|
75
|
+
|
|
76
|
+
statuses = Temporalio::Client::ActivityExecutionStatus
|
|
77
|
+
STATES.each do |name, state|
|
|
78
|
+
return state if statuses.const_defined?(name, false) && statuses.const_get(name, false) == status
|
|
79
|
+
end
|
|
80
|
+
:unknown
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "temporalio/client"
|
|
4
|
+
|
|
5
|
+
require_relative "search_attributes"
|
|
6
|
+
require_relative "visibility_query"
|
|
7
|
+
require_relative "workflow_types"
|
|
8
|
+
|
|
9
|
+
module ActiveJob
|
|
10
|
+
module Temporal
|
|
11
|
+
module StuckJobs
|
|
12
|
+
DEFAULT_LIMIT = 100
|
|
13
|
+
PROBLEMS_FILTER = "TemporalReportedProblems IN ('category=WorkflowTaskFailed', 'category=WorkflowTaskTimedOut')"
|
|
14
|
+
CAPABILITY_ERROR = "stuck job detection is not available: requires Temporal server >= 1.30 with " \
|
|
15
|
+
"system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute > 0"
|
|
16
|
+
|
|
17
|
+
module_function
|
|
18
|
+
|
|
19
|
+
# Lists running job workflows with repeated workflow task failures or timeouts.
|
|
20
|
+
#
|
|
21
|
+
# @param job_class [Class, String, nil] Filter by ActiveJob class.
|
|
22
|
+
# @param queue [String, nil] Filter by ActiveJob queue.
|
|
23
|
+
# @param limit [Integer] Maximum number of results.
|
|
24
|
+
# @param client [Temporalio::Client] Client for the namespace to inspect.
|
|
25
|
+
# @return [Array<Hash>] Workflow details, job metadata, and reported problems.
|
|
26
|
+
# @raise [ArgumentError] If the limit is invalid or a filter is unsafe.
|
|
27
|
+
# @raise [ActiveJob::Temporal::Error] If reported problems are unavailable.
|
|
28
|
+
# @note Requires Temporal Server >= 1.30 and a positive value for
|
|
29
|
+
# system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute.
|
|
30
|
+
# Temporal clears the problems on the next successful workflow task.
|
|
31
|
+
# The {ActiveJob::Temporal.cancel} method cannot fix a stuck workflow.
|
|
32
|
+
def list(job_class: nil, queue: nil, limit: DEFAULT_LIMIT, client: ActiveJob::Temporal.client)
|
|
33
|
+
VisibilityQuery.validate_limit!(limit)
|
|
34
|
+
query = visibility_query(job_class, queue)
|
|
35
|
+
check_capability!(client)
|
|
36
|
+
|
|
37
|
+
query_visibility { client.list_workflows(query).first(limit) }.map { |workflow| entry(workflow) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Counts running job workflows with repeated workflow task failures or timeouts.
|
|
41
|
+
#
|
|
42
|
+
# @param job_class [Class, String, nil] Filter by ActiveJob class.
|
|
43
|
+
# @param queue [String, nil] Filter by ActiveJob queue.
|
|
44
|
+
# @param client [Temporalio::Client] Client for the namespace to inspect.
|
|
45
|
+
# @return [Integer] Count from Temporal visibility.
|
|
46
|
+
# @raise [ArgumentError] If a filter is unsafe.
|
|
47
|
+
# @raise [ActiveJob::Temporal::Error] If reported problems are unavailable.
|
|
48
|
+
# @note Requires Temporal Server >= 1.30 and a positive value for
|
|
49
|
+
# system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute.
|
|
50
|
+
# Temporal clears the problems on the next successful workflow task.
|
|
51
|
+
# The {ActiveJob::Temporal.cancel} method cannot fix a stuck workflow.
|
|
52
|
+
def count(job_class: nil, queue: nil, client: ActiveJob::Temporal.client)
|
|
53
|
+
query = visibility_query(job_class, queue)
|
|
54
|
+
check_capability!(client)
|
|
55
|
+
|
|
56
|
+
query_visibility { client.count_workflows(query).count }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def visibility_query(job_class, queue)
|
|
60
|
+
workflow_types = [WorkflowTypes::ACTIVE_JOB, WorkflowTypes::DEAD_LETTER].map { |type| VisibilityQuery.quote(type) }
|
|
61
|
+
filters = ["ExecutionStatus='Running'", "WorkflowType IN (#{workflow_types.join(', ')})", PROBLEMS_FILTER]
|
|
62
|
+
unless job_class.nil?
|
|
63
|
+
class_name = job_class.is_a?(Class) ? job_class.name : job_class
|
|
64
|
+
filters << "#{attribute_name(:aj_class)}=#{VisibilityQuery.quote(class_name)}"
|
|
65
|
+
end
|
|
66
|
+
filters << "#{attribute_name(:aj_queue)}=#{VisibilityQuery.quote(queue)}" unless queue.nil?
|
|
67
|
+
filters.join(" AND ")
|
|
68
|
+
end
|
|
69
|
+
private_class_method :visibility_query
|
|
70
|
+
|
|
71
|
+
def check_capability!(client)
|
|
72
|
+
request = Temporalio::Api::WorkflowService::V1::DescribeNamespaceRequest.new(namespace: client.namespace)
|
|
73
|
+
response = client.workflow_service.describe_namespace(request)
|
|
74
|
+
return if response.namespace_info&.capabilities&.reported_problems_search_attribute
|
|
75
|
+
|
|
76
|
+
raise ActiveJob::Temporal::Error, CAPABILITY_ERROR
|
|
77
|
+
end
|
|
78
|
+
private_class_method :check_capability!
|
|
79
|
+
|
|
80
|
+
def query_visibility
|
|
81
|
+
yield
|
|
82
|
+
rescue Temporalio::Error::RPCError => e
|
|
83
|
+
raise unless e.code == Temporalio::Error::RPCError::Code::INVALID_ARGUMENT
|
|
84
|
+
|
|
85
|
+
# The capability check already passed, so this is usually an unregistered ajClass/ajQueue attribute.
|
|
86
|
+
raise ActiveJob::Temporal::Error.new("stuck job query rejected by Temporal: #{e.message}"), cause: e
|
|
87
|
+
end
|
|
88
|
+
private_class_method :query_visibility
|
|
89
|
+
|
|
90
|
+
def entry(workflow)
|
|
91
|
+
# Temporal returns "Datetime" for ajEnqueuedAt, which SDK 1.4.1 cannot decode as a typed key.
|
|
92
|
+
attributes = workflow.raw_info.search_attributes&.indexed_fields
|
|
93
|
+
{
|
|
94
|
+
workflow_id: workflow.id,
|
|
95
|
+
run_id: workflow.run_id,
|
|
96
|
+
workflow_type: workflow.workflow_type,
|
|
97
|
+
job_class: attribute_value(attributes, attribute_name(:aj_class)),
|
|
98
|
+
job_id: attribute_value(attributes, attribute_name(:aj_job_id)),
|
|
99
|
+
queue: attribute_value(attributes, attribute_name(:aj_queue)),
|
|
100
|
+
started_at: workflow.start_time,
|
|
101
|
+
problems: attribute_value(attributes, "TemporalReportedProblems") || []
|
|
102
|
+
}
|
|
103
|
+
end
|
|
104
|
+
private_class_method :entry
|
|
105
|
+
|
|
106
|
+
def attribute_value(attributes, name)
|
|
107
|
+
payload = attributes&.[](name)
|
|
108
|
+
Temporalio::Converters::PayloadConverter.default.from_payload(payload) if payload
|
|
109
|
+
end
|
|
110
|
+
private_class_method :attribute_value
|
|
111
|
+
|
|
112
|
+
def attribute_name(key)
|
|
113
|
+
SearchAttributes::SEARCH_ATTRIBUTE_KEY_DEFINITIONS.fetch(key).first
|
|
114
|
+
end
|
|
115
|
+
private_class_method :attribute_name
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -46,6 +46,10 @@ module ActiveJob
|
|
|
46
46
|
module TemporalOptions
|
|
47
47
|
extend ActiveSupport::Concern
|
|
48
48
|
|
|
49
|
+
included do
|
|
50
|
+
class_attribute :temporal_standalone_activity_enabled, instance_accessor: false, default: false
|
|
51
|
+
end
|
|
52
|
+
|
|
49
53
|
VALID_TIMEOUT_KEYS = %i[
|
|
50
54
|
start_to_close_timeout
|
|
51
55
|
schedule_to_close_timeout
|
|
@@ -55,6 +59,18 @@ module ActiveJob
|
|
|
55
59
|
|
|
56
60
|
# rubocop:disable-next Metrics/BlockLength
|
|
57
61
|
class_methods do
|
|
62
|
+
# Run direct enqueues as standalone Temporal activities.
|
|
63
|
+
# @param enabled [Boolean] Whether this class opts in
|
|
64
|
+
# rubocop:disable-next Style/OptionalBooleanParameter
|
|
65
|
+
def temporal_standalone_activity(enabled = true)
|
|
66
|
+
self.temporal_standalone_activity_enabled = enabled
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# @return [Boolean] Whether direct enqueues start standalone activities
|
|
70
|
+
def temporal_standalone_activity?
|
|
71
|
+
temporal_standalone_activity_enabled
|
|
72
|
+
end
|
|
73
|
+
|
|
58
74
|
# Define Temporal activity timeout options for this job class.
|
|
59
75
|
#
|
|
60
76
|
# @param options [Hash] Timeout configuration options
|
|
@@ -23,6 +23,12 @@ module ActiveJob
|
|
|
23
23
|
raise ArgumentError,
|
|
24
24
|
"visibility query values may only contain letters, numbers, underscore, hyphen, period, and colon"
|
|
25
25
|
end
|
|
26
|
+
|
|
27
|
+
def validate_limit!(limit)
|
|
28
|
+
return if limit.is_a?(Integer) && limit.positive?
|
|
29
|
+
|
|
30
|
+
raise ArgumentError, "limit must be a positive integer"
|
|
31
|
+
end
|
|
26
32
|
end
|
|
27
33
|
end
|
|
28
34
|
end
|
|
@@ -6,6 +6,7 @@ require_relative "dead_letter_payload_validation"
|
|
|
6
6
|
require_relative "job_payload_builder"
|
|
7
7
|
require_relative "observability"
|
|
8
8
|
require_relative "workflow_enqueuer_batch"
|
|
9
|
+
require_relative "standalone_activity"
|
|
9
10
|
require_relative "workflow_id_builder"
|
|
10
11
|
require_relative "workflow_types"
|
|
11
12
|
|
|
@@ -67,6 +68,8 @@ module ActiveJob
|
|
|
67
68
|
def enqueue(job, scheduled_at: nil)
|
|
68
69
|
validate_job_for_enqueueing(job)
|
|
69
70
|
scheduled_at = validate_scheduled_at!(scheduled_at)
|
|
71
|
+
return enqueue_standalone(job, scheduled_at) if StandaloneActivity.enabled?(job.class)
|
|
72
|
+
|
|
70
73
|
workflow_id = @workflow_id_builder.build(job)
|
|
71
74
|
payload = build_payload(job, workflow_id: workflow_id, scheduled_at: scheduled_at)
|
|
72
75
|
enqueue_with_payload(job, payload, workflow_id)
|
|
@@ -74,6 +77,30 @@ module ActiveJob
|
|
|
74
77
|
|
|
75
78
|
private
|
|
76
79
|
|
|
80
|
+
def enqueue_standalone(job, scheduled_at)
|
|
81
|
+
temporal_client = client
|
|
82
|
+
unless temporal_client.respond_to?(:start_activity)
|
|
83
|
+
raise ConfigurationError, "standalone activities require temporalio >= 1.5"
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
require_relative "activities/aj_runner_activity"
|
|
87
|
+
|
|
88
|
+
activity_id = WorkflowIdBuilder.default_for(job)
|
|
89
|
+
WorkflowIdBuilder.validate!(activity_id)
|
|
90
|
+
payload = @payload_builder.build(
|
|
91
|
+
job, scheduled_at: scheduled_at, encryption_context: encryption_context_for(activity_id), standalone: true
|
|
92
|
+
)
|
|
93
|
+
options = start_options(job, activity_id).merge(@payload_builder.standalone_activity_options(job))
|
|
94
|
+
options[:start_delay] = [scheduled_at - Time.now, 0].max if scheduled_at
|
|
95
|
+
if defined?(Temporalio::ActivityIDConflictPolicy::FAIL)
|
|
96
|
+
options[:id_conflict_policy] = Temporalio::ActivityIDConflictPolicy::FAIL
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
start_execution(job, payload, options, standalone: true) do
|
|
100
|
+
temporal_client.start_activity(Activities::AjRunnerActivity, payload, **options)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
77
104
|
def configured_workflow_id_generator
|
|
78
105
|
return unless @config.respond_to?(:workflow_id_generator)
|
|
79
106
|
|
|
@@ -85,14 +112,19 @@ module ActiveJob
|
|
|
85
112
|
def enqueue_with_payload(job, payload, workflow_id)
|
|
86
113
|
DeadLetterPayloadValidation.validate!(payload)
|
|
87
114
|
|
|
88
|
-
|
|
89
|
-
|
|
115
|
+
options = start_options(job, workflow_id)
|
|
116
|
+
options[:id_conflict_policy] = Temporalio::WorkflowIDConflictPolicy::FAIL
|
|
117
|
+
add_dead_letter_task_queue(payload, options.fetch(:task_queue))
|
|
90
118
|
|
|
91
|
-
options
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
119
|
+
start_execution(job, payload, options) do
|
|
120
|
+
client.start_workflow(WorkflowTypes::ACTIVE_JOB, payload, **options)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def start_options(job, id)
|
|
125
|
+
options = { id: id, task_queue: Adapter.resolve_task_queue(job, config: @config) }
|
|
126
|
+
priority = Adapter.resolve_priority(job, config: @config)
|
|
127
|
+
options[:priority] = priority if priority
|
|
96
128
|
|
|
97
129
|
# Add search attributes if configured
|
|
98
130
|
if @config.respond_to?(:enable_search_attributes) && @config.enable_search_attributes
|
|
@@ -100,7 +132,7 @@ module ActiveJob
|
|
|
100
132
|
options[:search_attributes] = search_attributes
|
|
101
133
|
end
|
|
102
134
|
|
|
103
|
-
|
|
135
|
+
options
|
|
104
136
|
end
|
|
105
137
|
|
|
106
138
|
def add_dead_letter_task_queue(payload, task_queue)
|
|
@@ -126,30 +158,25 @@ module ActiveJob
|
|
|
126
158
|
{ namespace: @config.namespace, workflow_id: workflow_id }
|
|
127
159
|
end
|
|
128
160
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
def start_workflow(job, payload, options)
|
|
132
|
-
workflow_class = WorkflowTypes::ACTIVE_JOB
|
|
133
|
-
handle = start_temporal_workflow(workflow_class, job, payload, options)
|
|
134
|
-
|
|
135
|
-
log_enqueued(job, options, payload, duplicate: false)
|
|
136
|
-
handle
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
def start_temporal_workflow(workflow_class, job, payload, options)
|
|
140
|
-
client.start_workflow(workflow_class, payload, **options)
|
|
161
|
+
def start_execution(job, payload, options, standalone: nil)
|
|
162
|
+
handle = yield
|
|
141
163
|
rescue StandardError => e
|
|
142
|
-
if
|
|
143
|
-
log_enqueued(job, options, payload, duplicate: true)
|
|
144
|
-
raise DuplicateEnqueueError, build_duplicate_enqueue_error_message(job, options)
|
|
164
|
+
if execution_already_started?(e)
|
|
165
|
+
log_enqueued(job, options, payload, duplicate: true, standalone: standalone)
|
|
166
|
+
raise DuplicateEnqueueError, build_duplicate_enqueue_error_message(job, options, standalone: standalone)
|
|
145
167
|
end
|
|
146
168
|
|
|
147
169
|
raise ActiveJob::EnqueueError.new(build_enqueue_error_message(job, e)), cause: e
|
|
170
|
+
else
|
|
171
|
+
log_enqueued(job, options, payload, duplicate: false, standalone: standalone)
|
|
172
|
+
handle
|
|
148
173
|
end
|
|
149
174
|
|
|
150
175
|
# Checks if error indicates workflow was already started (duplicate job_id).
|
|
151
176
|
# @api private
|
|
152
|
-
def
|
|
177
|
+
def execution_already_started?(error)
|
|
178
|
+
return true if defined?(Temporalio::Error::ActivityAlreadyStartedError) &&
|
|
179
|
+
error.is_a?(Temporalio::Error::ActivityAlreadyStartedError)
|
|
153
180
|
return true if defined?(Temporalio::Error::WorkflowAlreadyStartedError) &&
|
|
154
181
|
error.is_a?(Temporalio::Error::WorkflowAlreadyStartedError)
|
|
155
182
|
return true if defined?(Temporalio::Client::WorkflowAlreadyStartedError) &&
|
|
@@ -162,9 +189,11 @@ module ActiveJob
|
|
|
162
189
|
|
|
163
190
|
# Logs enqueue event with structured metadata.
|
|
164
191
|
# @api private
|
|
165
|
-
def log_enqueued(job, options, payload, duplicate:)
|
|
192
|
+
def log_enqueued(job, options, payload, duplicate:, standalone:)
|
|
166
193
|
attributes = enqueue_attributes(job, options, payload, duplicate: duplicate)
|
|
167
194
|
|
|
195
|
+
attributes[:standalone] = true if standalone
|
|
196
|
+
|
|
168
197
|
emit_enqueue_side_effect("log", attributes) do
|
|
169
198
|
Logger.log_event("workflow_enqueued", **attributes)
|
|
170
199
|
end
|
|
@@ -178,13 +207,15 @@ module ActiveJob
|
|
|
178
207
|
job,
|
|
179
208
|
workflow_id: options[:id],
|
|
180
209
|
task_queue: options[:task_queue],
|
|
181
|
-
duplicate: duplicate
|
|
210
|
+
duplicate: duplicate,
|
|
211
|
+
standalone: standalone
|
|
182
212
|
)
|
|
183
213
|
)
|
|
184
214
|
end
|
|
185
215
|
end
|
|
186
216
|
|
|
187
217
|
def enqueue_attributes(job, options, payload, duplicate:)
|
|
218
|
+
priority = options[:priority]
|
|
188
219
|
attributes = {
|
|
189
220
|
workflow_id: options[:id],
|
|
190
221
|
job_class: job.class.name,
|
|
@@ -195,7 +226,7 @@ module ActiveJob
|
|
|
195
226
|
}
|
|
196
227
|
attributes[:scheduled_at] = payload[:scheduled_at] if payload[:scheduled_at]
|
|
197
228
|
|
|
198
|
-
attributes
|
|
229
|
+
attributes.merge({ priority_key: priority&.priority_key, fairness_key: priority&.fairness_key }.compact)
|
|
199
230
|
end
|
|
200
231
|
|
|
201
232
|
def emit_enqueue_side_effect(side_effect, attributes)
|
|
@@ -224,12 +255,13 @@ module ActiveJob
|
|
|
224
255
|
)
|
|
225
256
|
end
|
|
226
257
|
|
|
227
|
-
def build_duplicate_enqueue_error_message(job, options)
|
|
258
|
+
def build_duplicate_enqueue_error_message(job, options, standalone:)
|
|
228
259
|
format(
|
|
229
|
-
"Job %<job_class>s (%<job_id>s) was already enqueued as
|
|
260
|
+
"Job %<job_class>s (%<job_id>s) was already enqueued as %<kind>s %<workflow_id>s",
|
|
230
261
|
job_class: job.class.name,
|
|
231
262
|
job_id: job.job_id,
|
|
232
|
-
workflow_id: options[:id]
|
|
263
|
+
workflow_id: options[:id],
|
|
264
|
+
kind: standalone ? "standalone activity" : "workflow"
|
|
233
265
|
)
|
|
234
266
|
end
|
|
235
267
|
|
|
@@ -3,10 +3,10 @@
|
|
|
3
3
|
require "time"
|
|
4
4
|
require "active_support/core_ext/hash/keys"
|
|
5
5
|
require "temporalio/error"
|
|
6
|
-
require "temporalio/retry_policy"
|
|
7
6
|
require "temporalio/workflow"
|
|
8
7
|
|
|
9
8
|
require_relative "../activities/rate_limit_activity"
|
|
9
|
+
require_relative "../retry_mapper"
|
|
10
10
|
require_relative "dead_letter_support"
|
|
11
11
|
require_relative "dead_letter_workflow"
|
|
12
12
|
require_relative "workflow_continue_as_new"
|
|
@@ -22,8 +22,6 @@ require_relative "workflow_versioning"
|
|
|
22
22
|
module ActiveJob
|
|
23
23
|
module Temporal
|
|
24
24
|
module Workflows
|
|
25
|
-
RETRY_POLICY_MAX_ATTEMPT_KEYS = [:maximum_attempts, "maximum_attempts", :max_attempts, "max_attempts"].freeze
|
|
26
|
-
|
|
27
25
|
# Deterministic orchestration workflow for ActiveJob execution.
|
|
28
26
|
#
|
|
29
27
|
# This workflow serves as the durable scheduling and orchestration layer for ActiveJob.
|
|
@@ -258,7 +256,7 @@ module ActiveJob
|
|
|
258
256
|
options.merge!(temporal_opts.symbolize_keys) if temporal_opts
|
|
259
257
|
|
|
260
258
|
retry_policy_hash = payload[:retry_policy] || payload["retry_policy"]
|
|
261
|
-
options[:retry_policy] =
|
|
259
|
+
options[:retry_policy] = RetryMapper.temporal_policy(retry_policy_hash) if retry_policy_hash
|
|
262
260
|
task_queue = payload[:activity_task_queue] || payload["activity_task_queue"]
|
|
263
261
|
options[:task_queue] = task_queue if task_queue
|
|
264
262
|
|
|
@@ -273,22 +271,6 @@ module ActiveJob
|
|
|
273
271
|
|
|
274
272
|
{ start_to_close_timeout: DEFAULT_START_TO_CLOSE_TIMEOUT }
|
|
275
273
|
end
|
|
276
|
-
|
|
277
|
-
# Builds Temporalio::RetryPolicy from hash.
|
|
278
|
-
# @api private
|
|
279
|
-
def build_retry_policy(hash)
|
|
280
|
-
# RetryMapper returns maximum_attempts, but Temporalio::RetryPolicy expects max_attempts
|
|
281
|
-
max_attempts_value = RETRY_POLICY_MAX_ATTEMPT_KEYS.filter_map { |key| hash[key] }.first
|
|
282
|
-
retry_policy_options = {
|
|
283
|
-
initial_interval: hash[:initial_interval] || hash["initial_interval"],
|
|
284
|
-
backoff_coefficient: hash[:backoff_coefficient] || hash["backoff_coefficient"],
|
|
285
|
-
max_interval: hash[:max_interval] || hash["max_interval"],
|
|
286
|
-
max_attempts: max_attempts_value,
|
|
287
|
-
non_retryable_error_types: hash[:non_retryable_error_types] || hash["non_retryable_error_types"]
|
|
288
|
-
}.compact
|
|
289
|
-
|
|
290
|
-
Temporalio::RetryPolicy.new(**retry_policy_options)
|
|
291
|
-
end
|
|
292
274
|
end
|
|
293
275
|
end
|
|
294
276
|
end
|
|
@@ -182,7 +182,7 @@ module ActiveJob
|
|
|
182
182
|
options.each_with_object({}) do |(key, value), normalized|
|
|
183
183
|
normalized_key = key.to_sym
|
|
184
184
|
normalized[normalized_key] = if normalized_key == :retry_policy && value.is_a?(Hash)
|
|
185
|
-
|
|
185
|
+
RetryMapper.temporal_policy(value)
|
|
186
186
|
else
|
|
187
187
|
value
|
|
188
188
|
end
|
data/lib/activejob/temporal.rb
CHANGED
|
@@ -48,6 +48,7 @@ require_relative "temporal/cancel"
|
|
|
48
48
|
require_relative "temporal/inspect"
|
|
49
49
|
require_relative "temporal/signal_query"
|
|
50
50
|
require_relative "temporal/dead_letter_queue"
|
|
51
|
+
require_relative "temporal/stuck_jobs"
|
|
51
52
|
|
|
52
53
|
module ActiveJob
|
|
53
54
|
# ActiveJob adapter for Temporal workflow orchestration.
|
|
@@ -287,6 +288,37 @@ module ActiveJob
|
|
|
287
288
|
DeadLetterQueue.discard(job_class, job_id, reason: reason)
|
|
288
289
|
end
|
|
289
290
|
|
|
291
|
+
# Lists running jobs with repeated workflow task failures or timeouts.
|
|
292
|
+
#
|
|
293
|
+
# @param job_class [Class, String, nil] Filter by ActiveJob class.
|
|
294
|
+
# @param queue [String, nil] Filter by ActiveJob queue.
|
|
295
|
+
# @param limit [Integer] Maximum number of results.
|
|
296
|
+
# @return [Array<Hash>] Workflow details, job metadata, and reported problems.
|
|
297
|
+
# @raise [ArgumentError] If the limit is invalid or a filter is unsafe.
|
|
298
|
+
# @raise [ActiveJob::Temporal::Error] If reported problems are unavailable.
|
|
299
|
+
# @note Requires Temporal Server >= 1.30 and a positive value for
|
|
300
|
+
# system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute.
|
|
301
|
+
# Temporal clears the problems on the next successful workflow task.
|
|
302
|
+
# The {cancel} method cannot fix a stuck workflow.
|
|
303
|
+
def stuck_jobs(job_class: nil, queue: nil, limit: StuckJobs::DEFAULT_LIMIT)
|
|
304
|
+
StuckJobs.list(job_class: job_class, queue: queue, limit: limit)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Counts running jobs with repeated workflow task failures or timeouts.
|
|
308
|
+
#
|
|
309
|
+
# @param job_class [Class, String, nil] Filter by ActiveJob class.
|
|
310
|
+
# @param queue [String, nil] Filter by ActiveJob queue.
|
|
311
|
+
# @return [Integer] Count from Temporal visibility.
|
|
312
|
+
# @raise [ArgumentError] If a filter is unsafe.
|
|
313
|
+
# @raise [ActiveJob::Temporal::Error] If reported problems are unavailable.
|
|
314
|
+
# @note Requires Temporal Server >= 1.30 and a positive value for
|
|
315
|
+
# system.numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute.
|
|
316
|
+
# Temporal clears the problems on the next successful workflow task.
|
|
317
|
+
# The {cancel} method cannot fix a stuck workflow.
|
|
318
|
+
def count_stuck_jobs(job_class: nil, queue: nil)
|
|
319
|
+
StuckJobs.count(job_class: job_class, queue: queue)
|
|
320
|
+
end
|
|
321
|
+
|
|
290
322
|
private
|
|
291
323
|
|
|
292
324
|
def client_mutex
|