activejob-temporal 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENTS.md +1 -0
- data/CHANGELOG.md +18 -0
- data/CLAUDE.md +274 -0
- data/CONTRIBUTING.md +69 -0
- data/README.md +34 -33
- data/activejob-temporal.gemspec +8 -12
- data/api/job_payload_schema.json +51 -6
- data/bin/temporal-worker +3 -8
- data/gemfiles/activejob_contract.gemfile +8 -0
- data/gemfiles/temporalio_contract.gemfile +7 -0
- data/lib/activejob/temporal/activities/aj_runner_activity.rb +75 -19
- data/lib/activejob/temporal/activities/dependency_status_activity.rb +36 -19
- data/lib/activejob/temporal/adapter.rb +4 -3
- data/lib/activejob/temporal/batch_enqueuer.rb +42 -33
- data/lib/activejob/temporal/bind_policy.rb +1 -1
- data/lib/activejob/temporal/cancel.rb +66 -29
- data/lib/activejob/temporal/certificate_watcher.rb +41 -6
- data/lib/activejob/temporal/client.rb +4 -3
- data/lib/activejob/temporal/conditional_enqueue.rb +2 -1
- data/lib/activejob/temporal/configurable.rb +56 -13
- data/lib/activejob/temporal/configuration.rb +139 -7
- data/lib/activejob/temporal/configured_job_compatibility.rb +91 -9
- data/lib/activejob/temporal/connection_worker_pool.rb +18 -1
- data/lib/activejob/temporal/dead_letter_queue.rb +64 -18
- data/lib/activejob/temporal/dependency_options.rb +126 -16
- data/lib/activejob/temporal/health_check_server.rb +14 -17
- data/lib/activejob/temporal/http_line_reader.rb +20 -2
- data/lib/activejob/temporal/http_request_failure_handling.rb +41 -0
- data/lib/activejob/temporal/inspect.rb +17 -7
- data/lib/activejob/temporal/job_id_validation.rb +41 -0
- data/lib/activejob/temporal/job_payload_dependencies.rb +23 -0
- data/lib/activejob/temporal/locales/en.yml +15 -3
- data/lib/activejob/temporal/metrics_server.rb +15 -18
- data/lib/activejob/temporal/middleware/chain.rb +7 -0
- data/lib/activejob/temporal/observability.rb +14 -2
- data/lib/activejob/temporal/payload.rb +64 -25
- data/lib/activejob/temporal/payload_encryption.rb +9 -1
- data/lib/activejob/temporal/payload_serializers.rb +3 -0
- data/lib/activejob/temporal/payload_storage.rb +4 -4
- data/lib/activejob/temporal/rails_environment_loader.rb +1 -8
- data/lib/activejob/temporal/reload_signal_queue.rb +19 -19
- data/lib/activejob/temporal/retry_handler_extractor.rb +18 -3
- data/lib/activejob/temporal/schedulable.rb +5 -7
- data/lib/activejob/temporal/schedule.rb +2 -2
- data/lib/activejob/temporal/signal_query.rb +35 -23
- data/lib/activejob/temporal/temporal_options.rb +26 -1
- data/lib/activejob/temporal/tls_file.rb +16 -16
- data/lib/activejob/temporal/transaction_safety.rb +102 -1
- data/lib/activejob/temporal/version.rb +1 -1
- data/lib/activejob/temporal/visibility_query.rb +16 -1
- data/lib/activejob/temporal/worker_pool.rb +13 -0
- data/lib/activejob/temporal/worker_runtime.rb +16 -0
- data/lib/activejob/temporal/workflow_enqueuer.rb +2 -1
- data/lib/activejob/temporal/workflow_types.rb +10 -0
- data/lib/activejob/temporal/workflows/aj_workflow.rb +17 -4
- data/lib/activejob/temporal/workflows/workflow_dependencies.rb +148 -15
- data/lib/activejob/temporal.rb +2 -13
- data/test/mutant_unit_test.rb +13 -0
- metadata +43 -48
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
require "temporalio/client/workflow_execution_status"
|
|
4
4
|
require "temporalio/error"
|
|
5
|
+
require_relative "job_id_validation"
|
|
5
6
|
require_relative "visibility_query"
|
|
6
7
|
require_relative "workflow_id_builder"
|
|
7
8
|
|
|
8
9
|
module ActiveJob
|
|
9
10
|
module Temporal
|
|
10
11
|
module Inspect
|
|
11
|
-
UUID_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i
|
|
12
12
|
JOB_CLASS_NAME_PATTERN = /\A[A-Z]\w*(?:::[A-Z]\w*)*\z/
|
|
13
13
|
WORKFLOW_STATES = {
|
|
14
14
|
Temporalio::Client::WorkflowExecutionStatus::RUNNING => :running,
|
|
@@ -26,7 +26,8 @@ module ActiveJob
|
|
|
26
26
|
validate_job_id!(job_id)
|
|
27
27
|
|
|
28
28
|
client = ActiveJob::Temporal.client
|
|
29
|
-
|
|
29
|
+
describe_schedule_execution_workflow(client, job_id) ||
|
|
30
|
+
describe_default_workflow(client, job_class, job_id) ||
|
|
30
31
|
describe_search_attribute_workflow(client, job_class, job_id)
|
|
31
32
|
rescue ArgumentError
|
|
32
33
|
raise
|
|
@@ -40,7 +41,9 @@ module ActiveJob
|
|
|
40
41
|
def completed?(job_class, job_id) = workflow_state?(job_class, job_id, :completed)
|
|
41
42
|
|
|
42
43
|
def failed?(job_class, job_id) = workflow_state?(job_class, job_id, :failed)
|
|
44
|
+
end
|
|
43
45
|
|
|
46
|
+
class << self
|
|
44
47
|
private
|
|
45
48
|
|
|
46
49
|
def workflow_state?(job_class, job_id, state)
|
|
@@ -58,11 +61,7 @@ module ActiveJob
|
|
|
58
61
|
end
|
|
59
62
|
|
|
60
63
|
def validate_job_id!(job_id)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
raise ArgumentError,
|
|
64
|
-
"Invalid job_id format: expected UUID (e.g., '550e8400-e29b-41d4-a716-446655440000'), " \
|
|
65
|
-
"got: #{job_id.inspect}"
|
|
64
|
+
JobIdValidation.validate!(job_id)
|
|
66
65
|
end
|
|
67
66
|
|
|
68
67
|
def describe_default_workflow(client, job_class, job_id)
|
|
@@ -73,6 +72,17 @@ module ActiveJob
|
|
|
73
72
|
nil
|
|
74
73
|
end
|
|
75
74
|
|
|
75
|
+
def describe_schedule_execution_workflow(client, job_id)
|
|
76
|
+
workflow_reference = JobIdValidation.schedule_execution_reference(job_id)
|
|
77
|
+
return unless workflow_reference
|
|
78
|
+
|
|
79
|
+
describe_workflow(client, workflow_reference)
|
|
80
|
+
rescue StandardError => e
|
|
81
|
+
raise unless rpc_not_found?(e)
|
|
82
|
+
|
|
83
|
+
nil
|
|
84
|
+
end
|
|
85
|
+
|
|
76
86
|
def describe_search_attribute_workflow(client, job_class, job_id)
|
|
77
87
|
workflow_reference = find_workflow_reference(client, job_class, job_id)
|
|
78
88
|
return unless workflow_reference
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "workflow_id_builder"
|
|
4
|
+
|
|
5
|
+
module ActiveJob
|
|
6
|
+
module Temporal
|
|
7
|
+
module JobIdValidation
|
|
8
|
+
MAX_JOB_ID_LENGTH = WorkflowIdBuilder::MAX_WORKFLOW_ID_LENGTH
|
|
9
|
+
CONTROL_CHARACTER_PATTERN = WorkflowIdBuilder::CONTROL_CHARACTER_PATTERN
|
|
10
|
+
SCHEDULE_WORKFLOW_ID_PREFIX = "ajschwf:"
|
|
11
|
+
|
|
12
|
+
module_function
|
|
13
|
+
|
|
14
|
+
def validate!(job_id)
|
|
15
|
+
unless job_id.is_a?(String)
|
|
16
|
+
raise ArgumentError, "job_id must be a String, got #{job_id.class}: #{job_id.inspect}"
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
raise ArgumentError, "job_id must be valid UTF-8" unless job_id.valid_encoding?
|
|
20
|
+
raise ArgumentError, "job_id must not be blank" if job_id.strip.empty?
|
|
21
|
+
|
|
22
|
+
if job_id.length > MAX_JOB_ID_LENGTH
|
|
23
|
+
raise ArgumentError, "job_id maximum length is #{MAX_JOB_ID_LENGTH} characters (got #{job_id.length})"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
return unless job_id.match?(CONTROL_CHARACTER_PATTERN)
|
|
27
|
+
|
|
28
|
+
raise ArgumentError, "job_id control characters are not allowed (got #{job_id.inspect})"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def schedule_execution_reference(job_id)
|
|
32
|
+
return unless job_id.start_with?(SCHEDULE_WORKFLOW_ID_PREFIX)
|
|
33
|
+
|
|
34
|
+
workflow_id, separator, run_id = job_id.rpartition(":")
|
|
35
|
+
return if separator.empty? || workflow_id.empty? || run_id.empty?
|
|
36
|
+
|
|
37
|
+
{ workflow_id: workflow_id, run_id: run_id }
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -16,6 +16,7 @@ module ActiveJob
|
|
|
16
16
|
payload[:dependencies] = dependencies.map { |dependency| enrich_dependency(dependency) }
|
|
17
17
|
policy = job.respond_to?(:temporal_dependency_failure_policy) ? job.temporal_dependency_failure_policy : :fail
|
|
18
18
|
payload[:dependency_failure_policy] = policy.to_s
|
|
19
|
+
payload[:dependency_wait] = dependency_wait_options(job)
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
def enrich_dependency(dependency)
|
|
@@ -35,6 +36,28 @@ module ActiveJob
|
|
|
35
36
|
WorkflowIdBuilder.validate!(workflow_id)
|
|
36
37
|
workflow_id
|
|
37
38
|
end
|
|
39
|
+
|
|
40
|
+
def dependency_wait_options(job)
|
|
41
|
+
default_dependency_wait_options.merge(job_dependency_wait_options(job))
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def default_dependency_wait_options
|
|
45
|
+
{
|
|
46
|
+
timeout: @config.dependency_wait_timeout.to_f,
|
|
47
|
+
initial_interval: @config.dependency_wait_initial_interval.to_f,
|
|
48
|
+
max_interval: @config.dependency_wait_max_interval.to_f,
|
|
49
|
+
backoff: @config.dependency_wait_backoff.to_f
|
|
50
|
+
}
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def job_dependency_wait_options(job)
|
|
54
|
+
return {} unless job.respond_to?(:temporal_dependency_wait)
|
|
55
|
+
|
|
56
|
+
dependency_wait = job.temporal_dependency_wait
|
|
57
|
+
return {} unless dependency_wait.is_a?(Hash)
|
|
58
|
+
|
|
59
|
+
dependency_wait
|
|
60
|
+
end
|
|
38
61
|
end
|
|
39
62
|
end
|
|
40
63
|
end
|
|
@@ -101,18 +101,30 @@ en:
|
|
|
101
101
|
not_a_duration: "must be a duration (e.g., 1.day or 6.hours), got: %{value}"
|
|
102
102
|
duration_not_positive: "must be positive (got: %{seconds} seconds). Use values like 1.hour or 7.days"
|
|
103
103
|
|
|
104
|
+
dependency_wait_timeout:
|
|
105
|
+
not_a_duration: "must be a duration (e.g., 10.minutes or 1.day), got: %{value}"
|
|
106
|
+
duration_not_positive: "must be positive (got: %{seconds} seconds). Use values like 1.minute or 1.day"
|
|
107
|
+
|
|
108
|
+
dependency_wait_initial_interval:
|
|
109
|
+
not_a_duration: "must be a duration (e.g., 5.seconds or 1.minute), got: %{value}"
|
|
110
|
+
duration_not_positive: "must be positive (got: %{seconds} seconds). Use values like 5.seconds or 1.minute"
|
|
111
|
+
|
|
112
|
+
dependency_wait_max_interval:
|
|
113
|
+
not_a_duration: "must be a duration (e.g., 30.seconds or 5.minutes), got: %{value}"
|
|
114
|
+
duration_not_positive: "must be positive (got: %{seconds} seconds). Use values like 30.seconds or 5.minutes"
|
|
115
|
+
|
|
104
116
|
tls_cert_path:
|
|
105
117
|
requires_key_path: "requires tls_key_path so the client certificate and private key rotate together"
|
|
106
118
|
invalid_path: "must be a non-empty file path, got: %{value}"
|
|
107
|
-
unreadable_path: "must
|
|
119
|
+
unreadable_path: "must resolve to a readable regular file, got: %{value}"
|
|
108
120
|
|
|
109
121
|
tls_key_path:
|
|
110
122
|
invalid_path: "must be a non-empty file path, got: %{value}"
|
|
111
|
-
unreadable_path: "must
|
|
123
|
+
unreadable_path: "must resolve to a readable regular file, got: %{value}"
|
|
112
124
|
|
|
113
125
|
tls_server_root_ca_cert_path:
|
|
114
126
|
invalid_path: "must be a non-empty file path, got: %{value}"
|
|
115
|
-
unreadable_path: "must
|
|
127
|
+
unreadable_path: "must resolve to a readable regular file, got: %{value}"
|
|
116
128
|
|
|
117
129
|
tls_domain:
|
|
118
130
|
blank: "must be present when configured"
|
|
@@ -5,18 +5,22 @@ require "socket"
|
|
|
5
5
|
|
|
6
6
|
require_relative "connection_worker_pool"
|
|
7
7
|
require_relative "bind_policy"
|
|
8
|
+
require_relative "http_request_failure_handling"
|
|
8
9
|
require_relative "http_line_reader"
|
|
9
10
|
|
|
10
11
|
module ActiveJob
|
|
11
12
|
module Temporal
|
|
12
13
|
class MetricsServer
|
|
13
14
|
include HttpLineReader
|
|
15
|
+
include HttpRequestFailureHandling
|
|
14
16
|
|
|
15
17
|
DEFAULT_BIND_ADDRESS = "127.0.0.1"
|
|
16
18
|
READ_TIMEOUT_SECONDS = 1
|
|
17
19
|
CONTENT_TYPE = "text/plain; version=0.0.4"
|
|
18
20
|
CONNECTION_WORKERS = 4
|
|
19
21
|
CONNECTION_QUEUE_SIZE = 16
|
|
22
|
+
REQUEST_FAILURE_EVENT = "metrics_request_failed"
|
|
23
|
+
REQUEST_FAILURE_FORMAT = :text
|
|
20
24
|
|
|
21
25
|
attr_reader :port, :bind_address
|
|
22
26
|
|
|
@@ -86,25 +90,24 @@ module ActiveJob
|
|
|
86
90
|
connection_pool.enqueue(server.accept)
|
|
87
91
|
rescue IOError, Errno::EBADF
|
|
88
92
|
break
|
|
93
|
+
rescue SystemCallError => e
|
|
94
|
+
# Transient accept failures (fd exhaustion, aborted connections) must not kill the listener.
|
|
95
|
+
ActiveJob::Temporal::Logger.error(
|
|
96
|
+
"metrics_accept_failed", error_class: e.class.name, message: e.message.to_s
|
|
97
|
+
)
|
|
98
|
+
sleep 0.05
|
|
89
99
|
end
|
|
90
100
|
ensure
|
|
91
101
|
@mutex.synchronize { @running = false if @server }
|
|
92
102
|
end
|
|
93
103
|
|
|
94
|
-
def serve_client(client)
|
|
95
|
-
handle_client(client)
|
|
96
|
-
rescue IOError, SystemCallError
|
|
97
|
-
nil
|
|
98
|
-
ensure
|
|
99
|
-
client&.close
|
|
100
|
-
end
|
|
101
|
-
|
|
102
104
|
def handle_client(client)
|
|
103
|
-
|
|
105
|
+
deadline = request_deadline
|
|
106
|
+
request_line = read_line(client, deadline)
|
|
104
107
|
return unless request_line
|
|
105
108
|
|
|
106
109
|
method, path = request_line.split.first(2)
|
|
107
|
-
drain_headers(client)
|
|
110
|
+
drain_headers(client, deadline)
|
|
108
111
|
|
|
109
112
|
unless method && path
|
|
110
113
|
write_text(client, 400, "bad_request\n")
|
|
@@ -121,13 +124,6 @@ module ActiveJob
|
|
|
121
124
|
end
|
|
122
125
|
end
|
|
123
126
|
|
|
124
|
-
def drain_headers(client)
|
|
125
|
-
loop do
|
|
126
|
-
line = read_line(client)
|
|
127
|
-
break if line.nil? || line == "\r\n" || line == "\n"
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
|
-
|
|
131
127
|
def write_text(client, status, payload, body: true)
|
|
132
128
|
response = "HTTP/1.1 #{status} #{reason_phrase(status)}\r\n"
|
|
133
129
|
response << "Content-Type: #{CONTENT_TYPE}\r\n"
|
|
@@ -142,7 +138,8 @@ module ActiveJob
|
|
|
142
138
|
200 => "OK",
|
|
143
139
|
400 => "Bad Request",
|
|
144
140
|
404 => "Not Found",
|
|
145
|
-
405 => "Method Not Allowed"
|
|
141
|
+
405 => "Method Not Allowed",
|
|
142
|
+
500 => "Internal Server Error"
|
|
146
143
|
}.fetch(status)
|
|
147
144
|
end
|
|
148
145
|
end
|
|
@@ -16,6 +16,13 @@ module ActiveJob
|
|
|
16
16
|
entries.each { |entry| add(entry) }
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
+
def initialize_copy(original)
|
|
20
|
+
super
|
|
21
|
+
@entries = original.instance_variable_get(:@entries).dup
|
|
22
|
+
@entry_indexes_by_key = original.instance_variable_get(:@entry_indexes_by_key).dup
|
|
23
|
+
@compiled_call_chain = compile_call_chain
|
|
24
|
+
end
|
|
25
|
+
|
|
19
26
|
def add(middleware, *args, **kwargs, &block)
|
|
20
27
|
key = entry_key(middleware, args, kwargs, block)
|
|
21
28
|
callable = build_callable(middleware, args, kwargs, block)
|
|
@@ -164,6 +164,13 @@ module ActiveJob
|
|
|
164
164
|
|
|
165
165
|
def initialize
|
|
166
166
|
@adapters = []
|
|
167
|
+
@stop_replaced_adapters = true
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def initialize_copy(original)
|
|
171
|
+
super
|
|
172
|
+
@adapters = original.adapters.dup
|
|
173
|
+
@stop_replaced_adapters = false
|
|
167
174
|
end
|
|
168
175
|
|
|
169
176
|
def use(name, **)
|
|
@@ -185,7 +192,7 @@ module ActiveJob
|
|
|
185
192
|
end
|
|
186
193
|
|
|
187
194
|
def reset!
|
|
188
|
-
adapters.each(&:stop!)
|
|
195
|
+
adapters.each(&:stop!) if @stop_replaced_adapters
|
|
189
196
|
adapters.clear
|
|
190
197
|
end
|
|
191
198
|
|
|
@@ -193,11 +200,16 @@ module ActiveJob
|
|
|
193
200
|
adapters.each(&:validate!)
|
|
194
201
|
end
|
|
195
202
|
|
|
203
|
+
def finalize_configuration_copy!
|
|
204
|
+
@stop_replaced_adapters = true
|
|
205
|
+
self
|
|
206
|
+
end
|
|
207
|
+
|
|
196
208
|
private
|
|
197
209
|
|
|
198
210
|
def replace_adapter(adapter)
|
|
199
211
|
previous_adapter = self.adapter(adapter.name)
|
|
200
|
-
previous_adapter&.stop!
|
|
212
|
+
previous_adapter&.stop! if @stop_replaced_adapters
|
|
201
213
|
adapters.delete(previous_adapter)
|
|
202
214
|
adapters << adapter
|
|
203
215
|
end
|
|
@@ -38,7 +38,10 @@ module ActiveJob
|
|
|
38
38
|
# job_class: "MyJob",
|
|
39
39
|
# job_id: "abc-123",
|
|
40
40
|
# queue_name: "default",
|
|
41
|
-
#
|
|
41
|
+
# active_job: {
|
|
42
|
+
# "arguments" => [{"_aj_serialized"=>"ActiveJob::Serializers::ObjectSerializer", ...}],
|
|
43
|
+
# ...
|
|
44
|
+
# },
|
|
42
45
|
# executions: 0,
|
|
43
46
|
# exception_executions: {},
|
|
44
47
|
# scheduled_at: "2025-10-29T12:00:00Z" # optional
|
|
@@ -83,7 +86,7 @@ module ActiveJob
|
|
|
83
86
|
# - :job_class [String] Fully-qualified job class name
|
|
84
87
|
# - :job_id [String] Unique job identifier
|
|
85
88
|
# - :queue_name [String] Target queue name
|
|
86
|
-
# - :
|
|
89
|
+
# - :active_job [Hash] ActiveJob serialized payload, including arguments
|
|
87
90
|
# - :executions [Integer] Current execution count (default 0)
|
|
88
91
|
# - :exception_executions [Hash] Exception execution counts (default {})
|
|
89
92
|
# - :scheduled_at [String] ISO8601 timestamp (optional)
|
|
@@ -98,7 +101,7 @@ module ActiveJob
|
|
|
98
101
|
# @example Basic job payload
|
|
99
102
|
# job = MyJob.new
|
|
100
103
|
# payload = Payload.from_job(job)
|
|
101
|
-
# # => { job_class: "MyJob", job_id: "...",
|
|
104
|
+
# # => { job_class: "MyJob", job_id: "...", active_job: { "arguments" => [...] }, ... }
|
|
102
105
|
#
|
|
103
106
|
# @example Scheduled job payload
|
|
104
107
|
# job = MyJob.new
|
|
@@ -155,19 +158,25 @@ module ActiveJob
|
|
|
155
158
|
final_payload = serializer_for(config).dump(payload)
|
|
156
159
|
final_payload[:scheduled_at] = scheduled_timestamp if scheduled_timestamp
|
|
157
160
|
final_payload = encrypt_payload_for_transport(final_payload, encrypt, config, encryption_context)
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
+
finalize_transport_payload(
|
|
162
|
+
final_payload,
|
|
163
|
+
metrics_payload: payload,
|
|
164
|
+
config: config,
|
|
165
|
+
offload: offload ? storage_metadata || {} : nil,
|
|
166
|
+
enforce_size: enforce_size
|
|
167
|
+
)
|
|
161
168
|
end
|
|
162
169
|
|
|
163
170
|
# Deserializes job arguments from a payload hash.
|
|
164
171
|
#
|
|
165
|
-
# Extracts the serialized arguments array from
|
|
166
|
-
#
|
|
172
|
+
# Extracts the serialized arguments array from ActiveJob's canonical
|
|
173
|
+
# serialized job data, falling back to legacy top-level arguments, then
|
|
174
|
+
# uses ActiveJob's built-in deserialization to reconstruct Ruby objects
|
|
167
175
|
# (including GlobalID references to ActiveRecord models).
|
|
168
176
|
#
|
|
169
|
-
# @param payload [Hash] Payload hash containing serialized
|
|
170
|
-
# @option payload [
|
|
177
|
+
# @param payload [Hash] Payload hash containing serialized ActiveJob data
|
|
178
|
+
# @option payload [Hash] :active_job Full ActiveJob serialized payload
|
|
179
|
+
# @option payload [Array] :arguments Legacy serialized arguments
|
|
171
180
|
#
|
|
172
181
|
# @return [Array] Deserialized arguments array ready for job.perform(*args)
|
|
173
182
|
#
|
|
@@ -175,13 +184,13 @@ module ActiveJob
|
|
|
175
184
|
# @raise [GlobalID::RecordNotFound] if a GlobalID reference points to a deleted record
|
|
176
185
|
#
|
|
177
186
|
# @example Deserialize arguments
|
|
178
|
-
# payload = {
|
|
187
|
+
# payload = { active_job: { "arguments" => [{"_aj_serialized"=>"..."}] } }
|
|
179
188
|
# args = Payload.deserialize_args(payload)
|
|
180
189
|
# # => [actual_ruby_object]
|
|
181
190
|
#
|
|
182
191
|
# @example GlobalID deserialization with deleted record
|
|
183
192
|
# begin
|
|
184
|
-
# payload = {
|
|
193
|
+
# payload = { active_job: { "arguments" => [{"_aj_globalid"=>"gid://app/User/999"}] } }
|
|
185
194
|
# args = Payload.deserialize_args(payload)
|
|
186
195
|
# rescue ActiveRecord::RecordNotFound => e
|
|
187
196
|
# # Record was deleted between enqueue and execution
|
|
@@ -192,7 +201,7 @@ module ActiveJob
|
|
|
192
201
|
end
|
|
193
202
|
|
|
194
203
|
def deserialize_payload_args(payload)
|
|
195
|
-
serialized_args = payload[:arguments] || payload["arguments"]
|
|
204
|
+
serialized_args = active_job_arguments(payload) || payload[:arguments] || payload["arguments"]
|
|
196
205
|
ActiveJob::Arguments.deserialize(serialized_args)
|
|
197
206
|
rescue ActiveJob::SerializationError, ActiveJob::Temporal::ConfigurationError
|
|
198
207
|
raise
|
|
@@ -216,12 +225,13 @@ module ActiveJob
|
|
|
216
225
|
encrypt_payload_if_configured(payload, config, encryption_context: encryption_context)
|
|
217
226
|
end
|
|
218
227
|
|
|
219
|
-
def offload_payload(payload, metadata:, config: ActiveJob::Temporal.config)
|
|
228
|
+
def offload_payload(payload, metadata:, config: ActiveJob::Temporal.config, byte_size: nil)
|
|
220
229
|
PayloadStorage.offload_if_needed(
|
|
221
230
|
payload,
|
|
222
231
|
config: config,
|
|
223
232
|
metadata: metadata,
|
|
224
|
-
workflow_control_fields: WORKFLOW_CONTROL_FIELDS
|
|
233
|
+
workflow_control_fields: WORKFLOW_CONTROL_FIELDS,
|
|
234
|
+
byte_size: byte_size
|
|
225
235
|
)
|
|
226
236
|
end
|
|
227
237
|
|
|
@@ -229,19 +239,19 @@ module ActiveJob
|
|
|
229
239
|
PayloadStorage.delete(payload, config: config)
|
|
230
240
|
end
|
|
231
241
|
|
|
232
|
-
def enforce_size!(payload, metrics_payload: payload, config: ActiveJob::Temporal.config)
|
|
233
|
-
|
|
242
|
+
def enforce_size!(payload, metrics_payload: payload, config: ActiveJob::Temporal.config, byte_size: nil)
|
|
243
|
+
bytes = byte_size || JSON.generate(payload).bytesize
|
|
234
244
|
max_size_kb = config.max_payload_size_kb || 250
|
|
235
245
|
size_limit_bytes = max_size_kb * 1024
|
|
236
|
-
actual_size_kb =
|
|
237
|
-
usage_ratio =
|
|
246
|
+
actual_size_kb = bytes / 1024.0
|
|
247
|
+
usage_ratio = bytes.to_f / size_limit_bytes
|
|
238
248
|
|
|
239
249
|
Observability.emit(
|
|
240
250
|
:payload_serialize,
|
|
241
|
-
Observability.attributes_from_payload(metrics_payload, bytes:
|
|
251
|
+
Observability.attributes_from_payload(metrics_payload, bytes: bytes)
|
|
242
252
|
)
|
|
243
253
|
log_payload_size(metrics_payload, actual_size_kb, max_size_kb, usage_ratio)
|
|
244
|
-
return if
|
|
254
|
+
return if bytes <= size_limit_bytes
|
|
245
255
|
|
|
246
256
|
message = format(
|
|
247
257
|
"Job payload size (%<actual>.1f KB) exceeds maximum allowed size (%<max>d KB). " \
|
|
@@ -262,8 +272,18 @@ module ActiveJob
|
|
|
262
272
|
encrypt_payload(payload, config: config, encryption_context: encryption_context)
|
|
263
273
|
end
|
|
264
274
|
|
|
265
|
-
|
|
266
|
-
|
|
275
|
+
# One JSON pass feeds both the storage threshold check and the size limit check.
|
|
276
|
+
def finalize_transport_payload(payload, metrics_payload:, config:, offload:, enforce_size:)
|
|
277
|
+
return payload unless offload || enforce_size
|
|
278
|
+
|
|
279
|
+
byte_size = JSON.generate(payload).bytesize
|
|
280
|
+
if offload
|
|
281
|
+
offloaded = offload_payload(payload, metadata: offload, config: config, byte_size: byte_size)
|
|
282
|
+
byte_size = nil unless offloaded.equal?(payload)
|
|
283
|
+
payload = offloaded
|
|
284
|
+
end
|
|
285
|
+
enforce_size!(payload, metrics_payload: metrics_payload, config: config, byte_size: byte_size) if enforce_size
|
|
286
|
+
payload
|
|
267
287
|
end
|
|
268
288
|
|
|
269
289
|
def decrypt_transport_payload(payload, config, encryption_context)
|
|
@@ -296,12 +316,24 @@ module ActiveJob
|
|
|
296
316
|
PayloadSerializers.fetch(config.payload_serializer)
|
|
297
317
|
end
|
|
298
318
|
|
|
299
|
-
def serializer_for_transport_payload(payload,
|
|
319
|
+
def serializer_for_transport_payload(payload, config)
|
|
300
320
|
payload_serializer = payload_serializer_name(payload)
|
|
301
321
|
validate_payload_serializer_version!(payload) if payload_serializer_metadata?(payload)
|
|
322
|
+
validate_payload_serializer_allowed!(payload_serializer, config)
|
|
302
323
|
PayloadSerializers.fetch(payload_serializer)
|
|
303
324
|
end
|
|
304
325
|
|
|
326
|
+
# The payload names its own serializer, so an attacker-supplied workflow input could
|
|
327
|
+
# otherwise select Marshal (arbitrary code execution) regardless of configuration.
|
|
328
|
+
def validate_payload_serializer_allowed!(payload_serializer, config)
|
|
329
|
+
return if PayloadSerializers::DATA_ONLY.include?(payload_serializer)
|
|
330
|
+
return if payload_serializer == PayloadSerializers.normalize_name(config.payload_serializer)
|
|
331
|
+
|
|
332
|
+
raise ActiveJob::SerializationError,
|
|
333
|
+
"Payload serializer #{payload_serializer.inspect} is not permitted by the configured " \
|
|
334
|
+
"payload_serializer (#{config.payload_serializer.inspect})"
|
|
335
|
+
end
|
|
336
|
+
|
|
305
337
|
def payload_serializer_name(payload)
|
|
306
338
|
serializer_name = payload[:payload_serializer] || payload["payload_serializer"]
|
|
307
339
|
return PayloadSerializers::JSON unless serializer_name
|
|
@@ -327,6 +359,13 @@ module ActiveJob
|
|
|
327
359
|
end
|
|
328
360
|
end
|
|
329
361
|
|
|
362
|
+
def active_job_arguments(payload)
|
|
363
|
+
active_job_payload = payload[:active_job] || payload["active_job"]
|
|
364
|
+
return unless active_job_payload.respond_to?(:[])
|
|
365
|
+
|
|
366
|
+
active_job_payload[:arguments] || active_job_payload["arguments"]
|
|
367
|
+
end
|
|
368
|
+
|
|
330
369
|
# Serializes job arguments using ActiveJob's built-in serializer.
|
|
331
370
|
# @api private
|
|
332
371
|
def serialize_arguments(arguments)
|
|
@@ -352,12 +391,12 @@ module ActiveJob
|
|
|
352
391
|
job_class: job.class.name,
|
|
353
392
|
job_id: job.job_id,
|
|
354
393
|
queue_name: job.queue_name,
|
|
355
|
-
arguments: serialize_arguments(job.arguments || []),
|
|
356
394
|
executions: job.executions || 0,
|
|
357
395
|
exception_executions: job.exception_executions || {}
|
|
358
396
|
}
|
|
359
397
|
active_job_payload = serialized_active_job(job)
|
|
360
398
|
payload[:active_job] = active_job_payload if active_job_payload
|
|
399
|
+
payload[:arguments] = serialize_arguments(job.arguments || []) unless active_job_payload
|
|
361
400
|
payload
|
|
362
401
|
end
|
|
363
402
|
|
|
@@ -32,7 +32,15 @@ module ActiveJob
|
|
|
32
32
|
|
|
33
33
|
def decrypt(payload, config, context: nil)
|
|
34
34
|
version = payload[:encrypted_payload_version] || payload["encrypted_payload_version"]
|
|
35
|
-
|
|
35
|
+
if version == LEGACY_VERSION
|
|
36
|
+
unless config.allow_legacy_encrypted_payloads
|
|
37
|
+
raise ActiveJob::SerializationError,
|
|
38
|
+
"Version 1 encrypted payloads are rejected; set allow_legacy_encrypted_payloads " \
|
|
39
|
+
"to accept payloads enqueued before context-bound encryption"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
return decrypt_legacy(payload, config)
|
|
43
|
+
end
|
|
36
44
|
return decrypt_v2(payload, config, context) if version == VERSION
|
|
37
45
|
|
|
38
46
|
raise ActiveJob::SerializationError, "Unsupported encrypted payload version: #{version.inspect}"
|
|
@@ -15,6 +15,9 @@ module ActiveJob
|
|
|
15
15
|
MESSAGE_PACK_ALIAS = :msgpack
|
|
16
16
|
MARSHAL = :marshal
|
|
17
17
|
SUPPORTED = [JSON, MESSAGE_PACK, MESSAGE_PACK_ALIAS, MARSHAL].freeze
|
|
18
|
+
# Serializers that cannot instantiate arbitrary objects, so a payload may select them
|
|
19
|
+
# even when the configuration names a different serializer.
|
|
20
|
+
DATA_ONLY = [JSON, MESSAGE_PACK].freeze
|
|
18
21
|
|
|
19
22
|
def fetch(name)
|
|
20
23
|
case normalize_name(name)
|
|
@@ -16,9 +16,9 @@ module ActiveJob
|
|
|
16
16
|
payload[:external_payload] == true || payload["external_payload"] == true
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
def offload_if_needed(payload, config:, metadata:, workflow_control_fields:)
|
|
19
|
+
def offload_if_needed(payload, config:, metadata:, workflow_control_fields:, byte_size: nil)
|
|
20
20
|
return payload unless configured?(config)
|
|
21
|
-
return payload unless payload_exceeds_threshold?(payload, config)
|
|
21
|
+
return payload unless payload_exceeds_threshold?(payload, config, byte_size)
|
|
22
22
|
|
|
23
23
|
reference = dump_payload(payload, config, metadata)
|
|
24
24
|
envelope = {
|
|
@@ -62,8 +62,8 @@ module ActiveJob
|
|
|
62
62
|
!config.payload_storage_adapter.nil? && !config.payload_storage_threshold_kb.nil?
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
-
def payload_exceeds_threshold?(payload, config)
|
|
66
|
-
JSON.generate(payload).bytesize > (config.payload_storage_threshold_kb * 1024)
|
|
65
|
+
def payload_exceeds_threshold?(payload, config, byte_size = nil)
|
|
66
|
+
(byte_size || JSON.generate(payload).bytesize) > (config.payload_storage_threshold_kb * 1024)
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def dump_payload(payload, config, metadata)
|
|
@@ -129,14 +129,7 @@ module ActiveJob
|
|
|
129
129
|
return unless Object.const_defined?(:Rails)
|
|
130
130
|
|
|
131
131
|
rails = Object.const_get(:Rails)
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
environment = rails.env
|
|
135
|
-
return unless environment.respond_to?(:development?) && environment.respond_to?(:test?)
|
|
136
|
-
return unless environment.development? || environment.test?
|
|
137
|
-
|
|
138
|
-
application = rails.application
|
|
139
|
-
application.eager_load! if application.respond_to?(:eager_load!)
|
|
132
|
+
rails.application.eager_load! if rails.env.development? || rails.env.test?
|
|
140
133
|
end
|
|
141
134
|
end
|
|
142
135
|
end
|
|
@@ -2,38 +2,38 @@
|
|
|
2
2
|
|
|
3
3
|
module ActiveJob
|
|
4
4
|
module Temporal
|
|
5
|
+
# Single-slot signal handoff between a trap handler and the reload thread.
|
|
6
|
+
#
|
|
7
|
+
# Backed by Thread::Queue because trap context forbids taking a Mutex.
|
|
5
8
|
class ReloadSignalQueue
|
|
6
|
-
POLL_INTERVAL_SECONDS = 0.05
|
|
7
|
-
|
|
8
9
|
def initialize
|
|
9
|
-
@
|
|
10
|
-
@closed = false
|
|
10
|
+
@queue = Thread::Queue.new
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
+
# Enqueues a reload signal unless one is already pending or the queue is closed.
|
|
14
|
+
#
|
|
15
|
+
# @param signal [String] Signal name
|
|
16
|
+
# @return [String, nil] The signal when enqueued, nil when coalesced or closed
|
|
13
17
|
def push(signal)
|
|
14
|
-
return if @closed ||
|
|
18
|
+
return nil if @queue.closed? || !@queue.empty?
|
|
15
19
|
|
|
16
|
-
@
|
|
20
|
+
@queue << signal
|
|
17
21
|
signal
|
|
22
|
+
rescue ClosedQueueError
|
|
23
|
+
nil
|
|
18
24
|
end
|
|
19
25
|
|
|
26
|
+
# Blocks until a signal is pending or the queue is closed.
|
|
27
|
+
#
|
|
28
|
+
# @return [String, nil] The pending signal, or nil once closed
|
|
20
29
|
def pop
|
|
21
|
-
|
|
22
|
-
return nil if @closed
|
|
23
|
-
|
|
24
|
-
if @pending_signal
|
|
25
|
-
signal = @pending_signal
|
|
26
|
-
@pending_signal = nil
|
|
27
|
-
return signal
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
sleep(POLL_INTERVAL_SECONDS)
|
|
31
|
-
end
|
|
30
|
+
@queue.pop
|
|
32
31
|
end
|
|
33
32
|
|
|
33
|
+
# @return [void]
|
|
34
34
|
def close
|
|
35
|
-
@
|
|
36
|
-
@
|
|
35
|
+
@queue.clear
|
|
36
|
+
@queue.close
|
|
37
37
|
end
|
|
38
38
|
end
|
|
39
39
|
end
|