activejob-temporal 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +130 -0
- data/LICENSE +21 -0
- data/README.md +198 -0
- data/activejob-temporal.gemspec +58 -0
- data/api/job_payload_schema.json +318 -0
- data/bin/temporal-worker +295 -0
- data/lib/activejob/temporal/active_job_handler_source.rb +84 -0
- data/lib/activejob/temporal/activities/aj_runner_activity.rb +454 -0
- data/lib/activejob/temporal/activities/best_effort_side_effects.rb +49 -0
- data/lib/activejob/temporal/activities/dependency_status_activity.rb +160 -0
- data/lib/activejob/temporal/activities/rate_limit_activity.rb +41 -0
- data/lib/activejob/temporal/adapter.rb +257 -0
- data/lib/activejob/temporal/audit_log.rb +118 -0
- data/lib/activejob/temporal/batch_enqueue_result.rb +110 -0
- data/lib/activejob/temporal/batch_enqueuer.rb +141 -0
- data/lib/activejob/temporal/bind_policy.rb +44 -0
- data/lib/activejob/temporal/cancel/batch_canceller.rb +154 -0
- data/lib/activejob/temporal/cancel/batch_summary.rb +45 -0
- data/lib/activejob/temporal/cancel.rb +236 -0
- data/lib/activejob/temporal/certificate_watcher.rb +76 -0
- data/lib/activejob/temporal/chain_options.rb +83 -0
- data/lib/activejob/temporal/child_workflow_options.rb +102 -0
- data/lib/activejob/temporal/client.rb +215 -0
- data/lib/activejob/temporal/conditional_enqueue.rb +56 -0
- data/lib/activejob/temporal/configurable.rb +55 -0
- data/lib/activejob/temporal/configuration.rb +981 -0
- data/lib/activejob/temporal/configured_job_compatibility.rb +44 -0
- data/lib/activejob/temporal/connection_worker_pool.rb +88 -0
- data/lib/activejob/temporal/dead_letter_payload_validation.rb +34 -0
- data/lib/activejob/temporal/dead_letter_queue.rb +163 -0
- data/lib/activejob/temporal/dependency_options.rb +134 -0
- data/lib/activejob/temporal/external_operation.rb +193 -0
- data/lib/activejob/temporal/health_check_server.rb +159 -0
- data/lib/activejob/temporal/http_line_reader.rb +36 -0
- data/lib/activejob/temporal/inspect.rb +184 -0
- data/lib/activejob/temporal/job_descriptor.rb +37 -0
- data/lib/activejob/temporal/job_payload_builder.rb +209 -0
- data/lib/activejob/temporal/job_payload_chain_builder.rb +106 -0
- data/lib/activejob/temporal/job_payload_child_workflows.rb +127 -0
- data/lib/activejob/temporal/job_payload_dependencies.rb +40 -0
- data/lib/activejob/temporal/job_payload_rate_limits.rb +53 -0
- data/lib/activejob/temporal/job_payload_workflow_interactions.rb +31 -0
- data/lib/activejob/temporal/job_tags.rb +40 -0
- data/lib/activejob/temporal/locales/en.yml +126 -0
- data/lib/activejob/temporal/logger.rb +214 -0
- data/lib/activejob/temporal/metrics_server.rb +150 -0
- data/lib/activejob/temporal/middleware/chain.rb +106 -0
- data/lib/activejob/temporal/middleware.rb +11 -0
- data/lib/activejob/temporal/observability/datadog.rb +167 -0
- data/lib/activejob/temporal/observability/opentelemetry.rb +107 -0
- data/lib/activejob/temporal/observability/prometheus.rb +271 -0
- data/lib/activejob/temporal/observability.rb +260 -0
- data/lib/activejob/temporal/payload.rb +415 -0
- data/lib/activejob/temporal/payload_encryption.rb +215 -0
- data/lib/activejob/temporal/payload_serializers/json.rb +23 -0
- data/lib/activejob/temporal/payload_serializers/marshal.rb +53 -0
- data/lib/activejob/temporal/payload_serializers/message_pack.rb +59 -0
- data/lib/activejob/temporal/payload_serializers.rb +37 -0
- data/lib/activejob/temporal/payload_storage.rb +103 -0
- data/lib/activejob/temporal/rails_environment_loader.rb +143 -0
- data/lib/activejob/temporal/rate_limit_options.rb +94 -0
- data/lib/activejob/temporal/rate_limiters/memory.rb +198 -0
- data/lib/activejob/temporal/reload_signal_queue.rb +40 -0
- data/lib/activejob/temporal/retry_handler_extractor.rb +361 -0
- data/lib/activejob/temporal/retry_mapper.rb +264 -0
- data/lib/activejob/temporal/schedulable.rb +60 -0
- data/lib/activejob/temporal/schedule.rb +181 -0
- data/lib/activejob/temporal/schedule_options.rb +105 -0
- data/lib/activejob/temporal/search_attributes.rb +173 -0
- data/lib/activejob/temporal/signal_query.rb +161 -0
- data/lib/activejob/temporal/signal_query_options.rb +106 -0
- data/lib/activejob/temporal/temporal_options.rb +114 -0
- data/lib/activejob/temporal/tls_file.rb +45 -0
- data/lib/activejob/temporal/transaction_safety.rb +39 -0
- data/lib/activejob/temporal/version.rb +7 -0
- data/lib/activejob/temporal/visibility_query.rb +13 -0
- data/lib/activejob/temporal/worker_client_reloader.rb +34 -0
- data/lib/activejob/temporal/worker_health.rb +117 -0
- data/lib/activejob/temporal/worker_pool.rb +408 -0
- data/lib/activejob/temporal/workflow_enqueuer.rb +271 -0
- data/lib/activejob/temporal/workflow_enqueuer_batch.rb +17 -0
- data/lib/activejob/temporal/workflow_id_builder.rb +155 -0
- data/lib/activejob/temporal/workflow_identity.rb +62 -0
- data/lib/activejob/temporal/workflows/aj_workflow.rb +282 -0
- data/lib/activejob/temporal/workflows/dead_letter_support.rb +134 -0
- data/lib/activejob/temporal/workflows/dead_letter_workflow.rb +114 -0
- data/lib/activejob/temporal/workflows/workflow_chaining.rb +194 -0
- data/lib/activejob/temporal/workflows/workflow_child_workflows.rb +140 -0
- data/lib/activejob/temporal/workflows/workflow_continue_as_new.rb +44 -0
- data/lib/activejob/temporal/workflows/workflow_dependencies.rb +115 -0
- data/lib/activejob/temporal/workflows/workflow_execution_steps.rb +22 -0
- data/lib/activejob/temporal/workflows/workflow_interactions.rb +215 -0
- data/lib/activejob/temporal/workflows/workflow_local_activities.rb +29 -0
- data/lib/activejob/temporal/workflows/workflow_nexus.rb +15 -0
- data/lib/activejob/temporal/workflows/workflow_versioning.rb +21 -0
- data/lib/activejob/temporal.rb +297 -0
- data/lib/activejob-temporal.rb +3 -0
- metadata +423 -0
|
@@ -0,0 +1,415 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
require "active_support"
|
|
6
|
+
require "active_support/core_ext/numeric/time"
|
|
7
|
+
require "active_support/core_ext/string/conversions"
|
|
8
|
+
require "active_job/serializers"
|
|
9
|
+
require "active_job/arguments"
|
|
10
|
+
|
|
11
|
+
require_relative "payload_encryption"
|
|
12
|
+
require_relative "payload_storage"
|
|
13
|
+
require_relative "payload_serializers"
|
|
14
|
+
require_relative "observability"
|
|
15
|
+
|
|
16
|
+
module ActiveJob
|
|
17
|
+
module Temporal
|
|
18
|
+
# rubocop:disable Metrics/ModuleLength
|
|
19
|
+
# Payload serialization and deserialization for ActiveJob.
|
|
20
|
+
#
|
|
21
|
+
# This module converts ActiveJob instances into JSON-serializable hash payloads
|
|
22
|
+
# for transmission to Temporal workflows and activities. It also handles argument
|
|
23
|
+
# deserialization back into Ruby objects.
|
|
24
|
+
#
|
|
25
|
+
# @note Payload Size Limit
|
|
26
|
+
# Temporal enforces a maximum payload size (configurable via `max_payload_size_kb`,
|
|
27
|
+
# default 250 KB). Large payloads will raise a SerializationError with a human-readable
|
|
28
|
+
# message indicating the actual size and the limit. Consider passing database IDs or
|
|
29
|
+
# S3 keys instead of large objects.
|
|
30
|
+
#
|
|
31
|
+
# @note GlobalID Serialization
|
|
32
|
+
# ActiveRecord models are automatically serialized using GlobalID. This requires the
|
|
33
|
+
# model to exist in the database at enqueue time and still exist at execution time.
|
|
34
|
+
# If the record is deleted, deserialization will fail.
|
|
35
|
+
#
|
|
36
|
+
# @example Payload structure
|
|
37
|
+
# {
|
|
38
|
+
# job_class: "MyJob",
|
|
39
|
+
# job_id: "abc-123",
|
|
40
|
+
# queue_name: "default",
|
|
41
|
+
# arguments: [{"_aj_serialized"=>"ActiveJob::Serializers::ObjectSerializer", ...}],
|
|
42
|
+
# executions: 0,
|
|
43
|
+
# exception_executions: {},
|
|
44
|
+
# scheduled_at: "2025-10-29T12:00:00Z" # optional
|
|
45
|
+
# }
|
|
46
|
+
#
|
|
47
|
+
# @see https://edgeapi.rubyonrails.org/classes/ActiveJob/Arguments.html ActiveJob Arguments Serialization
|
|
48
|
+
module Payload
|
|
49
|
+
extend self
|
|
50
|
+
|
|
51
|
+
PAYLOAD_WARNING_THRESHOLD = 0.8
|
|
52
|
+
PAYLOAD_NEAR_LIMIT_THRESHOLD = 0.9
|
|
53
|
+
WORKFLOW_CONTROL_FIELDS = %i[
|
|
54
|
+
scheduled_at
|
|
55
|
+
default_activity_options
|
|
56
|
+
retry_policy
|
|
57
|
+
temporal_options
|
|
58
|
+
workflow_identity
|
|
59
|
+
dead_letter
|
|
60
|
+
rate_limits
|
|
61
|
+
workflow_interactions
|
|
62
|
+
child_workflows
|
|
63
|
+
chain
|
|
64
|
+
dependencies
|
|
65
|
+
dependency_failure_policy
|
|
66
|
+
activity_task_queue
|
|
67
|
+
continue_as_new
|
|
68
|
+
workflow_state
|
|
69
|
+
local_activity_helpers
|
|
70
|
+
observability
|
|
71
|
+
schedule_id
|
|
72
|
+
schedule_workflow_id_prefix
|
|
73
|
+
schedule_execution_job_id
|
|
74
|
+
payload_encryption_context
|
|
75
|
+
].freeze
|
|
76
|
+
|
|
77
|
+
# Converts an ActiveJob instance into a serializable payload hash.
|
|
78
|
+
#
|
|
79
|
+
# @param job [ActiveJob::Base] The job instance to serialize
|
|
80
|
+
# @param scheduled_at [Time, String, nil] Optional scheduled execution time
|
|
81
|
+
#
|
|
82
|
+
# @return [Hash] Serialized payload with keys:
|
|
83
|
+
# - :job_class [String] Fully-qualified job class name
|
|
84
|
+
# - :job_id [String] Unique job identifier
|
|
85
|
+
# - :queue_name [String] Target queue name
|
|
86
|
+
# - :arguments [Array] Serialized job arguments (via ActiveJob::Arguments)
|
|
87
|
+
# - :executions [Integer] Current execution count (default 0)
|
|
88
|
+
# - :exception_executions [Hash] Exception execution counts (default {})
|
|
89
|
+
# - :scheduled_at [String] ISO8601 timestamp (optional)
|
|
90
|
+
#
|
|
91
|
+
# @raise [ActiveJob::SerializationError] if arguments cannot be serialized
|
|
92
|
+
# @raise [ActiveJob::SerializationError] if payload exceeds max_payload_size_kb (includes actual size in message)
|
|
93
|
+
# @raise [ArgumentError] if scheduled_at is not convertible to Time
|
|
94
|
+
# @raise [ArgumentError] if job is nil
|
|
95
|
+
# @raise [NoMethodError] if job does not respond to required attributes
|
|
96
|
+
# @raise [JSON::GeneratorError] if payload cannot be JSON-serialized
|
|
97
|
+
#
|
|
98
|
+
# @example Basic job payload
|
|
99
|
+
# job = MyJob.new
|
|
100
|
+
# payload = Payload.from_job(job)
|
|
101
|
+
# # => { job_class: "MyJob", job_id: "...", arguments: [...], ... }
|
|
102
|
+
#
|
|
103
|
+
# @example Scheduled job payload
|
|
104
|
+
# job = MyJob.new
|
|
105
|
+
# payload = Payload.from_job(job, scheduled_at: 1.hour.from_now)
|
|
106
|
+
# # => { ..., scheduled_at: "2025-10-29T13:00:00Z" }
|
|
107
|
+
#
|
|
108
|
+
# @example Handling payload size errors
|
|
109
|
+
# begin
|
|
110
|
+
# MyJob.perform_later(large_object)
|
|
111
|
+
# rescue ActiveJob::SerializationError => e
|
|
112
|
+
# Rails.logger.error("Payload too large: #{e.message}")
|
|
113
|
+
# # Recommendation: Pass ID instead of full object
|
|
114
|
+
# MyJob.perform_later(large_object.id)
|
|
115
|
+
# end
|
|
116
|
+
#
|
|
117
|
+
# @example GlobalID serialization (ActiveRecord models)
|
|
118
|
+
# user = User.find(123)
|
|
119
|
+
# MyJob.perform_later(user) # Serializes as GlobalID
|
|
120
|
+
# # Payload contains: { "_aj_globalid" => "gid://app/User/123" }
|
|
121
|
+
#
|
|
122
|
+
# @example Non-serializable object error
|
|
123
|
+
# begin
|
|
124
|
+
# MyJob.perform_later(File.open("/tmp/file.txt"))
|
|
125
|
+
# rescue ActiveJob::SerializationError => e
|
|
126
|
+
# # => "Unsupported argument type: File"
|
|
127
|
+
# Rails.logger.error(e.message)
|
|
128
|
+
# end
|
|
129
|
+
#
|
|
130
|
+
# @note Record Lifecycle Caveat
|
|
131
|
+
# When using GlobalID serialization for ActiveRecord models, the record MUST
|
|
132
|
+
# exist in the database at both enqueue time AND execution time. If the record
|
|
133
|
+
# is deleted before the job executes, deserialization will fail with
|
|
134
|
+
# ActiveRecord::RecordNotFound.
|
|
135
|
+
#
|
|
136
|
+
# @note Payload Size Optimization
|
|
137
|
+
# To reduce payload size, prefer passing database IDs instead of full ActiveRecord
|
|
138
|
+
# objects. For example, pass user.id instead of user. This is especially important
|
|
139
|
+
# for jobs with large argument lists or complex nested objects.
|
|
140
|
+
def from_job(
|
|
141
|
+
job,
|
|
142
|
+
scheduled_at: nil,
|
|
143
|
+
enforce_size: true,
|
|
144
|
+
config: ActiveJob::Temporal.config,
|
|
145
|
+
**options
|
|
146
|
+
)
|
|
147
|
+
encrypt = options.fetch(:encrypt, true)
|
|
148
|
+
offload = options.fetch(:offload, enforce_size)
|
|
149
|
+
encryption_context = options[:encryption_context]
|
|
150
|
+
storage_metadata = options[:storage_metadata]
|
|
151
|
+
payload = payload_from_job_attributes(job)
|
|
152
|
+
payload[:scheduled_at] = iso8601_timestamp(scheduled_at) if scheduled_at
|
|
153
|
+
|
|
154
|
+
scheduled_timestamp = payload.delete(:scheduled_at)
|
|
155
|
+
final_payload = serializer_for(config).dump(payload)
|
|
156
|
+
final_payload[:scheduled_at] = scheduled_timestamp if scheduled_timestamp
|
|
157
|
+
final_payload = encrypt_payload_for_transport(final_payload, encrypt, config, encryption_context)
|
|
158
|
+
final_payload = offload_payload_for_transport(final_payload, storage_metadata, config) if offload
|
|
159
|
+
enforce_size!(final_payload, metrics_payload: payload, config: config) if enforce_size
|
|
160
|
+
final_payload
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Deserializes job arguments from a payload hash.
|
|
164
|
+
#
|
|
165
|
+
# Extracts the serialized arguments array from the payload and uses
|
|
166
|
+
# ActiveJob's built-in deserialization to reconstruct Ruby objects
|
|
167
|
+
# (including GlobalID references to ActiveRecord models).
|
|
168
|
+
#
|
|
169
|
+
# @param payload [Hash] Payload hash containing serialized arguments
|
|
170
|
+
# @option payload [Array] :arguments Serialized arguments (required)
|
|
171
|
+
#
|
|
172
|
+
# @return [Array] Deserialized arguments array ready for job.perform(*args)
|
|
173
|
+
#
|
|
174
|
+
# @raise [ActiveJob::SerializationError] if deserialization fails
|
|
175
|
+
# @raise [GlobalID::RecordNotFound] if a GlobalID reference points to a deleted record
|
|
176
|
+
#
|
|
177
|
+
# @example Deserialize arguments
|
|
178
|
+
# payload = { arguments: [{"_aj_serialized"=>"..."}] }
|
|
179
|
+
# args = Payload.deserialize_args(payload)
|
|
180
|
+
# # => [actual_ruby_object]
|
|
181
|
+
#
|
|
182
|
+
# @example GlobalID deserialization with deleted record
|
|
183
|
+
# begin
|
|
184
|
+
# payload = { arguments: [{"_aj_globalid"=>"gid://app/User/999"}] }
|
|
185
|
+
# args = Payload.deserialize_args(payload)
|
|
186
|
+
# rescue ActiveRecord::RecordNotFound => e
|
|
187
|
+
# # Record was deleted between enqueue and execution
|
|
188
|
+
# Rails.logger.warn("Job argument no longer exists: #{e.message}")
|
|
189
|
+
# end
|
|
190
|
+
def deserialize_args(payload, config: ActiveJob::Temporal.config, encryption_context: nil)
|
|
191
|
+
deserialize_payload_args(deserialize_payload(payload, config: config, encryption_context: encryption_context))
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def deserialize_payload_args(payload)
|
|
195
|
+
serialized_args = payload[:arguments] || payload["arguments"]
|
|
196
|
+
ActiveJob::Arguments.deserialize(serialized_args)
|
|
197
|
+
rescue ActiveJob::SerializationError, ActiveJob::Temporal::ConfigurationError
|
|
198
|
+
raise
|
|
199
|
+
rescue StandardError => e
|
|
200
|
+
raise ActiveJob::SerializationError, e.message
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def deserialize_payload(payload, config: ActiveJob::Temporal.config, encryption_context: nil)
|
|
204
|
+
stored_payload = PayloadStorage.load(
|
|
205
|
+
payload,
|
|
206
|
+
config: config,
|
|
207
|
+
workflow_control_fields: WORKFLOW_CONTROL_FIELDS
|
|
208
|
+
)
|
|
209
|
+
transport_payload = decrypt_transport_payload(stored_payload, config, encryption_context)
|
|
210
|
+
|
|
211
|
+
execution_payload = serializer_for_transport_payload(transport_payload, config).load(transport_payload)
|
|
212
|
+
preserve_workflow_control_fields(transport_payload, execution_payload)
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def encrypt_payload(payload, config: ActiveJob::Temporal.config, encryption_context: nil)
|
|
216
|
+
encrypt_payload_if_configured(payload, config, encryption_context: encryption_context)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def offload_payload(payload, metadata:, config: ActiveJob::Temporal.config)
|
|
220
|
+
PayloadStorage.offload_if_needed(
|
|
221
|
+
payload,
|
|
222
|
+
config: config,
|
|
223
|
+
metadata: metadata,
|
|
224
|
+
workflow_control_fields: WORKFLOW_CONTROL_FIELDS
|
|
225
|
+
)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def delete_external_payload(payload, config: ActiveJob::Temporal.config)
|
|
229
|
+
PayloadStorage.delete(payload, config: config)
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
def enforce_size!(payload, metrics_payload: payload, config: ActiveJob::Temporal.config)
|
|
233
|
+
json = JSON.generate(payload)
|
|
234
|
+
max_size_kb = config.max_payload_size_kb || 250
|
|
235
|
+
size_limit_bytes = max_size_kb * 1024
|
|
236
|
+
actual_size_kb = json.bytesize / 1024.0
|
|
237
|
+
usage_ratio = json.bytesize.to_f / size_limit_bytes
|
|
238
|
+
|
|
239
|
+
Observability.emit(
|
|
240
|
+
:payload_serialize,
|
|
241
|
+
Observability.attributes_from_payload(metrics_payload, bytes: json.bytesize)
|
|
242
|
+
)
|
|
243
|
+
log_payload_size(metrics_payload, actual_size_kb, max_size_kb, usage_ratio)
|
|
244
|
+
return if json.bytesize <= size_limit_bytes
|
|
245
|
+
|
|
246
|
+
message = format(
|
|
247
|
+
"Job payload size (%<actual>.1f KB) exceeds maximum allowed size (%<max>d KB). " \
|
|
248
|
+
"Consider reducing argument size or using references (e.g., database IDs).",
|
|
249
|
+
actual: actual_size_kb,
|
|
250
|
+
max: max_size_kb
|
|
251
|
+
)
|
|
252
|
+
raise ActiveJob::SerializationError, message
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
private
|
|
256
|
+
|
|
257
|
+
# Plaintext copies keep workflow replay config-free; ciphertext keeps activities
|
|
258
|
+
# from trusting tampered envelope controls after decryption.
|
|
259
|
+
def encrypt_payload_for_transport(payload, encrypt, config, encryption_context)
|
|
260
|
+
return payload unless encrypt
|
|
261
|
+
|
|
262
|
+
encrypt_payload(payload, config: config, encryption_context: encryption_context)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
def offload_payload_for_transport(payload, storage_metadata, config)
|
|
266
|
+
offload_payload(payload, metadata: storage_metadata || {}, config: config)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
def decrypt_transport_payload(payload, config, encryption_context)
|
|
270
|
+
return payload unless PayloadEncryption.encrypted?(payload)
|
|
271
|
+
|
|
272
|
+
decrypted_payload = PayloadEncryption.decrypt(payload, config, context: encryption_context)
|
|
273
|
+
preserve_workflow_control_fields(payload, decrypted_payload, fields: %i[scheduled_at])
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def encrypt_payload_if_configured(payload, config, encryption_context:)
|
|
277
|
+
return payload unless config.encrypt_payload
|
|
278
|
+
|
|
279
|
+
execution_payload = payload.except(:scheduled_at)
|
|
280
|
+
encrypted_payload = PayloadEncryption.encrypt(execution_payload, config, context: encryption_context)
|
|
281
|
+
copy_payload_serializer_metadata(payload, encrypted_payload)
|
|
282
|
+
preserve_workflow_control_fields(payload, encrypted_payload)
|
|
283
|
+
encrypted_payload
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def preserve_workflow_control_fields(source_payload, decrypted_payload, fields: WORKFLOW_CONTROL_FIELDS)
|
|
287
|
+
fields.each do |key|
|
|
288
|
+
value = source_payload[key] || source_payload[key.to_s]
|
|
289
|
+
decrypted_payload[key] = value if value
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
decrypted_payload
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def serializer_for(config)
|
|
296
|
+
PayloadSerializers.fetch(config.payload_serializer)
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def serializer_for_transport_payload(payload, _config)
|
|
300
|
+
payload_serializer = payload_serializer_name(payload)
|
|
301
|
+
validate_payload_serializer_version!(payload) if payload_serializer_metadata?(payload)
|
|
302
|
+
PayloadSerializers.fetch(payload_serializer)
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def payload_serializer_name(payload)
|
|
306
|
+
serializer_name = payload[:payload_serializer] || payload["payload_serializer"]
|
|
307
|
+
return PayloadSerializers::JSON unless serializer_name
|
|
308
|
+
|
|
309
|
+
PayloadSerializers.normalize_name(serializer_name)
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def payload_serializer_metadata?(payload)
|
|
313
|
+
payload.key?(:payload_serializer) || payload.key?("payload_serializer")
|
|
314
|
+
end
|
|
315
|
+
|
|
316
|
+
def validate_payload_serializer_version!(payload)
|
|
317
|
+
version = payload[:payload_serializer_version] || payload["payload_serializer_version"]
|
|
318
|
+
return if version == PayloadSerializers::ENVELOPE_VERSION
|
|
319
|
+
|
|
320
|
+
raise ActiveJob::SerializationError, "Unsupported payload serializer version: #{version.inspect}"
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def copy_payload_serializer_metadata(source_payload, encrypted_payload)
|
|
324
|
+
%i[payload_serializer payload_serializer_version].each do |key|
|
|
325
|
+
value = source_payload[key] || source_payload[key.to_s]
|
|
326
|
+
encrypted_payload[key] = value if value
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Serializes job arguments using ActiveJob's built-in serializer.
|
|
331
|
+
# @api private
|
|
332
|
+
def serialize_arguments(arguments)
|
|
333
|
+
ActiveJob::Arguments.serialize(arguments)
|
|
334
|
+
rescue StandardError => e
|
|
335
|
+
raise ActiveJob::SerializationError, e.message
|
|
336
|
+
end
|
|
337
|
+
|
|
338
|
+
def serialized_active_job(job)
|
|
339
|
+
return unless job.respond_to?(:serialize)
|
|
340
|
+
|
|
341
|
+
original_arguments = job.arguments
|
|
342
|
+
job.arguments = [] if original_arguments.nil? && job.respond_to?(:arguments=)
|
|
343
|
+
job.serialize
|
|
344
|
+
ensure
|
|
345
|
+
if defined?(original_arguments) && original_arguments.nil? && job.respond_to?(:arguments=)
|
|
346
|
+
job.arguments = original_arguments
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
def payload_from_job_attributes(job)
|
|
351
|
+
payload = {
|
|
352
|
+
job_class: job.class.name,
|
|
353
|
+
job_id: job.job_id,
|
|
354
|
+
queue_name: job.queue_name,
|
|
355
|
+
arguments: serialize_arguments(job.arguments || []),
|
|
356
|
+
executions: job.executions || 0,
|
|
357
|
+
exception_executions: job.exception_executions || {}
|
|
358
|
+
}
|
|
359
|
+
active_job_payload = serialized_active_job(job)
|
|
360
|
+
payload[:active_job] = active_job_payload if active_job_payload
|
|
361
|
+
payload
|
|
362
|
+
end
|
|
363
|
+
|
|
364
|
+
# Converts a value to ISO8601 timestamp string.
|
|
365
|
+
# @api private
|
|
366
|
+
def iso8601_timestamp(value)
|
|
367
|
+
return value if value.is_a?(String) && valid_iso8601?(value)
|
|
368
|
+
|
|
369
|
+
timestamp = if value.respond_to?(:iso8601)
|
|
370
|
+
value
|
|
371
|
+
elsif value.respond_to?(:to_time)
|
|
372
|
+
value.to_time
|
|
373
|
+
else
|
|
374
|
+
raise ArgumentError, "scheduled_at must be convertible to Time"
|
|
375
|
+
end
|
|
376
|
+
raise ArgumentError, "scheduled_at must be convertible to Time" unless timestamp.respond_to?(:iso8601)
|
|
377
|
+
|
|
378
|
+
timestamp.iso8601
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
def log_payload_size(payload, actual_size_kb, max_size_kb, usage_ratio)
|
|
382
|
+
return if usage_ratio < PAYLOAD_WARNING_THRESHOLD
|
|
383
|
+
|
|
384
|
+
attributes = payload_size_log_attributes(payload, actual_size_kb, max_size_kb, usage_ratio)
|
|
385
|
+
|
|
386
|
+
if usage_ratio > 1.0
|
|
387
|
+
ActiveJob::Temporal::Logger.error("payload_size_exceeded", attributes)
|
|
388
|
+
elsif usage_ratio >= PAYLOAD_NEAR_LIMIT_THRESHOLD
|
|
389
|
+
ActiveJob::Temporal::Logger.warn("payload_size_near_limit", attributes)
|
|
390
|
+
elsif usage_ratio >= PAYLOAD_WARNING_THRESHOLD
|
|
391
|
+
ActiveJob::Temporal::Logger.info("payload_size_large", attributes)
|
|
392
|
+
end
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
def payload_size_log_attributes(payload, actual_size_kb, max_size_kb, usage_ratio)
|
|
396
|
+
{
|
|
397
|
+
job_class: payload[:job_class] || payload["job_class"],
|
|
398
|
+
size_kb: actual_size_kb.round(1),
|
|
399
|
+
limit_kb: max_size_kb,
|
|
400
|
+
percentage: (usage_ratio * 100).round(1)
|
|
401
|
+
}
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
# Checks if a string is valid ISO8601 format.
|
|
405
|
+
# @api private
|
|
406
|
+
def valid_iso8601?(value)
|
|
407
|
+
Time.iso8601(value)
|
|
408
|
+
true
|
|
409
|
+
rescue ArgumentError
|
|
410
|
+
false
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
# rubocop:enable Metrics/ModuleLength
|
|
414
|
+
end
|
|
415
|
+
end
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support/message_encryptor"
|
|
4
|
+
require "base64"
|
|
5
|
+
require "json"
|
|
6
|
+
require "openssl"
|
|
7
|
+
require "securerandom"
|
|
8
|
+
require "time"
|
|
9
|
+
|
|
10
|
+
module ActiveJob
|
|
11
|
+
module Temporal
|
|
12
|
+
module PayloadEncryption
|
|
13
|
+
extend self
|
|
14
|
+
|
|
15
|
+
CIPHER = "aes-256-gcm"
|
|
16
|
+
LEGACY_VERSION = 1
|
|
17
|
+
VERSION = 2
|
|
18
|
+
DEFAULT_KEY_ID = "primary"
|
|
19
|
+
KEY_ID_PATTERN = /\A[A-Za-z0-9_.:-]{1,128}\z/
|
|
20
|
+
V2_IV_BYTES = 12
|
|
21
|
+
KeyEntry = Struct.new(:id, :key, :decrypt_until, keyword_init: true)
|
|
22
|
+
|
|
23
|
+
def encrypted?(payload)
|
|
24
|
+
payload[:encrypted_payload] == true || payload["encrypted_payload"] == true
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def encrypt(payload, config, context: nil)
|
|
28
|
+
return encrypt_legacy(payload, config) unless context
|
|
29
|
+
|
|
30
|
+
encrypt_v2(payload, config, context)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def decrypt(payload, config, context: nil)
|
|
34
|
+
version = payload[:encrypted_payload_version] || payload["encrypted_payload_version"]
|
|
35
|
+
return decrypt_legacy(payload, config) if version == LEGACY_VERSION
|
|
36
|
+
return decrypt_v2(payload, config, context) if version == VERSION
|
|
37
|
+
|
|
38
|
+
raise ActiveJob::SerializationError, "Unsupported encrypted payload version: #{version.inspect}"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def key_length
|
|
42
|
+
ActiveSupport::MessageEncryptor.key_len(CIPHER)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def decode_key(value)
|
|
46
|
+
Base64.strict_decode64(value.to_s)
|
|
47
|
+
rescue ArgumentError
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def valid_key?(value)
|
|
52
|
+
!key_entry(value, fallback_id: DEFAULT_KEY_ID).nil?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
private
|
|
56
|
+
|
|
57
|
+
def encrypt_legacy(payload, config)
|
|
58
|
+
{
|
|
59
|
+
encrypted_payload: true,
|
|
60
|
+
encrypted_payload_version: LEGACY_VERSION,
|
|
61
|
+
encrypted_data: encryptor(config).encrypt_and_sign(payload)
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def encrypt_v2(payload, config, context)
|
|
66
|
+
key_entry = primary_key_entry(config)
|
|
67
|
+
iv = SecureRandom.random_bytes(V2_IV_BYTES)
|
|
68
|
+
cipher = OpenSSL::Cipher.new(CIPHER)
|
|
69
|
+
cipher.encrypt
|
|
70
|
+
cipher.key = key_entry.key
|
|
71
|
+
cipher.iv = iv
|
|
72
|
+
cipher.auth_data = authenticated_data(context, key_entry.id)
|
|
73
|
+
encrypted_data = cipher.update(JSON.generate(payload)) + cipher.final
|
|
74
|
+
|
|
75
|
+
{
|
|
76
|
+
encrypted_payload: true,
|
|
77
|
+
encrypted_payload_version: VERSION,
|
|
78
|
+
encrypted_key_id: key_entry.id,
|
|
79
|
+
encrypted_data: Base64.strict_encode64(encrypted_data),
|
|
80
|
+
encrypted_iv: Base64.strict_encode64(iv),
|
|
81
|
+
encrypted_auth_tag: Base64.strict_encode64(cipher.auth_tag)
|
|
82
|
+
}
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def decrypt_legacy(payload, config)
|
|
86
|
+
encrypted_data = payload[:encrypted_data] || payload["encrypted_data"]
|
|
87
|
+
normalize_top_level_keys(encryptor(config).decrypt_and_verify(encrypted_data))
|
|
88
|
+
rescue ActiveSupport::MessageEncryptor::InvalidMessage
|
|
89
|
+
raise ActiveJob::SerializationError, "Unable to decrypt ActiveJob::Temporal payload"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def decrypt_v2(payload, config, context)
|
|
93
|
+
key_id = payload[:encrypted_key_id] || payload["encrypted_key_id"]
|
|
94
|
+
key_entry = key_entry_for_id(config, key_id)
|
|
95
|
+
plaintext = decrypt_v2_data(payload, key_entry, authenticated_data(context, key_id))
|
|
96
|
+
normalize_top_level_keys(JSON.parse(plaintext))
|
|
97
|
+
rescue ActiveJob::SerializationError
|
|
98
|
+
raise
|
|
99
|
+
rescue OpenSSL::Cipher::CipherError, JSON::ParserError, ArgumentError
|
|
100
|
+
raise ActiveJob::SerializationError, "Unable to decrypt ActiveJob::Temporal payload"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def decrypt_v2_data(payload, key_entry, auth_data)
|
|
104
|
+
cipher = OpenSSL::Cipher.new(CIPHER)
|
|
105
|
+
cipher.decrypt
|
|
106
|
+
cipher.key = key_entry.key
|
|
107
|
+
cipher.iv = decode_envelope_field(payload, :encrypted_iv)
|
|
108
|
+
cipher.auth_tag = decode_envelope_field(payload, :encrypted_auth_tag)
|
|
109
|
+
cipher.auth_data = auth_data
|
|
110
|
+
cipher.update(decode_envelope_field(payload, :encrypted_data)) + cipher.final
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def decode_envelope_field(payload, key)
|
|
114
|
+
value = payload[key] || payload[key.to_s]
|
|
115
|
+
Base64.strict_decode64(value.to_s)
|
|
116
|
+
rescue ArgumentError
|
|
117
|
+
raise ActiveJob::SerializationError, "Unable to decrypt ActiveJob::Temporal payload"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def encryptor(config)
|
|
121
|
+
primary_key = primary_key_entry(config).key
|
|
122
|
+
encryptor = ActiveSupport::MessageEncryptor.new(primary_key, cipher: CIPHER, serializer: :json)
|
|
123
|
+
|
|
124
|
+
old_key_entries(config).each do |key|
|
|
125
|
+
next if expired_key?(key)
|
|
126
|
+
|
|
127
|
+
encryptor.rotate(key.key, cipher: CIPHER, serializer: :json)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
encryptor
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def primary_key_entry(config)
|
|
134
|
+
key_entry(config.encryption_key, fallback_id: DEFAULT_KEY_ID) ||
|
|
135
|
+
raise(ConfigurationError, "Encryption keys must be Base64-encoded #{key_length}-byte values")
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def key_entry_for_id(config, key_id)
|
|
139
|
+
matching_key = ([primary_key_entry(config)] + old_key_entries(config)).find { |entry| entry.id == key_id }
|
|
140
|
+
raise ActiveJob::SerializationError, "Unknown encrypted payload key id: #{key_id.inspect}" unless matching_key
|
|
141
|
+
|
|
142
|
+
if expired_key?(matching_key)
|
|
143
|
+
raise ActiveJob::SerializationError, "Encrypted payload key id #{key_id.inspect} is expired"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
matching_key
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def expired_key?(key_entry)
|
|
150
|
+
key_entry.decrypt_until && key_entry.decrypt_until < Time.now.utc
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def old_key_entries(config)
|
|
154
|
+
config.encryption_old_keys.filter_map { |key| key_entry(key, fallback_id: nil) }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def key_entry(value, fallback_id:)
|
|
158
|
+
id, raw_key, decrypt_until = key_parts(value, fallback_id)
|
|
159
|
+
return if value.is_a?(Hash) && id.to_s.empty?
|
|
160
|
+
return unless id.nil? || id.to_s.match?(KEY_ID_PATTERN)
|
|
161
|
+
|
|
162
|
+
decoded_key = decode_key(raw_key)
|
|
163
|
+
return unless decoded_key && decoded_key.bytesize == key_length
|
|
164
|
+
|
|
165
|
+
parsed_decrypt_until = parse_decrypt_until(decrypt_until)
|
|
166
|
+
return if decrypt_until && !parsed_decrypt_until
|
|
167
|
+
|
|
168
|
+
KeyEntry.new(id: id, key: decoded_key, decrypt_until: parsed_decrypt_until)
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
def key_parts(value, fallback_id)
|
|
172
|
+
return [fallback_id, value, nil] unless value.is_a?(Hash)
|
|
173
|
+
|
|
174
|
+
[
|
|
175
|
+
value[:id] || value["id"],
|
|
176
|
+
value[:key] || value["key"],
|
|
177
|
+
value[:decrypt_until] || value["decrypt_until"]
|
|
178
|
+
]
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def parse_decrypt_until(value)
|
|
182
|
+
return unless value
|
|
183
|
+
return value.utc if value.respond_to?(:utc)
|
|
184
|
+
|
|
185
|
+
Time.parse(value.to_s).utc
|
|
186
|
+
rescue ArgumentError
|
|
187
|
+
nil
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def authenticated_data(context, key_id)
|
|
191
|
+
JSON.generate(
|
|
192
|
+
encrypted_key_id: key_id,
|
|
193
|
+
encrypted_payload_version: VERSION,
|
|
194
|
+
namespace: context_value(context, :namespace),
|
|
195
|
+
workflow_id: context_value(context, :workflow_id)
|
|
196
|
+
)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def context_value(context, key)
|
|
200
|
+
raise ActiveJob::SerializationError, "Encrypted payload context requires #{key}" unless context.respond_to?(:[])
|
|
201
|
+
|
|
202
|
+
value = context[key] || context[key.to_s]
|
|
203
|
+
return value.to_s unless value.to_s.empty?
|
|
204
|
+
|
|
205
|
+
raise ActiveJob::SerializationError, "Encrypted payload context requires #{key}"
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def normalize_top_level_keys(payload)
|
|
209
|
+
payload.each_with_object({}) do |(key, value), normalized|
|
|
210
|
+
normalized[key.to_sym] = value
|
|
211
|
+
end
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module ActiveJob
|
|
4
|
+
module Temporal
|
|
5
|
+
module PayloadSerializers
|
|
6
|
+
module Json
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def dump(payload)
|
|
10
|
+
payload
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def load(payload)
|
|
14
|
+
payload
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def envelope?(_payload)
|
|
18
|
+
false
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|