joobq 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +21 -0
- data/README.md +20 -0
- data/lib/joobq.rb +1528 -0
- metadata +49 -0
data/lib/joobq.rb
ADDED
|
@@ -0,0 +1,1528 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
# Code generated by joobqui-selfhosted sdk:generate (v0.1.0) from JoobQ Polyglot SDK Gateway API. DO NOT EDIT.
|
|
3
|
+
# Supported Gateway Operations:
|
|
4
|
+
# - POST /api/v1/sdk/batches: Create a Pro batch of jobs with completion callbacks, filled and sealed in one call (Pro)
|
|
5
|
+
# - GET /api/v1/sdk/batches/{id}: Read a Pro batch's progress: pending, succeeded and failed members (Pro)
|
|
6
|
+
# - POST /api/v1/sdk/blueprints: Register a workflow blueprint, versioned by its content hash
|
|
7
|
+
# - POST /api/v1/sdk/jobs: Enqueue a background job onto a queue
|
|
8
|
+
# - POST /api/v1/sdk/jobs/batch: Enqueue up to 500 background jobs in one call
|
|
9
|
+
# - GET /api/v1/sdk/openapi.json: Describe the Polyglot SDK Gateway API as an OpenAPI 3.1 document
|
|
10
|
+
# - POST /api/v1/sdk/rate-limit: Reserve one slot of a sliding-window rate limit (Pro)
|
|
11
|
+
# - POST /api/v1/sdk/schedules: Register a code-declared cron/recurring job or workflow schedule, idempotent by name
|
|
12
|
+
# - POST /api/v1/sdk/tasks/ack: Settle up to 100 task leases in one call, each completed or failed
|
|
13
|
+
# - POST /api/v1/sdk/tasks/heartbeat: Extend every lease a worker holds in one call and check each for cancellation
|
|
14
|
+
# - POST /api/v1/sdk/tasks/poll: Claim due jobs and workflow steps from unpaused queues
|
|
15
|
+
# - POST /api/v1/sdk/tasks/{token}/complete: Commit task completion and output
|
|
16
|
+
# - POST /api/v1/sdk/tasks/{token}/fail: Record structured polyglot task failure and stack frames
|
|
17
|
+
# - POST /api/v1/sdk/tasks/{token}/heartbeat: Extend an active task lease and check for cancellation
|
|
18
|
+
# - POST /api/v1/sdk/workers/register: Register or heartbeat a polyglot SDK worker
|
|
19
|
+
# - POST /api/v1/sdk/workers/{id}/deregister: Deregister a polyglot SDK worker on shutdown
|
|
20
|
+
# - POST /api/v1/sdk/workflows: Start a durable workflow execution
|
|
21
|
+
# - GET /api/v1/sdk/workflows/{id}: Describe one workflow run: status, output, failure and search attributes
|
|
22
|
+
# - GET /api/v1/sdk/workflows/{id}/history: The events of a workflow run, after after_seq, at most limit
|
|
23
|
+
# - POST /api/v1/sdk/workflows/{id}/query: Ask a workflow run a read-only question
|
|
24
|
+
# - GET /api/v1/sdk/workflows/{id}/result: The outcome of a workflow, following continue-as-new; waits up to wait_ms for it to close
|
|
25
|
+
# - POST /api/v1/sdk/workflows/{id}/search-attributes: Set and remove search attributes of a workflow's open run
|
|
26
|
+
# - POST /api/v1/sdk/workflows/{id}/signal: Deliver an asynchronous signal to a running workflow
|
|
27
|
+
# - POST /api/v1/sdk/workflows/{id}/updates: Send an update to a running workflow and wait for its outcome
|
|
28
|
+
# NOTE: Queue & workflow management (pause, resume, redrive, cancel, terminate) is intentionally
|
|
29
|
+
# excluded from SDKs and must be performed via the JoobQUI Operator Console (/ui/).
|
|
30
|
+
|
|
31
|
+
require "json"
|
|
32
|
+
require "net/http"
|
|
33
|
+
require "socket"
|
|
34
|
+
require "uri"
|
|
35
|
+
|
|
36
|
+
module JoobQ
|
|
37
|
+
SDK_VERSION = "0.1.0"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class NonRetryableError < StandardError; end
|
|
41
|
+
|
|
42
|
+
# The gateway answered with an error status.
|
|
43
|
+
class GatewayError < StandardError
|
|
44
|
+
attr_reader :status
|
|
45
|
+
|
|
46
|
+
def initialize(status, body)
|
|
47
|
+
super("JoobQ Gateway error (#{status}): #{body.to_s[0, 300]}")
|
|
48
|
+
@status = status
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# `non_retryable` lists error types (the exception's class name, as the worker reports it in `error_class`) that end
|
|
53
|
+
# the step without another attempt.
|
|
54
|
+
RetryPolicy = Struct.new(:attempts, :initial_ms, :backoff, :maximum_ms, :non_retryable, keyword_init: true)
|
|
55
|
+
Compensation = Struct.new(:handler, :input, :activity, keyword_init: true)
|
|
56
|
+
# Puts a node on one side of a change (versioning): it runs only for runs whose version of `change` is within
|
|
57
|
+
# `min..max` (open ended when nil), and is skipped otherwise. Made with `gate`.
|
|
58
|
+
Gate = Struct.new(:change, :min, :max, keyword_init: true)
|
|
59
|
+
# An update handler declared with `WorkflowBlueprint#update`.
|
|
60
|
+
UpdateDef = Struct.new(:name, :handler, :validate, :message, :input, :result, :timeout_ms, keyword_init: true)
|
|
61
|
+
DataRef = Struct.new(:expr) do
|
|
62
|
+
def to_s
|
|
63
|
+
expr
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
module DSL
|
|
68
|
+
# A step's retry policy, applied by the engine. `non_retryable` names error types (class names) that are not retried.
|
|
69
|
+
def retry_policy(attempts: 3, initial_ms: 1000, backoff: 2.0, maximum_ms: 30_000, non_retryable: [])
|
|
70
|
+
RetryPolicy.new(attempts: attempts, initial_ms: initial_ms, backoff: backoff, maximum_ms: maximum_ms,
|
|
71
|
+
non_retryable: Array(non_retryable).map(&:to_s))
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def compensation(handler, input: nil)
|
|
75
|
+
Compensation.new(handler: JoobQ.normalize_callable(handler), input: input, activity: JoobQ.activity_name(handler))
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def input(key = nil)
|
|
79
|
+
key.nil? ? DataRef.new("${input}") : DataRef.new("${input.#{key}}")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def output(step_name, key = nil)
|
|
83
|
+
key.nil? ? DataRef.new("${steps.#{step_name}.output}") : DataRef.new("${steps.#{step_name}.output.#{key}}")
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
# The input of the update being handled (`${update.input}`, or `${update.input.<key>}`): for an update's
|
|
87
|
+
# `validate`, `input` and `result`.
|
|
88
|
+
def update_input(key = nil)
|
|
89
|
+
key.nil? ? DataRef.new("${update.input}") : DataRef.new("${update.input.#{key}}")
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# A gate on the change `change` (see `WorkflowBlueprint#change`), for the `gate:` option of `step`, `wait_for`,
|
|
93
|
+
# `timer` and `child`: the node runs only for runs whose version of the change is within `min..max`. At least one
|
|
94
|
+
# bound is required (ArgumentError otherwise).
|
|
95
|
+
def gate(change, min: nil, max: nil)
|
|
96
|
+
raise ArgumentError, "gate on #{change.to_s.inspect} needs a min or a max" if min.nil? && max.nil?
|
|
97
|
+
|
|
98
|
+
Gate.new(change: change.to_s, min: min, max: max)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def ref(path)
|
|
102
|
+
clean = path.to_s.strip
|
|
103
|
+
clean.start_with?("${") && clean.end_with?("}") ? DataRef.new(clean) : DataRef.new("${#{clean}}")
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def workflow(name, version: 1, timeout_ms: nil, task_queue: "workflows", &block)
|
|
107
|
+
JoobQ.workflow(name, version: version, timeout_ms: timeout_ms, task_queue: task_queue, &block)
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
extend DSL
|
|
112
|
+
|
|
113
|
+
# The `${...}` references a step input makes (a blueprint sends these, never the literal values around them).
|
|
114
|
+
def self.collect_references(spec, found = [])
|
|
115
|
+
case spec
|
|
116
|
+
when DataRef then found << spec.expr unless found.include?(spec.expr)
|
|
117
|
+
when Hash then spec.each_value { |v| collect_references(v, found) }
|
|
118
|
+
when Array then spec.each { |v| collect_references(v, found) }
|
|
119
|
+
end
|
|
120
|
+
found
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# A readable name for a step's activity: its class or job name; nil for a block or lambda.
|
|
124
|
+
def self.activity_name(target)
|
|
125
|
+
return target.job_name if target.respond_to?(:job_name)
|
|
126
|
+
return target.name if target.is_a?(Class) || target.is_a?(Module)
|
|
127
|
+
return target.name.to_s if target.is_a?(Method)
|
|
128
|
+
return nil if target.nil? || target.is_a?(Proc)
|
|
129
|
+
|
|
130
|
+
target.class.name
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def self.normalize_callable(target)
|
|
134
|
+
return target if target.respond_to?(:call)
|
|
135
|
+
return ->(payload, ctx) { target.perform(payload, ctx) } if target.respond_to?(:perform)
|
|
136
|
+
if target.is_a?(Class) && target.method_defined?(:perform)
|
|
137
|
+
return ->(payload, ctx) { target.new.perform(payload, ctx) }
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
raise ArgumentError, "Expected callable or Job/Activity with .perform(payload, ctx), got #{target.inspect}"
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def self.lookup_path(root, dotted)
|
|
144
|
+
cur = root
|
|
145
|
+
dotted.to_s.split(".").each do |part|
|
|
146
|
+
next if part.empty?
|
|
147
|
+
return nil unless cur.is_a?(Hash)
|
|
148
|
+
|
|
149
|
+
cur = cur[part] || cur[part.to_sym]
|
|
150
|
+
return nil if cur.nil?
|
|
151
|
+
end
|
|
152
|
+
cur
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def self.resolve_references(spec, workflow_input, checkpoints)
|
|
156
|
+
return workflow_input if spec.nil?
|
|
157
|
+
|
|
158
|
+
case spec
|
|
159
|
+
when DataRef
|
|
160
|
+
expr = spec.expr
|
|
161
|
+
return workflow_input if expr == "${input}"
|
|
162
|
+
if expr.start_with?("${input.") && expr.end_with?("}")
|
|
163
|
+
return lookup_path(workflow_input, expr[8..-2])
|
|
164
|
+
end
|
|
165
|
+
if expr.start_with?("${steps.") && expr.end_with?("}")
|
|
166
|
+
parts = expr[8..-2].split(".")
|
|
167
|
+
step_name = parts[0]
|
|
168
|
+
step_out = checkpoints[step_name] || checkpoints[step_name.to_sym]
|
|
169
|
+
return step_out if parts.length <= 2
|
|
170
|
+
return lookup_path(step_out, parts[2..].join("."))
|
|
171
|
+
end
|
|
172
|
+
return checkpoints.dup if expr == "${steps}"
|
|
173
|
+
if expr.start_with?("${") && expr.end_with?("}")
|
|
174
|
+
return lookup_path({ "input" => workflow_input, "steps" => checkpoints }, expr[2..-2])
|
|
175
|
+
end
|
|
176
|
+
expr
|
|
177
|
+
when Hash
|
|
178
|
+
spec.each_with_object({}) do |(k, v), acc|
|
|
179
|
+
acc[k.to_s] = resolve_references(v, workflow_input, checkpoints)
|
|
180
|
+
end
|
|
181
|
+
when Array
|
|
182
|
+
spec.map { |v| resolve_references(v, workflow_input, checkpoints) }
|
|
183
|
+
else
|
|
184
|
+
spec
|
|
185
|
+
end
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# A child's or continued run's input as the engine resolves it: references (`input`, `output`, `ref`) in hashes and
|
|
189
|
+
# arrays, turned into their `${...}` strings. A literal value is refused (ArgumentError): compute a constant in a step
|
|
190
|
+
# and refer to its output.
|
|
191
|
+
def self.template(spec, where)
|
|
192
|
+
case spec
|
|
193
|
+
when DataRef then spec.expr
|
|
194
|
+
when Hash then spec.each_with_object({}) { |(k, v), acc| acc[k.to_s] = template(v, "#{where}.#{k}") }
|
|
195
|
+
when Array then spec.each_with_index.map { |v, i| template(v, "#{where}[#{i}]") }
|
|
196
|
+
else
|
|
197
|
+
raise ArgumentError, "#{where}: #{spec.inspect} is not a reference; the engine resolves this input, so only references are sent"
|
|
198
|
+
end
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def self.extract_step_dependencies(spec)
|
|
202
|
+
deps = []
|
|
203
|
+
visit = lambda do |node|
|
|
204
|
+
case node
|
|
205
|
+
when DataRef
|
|
206
|
+
if node.expr.start_with?("${steps.")
|
|
207
|
+
step_name = node.expr[8..-2].split(".").first
|
|
208
|
+
deps << step_name if step_name && !deps.include?(step_name)
|
|
209
|
+
end
|
|
210
|
+
when Hash
|
|
211
|
+
node.each_value { |v| visit.call(v) }
|
|
212
|
+
when Array
|
|
213
|
+
node.each { |v| visit.call(v) }
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
visit.call(spec)
|
|
217
|
+
deps
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
module Job
|
|
221
|
+
def self.included(base)
|
|
222
|
+
base.extend(ClassMethods)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
module ClassMethods
|
|
226
|
+
def job_name(val = nil)
|
|
227
|
+
@job_name = val.to_s unless val.nil?
|
|
228
|
+
@job_name || name.split("::").last
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def queue(val = nil)
|
|
232
|
+
@queue = val.to_s unless val.nil?
|
|
233
|
+
@queue || "default"
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def retries(val = nil)
|
|
237
|
+
@retries = val.to_i unless val.nil?
|
|
238
|
+
@retries.nil? ? 3 : @retries
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def expires_ms(val = nil)
|
|
242
|
+
@expires_ms = val.to_i unless val.nil?
|
|
243
|
+
@expires_ms
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def encrypted_fields(*fields)
|
|
247
|
+
@encrypted_fields = fields.flatten.map(&:to_s) unless fields.empty?
|
|
248
|
+
@encrypted_fields || []
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def unique_ttl_ms(val = nil)
|
|
252
|
+
@unique_ttl_ms = val.to_i unless val.nil?
|
|
253
|
+
@unique_ttl_ms
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def enqueue(client, payload = {}, **opts)
|
|
257
|
+
client.enqueue(self, payload, **opts)
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
class WorkflowBlueprint
|
|
263
|
+
include DSL
|
|
264
|
+
|
|
265
|
+
attr_reader :name, :version, :timeout_ms, :task_queue, :steps, :queries, :continuation, :changes, :updates
|
|
266
|
+
|
|
267
|
+
# `timeout_ms` is the whole run's deadline (none by default), enforced by the engine.
|
|
268
|
+
def initialize(name, version: 1, timeout_ms: nil, task_queue: "workflows")
|
|
269
|
+
@name = name.to_s
|
|
270
|
+
@version = version
|
|
271
|
+
@timeout_ms = timeout_ms
|
|
272
|
+
@task_queue = task_queue
|
|
273
|
+
@steps = []
|
|
274
|
+
@queries = {}
|
|
275
|
+
@declared_steps = {}
|
|
276
|
+
@continuation = nil
|
|
277
|
+
@changes = {}
|
|
278
|
+
@updates = {}
|
|
279
|
+
@in_scope = false
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def query(name, expr)
|
|
283
|
+
@queries[name.to_s] = expr
|
|
284
|
+
self
|
|
285
|
+
end
|
|
286
|
+
|
|
287
|
+
# A step. `after` names earlier steps it also waits for (on top of what precedes it, e.g. a sibling in a `parallel`
|
|
288
|
+
# group); `timeout_ms` limits one attempt, `heartbeat_ms` how long the handler may go without calling
|
|
289
|
+
# `ctx.heartbeat`, `schedule_to_start_ms` how long the step may wait for a worker; `only_if` (a reference) runs the
|
|
290
|
+
# step only when it is truthy, else the step is skipped; `gate` (see `gate`) runs it only on one side of a change.
|
|
291
|
+
# The engine enforces them.
|
|
292
|
+
def step(name, activity = nil, input: nil, retry_policy: nil, retry: nil, compensate: nil, after: [], timeout_ms: nil,
|
|
293
|
+
heartbeat_ms: nil, schedule_to_start_ms: nil, only_if: nil, gate: nil, &block)
|
|
294
|
+
step_name = name.to_s
|
|
295
|
+
after = Array(after).map(&:to_s)
|
|
296
|
+
check_condition!(step_name, only_if)
|
|
297
|
+
check_gate!(step_name, gate)
|
|
298
|
+
validate_and_register!(step_name, input, after)
|
|
299
|
+
callable = JoobQ.normalize_callable(activity || block)
|
|
300
|
+
@steps << {
|
|
301
|
+
kind: "step",
|
|
302
|
+
name: step_name,
|
|
303
|
+
handler: callable,
|
|
304
|
+
activity: JoobQ.activity_name(activity),
|
|
305
|
+
input_spec: input,
|
|
306
|
+
retry: retry_policy || binding.local_variable_get(:retry),
|
|
307
|
+
compensate: compensate,
|
|
308
|
+
after: after,
|
|
309
|
+
timeout_ms: timeout_ms,
|
|
310
|
+
heartbeat_ms: heartbeat_ms,
|
|
311
|
+
schedule_to_start_ms: schedule_to_start_ms,
|
|
312
|
+
only_if: only_if,
|
|
313
|
+
gate: gate
|
|
314
|
+
}
|
|
315
|
+
self
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
# Waits for a signal. `after` names earlier steps it also waits for, `only_if` (a reference) skips the wait unless
|
|
319
|
+
# it is truthy, `compensate` undoes it when the run fails (its input defaults to the signal's payload), `gate` as on
|
|
320
|
+
# `step`.
|
|
321
|
+
def wait_for(name, signal:, timeout_ms: 600_000, after: [], only_if: nil, compensate: nil, gate: nil)
|
|
322
|
+
step_name = name.to_s
|
|
323
|
+
after = Array(after).map(&:to_s)
|
|
324
|
+
check_condition!(step_name, only_if)
|
|
325
|
+
check_gate!(step_name, gate)
|
|
326
|
+
validate_and_register!(step_name, nil, after)
|
|
327
|
+
@steps << { kind: "wait_for", name: step_name, signal: signal.to_s, timeout_ms: timeout_ms, after: after,
|
|
328
|
+
only_if: only_if, compensate: compensate, gate: gate }
|
|
329
|
+
self
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
# Sleeps `for_ms` durably. `after` names earlier steps it also waits for, `only_if` (a reference) skips the timer
|
|
333
|
+
# unless it is truthy, `compensate` undoes it when the run fails (its input defaults to the timer's output), `gate`
|
|
334
|
+
# as on `step`.
|
|
335
|
+
def timer(name, for_ms:, after: [], only_if: nil, compensate: nil, gate: nil)
|
|
336
|
+
step_name = name.to_s
|
|
337
|
+
after = Array(after).map(&:to_s)
|
|
338
|
+
check_condition!(step_name, only_if)
|
|
339
|
+
check_gate!(step_name, gate)
|
|
340
|
+
validate_and_register!(step_name, nil, after)
|
|
341
|
+
@steps << { kind: "timer", name: step_name, duration_ms: for_ms, after: after, only_if: only_if, compensate: compensate,
|
|
342
|
+
gate: gate }
|
|
343
|
+
self
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
# Runs another workflow (a blueprint or its name) as a node; its output is the child run's output, its failure the
|
|
347
|
+
# node's. The engine starts the child and resolves `input` (references only: `input`, `output`, `ref`, in hashes and
|
|
348
|
+
# arrays; the parent run's input when omitted). A child is not retried and has no timeout of its own: give the child
|
|
349
|
+
# workflow's steps retries, and the child workflow a `timeout_ms`. `compensate` undoes it when the run fails (its
|
|
350
|
+
# input defaults to the child's output), `after`, `only_if` and `gate` as on `step`. The child workflow must be
|
|
351
|
+
# registered before a run reaches this node.
|
|
352
|
+
def child(name, workflow, input: nil, compensate: nil, after: [], only_if: nil, gate: nil)
|
|
353
|
+
step_name = name.to_s
|
|
354
|
+
after = Array(after).map(&:to_s)
|
|
355
|
+
JoobQ.template(input, "#{step_name}.input") unless input.nil?
|
|
356
|
+
check_condition!(step_name, only_if)
|
|
357
|
+
check_gate!(step_name, gate)
|
|
358
|
+
validate_and_register!(step_name, input, after)
|
|
359
|
+
child_name = workflow.is_a?(WorkflowBlueprint) ? workflow.name : workflow.to_s
|
|
360
|
+
@steps << { kind: "child", name: step_name, workflow: child_name, input_spec: input, compensate: compensate,
|
|
361
|
+
after: after, only_if: only_if, gate: gate }
|
|
362
|
+
self
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
# When the run ends (and `only_if`, a reference, is truthy), it closes `continued_as_new` and a fresh run of this
|
|
366
|
+
# workflow starts with `input` (references only, resolved by the engine; the same input when omitted), at most
|
|
367
|
+
# `max_runs` runs; `max_events` bounds a run's history. The engine enforces it.
|
|
368
|
+
def continue_as_new(input: nil, only_if: nil, max_runs: nil, max_events: nil)
|
|
369
|
+
policy = {}
|
|
370
|
+
policy["input"] = JoobQ.template(input, "continue_as_new.input") unless input.nil?
|
|
371
|
+
unless only_if.nil?
|
|
372
|
+
raise ArgumentError, "continue_as_new: only_if must be a reference (input, output, ref)" unless only_if.is_a?(DataRef)
|
|
373
|
+
|
|
374
|
+
policy["only_if"] = only_if.expr
|
|
375
|
+
end
|
|
376
|
+
policy["max_runs"] = max_runs unless max_runs.nil?
|
|
377
|
+
policy["max_events"] = max_events unless max_events.nil?
|
|
378
|
+
@continuation = policy
|
|
379
|
+
self
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
# Declares a change (versioning, like Temporal's getVersion): runs that reach a gated node fresh take `max`, runs
|
|
383
|
+
# that went past it on the old path keep 0; nodes pick a side with `gate: gate(id, min: ..., max: ...)`. Raise `min`
|
|
384
|
+
# only once no open run is on an older path. `min` above `max` is an ArgumentError.
|
|
385
|
+
def change(id, min: 0, max: 1)
|
|
386
|
+
raise ArgumentError, "change #{id.to_s.inspect} in workflow '#{@name}': min is above max" if min > max
|
|
387
|
+
|
|
388
|
+
@changes[id.to_s] = { "min" => min, "max" => max }
|
|
389
|
+
self
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# An update handler (`Client#update_workflow`): `validate` (a reference, e.g. `update_input("name")`) must be truthy
|
|
393
|
+
# or the update is rejected with `message`; then `handler` runs as a step of the run (its payload is `input`, by
|
|
394
|
+
# default the update's input, and its return value is the result), or, without one, `result` (a template of
|
|
395
|
+
# references) is the result; `timeout_ms` limits the handler. References may read the run (`input`, `output`) and
|
|
396
|
+
# `update_input`. A duplicate name, a `validate` that is not a reference, or a literal in `input`/`result` is an
|
|
397
|
+
# ArgumentError.
|
|
398
|
+
def update(name, handler = nil, validate: nil, message: nil, input: nil, result: nil, timeout_ms: nil, &block)
|
|
399
|
+
update_name = name.to_s
|
|
400
|
+
raise ArgumentError, "Duplicate update '#{update_name}' in workflow '#{@name}'" if @updates.key?(update_name)
|
|
401
|
+
unless validate.nil? || validate.is_a?(DataRef)
|
|
402
|
+
raise ArgumentError, "update '#{update_name}': validate must be a reference (update_input, input, output, ref)"
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
JoobQ.template(input, "update #{update_name}.input") unless input.nil?
|
|
406
|
+
JoobQ.template(result, "update #{update_name}.result") unless result.nil?
|
|
407
|
+
target = handler || block
|
|
408
|
+
fn = target.nil? ? nil : JoobQ.normalize_callable(target)
|
|
409
|
+
@updates[update_name] = UpdateDef.new(name: update_name, handler: fn, validate: validate, message: message,
|
|
410
|
+
input: input, result: result, timeout_ms: timeout_ms)
|
|
411
|
+
self
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# A cancellation scope: its steps (declared in the block) run in sequence and can be cancelled as a unit (from the
|
|
415
|
+
# console or API, or after `timeout_ms` from its first step's start) while the rest of the run goes on; a cancelled
|
|
416
|
+
# step counts as finished with a null output. A `non_cancellable` scope holds cleanup that still runs when the
|
|
417
|
+
# workflow is cancelled, times out or fails (it cannot have a timeout). A running step sees a cancel through
|
|
418
|
+
# `ctx.heartbeat`. Scopes do not nest, and a scope needs at least one step (ArgumentError otherwise).
|
|
419
|
+
def scope(name, timeout_ms: nil, non_cancellable: false, &block)
|
|
420
|
+
scope_name = name.to_s
|
|
421
|
+
if @in_scope
|
|
422
|
+
raise ArgumentError, "scope '#{scope_name}' in workflow '#{@name}' is inside another scope: scopes do not nest"
|
|
423
|
+
end
|
|
424
|
+
if non_cancellable && !timeout_ms.nil?
|
|
425
|
+
raise ArgumentError, "scope '#{scope_name}' is non_cancellable and cannot time out"
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
sub = WorkflowBlueprint.new("#{@name}:scope", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
|
|
429
|
+
@declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
|
|
430
|
+
sub.send(:enter_scope!)
|
|
431
|
+
sub.instance_eval(&block) if block
|
|
432
|
+
raise ArgumentError, "scope '#{scope_name}' in workflow '#{@name}' has no steps" if sub.steps.empty?
|
|
433
|
+
|
|
434
|
+
sub.send(:declared_keys).each { |k| mark_declared!(k) }
|
|
435
|
+
@steps << { kind: "scope", name: scope_name, children: sub.steps, timeout_ms: timeout_ms,
|
|
436
|
+
non_cancellable: non_cancellable ? true : false }
|
|
437
|
+
self
|
|
438
|
+
end
|
|
439
|
+
|
|
440
|
+
def parallel(&block)
|
|
441
|
+
sub = WorkflowBlueprint.new("#{@name}:parallel", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
|
|
442
|
+
@declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
|
|
443
|
+
sub.send(:enter_scope!) if @in_scope
|
|
444
|
+
sub.instance_eval(&block)
|
|
445
|
+
sub.send(:declared_keys).each { |k| mark_declared!(k) }
|
|
446
|
+
@steps << { kind: "parallel", name: "parallel_#{@steps.length + 1}", children: sub.steps }
|
|
447
|
+
self
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def sequence(&block)
|
|
451
|
+
sub = WorkflowBlueprint.new("#{@name}:sequence", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
|
|
452
|
+
@declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
|
|
453
|
+
sub.send(:enter_scope!) if @in_scope
|
|
454
|
+
sub.instance_eval(&block)
|
|
455
|
+
sub.send(:declared_keys).each { |k| mark_declared!(k) }
|
|
456
|
+
@steps << { kind: "sequence", name: "sequence_#{@steps.length + 1}", children: sub.steps }
|
|
457
|
+
self
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
# The workflow's structure as data, as the worker registers it with the gateway (`CONTRACT.md`).
|
|
461
|
+
def to_blueprint
|
|
462
|
+
blueprint = {
|
|
463
|
+
"name" => @name,
|
|
464
|
+
"version" => @version,
|
|
465
|
+
"task_queue" => @task_queue,
|
|
466
|
+
"queries" => @queries.transform_values(&:to_s),
|
|
467
|
+
"nodes" => @steps.map { |s| blueprint_node(s) }
|
|
468
|
+
}
|
|
469
|
+
blueprint["timeout_ms"] = @timeout_ms unless @timeout_ms.nil? # the whole run's deadline, enforced by the engine
|
|
470
|
+
blueprint["continue_as_new"] = @continuation.dup unless @continuation.nil?
|
|
471
|
+
blueprint["changes"] = @changes.map { |id, bounds| { "id" => id }.merge(bounds) } unless @changes.empty?
|
|
472
|
+
blueprint["updates"] = @updates.transform_values { |u| update_spec(u) } unless @updates.empty?
|
|
473
|
+
blueprint
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
private def update_spec(update)
|
|
477
|
+
spec = {}
|
|
478
|
+
spec["handler"] = true unless update.handler.nil?
|
|
479
|
+
spec["validate"] = update.validate.expr unless update.validate.nil?
|
|
480
|
+
spec["message"] = update.message unless update.message.nil? || update.message.to_s.empty?
|
|
481
|
+
spec["input"] = JoobQ.template(update.input, "update #{update.name}.input") unless update.input.nil?
|
|
482
|
+
spec["result"] = JoobQ.template(update.result, "update #{update.name}.result") unless update.result.nil?
|
|
483
|
+
spec["timeout_ms"] = update.timeout_ms unless update.timeout_ms.nil?
|
|
484
|
+
spec
|
|
485
|
+
end
|
|
486
|
+
|
|
487
|
+
private def blueprint_node(node)
|
|
488
|
+
out = { "name" => node[:name], "kind" => node[:kind] }
|
|
489
|
+
case node[:kind]
|
|
490
|
+
when "parallel", "sequence", "scope"
|
|
491
|
+
out["children"] = node[:children].map { |c| blueprint_node(c) }
|
|
492
|
+
if node[:kind] == "scope"
|
|
493
|
+
out["timeout_ms"] = node[:timeout_ms] unless node[:timeout_ms].nil?
|
|
494
|
+
out["non_cancellable"] = true if node[:non_cancellable]
|
|
495
|
+
end
|
|
496
|
+
when "wait_for"
|
|
497
|
+
out.merge!("signal" => node[:signal], "timeout_ms" => node[:timeout_ms])
|
|
498
|
+
when "timer"
|
|
499
|
+
out["for_ms"] = node[:duration_ms]
|
|
500
|
+
when "child"
|
|
501
|
+
out["workflow"] = node[:workflow]
|
|
502
|
+
out["input"] = JoobQ.template(node[:input_spec], "#{node[:name]}.input") unless node[:input_spec].nil?
|
|
503
|
+
else
|
|
504
|
+
out["activity"] = node[:activity]
|
|
505
|
+
out["references"] = node[:input_spec].nil? ? ["${input}"] : JoobQ.collect_references(node[:input_spec])
|
|
506
|
+
%i[timeout_ms heartbeat_ms schedule_to_start_ms].each do |key|
|
|
507
|
+
out[key.to_s] = node[key] unless node[key].nil?
|
|
508
|
+
end
|
|
509
|
+
out["retry"] = blueprint_retry(node[:retry]) if node[:retry]
|
|
510
|
+
end
|
|
511
|
+
unless %w[parallel sequence scope].include?(node[:kind]) # a step, wait, timer or child can be undone, skipped, gated
|
|
512
|
+
if (c = node[:compensate])
|
|
513
|
+
out["compensate"] = { "activity" => c.activity,
|
|
514
|
+
"references" => JoobQ.collect_references(c.input.nil? ? output(node[:name]) : c.input) }
|
|
515
|
+
end
|
|
516
|
+
out["only_if"] = node[:only_if].expr unless node[:only_if].nil?
|
|
517
|
+
if (g = node[:gate])
|
|
518
|
+
gate_out = { "change" => g.change }
|
|
519
|
+
gate_out["min"] = g.min unless g.min.nil?
|
|
520
|
+
gate_out["max"] = g.max unless g.max.nil?
|
|
521
|
+
out["gate"] = gate_out
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
out["after"] = node[:after].dup unless node[:after].nil? || node[:after].empty?
|
|
525
|
+
out
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
private def blueprint_retry(r)
|
|
529
|
+
out = { "attempts" => r.attempts, "initial_ms" => r.initial_ms, "backoff" => r.backoff, "maximum_ms" => r.maximum_ms }
|
|
530
|
+
out["non_retryable"] = r.non_retryable.map(&:to_s) unless r.non_retryable.nil? || r.non_retryable.empty?
|
|
531
|
+
out
|
|
532
|
+
end
|
|
533
|
+
|
|
534
|
+
# The handlers of this workflow's steps, by the activity name the engine sends them under (`<workflow>/<step>`,
|
|
535
|
+
# compensations `<workflow>/<node>/compensate`, for waits, timers and children too; a child has no handler of its
|
|
536
|
+
# own: the engine runs the child workflow; update handlers `<workflow>/@update/<name>`, which receive the task's
|
|
537
|
+
# payload as is: the engine resolved their input). Each runs its step once: the engine conducts the run (it
|
|
538
|
+
# retries, compensates and waits). A step's input is resolved against the run's input (the task's payload) and the
|
|
539
|
+
# outputs of the steps that completed (the task's checkpoints).
|
|
540
|
+
def activities
|
|
541
|
+
found = {}
|
|
542
|
+
visit = lambda do |nodes|
|
|
543
|
+
nodes.each do |node|
|
|
544
|
+
case node[:kind]
|
|
545
|
+
when "sequence", "parallel", "scope"
|
|
546
|
+
visit.call(node[:children])
|
|
547
|
+
else
|
|
548
|
+
found[JoobQ.activity_task_name(@name, node[:name])] = step_activity(node) if node[:kind] == "step"
|
|
549
|
+
if (comp = node[:compensate]) # a step, wait, timer or child can be undone
|
|
550
|
+
found[JoobQ.activity_task_name(@name, node[:name], compensate: true)] = compensation_activity(node, comp)
|
|
551
|
+
end
|
|
552
|
+
end
|
|
553
|
+
end
|
|
554
|
+
end
|
|
555
|
+
visit.call(@steps)
|
|
556
|
+
@updates.each_value do |u|
|
|
557
|
+
found[JoobQ.update_task_name(@name, u.name)] = u.handler unless u.handler.nil?
|
|
558
|
+
end
|
|
559
|
+
found
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
private def step_activity(node)
|
|
563
|
+
lambda do |workflow_input, ctx|
|
|
564
|
+
node[:handler].call(JoobQ.resolve_references(node[:input_spec], workflow_input, ctx.task["checkpoints"] || {}), ctx)
|
|
565
|
+
end
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
private def compensation_activity(node, comp)
|
|
569
|
+
lambda do |workflow_input, ctx|
|
|
570
|
+
spec = comp.input.nil? ? output(node[:name]) : comp.input
|
|
571
|
+
comp.handler.call(JoobQ.resolve_references(spec, workflow_input, ctx.task["checkpoints"] || {}), ctx)
|
|
572
|
+
end
|
|
573
|
+
end
|
|
574
|
+
|
|
575
|
+
private
|
|
576
|
+
|
|
577
|
+
def mark_declared!(key)
|
|
578
|
+
@declared_steps[key.to_s] = true
|
|
579
|
+
end
|
|
580
|
+
|
|
581
|
+
def declared_keys
|
|
582
|
+
@declared_steps.keys
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
def enter_scope!
|
|
586
|
+
@in_scope = true
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
def check_gate!(step_name, gate)
|
|
590
|
+
return if gate.nil? || gate.is_a?(Gate)
|
|
591
|
+
|
|
592
|
+
raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': gate must be made with gate(change, min:, max:)"
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
# `only_if` must be a reference to the input or to an earlier step.
|
|
596
|
+
def check_condition!(step_name, only_if)
|
|
597
|
+
return if only_if.nil?
|
|
598
|
+
unless only_if.is_a?(DataRef)
|
|
599
|
+
raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': only_if must be a reference (input, output, ref)"
|
|
600
|
+
end
|
|
601
|
+
|
|
602
|
+
JoobQ.extract_step_dependencies(only_if).each do |dep|
|
|
603
|
+
unless @declared_steps.key?(dep)
|
|
604
|
+
raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': only_if refers to unknown or downstream step '#{dep}'"
|
|
605
|
+
end
|
|
606
|
+
end
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
def validate_and_register!(step_name, input_spec, after = [])
|
|
610
|
+
raise ArgumentError, "Duplicate step '#{step_name}' in workflow '#{@name}'" if @declared_steps.key?(step_name)
|
|
611
|
+
|
|
612
|
+
after.each do |dep|
|
|
613
|
+
unless @declared_steps.key?(dep)
|
|
614
|
+
raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}' waits (after:) for unknown or downstream step '#{dep}'"
|
|
615
|
+
end
|
|
616
|
+
end
|
|
617
|
+
JoobQ.extract_step_dependencies(input_spec).each do |dep|
|
|
618
|
+
unless @declared_steps.key?(dep)
|
|
619
|
+
raise ArgumentError, "Step '#{step_name}' references unknown or downstream step '#{dep}'"
|
|
620
|
+
end
|
|
621
|
+
end
|
|
622
|
+
mark_declared!(step_name)
|
|
623
|
+
end
|
|
624
|
+
end
|
|
625
|
+
|
|
626
|
+
# The activity the engine sends a step (or its compensation) of an SDK workflow under.
|
|
627
|
+
def self.activity_task_name(workflow_name, step, compensate: false)
|
|
628
|
+
compensate ? "#{workflow_name}/#{step}/compensate" : "#{workflow_name}/#{step}"
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
# The activity the engine sends an update handler of an SDK workflow under.
|
|
632
|
+
def self.update_task_name(workflow_name, update)
|
|
633
|
+
"#{workflow_name}/@update/#{update}"
|
|
634
|
+
end
|
|
635
|
+
|
|
636
|
+
# A workflow blueprint. `timeout_ms` is the whole run's deadline (none by default), enforced by the engine.
|
|
637
|
+
def self.workflow(name, version: 1, timeout_ms: nil, task_queue: "workflows", &block)
|
|
638
|
+
bp = WorkflowBlueprint.new(name, version: version, timeout_ms: timeout_ms, task_queue: task_queue)
|
|
639
|
+
bp.instance_eval(&block) if block
|
|
640
|
+
bp
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
class ExecutionContext
|
|
644
|
+
attr_reader :task, :worker_id, :client, :context, :lease_ttl_ms
|
|
645
|
+
|
|
646
|
+
def initialize(task, worker_id, client, lease_ttl_ms = 30_000, transport = nil)
|
|
647
|
+
@transport = transport
|
|
648
|
+
@task = task
|
|
649
|
+
@worker_id = worker_id
|
|
650
|
+
@client = client
|
|
651
|
+
@lease_ttl_ms = lease_ttl_ms
|
|
652
|
+
@context = (task["context"] || {}).dup
|
|
653
|
+
@last_beat_ms = nil
|
|
654
|
+
@beat_cancel = false
|
|
655
|
+
end
|
|
656
|
+
|
|
657
|
+
def attempt
|
|
658
|
+
(@task["attempt"] || 1).to_i
|
|
659
|
+
end
|
|
660
|
+
|
|
661
|
+
def dedupe_key
|
|
662
|
+
wf = @task["workflow_id"] || @task["id"]
|
|
663
|
+
st = @task["step_id"] || @task["name"]
|
|
664
|
+
"#{wf}:#{st}:#{attempt}"
|
|
665
|
+
end
|
|
666
|
+
|
|
667
|
+
# An engine step's heartbeat timeout (`task["heartbeat_ms"]`, nil for most tasks): call `heartbeat` more often.
|
|
668
|
+
def heartbeat_ms
|
|
669
|
+
@task["heartbeat_ms"]
|
|
670
|
+
end
|
|
671
|
+
|
|
672
|
+
# True when the run was cancelled. In a running worker it answers what the last lease renewal said without waiting
|
|
673
|
+
# for the gateway; the worker renews every lease on its own. For a step with `heartbeat_ms` it also
|
|
674
|
+
# tells the engine the step is alive (at most every third of `heartbeat_ms`): the worker's own renewals do not, so a
|
|
675
|
+
# handler that stops calling this is timed out.
|
|
676
|
+
def heartbeat(lease_ttl_ms: @lease_ttl_ms)
|
|
677
|
+
if (beat_ms = heartbeat_ms) && beat_ms.to_i.positive?
|
|
678
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
|
|
679
|
+
if @last_beat_ms.nil? || now - @last_beat_ms >= [100, beat_ms.to_i / 3].max
|
|
680
|
+
@last_beat_ms = now
|
|
681
|
+
begin
|
|
682
|
+
res = @client.heartbeat_task(@task["lease_token"], worker_id: @worker_id, lease_ttl_ms: lease_ttl_ms)
|
|
683
|
+
@beat_cancel = res.is_a?(Hash) && res["cancel_requested"] == true
|
|
684
|
+
rescue StandardError
|
|
685
|
+
nil # the next window's beat tries again; the engine's deadline decides
|
|
686
|
+
end
|
|
687
|
+
end
|
|
688
|
+
return true if @beat_cancel
|
|
689
|
+
end
|
|
690
|
+
return @transport.cancel_requested?(@task["lease_token"]) if @transport
|
|
691
|
+
|
|
692
|
+
res = @client.heartbeat_task(@task["lease_token"], worker_id: @worker_id, lease_ttl_ms: lease_ttl_ms)
|
|
693
|
+
res["cancel_requested"] == true
|
|
694
|
+
end
|
|
695
|
+
|
|
696
|
+
def enqueue(job_cls, payload = {}, context: {}, **opts)
|
|
697
|
+
@client.enqueue(job_cls, payload, context: @context.merge(context), **opts)
|
|
698
|
+
end
|
|
699
|
+
|
|
700
|
+
def enqueue_job(queue:, job_type:, payload: {}, context: {}, **opts)
|
|
701
|
+
@client.enqueue_job(queue: queue, job_type: job_type, payload: payload, context: @context.merge(context), **opts)
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
def start(workflow_def, input: nil, context: {}, **opts)
|
|
705
|
+
@client.start(workflow_def, input: input, context: @context.merge(context), **opts)
|
|
706
|
+
end
|
|
707
|
+
|
|
708
|
+
def start_workflow(name:, input: nil, context: {}, **opts)
|
|
709
|
+
@client.start_workflow(name: name, input: input, context: @context.merge(context), **opts)
|
|
710
|
+
end
|
|
711
|
+
|
|
712
|
+
def register_schedule(name, job_type, queue, **opts)
|
|
713
|
+
@client.register_schedule(name, job_type, queue, **opts)
|
|
714
|
+
end
|
|
715
|
+
|
|
716
|
+
def rate_limit(key, limit, period_ms)
|
|
717
|
+
@client.rate_limit(key, limit, period_ms)
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
# `Client#create_batch`; the members and callbacks carry this handler's context.
|
|
721
|
+
def create_batch(jobs, context: {}, **opts)
|
|
722
|
+
@client.create_batch(jobs, context: @context.merge(context), **opts)
|
|
723
|
+
end
|
|
724
|
+
|
|
725
|
+
def get_batch(id)
|
|
726
|
+
@client.get_batch(id)
|
|
727
|
+
end
|
|
728
|
+
|
|
729
|
+
def describe_workflow(id)
|
|
730
|
+
@client.describe_workflow(id)
|
|
731
|
+
end
|
|
732
|
+
|
|
733
|
+
def workflow_result(id, wait_ms: 0)
|
|
734
|
+
@client.workflow_result(id, wait_ms: wait_ms)
|
|
735
|
+
end
|
|
736
|
+
|
|
737
|
+
def workflow_history(id, after_seq: 0, limit: 200)
|
|
738
|
+
@client.workflow_history(id, after_seq: after_seq, limit: limit)
|
|
739
|
+
end
|
|
740
|
+
|
|
741
|
+
def query_workflow(id, name)
|
|
742
|
+
@client.query_workflow(id, name)
|
|
743
|
+
end
|
|
744
|
+
|
|
745
|
+
def signal_workflow(id:, name:, payload: nil)
|
|
746
|
+
@client.signal_workflow(id: id, name: name, payload: payload)
|
|
747
|
+
end
|
|
748
|
+
|
|
749
|
+
def update_workflow(id, name, input: nil, update_id: nil, wait_ms: nil)
|
|
750
|
+
@client.update_workflow(id, name, input: input, update_id: update_id, wait_ms: wait_ms)
|
|
751
|
+
end
|
|
752
|
+
|
|
753
|
+
def upsert_workflow_attributes(id, set: nil, remove: nil)
|
|
754
|
+
@client.upsert_workflow_attributes(id, set: set, remove: remove)
|
|
755
|
+
end
|
|
756
|
+
end
|
|
757
|
+
|
|
758
|
+
# The fields that report `error` to the gateway (`fail_task`, and a "fail" item of `ack_tasks`).
|
|
759
|
+
def self.failure(error, retryable: true)
|
|
760
|
+
frames = Array(error.backtrace).map do |line|
|
|
761
|
+
if line =~ /\A(.+):(\d+):in [`'](.+)'\z/
|
|
762
|
+
{ file: Regexp.last_match(1), line: Regexp.last_match(2).to_i, function: Regexp.last_match(3) }
|
|
763
|
+
else
|
|
764
|
+
{ file: line, line: 0, function: "unknown" }
|
|
765
|
+
end
|
|
766
|
+
end
|
|
767
|
+
message = error.message.to_s
|
|
768
|
+
message = error.class.name if message.empty?
|
|
769
|
+
{
|
|
770
|
+
error_class: error.class.name,
|
|
771
|
+
error_message: message[0, 4000],
|
|
772
|
+
retryable: retryable,
|
|
773
|
+
sdk_language: "ruby",
|
|
774
|
+
stack_frames: frames,
|
|
775
|
+
raw_stacktrace: Array(error.backtrace).join("\n")[0, 4000]
|
|
776
|
+
}
|
|
777
|
+
end
|
|
778
|
+
|
|
779
|
+
class Client
|
|
780
|
+
attr_reader :gateway_url, :token
|
|
781
|
+
# Seconds one gateway call may take. A worker lowers it (`Worker#initialize`): a call that hangs for longer than a
|
|
782
|
+
# lease would let every lease the worker holds expire while its tasks still run.
|
|
783
|
+
attr_accessor :request_timeout
|
|
784
|
+
|
|
785
|
+
def initialize(gateway_url:, token:, context_provider: nil)
|
|
786
|
+
@gateway_url = gateway_url.sub(%r{/+\z}, "")
|
|
787
|
+
@token = token
|
|
788
|
+
@request_timeout = 60
|
|
789
|
+
@context_provider = context_provider
|
|
790
|
+
@uri = URI.parse(@gateway_url)
|
|
791
|
+
@connection_key = :"joobq_http_#{object_id}" # one kept-alive connection per thread
|
|
792
|
+
end
|
|
793
|
+
|
|
794
|
+
# Enqueues many jobs with POST /sdk/jobs/batch (500 per call). Each item is `[JobClass, payload]`,
|
|
795
|
+
# `[JobClass, payload, { delay_ms:, jid:, unique_key: }]` or a hash of `enqueue_job` arguments. Answers one
|
|
796
|
+
# `{"jid", "queue", "status"}` per job, in order.
|
|
797
|
+
def enqueue_many(jobs, context: {})
|
|
798
|
+
bodies = jobs.map { |item| item_body(item, context) }
|
|
799
|
+
bodies.each_slice(500).flat_map { |slice| post("/api/v1/sdk/jobs/batch", { jobs: slice })["jobs"] || [] }
|
|
800
|
+
end
|
|
801
|
+
|
|
802
|
+
# Creates a Pro batch with POST /sdk/batches: 1 to 1000 member jobs (the shapes `enqueue_many` takes, without a
|
|
803
|
+
# `unique_key`) and callback jobs (the same shapes). `on_failure` is enqueued at the first dead member,
|
|
804
|
+
# `on_complete` once every member completed or was dead-lettered, `on_success` then too when none was dead; each
|
|
805
|
+
# once, with `{"batch" => {id, kind, total, succeeded, failed}}` merged into its payload. Answers
|
|
806
|
+
# `{"batch_id", "total", "jobs"}`; 402 pro_required without a Pro licence.
|
|
807
|
+
def create_batch(jobs, on_complete: nil, on_success: nil, on_failure: nil, description: nil, context: {})
|
|
808
|
+
body = { description: description, jobs: jobs.map { |item| batch_item_body(item, context) } }
|
|
809
|
+
{ on_complete: on_complete, on_success: on_success, on_failure: on_failure }.each do |kind, callback|
|
|
810
|
+
body[kind] = batch_item_body(callback, context) unless callback.nil?
|
|
811
|
+
end
|
|
812
|
+
post("/api/v1/sdk/batches", body)
|
|
813
|
+
end
|
|
814
|
+
|
|
815
|
+
# A Pro batch's progress with GET /sdk/batches/:id: `{"batch_id", "description", "status", "total", "pending",
|
|
816
|
+
# "succeeded", "failed", "created_at", "finished_at"}`; `status` is pending, then success (no member dead) or
|
|
817
|
+
# complete.
|
|
818
|
+
def get_batch(id)
|
|
819
|
+
get("/api/v1/sdk/batches/#{segment(id)}")
|
|
820
|
+
end
|
|
821
|
+
|
|
822
|
+
# One member or callback of `create_batch`: the batch's `context` merged key by key under the job's own (the job's
|
|
823
|
+
# keys win).
|
|
824
|
+
def batch_item_body(item, context)
|
|
825
|
+
base = context || {}
|
|
826
|
+
if item.is_a?(Hash)
|
|
827
|
+
spec = item.transform_keys(&:to_sym)
|
|
828
|
+
item_body(spec.merge(context: base.merge(spec[:context] || {})), base)
|
|
829
|
+
else
|
|
830
|
+
job_cls, payload, opts = item
|
|
831
|
+
opts = (opts || {}).transform_keys(&:to_sym)
|
|
832
|
+
item_body([job_cls, payload, opts.merge(context: base.merge(opts[:context] || {}))], base)
|
|
833
|
+
end
|
|
834
|
+
end
|
|
835
|
+
|
|
836
|
+
# One job of `enqueue_many` or `create_batch`: a hash of `enqueue_job` arguments or `[JobClass, payload, opts]`.
|
|
837
|
+
def item_body(item, context)
|
|
838
|
+
if item.is_a?(Hash)
|
|
839
|
+
spec = item.transform_keys(&:to_sym)
|
|
840
|
+
job_body(context: context, **spec)
|
|
841
|
+
else
|
|
842
|
+
job_cls, payload, opts = item
|
|
843
|
+
opts = (opts || {}).transform_keys(&:to_sym)
|
|
844
|
+
job_body(queue: job_cls.queue, job_type: job_cls.job_name, payload: payload || {},
|
|
845
|
+
context: opts.fetch(:context, context), jid: opts[:jid], max_retries: job_cls.retries,
|
|
846
|
+
delay_ms: opts.fetch(:delay_ms, 0), run_at_ms: opts[:run_at_ms],
|
|
847
|
+
expires_ms: opts[:expires_ms], timeout_ms: opts[:timeout_ms],
|
|
848
|
+
encrypt_fields: job_cls.encrypted_fields,
|
|
849
|
+
unique_key: opts[:unique_key], unique_mode: opts.fetch(:unique_mode, "while_locked"),
|
|
850
|
+
unique_ttl_ms: job_cls.unique_ttl_ms)
|
|
851
|
+
end
|
|
852
|
+
end
|
|
853
|
+
|
|
854
|
+
# `enqueue` on a thread of its own: `value` on the returned thread waits for `{"jid", "queue", "status"}` (or raises
|
|
855
|
+
# the error). `enqueue` is the call that answers only once the job is stored.
|
|
856
|
+
def enqueue_async(job_cls, payload = {}, **opts)
|
|
857
|
+
Thread.new { enqueue(job_cls, payload, **opts) }.tap { |t| t.report_on_exception = false }
|
|
858
|
+
end
|
|
859
|
+
|
|
860
|
+
def enqueue(job_cls, payload = {}, context: {}, jid: nil, delay_ms: 0, run_at_ms: nil, expires_ms: nil, timeout_ms: nil, unique_key: nil, unique_mode: "while_locked", unique_ttl_ms: nil)
|
|
861
|
+
enqueue_job(
|
|
862
|
+
queue: job_cls.queue,
|
|
863
|
+
job_type: job_cls.job_name,
|
|
864
|
+
payload: payload,
|
|
865
|
+
context: context,
|
|
866
|
+
jid: jid,
|
|
867
|
+
max_retries: job_cls.retries,
|
|
868
|
+
delay_ms: delay_ms,
|
|
869
|
+
run_at_ms: run_at_ms,
|
|
870
|
+
expires_ms: expires_ms,
|
|
871
|
+
timeout_ms: timeout_ms,
|
|
872
|
+
encrypt_fields: job_cls.encrypted_fields,
|
|
873
|
+
unique_key: unique_key,
|
|
874
|
+
unique_mode: unique_mode,
|
|
875
|
+
unique_ttl_ms: unique_ttl_ms.nil? ? job_cls.unique_ttl_ms : unique_ttl_ms
|
|
876
|
+
)
|
|
877
|
+
end
|
|
878
|
+
|
|
879
|
+
def enqueue_job(**opts)
|
|
880
|
+
post("/api/v1/sdk/jobs", job_body(**opts))
|
|
881
|
+
end
|
|
882
|
+
|
|
883
|
+
def job_body(queue:, job_type:, payload: {}, context: {}, jid: nil, max_retries: 3, delay_ms: 0, run_at_ms: nil, expires_ms: nil, timeout_ms: nil, encrypt_fields: [], unique_key: nil, unique_mode: "while_locked", unique_ttl_ms: nil)
|
|
884
|
+
{
|
|
885
|
+
queue: queue,
|
|
886
|
+
job_type: job_type,
|
|
887
|
+
payload: payload,
|
|
888
|
+
context: resolve_context(context),
|
|
889
|
+
jid: jid,
|
|
890
|
+
max_retries: max_retries,
|
|
891
|
+
delay_ms: delay_ms,
|
|
892
|
+
run_at_ms: run_at_ms,
|
|
893
|
+
expires_ms: expires_ms,
|
|
894
|
+
timeout_ms: timeout_ms,
|
|
895
|
+
encrypt_fields: encrypt_fields,
|
|
896
|
+
unique_key: unique_key,
|
|
897
|
+
unique_mode: unique_mode,
|
|
898
|
+
unique_ttl_ms: unique_ttl_ms
|
|
899
|
+
}
|
|
900
|
+
end
|
|
901
|
+
|
|
902
|
+
def start(workflow_def, input: nil, context: {}, id: nil, search_attributes: nil, build: nil)
|
|
903
|
+
start_workflow(
|
|
904
|
+
name: workflow_def.name,
|
|
905
|
+
input: input,
|
|
906
|
+
context: context,
|
|
907
|
+
id: id,
|
|
908
|
+
search_attributes: search_attributes,
|
|
909
|
+
build: build
|
|
910
|
+
)
|
|
911
|
+
end
|
|
912
|
+
|
|
913
|
+
def start_workflow(name:, input: nil, context: {}, id: nil, search_attributes: nil, build: nil)
|
|
914
|
+
post("/api/v1/sdk/workflows", {
|
|
915
|
+
name: name,
|
|
916
|
+
input: input,
|
|
917
|
+
context: resolve_context(context),
|
|
918
|
+
id: id,
|
|
919
|
+
search_attributes: search_attributes,
|
|
920
|
+
build: build
|
|
921
|
+
})
|
|
922
|
+
end
|
|
923
|
+
|
|
924
|
+
# Registers a recurring schedule with POST /sdk/schedules. Pass exactly one of `cron:` or `every:`
|
|
925
|
+
# (`every` is "<count> <unit>", e.g. "3 seconds"). Answers `{"id", "name", "kind", "created"}`.
|
|
926
|
+
def register_schedule(name, job_type, queue, cron: nil, every: nil, timezone: "UTC", payload: nil, max_retries: 3, encrypt_fields: [], unique_key: nil)
|
|
927
|
+
post("/api/v1/sdk/schedules", {
|
|
928
|
+
name: name,
|
|
929
|
+
kind: "job",
|
|
930
|
+
cron: cron,
|
|
931
|
+
every: every,
|
|
932
|
+
timezone: timezone,
|
|
933
|
+
queue: queue,
|
|
934
|
+
job_type: job_type,
|
|
935
|
+
payload: payload || {},
|
|
936
|
+
max_retries: max_retries,
|
|
937
|
+
encrypt_fields: encrypt_fields,
|
|
938
|
+
unique_key: unique_key
|
|
939
|
+
})
|
|
940
|
+
end
|
|
941
|
+
|
|
942
|
+
# Consumes one unit of a Pro rate-limit window with POST /sdk/rate-limit. Answers
|
|
943
|
+
# `{"allowed" => Bool, "retry_after_ms" => Int}`.
|
|
944
|
+
def rate_limit(key, limit, period_ms)
|
|
945
|
+
post("/api/v1/sdk/rate-limit", { key: key, limit: limit, period_ms: period_ms })
|
|
946
|
+
end
|
|
947
|
+
|
|
948
|
+
def signal_workflow(id:, name:, payload: nil)
|
|
949
|
+
post("/api/v1/sdk/workflows/#{segment(id)}/signal", {
|
|
950
|
+
name: name,
|
|
951
|
+
payload: payload
|
|
952
|
+
})
|
|
953
|
+
end
|
|
954
|
+
|
|
955
|
+
# Sends an update to a running workflow's update handler `name` (Pro; 402 without a licence) with
|
|
956
|
+
# POST /sdk/workflows/:id/updates and waits up to `wait_ms` (100 to 30000, default 10000) for its outcome. Answers
|
|
957
|
+
# `{"update_id", "name", "status", "result", "error", "reason", "message"}`: `status` is completed, rejected
|
|
958
|
+
# (validate failed: `reason`, `message`), failed, or accepted (a 202) when the handler had not answered in time
|
|
959
|
+
# (send again with the same `update_id` to wait for the same update).
|
|
960
|
+
def update_workflow(id, name, input: nil, update_id: nil, wait_ms: nil)
|
|
961
|
+
body = { name: name, input: input }
|
|
962
|
+
body[:update_id] = update_id unless update_id.nil?
|
|
963
|
+
body[:wait_ms] = Integer(wait_ms) unless wait_ms.nil?
|
|
964
|
+
post("/api/v1/sdk/workflows/#{segment(id)}/updates", body)
|
|
965
|
+
end
|
|
966
|
+
|
|
967
|
+
# One run (Pro; 402 without a licence) with GET /sdk/workflows/:id. Answers `{"id", "name", "status", "closed",
|
|
968
|
+
# "run", "started_at", "closed_at", "continued_to", "output", "failure", "attributes", "queries"}`; payload fields
|
|
969
|
+
# with secret-looking names are masked, as in the console.
|
|
970
|
+
def describe_workflow(id)
|
|
971
|
+
get("/api/v1/sdk/workflows/#{segment(id)}")
|
|
972
|
+
end
|
|
973
|
+
|
|
974
|
+
# The run at the end of the workflow's continue-as-new chain (Pro; 402 without a licence) with
|
|
975
|
+
# GET /sdk/workflows/:id/result, waiting up to `wait_ms` (at most 25000) for it to close. Same shape as
|
|
976
|
+
# `describe_workflow`; `"closed" => false` means it was still open when the wait ended.
|
|
977
|
+
def workflow_result(id, wait_ms: 0)
|
|
978
|
+
get("/api/v1/sdk/workflows/#{segment(id)}/result?wait_ms=#{Integer(wait_ms)}")
|
|
979
|
+
end
|
|
980
|
+
|
|
981
|
+
# The run's events after `after_seq`, at most `limit`, oldest first (Pro; 402 without a licence), with
|
|
982
|
+
# GET /sdk/workflows/:id/history. Answers the array of events.
|
|
983
|
+
def workflow_history(id, after_seq: 0, limit: 200)
|
|
984
|
+
res = get("/api/v1/sdk/workflows/#{segment(id)}/history?after_seq=#{Integer(after_seq)}&limit=#{Integer(limit)}")
|
|
985
|
+
res["items"] || []
|
|
986
|
+
end
|
|
987
|
+
|
|
988
|
+
# Asks the run a read-only question (Pro; 402 without a licence) with POST /sdk/workflows/:id/query. Answers the
|
|
989
|
+
# query's result (any JSON value).
|
|
990
|
+
def query_workflow(id, name)
|
|
991
|
+
post("/api/v1/sdk/workflows/#{segment(id)}/query", { name: name })["result"]
|
|
992
|
+
end
|
|
993
|
+
|
|
994
|
+
# Sets and/or removes search attributes of the workflow's open run (Pro; 402 without a licence) with
|
|
995
|
+
# POST /sdk/workflows/:id/search-attributes. Answers `{"id", "attributes"}`.
|
|
996
|
+
def upsert_workflow_attributes(id, set: nil, remove: nil)
|
|
997
|
+
post("/api/v1/sdk/workflows/#{segment(id)}/search-attributes", { set: set, remove: remove })
|
|
998
|
+
end
|
|
999
|
+
|
|
1000
|
+
# Sends a workflow's blueprint; the gateway keeps each new content hash (`created`) and ignores a known one.
|
|
1001
|
+
def register_blueprint(workflow_def, worker_id: nil)
|
|
1002
|
+
post("/api/v1/sdk/blueprints", {
|
|
1003
|
+
name: workflow_def.name,
|
|
1004
|
+
sdk_language: "ruby",
|
|
1005
|
+
worker_id: worker_id,
|
|
1006
|
+
blueprint: workflow_def.to_blueprint
|
|
1007
|
+
})
|
|
1008
|
+
end
|
|
1009
|
+
|
|
1010
|
+
def register_worker(worker_id:, task_queues:, concurrency: 4, service_name: nil, team: nil)
|
|
1011
|
+
post("/api/v1/sdk/workers/register", {
|
|
1012
|
+
worker_id: worker_id,
|
|
1013
|
+
sdk_language: "ruby",
|
|
1014
|
+
sdk_version: SDK_VERSION,
|
|
1015
|
+
service_name: service_name,
|
|
1016
|
+
team: team || ENV["JOOBQ_TEAM"],
|
|
1017
|
+
host: Socket.gethostname,
|
|
1018
|
+
pid: Process.pid,
|
|
1019
|
+
task_queues: task_queues,
|
|
1020
|
+
concurrency: concurrency,
|
|
1021
|
+
delivery_mode: "poll"
|
|
1022
|
+
})
|
|
1023
|
+
end
|
|
1024
|
+
|
|
1025
|
+
def deregister_worker(worker_id:)
|
|
1026
|
+
post("/api/v1/sdk/workers/#{URI.encode_www_form_component(worker_id)}/deregister", {})
|
|
1027
|
+
end
|
|
1028
|
+
|
|
1029
|
+
def poll_tasks(worker_id:, task_queues:, max_tasks: 1, timeout_ms: 0, lease_ttl_ms: 30_000)
|
|
1030
|
+
res = post("/api/v1/sdk/tasks/poll", {
|
|
1031
|
+
worker_id: worker_id,
|
|
1032
|
+
task_queues: task_queues,
|
|
1033
|
+
max_tasks: max_tasks,
|
|
1034
|
+
timeout_ms: timeout_ms,
|
|
1035
|
+
lease_ttl_ms: lease_ttl_ms
|
|
1036
|
+
})
|
|
1037
|
+
res["tasks"] || []
|
|
1038
|
+
end
|
|
1039
|
+
|
|
1040
|
+
def heartbeat_task(lease_token, worker_id:, lease_ttl_ms: 30_000)
|
|
1041
|
+
post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/heartbeat", {
|
|
1042
|
+
worker_id: worker_id,
|
|
1043
|
+
lease_ttl_ms: lease_ttl_ms
|
|
1044
|
+
})
|
|
1045
|
+
end
|
|
1046
|
+
|
|
1047
|
+
def complete_task(lease_token, worker_id:, output: nil)
|
|
1048
|
+
post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/complete", {
|
|
1049
|
+
worker_id: worker_id,
|
|
1050
|
+
output: output
|
|
1051
|
+
})
|
|
1052
|
+
end
|
|
1053
|
+
|
|
1054
|
+
def fail_task(lease_token, worker_id:, error:, retryable: true)
|
|
1055
|
+
post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/fail",
|
|
1056
|
+
{ worker_id: worker_id }.merge(JoobQ.failure(error, retryable: retryable)))
|
|
1057
|
+
end
|
|
1058
|
+
|
|
1059
|
+
# Settles up to 100 leases in one call (POST /sdk/tasks/ack): items are `{token:, outcome: "complete", output:}`
|
|
1060
|
+
# or `{token:, outcome: "fail", **JoobQ.failure(error)}`; each gets its own result.
|
|
1061
|
+
def ack_tasks(worker_id:, items:)
|
|
1062
|
+
post("/api/v1/sdk/tasks/ack", { worker_id: worker_id, items: items })
|
|
1063
|
+
end
|
|
1064
|
+
|
|
1065
|
+
# Renews many leases in one call (POST /sdk/tasks/heartbeat): `leases` are `{token:}`.
|
|
1066
|
+
def heartbeat_tasks(worker_id:, leases:, lease_ttl_ms: 30_000)
|
|
1067
|
+
post("/api/v1/sdk/tasks/heartbeat", { worker_id: worker_id, lease_ttl_ms: lease_ttl_ms, leases: leases })
|
|
1068
|
+
end
|
|
1069
|
+
|
|
1070
|
+
private
|
|
1071
|
+
|
|
1072
|
+
def resolve_context(explicit)
|
|
1073
|
+
ambient = @context_provider ? (@context_provider.call || {}) : {}
|
|
1074
|
+
ambient.merge(explicit || {})
|
|
1075
|
+
end
|
|
1076
|
+
|
|
1077
|
+
# This thread's kept-alive connection to the gateway (Net::HTTP honours the proxy environment variables).
|
|
1078
|
+
def connection
|
|
1079
|
+
http = Thread.current[@connection_key]
|
|
1080
|
+
return http if http&.started?
|
|
1081
|
+
|
|
1082
|
+
http = Net::HTTP.new(@uri.hostname, @uri.port)
|
|
1083
|
+
http.use_ssl = @uri.scheme == "https"
|
|
1084
|
+
http.open_timeout = 10
|
|
1085
|
+
http.read_timeout = @request_timeout
|
|
1086
|
+
http.write_timeout = @request_timeout
|
|
1087
|
+
http.keep_alive_timeout = 30
|
|
1088
|
+
http.start
|
|
1089
|
+
Thread.current[@connection_key] = http
|
|
1090
|
+
end
|
|
1091
|
+
|
|
1092
|
+
def drop_connection
|
|
1093
|
+
http = Thread.current[@connection_key]
|
|
1094
|
+
Thread.current[@connection_key] = nil
|
|
1095
|
+
http&.finish if http&.started?
|
|
1096
|
+
rescue IOError
|
|
1097
|
+
nil
|
|
1098
|
+
end
|
|
1099
|
+
|
|
1100
|
+
# A workflow id as one percent-encoded URL path segment.
|
|
1101
|
+
def segment(value)
|
|
1102
|
+
URI.encode_www_form_component(value.to_s).gsub("+", "%20")
|
|
1103
|
+
end
|
|
1104
|
+
|
|
1105
|
+
def post(path, body)
|
|
1106
|
+
req = Net::HTTP::Post.new("#{@uri.path}#{path}")
|
|
1107
|
+
req["Content-Type"] = "application/json"
|
|
1108
|
+
req.body = JSON.generate(body)
|
|
1109
|
+
send_request(req)
|
|
1110
|
+
end
|
|
1111
|
+
|
|
1112
|
+
def get(path)
|
|
1113
|
+
send_request(Net::HTTP::Get.new("#{@uri.path}#{path}"))
|
|
1114
|
+
end
|
|
1115
|
+
|
|
1116
|
+
def send_request(req)
|
|
1117
|
+
req["Authorization"] = "Bearer #{@token}"
|
|
1118
|
+
attempts = 0
|
|
1119
|
+
begin
|
|
1120
|
+
reused = !Thread.current[@connection_key].nil?
|
|
1121
|
+
res = connection.request(req)
|
|
1122
|
+
rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError => e
|
|
1123
|
+
drop_connection
|
|
1124
|
+
attempts += 1
|
|
1125
|
+
# The gateway closed the kept-alive connection while it was idle: once more, on a new one.
|
|
1126
|
+
retry if reused && attempts == 1
|
|
1127
|
+
raise e
|
|
1128
|
+
rescue StandardError
|
|
1129
|
+
drop_connection
|
|
1130
|
+
raise
|
|
1131
|
+
end
|
|
1132
|
+
drop_connection if res["connection"].to_s.downcase == "close"
|
|
1133
|
+
raise GatewayError.new(res.code.to_i, res.body) unless res.code.to_i.between?(200, 299)
|
|
1134
|
+
|
|
1135
|
+
res.body.to_s.empty? ? {} : JSON.parse(res.body)
|
|
1136
|
+
end
|
|
1137
|
+
end
|
|
1138
|
+
|
|
1139
|
+
# Every gateway call of a running worker, on threads of its own, so a handler thread never waits for HTTP: a poller
|
|
1140
|
+
# keeps a small buffer of leased tasks (free slots plus `prefetch`), an acker sends the outcomes handlers queue
|
|
1141
|
+
# (POST /sdk/tasks/ack, up to 100 per call after at most `ack_linger_ms`, retried while the gateway is unavailable; a
|
|
1142
|
+
# task stays leased until its outcome is acknowledged; when `ack_buffer` outcomes wait, handlers wait), and a
|
|
1143
|
+
# heartbeater renews every lease the worker holds in one call (POST /sdk/tasks/heartbeat).
|
|
1144
|
+
class Transport
|
|
1145
|
+
attr_reader :ready, :lock, :changed
|
|
1146
|
+
attr_accessor :held
|
|
1147
|
+
|
|
1148
|
+
def initialize(ack_buffer)
|
|
1149
|
+
@ready = Queue.new
|
|
1150
|
+
@acks = SizedQueue.new(ack_buffer)
|
|
1151
|
+
@lock = Mutex.new
|
|
1152
|
+
@changed = ConditionVariable.new
|
|
1153
|
+
@held = 0 # buffered + running
|
|
1154
|
+
@leased = {}
|
|
1155
|
+
@cancelled = {}
|
|
1156
|
+
@beat_wake = Queue.new
|
|
1157
|
+
@ack_done = false
|
|
1158
|
+
@give_up = false
|
|
1159
|
+
@beat_stop = false
|
|
1160
|
+
end
|
|
1161
|
+
|
|
1162
|
+
attr_writer :ack_done, :give_up
|
|
1163
|
+
|
|
1164
|
+
# Ends the heartbeater at once instead of at its next renewal.
|
|
1165
|
+
def stop_heartbeat
|
|
1166
|
+
@beat_stop = true
|
|
1167
|
+
@beat_wake << true
|
|
1168
|
+
end
|
|
1169
|
+
|
|
1170
|
+
def give_up?
|
|
1171
|
+
@give_up
|
|
1172
|
+
end
|
|
1173
|
+
|
|
1174
|
+
def leased(tasks)
|
|
1175
|
+
@lock.synchronize do
|
|
1176
|
+
tasks.each { |t| @leased[t["lease_token"]] = true }
|
|
1177
|
+
@held += tasks.size
|
|
1178
|
+
end
|
|
1179
|
+
tasks.each { |t| @ready << t }
|
|
1180
|
+
end
|
|
1181
|
+
|
|
1182
|
+
def settle(item)
|
|
1183
|
+
loop do
|
|
1184
|
+
return if @acks.push(item, timeout: 0.5)
|
|
1185
|
+
return if @give_up # left to its lease
|
|
1186
|
+
end
|
|
1187
|
+
end
|
|
1188
|
+
|
|
1189
|
+
def cancel_requested?(token)
|
|
1190
|
+
@lock.synchronize { @cancelled[token] == true }
|
|
1191
|
+
end
|
|
1192
|
+
|
|
1193
|
+
def release(token)
|
|
1194
|
+
@lock.synchronize do
|
|
1195
|
+
@leased.delete(token)
|
|
1196
|
+
@cancelled.delete(token)
|
|
1197
|
+
end
|
|
1198
|
+
end
|
|
1199
|
+
|
|
1200
|
+
def ack(client, worker_id, linger)
|
|
1201
|
+
loop do
|
|
1202
|
+
first = @acks.pop(timeout: 0.1)
|
|
1203
|
+
if first.nil?
|
|
1204
|
+
return if @ack_done
|
|
1205
|
+
|
|
1206
|
+
next
|
|
1207
|
+
end
|
|
1208
|
+
batch = [first]
|
|
1209
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + linger
|
|
1210
|
+
while batch.size < 100
|
|
1211
|
+
left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
1212
|
+
more = left.positive? ? @acks.pop(timeout: left) : (@acks.pop(true) rescue nil)
|
|
1213
|
+
break if more.nil?
|
|
1214
|
+
|
|
1215
|
+
batch << more
|
|
1216
|
+
end
|
|
1217
|
+
send_acks(client, worker_id, batch)
|
|
1218
|
+
end
|
|
1219
|
+
end
|
|
1220
|
+
|
|
1221
|
+
def send_acks(client, worker_id, batch)
|
|
1222
|
+
delay = 0.25
|
|
1223
|
+
loop do
|
|
1224
|
+
begin
|
|
1225
|
+
client.ack_tasks(worker_id: worker_id, items: batch)
|
|
1226
|
+
break # a refused item (its lease is gone) is not sent again either
|
|
1227
|
+
rescue GatewayError => e
|
|
1228
|
+
if e.status == 413 && batch.size > 1 # too large for one call: in halves
|
|
1229
|
+
half = batch.size / 2
|
|
1230
|
+
send_acks(client, worker_id, batch[0, half])
|
|
1231
|
+
send_acks(client, worker_id, batch[half..])
|
|
1232
|
+
return
|
|
1233
|
+
end
|
|
1234
|
+
if e.status < 500 && e.status != 429
|
|
1235
|
+
warn "joobq: the gateway refused #{batch.size} outcomes: #{e.message}"
|
|
1236
|
+
break
|
|
1237
|
+
end
|
|
1238
|
+
rescue StandardError
|
|
1239
|
+
nil
|
|
1240
|
+
end
|
|
1241
|
+
break if @give_up # shutting down: the leases bring these tasks back
|
|
1242
|
+
|
|
1243
|
+
sleep delay
|
|
1244
|
+
delay = [delay * 2, 5.0].min
|
|
1245
|
+
end
|
|
1246
|
+
batch.each { |item| release(item[:token]) }
|
|
1247
|
+
end
|
|
1248
|
+
|
|
1249
|
+
def heartbeat(client, worker_id, lease_ttl_ms, ®ister)
|
|
1250
|
+
every = [lease_ttl_ms / 3000.0, 0.5].max
|
|
1251
|
+
now = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
1252
|
+
last_register = now.call
|
|
1253
|
+
until @beat_stop
|
|
1254
|
+
@beat_wake.pop(timeout: every)
|
|
1255
|
+
return if @beat_stop
|
|
1256
|
+
|
|
1257
|
+
tokens = @lock.synchronize { @leased.keys }
|
|
1258
|
+
tokens.each_slice(500) do |chunk|
|
|
1259
|
+
res = client.heartbeat_tasks(worker_id: worker_id, leases: chunk.map { |t| { token: t } }, lease_ttl_ms: lease_ttl_ms)
|
|
1260
|
+
@lock.synchronize do
|
|
1261
|
+
Array(res["leases"]).each do |lease|
|
|
1262
|
+
token = lease["lease_token"]
|
|
1263
|
+
if lease["error"]
|
|
1264
|
+
@leased.delete(token) # gone: its outcome will be refused, nothing to renew
|
|
1265
|
+
elsif @leased.key?(token)
|
|
1266
|
+
@cancelled[token] = lease["cancel_requested"] == true
|
|
1267
|
+
end
|
|
1268
|
+
end
|
|
1269
|
+
end
|
|
1270
|
+
rescue StandardError
|
|
1271
|
+
nil # renewed with the next heartbeat
|
|
1272
|
+
end
|
|
1273
|
+
if now.call - last_register > 10
|
|
1274
|
+
register.call
|
|
1275
|
+
last_register = now.call
|
|
1276
|
+
end
|
|
1277
|
+
end
|
|
1278
|
+
end
|
|
1279
|
+
end
|
|
1280
|
+
|
|
1281
|
+
# In-app worker: runs up to `concurrency` jobs and workflow steps at once on threads inside the booted application,
|
|
1282
|
+
# and on SIGTERM/SIGINT (or `stop`) drains the running tasks, sends their outcomes and deregisters. Its gateway calls
|
|
1283
|
+
# run beside the handlers (`Transport`), never in their way.
|
|
1284
|
+
class Worker
|
|
1285
|
+
attr_reader :task_queues
|
|
1286
|
+
|
|
1287
|
+
def initialize(client:, worker_id:, task_queues: [], concurrency: 4, service_name: nil, team: nil,
|
|
1288
|
+
lease_ttl_ms: 30_000, poll_timeout_ms: 5_000, drain_timeout: 30.0, prefetch: nil, ack_buffer: nil,
|
|
1289
|
+
ack_linger_ms: 5)
|
|
1290
|
+
@client = client
|
|
1291
|
+
@worker_id = worker_id
|
|
1292
|
+
@task_queues = Array(task_queues).dup
|
|
1293
|
+
@concurrency = [concurrency.to_i, 1].max
|
|
1294
|
+
@service_name = service_name
|
|
1295
|
+
@team = team
|
|
1296
|
+
@lease_ttl_ms = lease_ttl_ms
|
|
1297
|
+
@poll_timeout_ms = poll_timeout_ms
|
|
1298
|
+
@drain_timeout = drain_timeout
|
|
1299
|
+
# One hung call must not outlast the leases: a poll waits `poll_timeout_ms` at the gateway, and a sixth of a lease
|
|
1300
|
+
# on top keeps a renewal that hangs well inside the lease (10 s with the defaults).
|
|
1301
|
+
bound = (poll_timeout_ms / 1000.0) + (lease_ttl_ms / 6000.0)
|
|
1302
|
+
current = client.respond_to?(:request_timeout) ? client.request_timeout : nil
|
|
1303
|
+
client.request_timeout = bound if current.is_a?(Numeric) && current > bound
|
|
1304
|
+
# Leases held beyond the free slots, so a handler that finishes finds its next task at once.
|
|
1305
|
+
@prefetch = [prefetch.nil? ? [@concurrency / 2, 1].max : prefetch.to_i, 0].max
|
|
1306
|
+
# Outcomes queued for the acker before handlers wait for it.
|
|
1307
|
+
@ack_buffer = [ack_buffer.nil? ? 4 * @concurrency : ack_buffer.to_i, 1].max
|
|
1308
|
+
@ack_linger = [ack_linger_ms.to_f, 0].max / 1000.0
|
|
1309
|
+
@handlers = {}
|
|
1310
|
+
@middlewares = []
|
|
1311
|
+
@stopping = false
|
|
1312
|
+
@workflows = {}
|
|
1313
|
+
@blueprints_sent = {}
|
|
1314
|
+
@transport = nil
|
|
1315
|
+
end
|
|
1316
|
+
|
|
1317
|
+
def use(&block)
|
|
1318
|
+
@middlewares << block
|
|
1319
|
+
self
|
|
1320
|
+
end
|
|
1321
|
+
|
|
1322
|
+
def register_job(job_cls)
|
|
1323
|
+
@task_queues << job_cls.queue if job_cls.respond_to?(:queue) && !@task_queues.include?(job_cls.queue)
|
|
1324
|
+
@handlers[job_cls.job_name] = JoobQ.normalize_callable(job_cls)
|
|
1325
|
+
self
|
|
1326
|
+
end
|
|
1327
|
+
|
|
1328
|
+
def register_workflow(workflow_def)
|
|
1329
|
+
if workflow_def.respond_to?(:task_queue) && !@task_queues.include?(workflow_def.task_queue)
|
|
1330
|
+
@task_queues << workflow_def.task_queue
|
|
1331
|
+
end
|
|
1332
|
+
@workflows[workflow_def.name] = workflow_def
|
|
1333
|
+
# The engine conducts the workflow's runs and sends each step as its own task.
|
|
1334
|
+
@handlers.merge!(workflow_def.activities) if workflow_def.respond_to?(:activities)
|
|
1335
|
+
self
|
|
1336
|
+
end
|
|
1337
|
+
|
|
1338
|
+
def handle_job(job_type, &block)
|
|
1339
|
+
@handlers[job_type] = block
|
|
1340
|
+
self
|
|
1341
|
+
end
|
|
1342
|
+
|
|
1343
|
+
# Says once, on stderr, that the gateway refuses this worker (a wrong or rotated token, no Pro key): it goes on
|
|
1344
|
+
# retrying, and without this line it would sit idle with nothing in its output.
|
|
1345
|
+
def report_refusal(what, error)
|
|
1346
|
+
status = error.respond_to?(:status) ? error.status : nil
|
|
1347
|
+
return unless status.is_a?(Integer) && status >= 400 && status < 500 && ![408, 429].include?(status)
|
|
1348
|
+
return if @refused == status
|
|
1349
|
+
|
|
1350
|
+
@refused = status
|
|
1351
|
+
warn "joobq: the gateway refuses this worker (#{what}): #{error.message}. Retrying; check the SDK token and the server's licence."
|
|
1352
|
+
end
|
|
1353
|
+
|
|
1354
|
+
def process_task(task)
|
|
1355
|
+
token = task["lease_token"]
|
|
1356
|
+
handler = @handlers[task["name"]]
|
|
1357
|
+
unless handler
|
|
1358
|
+
settle(token, { outcome: "fail" }.merge(JoobQ.failure(NonRetryableError.new("No handler registered for #{task['name']}"), retryable: false)))
|
|
1359
|
+
return { "status" => "failed" }
|
|
1360
|
+
end
|
|
1361
|
+
|
|
1362
|
+
# Everything that can raise for this task is inside the begin: a task with no outcome keeps its lease renewed for
|
|
1363
|
+
# ever. Exception, not StandardError, so that a handler raising a ScriptError (NotImplementedError in a stub)
|
|
1364
|
+
# fails its task instead of killing its thread; signals still pass.
|
|
1365
|
+
begin
|
|
1366
|
+
ctx = ExecutionContext.new(task, @worker_id, @client, @lease_ttl_ms, @transport)
|
|
1367
|
+
|
|
1368
|
+
runner = @middlewares.reverse.inject(-> { handler.call(task["payload"], ctx) }) do |next_fn, mw|
|
|
1369
|
+
-> { mw.call(ctx, next_fn) }
|
|
1370
|
+
end
|
|
1371
|
+
|
|
1372
|
+
output = runner.call
|
|
1373
|
+
rescue SignalException
|
|
1374
|
+
raise
|
|
1375
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
1376
|
+
retryable = !e.is_a?(NonRetryableError)
|
|
1377
|
+
settle(token, { outcome: "fail" }.merge(JoobQ.failure(e, retryable: retryable)))
|
|
1378
|
+
return { "status" => "failed" }
|
|
1379
|
+
end
|
|
1380
|
+
settle(token, { outcome: "complete", output: output })
|
|
1381
|
+
{ "status" => "completed", "output" => output }
|
|
1382
|
+
end
|
|
1383
|
+
|
|
1384
|
+
# Stops polling; `start` returns once the running tasks have finished and their outcomes are acknowledged (or
|
|
1385
|
+
# `drain_timeout` passed).
|
|
1386
|
+
def stop
|
|
1387
|
+
@stopping = true
|
|
1388
|
+
end
|
|
1389
|
+
|
|
1390
|
+
# Runs until SIGTERM/SIGINT or `stop`, then drains the running tasks, sends their outcomes and deregisters.
|
|
1391
|
+
def start(install_signal_handlers: true)
|
|
1392
|
+
if install_signal_handlers
|
|
1393
|
+
%w[TERM INT].each { |sig| Signal.trap(sig) { @stopping = true } }
|
|
1394
|
+
end
|
|
1395
|
+
backoff = 0.25
|
|
1396
|
+
until @stopping
|
|
1397
|
+
begin
|
|
1398
|
+
register
|
|
1399
|
+
break
|
|
1400
|
+
rescue StandardError
|
|
1401
|
+
sleep(backoff)
|
|
1402
|
+
backoff = [backoff * 2, 5.0].min
|
|
1403
|
+
end
|
|
1404
|
+
end
|
|
1405
|
+
t = Transport.new(@ack_buffer)
|
|
1406
|
+
@transport = t
|
|
1407
|
+
threads = Array.new(@concurrency) do
|
|
1408
|
+
Thread.new do
|
|
1409
|
+
while (task = t.ready.pop)
|
|
1410
|
+
begin
|
|
1411
|
+
process_task(task)
|
|
1412
|
+
rescue StandardError => e
|
|
1413
|
+
warn "joobq: task #{task['id']} could not be settled: #{e.message}"
|
|
1414
|
+
ensure
|
|
1415
|
+
t.lock.synchronize do
|
|
1416
|
+
t.held -= 1
|
|
1417
|
+
t.changed.broadcast
|
|
1418
|
+
end
|
|
1419
|
+
end
|
|
1420
|
+
end
|
|
1421
|
+
end
|
|
1422
|
+
end
|
|
1423
|
+
acker = Thread.new { t.ack(@client, @worker_id, @ack_linger) }
|
|
1424
|
+
beat = Thread.new do
|
|
1425
|
+
t.heartbeat(@client, @worker_id, @lease_ttl_ms) do
|
|
1426
|
+
register unless @stopping
|
|
1427
|
+
rescue StandardError
|
|
1428
|
+
nil
|
|
1429
|
+
end
|
|
1430
|
+
end
|
|
1431
|
+
|
|
1432
|
+
capacity = @concurrency + @prefetch
|
|
1433
|
+
backoff = 0.25
|
|
1434
|
+
until @stopping
|
|
1435
|
+
room = t.lock.synchronize do
|
|
1436
|
+
t.changed.wait(t.lock, 0.5) while t.held >= capacity && !@stopping
|
|
1437
|
+
capacity - t.held
|
|
1438
|
+
end
|
|
1439
|
+
break if @stopping
|
|
1440
|
+
|
|
1441
|
+
begin
|
|
1442
|
+
tasks = @client.poll_tasks(worker_id: @worker_id, task_queues: @task_queues, max_tasks: [room, 100].min,
|
|
1443
|
+
timeout_ms: @poll_timeout_ms, lease_ttl_ms: @lease_ttl_ms)
|
|
1444
|
+
backoff = 0.25
|
|
1445
|
+
@refused = nil
|
|
1446
|
+
rescue StandardError => e
|
|
1447
|
+
report_refusal("poll for tasks", e)
|
|
1448
|
+
sleep(backoff + rand * 0.1)
|
|
1449
|
+
backoff = [backoff * 2, 5.0].min
|
|
1450
|
+
next
|
|
1451
|
+
end
|
|
1452
|
+
t.leased(tasks)
|
|
1453
|
+
end
|
|
1454
|
+
|
|
1455
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @drain_timeout
|
|
1456
|
+
left = -> { deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC) }
|
|
1457
|
+
t.lock.synchronize { t.changed.wait(t.lock, [left.call, 0.05].min) while t.held.positive? && left.call.positive? }
|
|
1458
|
+
threads.size.times { t.ready << nil }
|
|
1459
|
+
t.ack_done = true
|
|
1460
|
+
unless acker.join([left.call, 0.01].max)
|
|
1461
|
+
t.give_up = true
|
|
1462
|
+
acker.join(5)
|
|
1463
|
+
end
|
|
1464
|
+
t.give_up = true
|
|
1465
|
+
t.stop_heartbeat
|
|
1466
|
+
beat.join(5)
|
|
1467
|
+
@transport = nil
|
|
1468
|
+
begin
|
|
1469
|
+
@client.deregister_worker(worker_id: @worker_id)
|
|
1470
|
+
rescue StandardError
|
|
1471
|
+
nil
|
|
1472
|
+
end
|
|
1473
|
+
end
|
|
1474
|
+
alias run start
|
|
1475
|
+
|
|
1476
|
+
private
|
|
1477
|
+
|
|
1478
|
+
# Hands the outcome to the acker (waiting while its queue is full); outside `start` it is sent here.
|
|
1479
|
+
def settle(token, item)
|
|
1480
|
+
item = item.merge(token: token)
|
|
1481
|
+
if @transport
|
|
1482
|
+
@transport.settle(item)
|
|
1483
|
+
else
|
|
1484
|
+
report { @client.ack_tasks(worker_id: @worker_id, items: [item]) }
|
|
1485
|
+
end
|
|
1486
|
+
end
|
|
1487
|
+
|
|
1488
|
+
def register
|
|
1489
|
+
register_blueprints # first: a worker that shows up has its blueprints in
|
|
1490
|
+
@client.register_worker(worker_id: @worker_id, task_queues: @task_queues, concurrency: @concurrency,
|
|
1491
|
+
service_name: @service_name, team: @team)
|
|
1492
|
+
end
|
|
1493
|
+
|
|
1494
|
+
# Registers every workflow's blueprint once; one the gateway refuses (4xx) is logged and not sent again.
|
|
1495
|
+
def register_blueprints
|
|
1496
|
+
@workflows.each do |name, wf|
|
|
1497
|
+
next if @blueprints_sent[name]
|
|
1498
|
+
|
|
1499
|
+
begin
|
|
1500
|
+
@client.register_blueprint(wf, worker_id: @worker_id)
|
|
1501
|
+
rescue GatewayError => e
|
|
1502
|
+
raise if e.status >= 500
|
|
1503
|
+
|
|
1504
|
+
warn "joobq: blueprint of #{name} refused: #{e.message}"
|
|
1505
|
+
end
|
|
1506
|
+
@blueprints_sent[name] = true
|
|
1507
|
+
end
|
|
1508
|
+
end
|
|
1509
|
+
|
|
1510
|
+
# Sends an outcome, retrying while the gateway is unavailable; a lost outcome is left to the lease.
|
|
1511
|
+
def report
|
|
1512
|
+
delay = 0.25
|
|
1513
|
+
6.times do
|
|
1514
|
+
yield
|
|
1515
|
+
return
|
|
1516
|
+
rescue GatewayError => e
|
|
1517
|
+
return if e.status < 500
|
|
1518
|
+
|
|
1519
|
+
sleep delay
|
|
1520
|
+
delay = [delay * 2, 5.0].min
|
|
1521
|
+
rescue StandardError
|
|
1522
|
+
sleep delay
|
|
1523
|
+
delay = [delay * 2, 5.0].min
|
|
1524
|
+
end
|
|
1525
|
+
end
|
|
1526
|
+
end
|
|
1527
|
+
|
|
1528
|
+
end
|