joobq 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +20 -0
  4. data/lib/joobq.rb +1528 -0
  5. metadata +49 -0
data/lib/joobq.rb ADDED
@@ -0,0 +1,1528 @@
1
+ # frozen_string_literal: true
2
+ # Code generated by joobqui-selfhosted sdk:generate (v0.1.0) from JoobQ Polyglot SDK Gateway API. DO NOT EDIT.
3
+ # Supported Gateway Operations:
4
+ # - POST /api/v1/sdk/batches: Create a Pro batch of jobs with completion callbacks, filled and sealed in one call (Pro)
5
+ # - GET /api/v1/sdk/batches/{id}: Read a Pro batch's progress: pending, succeeded and failed members (Pro)
6
+ # - POST /api/v1/sdk/blueprints: Register a workflow blueprint, versioned by its content hash
7
+ # - POST /api/v1/sdk/jobs: Enqueue a background job onto a queue
8
+ # - POST /api/v1/sdk/jobs/batch: Enqueue up to 500 background jobs in one call
9
+ # - GET /api/v1/sdk/openapi.json: Describe the Polyglot SDK Gateway API as an OpenAPI 3.1 document
10
+ # - POST /api/v1/sdk/rate-limit: Reserve one slot of a sliding-window rate limit (Pro)
11
+ # - POST /api/v1/sdk/schedules: Register a code-declared cron/recurring job or workflow schedule, idempotent by name
12
+ # - POST /api/v1/sdk/tasks/ack: Settle up to 100 task leases in one call, each completed or failed
13
+ # - POST /api/v1/sdk/tasks/heartbeat: Extend every lease a worker holds in one call and check each for cancellation
14
+ # - POST /api/v1/sdk/tasks/poll: Claim due jobs and workflow steps from unpaused queues
15
+ # - POST /api/v1/sdk/tasks/{token}/complete: Commit task completion and output
16
+ # - POST /api/v1/sdk/tasks/{token}/fail: Record structured polyglot task failure and stack frames
17
+ # - POST /api/v1/sdk/tasks/{token}/heartbeat: Extend an active task lease and check for cancellation
18
+ # - POST /api/v1/sdk/workers/register: Register or heartbeat a polyglot SDK worker
19
+ # - POST /api/v1/sdk/workers/{id}/deregister: Deregister a polyglot SDK worker on shutdown
20
+ # - POST /api/v1/sdk/workflows: Start a durable workflow execution
21
+ # - GET /api/v1/sdk/workflows/{id}: Describe one workflow run: status, output, failure and search attributes
22
+ # - GET /api/v1/sdk/workflows/{id}/history: The events of a workflow run, after after_seq, at most limit
23
+ # - POST /api/v1/sdk/workflows/{id}/query: Ask a workflow run a read-only question
24
+ # - GET /api/v1/sdk/workflows/{id}/result: The outcome of a workflow, following continue-as-new; waits up to wait_ms for it to close
25
+ # - POST /api/v1/sdk/workflows/{id}/search-attributes: Set and remove search attributes of a workflow's open run
26
+ # - POST /api/v1/sdk/workflows/{id}/signal: Deliver an asynchronous signal to a running workflow
27
+ # - POST /api/v1/sdk/workflows/{id}/updates: Send an update to a running workflow and wait for its outcome
28
+ # NOTE: Queue & workflow management (pause, resume, redrive, cancel, terminate) is intentionally
29
+ # excluded from SDKs and must be performed via the JoobQUI Operator Console (/ui/).
30
+
31
+ require "json"
32
+ require "net/http"
33
+ require "socket"
34
+ require "uri"
35
+
36
+ module JoobQ
37
+ SDK_VERSION = "0.1.0"
38
+
39
+
40
+ class NonRetryableError < StandardError; end
41
+
42
+ # The gateway answered with an error status.
43
+ class GatewayError < StandardError
44
+ attr_reader :status
45
+
46
+ def initialize(status, body)
47
+ super("JoobQ Gateway error (#{status}): #{body.to_s[0, 300]}")
48
+ @status = status
49
+ end
50
+ end
51
+
52
+ # `non_retryable` lists error types (the exception's class name, as the worker reports it in `error_class`) that end
53
+ # the step without another attempt.
54
+ RetryPolicy = Struct.new(:attempts, :initial_ms, :backoff, :maximum_ms, :non_retryable, keyword_init: true)
55
+ Compensation = Struct.new(:handler, :input, :activity, keyword_init: true)
56
+ # Puts a node on one side of a change (versioning): it runs only for runs whose version of `change` is within
57
+ # `min..max` (open ended when nil), and is skipped otherwise. Made with `gate`.
58
+ Gate = Struct.new(:change, :min, :max, keyword_init: true)
59
+ # An update handler declared with `WorkflowBlueprint#update`.
60
+ UpdateDef = Struct.new(:name, :handler, :validate, :message, :input, :result, :timeout_ms, keyword_init: true)
61
+ DataRef = Struct.new(:expr) do
62
+ def to_s
63
+ expr
64
+ end
65
+ end
66
+
67
+ module DSL
68
+ # A step's retry policy, applied by the engine. `non_retryable` names error types (class names) that are not retried.
69
+ def retry_policy(attempts: 3, initial_ms: 1000, backoff: 2.0, maximum_ms: 30_000, non_retryable: [])
70
+ RetryPolicy.new(attempts: attempts, initial_ms: initial_ms, backoff: backoff, maximum_ms: maximum_ms,
71
+ non_retryable: Array(non_retryable).map(&:to_s))
72
+ end
73
+
74
+ def compensation(handler, input: nil)
75
+ Compensation.new(handler: JoobQ.normalize_callable(handler), input: input, activity: JoobQ.activity_name(handler))
76
+ end
77
+
78
+ def input(key = nil)
79
+ key.nil? ? DataRef.new("${input}") : DataRef.new("${input.#{key}}")
80
+ end
81
+
82
+ def output(step_name, key = nil)
83
+ key.nil? ? DataRef.new("${steps.#{step_name}.output}") : DataRef.new("${steps.#{step_name}.output.#{key}}")
84
+ end
85
+
86
+ # The input of the update being handled (`${update.input}`, or `${update.input.<key>}`): for an update's
87
+ # `validate`, `input` and `result`.
88
+ def update_input(key = nil)
89
+ key.nil? ? DataRef.new("${update.input}") : DataRef.new("${update.input.#{key}}")
90
+ end
91
+
92
+ # A gate on the change `change` (see `WorkflowBlueprint#change`), for the `gate:` option of `step`, `wait_for`,
93
+ # `timer` and `child`: the node runs only for runs whose version of the change is within `min..max`. At least one
94
+ # bound is required (ArgumentError otherwise).
95
+ def gate(change, min: nil, max: nil)
96
+ raise ArgumentError, "gate on #{change.to_s.inspect} needs a min or a max" if min.nil? && max.nil?
97
+
98
+ Gate.new(change: change.to_s, min: min, max: max)
99
+ end
100
+
101
+ def ref(path)
102
+ clean = path.to_s.strip
103
+ clean.start_with?("${") && clean.end_with?("}") ? DataRef.new(clean) : DataRef.new("${#{clean}}")
104
+ end
105
+
106
+ def workflow(name, version: 1, timeout_ms: nil, task_queue: "workflows", &block)
107
+ JoobQ.workflow(name, version: version, timeout_ms: timeout_ms, task_queue: task_queue, &block)
108
+ end
109
+ end
110
+
111
+ extend DSL
112
+
113
+ # The `${...}` references a step input makes (a blueprint sends these, never the literal values around them).
114
+ def self.collect_references(spec, found = [])
115
+ case spec
116
+ when DataRef then found << spec.expr unless found.include?(spec.expr)
117
+ when Hash then spec.each_value { |v| collect_references(v, found) }
118
+ when Array then spec.each { |v| collect_references(v, found) }
119
+ end
120
+ found
121
+ end
122
+
123
+ # A readable name for a step's activity: its class or job name; nil for a block or lambda.
124
+ def self.activity_name(target)
125
+ return target.job_name if target.respond_to?(:job_name)
126
+ return target.name if target.is_a?(Class) || target.is_a?(Module)
127
+ return target.name.to_s if target.is_a?(Method)
128
+ return nil if target.nil? || target.is_a?(Proc)
129
+
130
+ target.class.name
131
+ end
132
+
133
+ def self.normalize_callable(target)
134
+ return target if target.respond_to?(:call)
135
+ return ->(payload, ctx) { target.perform(payload, ctx) } if target.respond_to?(:perform)
136
+ if target.is_a?(Class) && target.method_defined?(:perform)
137
+ return ->(payload, ctx) { target.new.perform(payload, ctx) }
138
+ end
139
+
140
+ raise ArgumentError, "Expected callable or Job/Activity with .perform(payload, ctx), got #{target.inspect}"
141
+ end
142
+
143
+ def self.lookup_path(root, dotted)
144
+ cur = root
145
+ dotted.to_s.split(".").each do |part|
146
+ next if part.empty?
147
+ return nil unless cur.is_a?(Hash)
148
+
149
+ cur = cur[part] || cur[part.to_sym]
150
+ return nil if cur.nil?
151
+ end
152
+ cur
153
+ end
154
+
155
+ def self.resolve_references(spec, workflow_input, checkpoints)
156
+ return workflow_input if spec.nil?
157
+
158
+ case spec
159
+ when DataRef
160
+ expr = spec.expr
161
+ return workflow_input if expr == "${input}"
162
+ if expr.start_with?("${input.") && expr.end_with?("}")
163
+ return lookup_path(workflow_input, expr[8..-2])
164
+ end
165
+ if expr.start_with?("${steps.") && expr.end_with?("}")
166
+ parts = expr[8..-2].split(".")
167
+ step_name = parts[0]
168
+ step_out = checkpoints[step_name] || checkpoints[step_name.to_sym]
169
+ return step_out if parts.length <= 2
170
+ return lookup_path(step_out, parts[2..].join("."))
171
+ end
172
+ return checkpoints.dup if expr == "${steps}"
173
+ if expr.start_with?("${") && expr.end_with?("}")
174
+ return lookup_path({ "input" => workflow_input, "steps" => checkpoints }, expr[2..-2])
175
+ end
176
+ expr
177
+ when Hash
178
+ spec.each_with_object({}) do |(k, v), acc|
179
+ acc[k.to_s] = resolve_references(v, workflow_input, checkpoints)
180
+ end
181
+ when Array
182
+ spec.map { |v| resolve_references(v, workflow_input, checkpoints) }
183
+ else
184
+ spec
185
+ end
186
+ end
187
+
188
+ # A child's or continued run's input as the engine resolves it: references (`input`, `output`, `ref`) in hashes and
189
+ # arrays, turned into their `${...}` strings. A literal value is refused (ArgumentError): compute a constant in a step
190
+ # and refer to its output.
191
+ def self.template(spec, where)
192
+ case spec
193
+ when DataRef then spec.expr
194
+ when Hash then spec.each_with_object({}) { |(k, v), acc| acc[k.to_s] = template(v, "#{where}.#{k}") }
195
+ when Array then spec.each_with_index.map { |v, i| template(v, "#{where}[#{i}]") }
196
+ else
197
+ raise ArgumentError, "#{where}: #{spec.inspect} is not a reference; the engine resolves this input, so only references are sent"
198
+ end
199
+ end
200
+
201
+ def self.extract_step_dependencies(spec)
202
+ deps = []
203
+ visit = lambda do |node|
204
+ case node
205
+ when DataRef
206
+ if node.expr.start_with?("${steps.")
207
+ step_name = node.expr[8..-2].split(".").first
208
+ deps << step_name if step_name && !deps.include?(step_name)
209
+ end
210
+ when Hash
211
+ node.each_value { |v| visit.call(v) }
212
+ when Array
213
+ node.each { |v| visit.call(v) }
214
+ end
215
+ end
216
+ visit.call(spec)
217
+ deps
218
+ end
219
+
220
+ module Job
221
+ def self.included(base)
222
+ base.extend(ClassMethods)
223
+ end
224
+
225
+ module ClassMethods
226
+ def job_name(val = nil)
227
+ @job_name = val.to_s unless val.nil?
228
+ @job_name || name.split("::").last
229
+ end
230
+
231
+ def queue(val = nil)
232
+ @queue = val.to_s unless val.nil?
233
+ @queue || "default"
234
+ end
235
+
236
+ def retries(val = nil)
237
+ @retries = val.to_i unless val.nil?
238
+ @retries.nil? ? 3 : @retries
239
+ end
240
+
241
+ def expires_ms(val = nil)
242
+ @expires_ms = val.to_i unless val.nil?
243
+ @expires_ms
244
+ end
245
+
246
+ def encrypted_fields(*fields)
247
+ @encrypted_fields = fields.flatten.map(&:to_s) unless fields.empty?
248
+ @encrypted_fields || []
249
+ end
250
+
251
+ def unique_ttl_ms(val = nil)
252
+ @unique_ttl_ms = val.to_i unless val.nil?
253
+ @unique_ttl_ms
254
+ end
255
+
256
+ def enqueue(client, payload = {}, **opts)
257
+ client.enqueue(self, payload, **opts)
258
+ end
259
+ end
260
+ end
261
+
262
+ class WorkflowBlueprint
263
+ include DSL
264
+
265
+ attr_reader :name, :version, :timeout_ms, :task_queue, :steps, :queries, :continuation, :changes, :updates
266
+
267
+ # `timeout_ms` is the whole run's deadline (none by default), enforced by the engine.
268
+ def initialize(name, version: 1, timeout_ms: nil, task_queue: "workflows")
269
+ @name = name.to_s
270
+ @version = version
271
+ @timeout_ms = timeout_ms
272
+ @task_queue = task_queue
273
+ @steps = []
274
+ @queries = {}
275
+ @declared_steps = {}
276
+ @continuation = nil
277
+ @changes = {}
278
+ @updates = {}
279
+ @in_scope = false
280
+ end
281
+
282
+ def query(name, expr)
283
+ @queries[name.to_s] = expr
284
+ self
285
+ end
286
+
287
+ # A step. `after` names earlier steps it also waits for (on top of what precedes it, e.g. a sibling in a `parallel`
288
+ # group); `timeout_ms` limits one attempt, `heartbeat_ms` how long the handler may go without calling
289
+ # `ctx.heartbeat`, `schedule_to_start_ms` how long the step may wait for a worker; `only_if` (a reference) runs the
290
+ # step only when it is truthy, else the step is skipped; `gate` (see `gate`) runs it only on one side of a change.
291
+ # The engine enforces them.
292
+ def step(name, activity = nil, input: nil, retry_policy: nil, retry: nil, compensate: nil, after: [], timeout_ms: nil,
293
+ heartbeat_ms: nil, schedule_to_start_ms: nil, only_if: nil, gate: nil, &block)
294
+ step_name = name.to_s
295
+ after = Array(after).map(&:to_s)
296
+ check_condition!(step_name, only_if)
297
+ check_gate!(step_name, gate)
298
+ validate_and_register!(step_name, input, after)
299
+ callable = JoobQ.normalize_callable(activity || block)
300
+ @steps << {
301
+ kind: "step",
302
+ name: step_name,
303
+ handler: callable,
304
+ activity: JoobQ.activity_name(activity),
305
+ input_spec: input,
306
+ retry: retry_policy || binding.local_variable_get(:retry),
307
+ compensate: compensate,
308
+ after: after,
309
+ timeout_ms: timeout_ms,
310
+ heartbeat_ms: heartbeat_ms,
311
+ schedule_to_start_ms: schedule_to_start_ms,
312
+ only_if: only_if,
313
+ gate: gate
314
+ }
315
+ self
316
+ end
317
+
318
+ # Waits for a signal. `after` names earlier steps it also waits for, `only_if` (a reference) skips the wait unless
319
+ # it is truthy, `compensate` undoes it when the run fails (its input defaults to the signal's payload), `gate` as on
320
+ # `step`.
321
+ def wait_for(name, signal:, timeout_ms: 600_000, after: [], only_if: nil, compensate: nil, gate: nil)
322
+ step_name = name.to_s
323
+ after = Array(after).map(&:to_s)
324
+ check_condition!(step_name, only_if)
325
+ check_gate!(step_name, gate)
326
+ validate_and_register!(step_name, nil, after)
327
+ @steps << { kind: "wait_for", name: step_name, signal: signal.to_s, timeout_ms: timeout_ms, after: after,
328
+ only_if: only_if, compensate: compensate, gate: gate }
329
+ self
330
+ end
331
+
332
+ # Sleeps `for_ms` durably. `after` names earlier steps it also waits for, `only_if` (a reference) skips the timer
333
+ # unless it is truthy, `compensate` undoes it when the run fails (its input defaults to the timer's output), `gate`
334
+ # as on `step`.
335
+ def timer(name, for_ms:, after: [], only_if: nil, compensate: nil, gate: nil)
336
+ step_name = name.to_s
337
+ after = Array(after).map(&:to_s)
338
+ check_condition!(step_name, only_if)
339
+ check_gate!(step_name, gate)
340
+ validate_and_register!(step_name, nil, after)
341
+ @steps << { kind: "timer", name: step_name, duration_ms: for_ms, after: after, only_if: only_if, compensate: compensate,
342
+ gate: gate }
343
+ self
344
+ end
345
+
346
+ # Runs another workflow (a blueprint or its name) as a node; its output is the child run's output, its failure the
347
+ # node's. The engine starts the child and resolves `input` (references only: `input`, `output`, `ref`, in hashes and
348
+ # arrays; the parent run's input when omitted). A child is not retried and has no timeout of its own: give the child
349
+ # workflow's steps retries, and the child workflow a `timeout_ms`. `compensate` undoes it when the run fails (its
350
+ # input defaults to the child's output), `after`, `only_if` and `gate` as on `step`. The child workflow must be
351
+ # registered before a run reaches this node.
352
+ def child(name, workflow, input: nil, compensate: nil, after: [], only_if: nil, gate: nil)
353
+ step_name = name.to_s
354
+ after = Array(after).map(&:to_s)
355
+ JoobQ.template(input, "#{step_name}.input") unless input.nil?
356
+ check_condition!(step_name, only_if)
357
+ check_gate!(step_name, gate)
358
+ validate_and_register!(step_name, input, after)
359
+ child_name = workflow.is_a?(WorkflowBlueprint) ? workflow.name : workflow.to_s
360
+ @steps << { kind: "child", name: step_name, workflow: child_name, input_spec: input, compensate: compensate,
361
+ after: after, only_if: only_if, gate: gate }
362
+ self
363
+ end
364
+
365
+ # When the run ends (and `only_if`, a reference, is truthy), it closes `continued_as_new` and a fresh run of this
366
+ # workflow starts with `input` (references only, resolved by the engine; the same input when omitted), at most
367
+ # `max_runs` runs; `max_events` bounds a run's history. The engine enforces it.
368
+ def continue_as_new(input: nil, only_if: nil, max_runs: nil, max_events: nil)
369
+ policy = {}
370
+ policy["input"] = JoobQ.template(input, "continue_as_new.input") unless input.nil?
371
+ unless only_if.nil?
372
+ raise ArgumentError, "continue_as_new: only_if must be a reference (input, output, ref)" unless only_if.is_a?(DataRef)
373
+
374
+ policy["only_if"] = only_if.expr
375
+ end
376
+ policy["max_runs"] = max_runs unless max_runs.nil?
377
+ policy["max_events"] = max_events unless max_events.nil?
378
+ @continuation = policy
379
+ self
380
+ end
381
+
382
+ # Declares a change (versioning, like Temporal's getVersion): runs that reach a gated node fresh take `max`, runs
383
+ # that went past it on the old path keep 0; nodes pick a side with `gate: gate(id, min: ..., max: ...)`. Raise `min`
384
+ # only once no open run is on an older path. `min` above `max` is an ArgumentError.
385
+ def change(id, min: 0, max: 1)
386
+ raise ArgumentError, "change #{id.to_s.inspect} in workflow '#{@name}': min is above max" if min > max
387
+
388
+ @changes[id.to_s] = { "min" => min, "max" => max }
389
+ self
390
+ end
391
+
392
+ # An update handler (`Client#update_workflow`): `validate` (a reference, e.g. `update_input("name")`) must be truthy
393
+ # or the update is rejected with `message`; then `handler` runs as a step of the run (its payload is `input`, by
394
+ # default the update's input, and its return value is the result), or, without one, `result` (a template of
395
+ # references) is the result; `timeout_ms` limits the handler. References may read the run (`input`, `output`) and
396
+ # `update_input`. A duplicate name, a `validate` that is not a reference, or a literal in `input`/`result` is an
397
+ # ArgumentError.
398
+ def update(name, handler = nil, validate: nil, message: nil, input: nil, result: nil, timeout_ms: nil, &block)
399
+ update_name = name.to_s
400
+ raise ArgumentError, "Duplicate update '#{update_name}' in workflow '#{@name}'" if @updates.key?(update_name)
401
+ unless validate.nil? || validate.is_a?(DataRef)
402
+ raise ArgumentError, "update '#{update_name}': validate must be a reference (update_input, input, output, ref)"
403
+ end
404
+
405
+ JoobQ.template(input, "update #{update_name}.input") unless input.nil?
406
+ JoobQ.template(result, "update #{update_name}.result") unless result.nil?
407
+ target = handler || block
408
+ fn = target.nil? ? nil : JoobQ.normalize_callable(target)
409
+ @updates[update_name] = UpdateDef.new(name: update_name, handler: fn, validate: validate, message: message,
410
+ input: input, result: result, timeout_ms: timeout_ms)
411
+ self
412
+ end
413
+
414
+ # A cancellation scope: its steps (declared in the block) run in sequence and can be cancelled as a unit (from the
415
+ # console or API, or after `timeout_ms` from its first step's start) while the rest of the run goes on; a cancelled
416
+ # step counts as finished with a null output. A `non_cancellable` scope holds cleanup that still runs when the
417
+ # workflow is cancelled, times out or fails (it cannot have a timeout). A running step sees a cancel through
418
+ # `ctx.heartbeat`. Scopes do not nest, and a scope needs at least one step (ArgumentError otherwise).
419
+ def scope(name, timeout_ms: nil, non_cancellable: false, &block)
420
+ scope_name = name.to_s
421
+ if @in_scope
422
+ raise ArgumentError, "scope '#{scope_name}' in workflow '#{@name}' is inside another scope: scopes do not nest"
423
+ end
424
+ if non_cancellable && !timeout_ms.nil?
425
+ raise ArgumentError, "scope '#{scope_name}' is non_cancellable and cannot time out"
426
+ end
427
+
428
+ sub = WorkflowBlueprint.new("#{@name}:scope", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
429
+ @declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
430
+ sub.send(:enter_scope!)
431
+ sub.instance_eval(&block) if block
432
+ raise ArgumentError, "scope '#{scope_name}' in workflow '#{@name}' has no steps" if sub.steps.empty?
433
+
434
+ sub.send(:declared_keys).each { |k| mark_declared!(k) }
435
+ @steps << { kind: "scope", name: scope_name, children: sub.steps, timeout_ms: timeout_ms,
436
+ non_cancellable: non_cancellable ? true : false }
437
+ self
438
+ end
439
+
440
+ def parallel(&block)
441
+ sub = WorkflowBlueprint.new("#{@name}:parallel", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
442
+ @declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
443
+ sub.send(:enter_scope!) if @in_scope
444
+ sub.instance_eval(&block)
445
+ sub.send(:declared_keys).each { |k| mark_declared!(k) }
446
+ @steps << { kind: "parallel", name: "parallel_#{@steps.length + 1}", children: sub.steps }
447
+ self
448
+ end
449
+
450
+ def sequence(&block)
451
+ sub = WorkflowBlueprint.new("#{@name}:sequence", version: @version, timeout_ms: @timeout_ms, task_queue: @task_queue)
452
+ @declared_steps.each_key { |k| sub.send(:mark_declared!, k) }
453
+ sub.send(:enter_scope!) if @in_scope
454
+ sub.instance_eval(&block)
455
+ sub.send(:declared_keys).each { |k| mark_declared!(k) }
456
+ @steps << { kind: "sequence", name: "sequence_#{@steps.length + 1}", children: sub.steps }
457
+ self
458
+ end
459
+
460
+ # The workflow's structure as data, as the worker registers it with the gateway (`CONTRACT.md`).
461
+ def to_blueprint
462
+ blueprint = {
463
+ "name" => @name,
464
+ "version" => @version,
465
+ "task_queue" => @task_queue,
466
+ "queries" => @queries.transform_values(&:to_s),
467
+ "nodes" => @steps.map { |s| blueprint_node(s) }
468
+ }
469
+ blueprint["timeout_ms"] = @timeout_ms unless @timeout_ms.nil? # the whole run's deadline, enforced by the engine
470
+ blueprint["continue_as_new"] = @continuation.dup unless @continuation.nil?
471
+ blueprint["changes"] = @changes.map { |id, bounds| { "id" => id }.merge(bounds) } unless @changes.empty?
472
+ blueprint["updates"] = @updates.transform_values { |u| update_spec(u) } unless @updates.empty?
473
+ blueprint
474
+ end
475
+
476
+ private def update_spec(update)
477
+ spec = {}
478
+ spec["handler"] = true unless update.handler.nil?
479
+ spec["validate"] = update.validate.expr unless update.validate.nil?
480
+ spec["message"] = update.message unless update.message.nil? || update.message.to_s.empty?
481
+ spec["input"] = JoobQ.template(update.input, "update #{update.name}.input") unless update.input.nil?
482
+ spec["result"] = JoobQ.template(update.result, "update #{update.name}.result") unless update.result.nil?
483
+ spec["timeout_ms"] = update.timeout_ms unless update.timeout_ms.nil?
484
+ spec
485
+ end
486
+
487
+ private def blueprint_node(node)
488
+ out = { "name" => node[:name], "kind" => node[:kind] }
489
+ case node[:kind]
490
+ when "parallel", "sequence", "scope"
491
+ out["children"] = node[:children].map { |c| blueprint_node(c) }
492
+ if node[:kind] == "scope"
493
+ out["timeout_ms"] = node[:timeout_ms] unless node[:timeout_ms].nil?
494
+ out["non_cancellable"] = true if node[:non_cancellable]
495
+ end
496
+ when "wait_for"
497
+ out.merge!("signal" => node[:signal], "timeout_ms" => node[:timeout_ms])
498
+ when "timer"
499
+ out["for_ms"] = node[:duration_ms]
500
+ when "child"
501
+ out["workflow"] = node[:workflow]
502
+ out["input"] = JoobQ.template(node[:input_spec], "#{node[:name]}.input") unless node[:input_spec].nil?
503
+ else
504
+ out["activity"] = node[:activity]
505
+ out["references"] = node[:input_spec].nil? ? ["${input}"] : JoobQ.collect_references(node[:input_spec])
506
+ %i[timeout_ms heartbeat_ms schedule_to_start_ms].each do |key|
507
+ out[key.to_s] = node[key] unless node[key].nil?
508
+ end
509
+ out["retry"] = blueprint_retry(node[:retry]) if node[:retry]
510
+ end
511
+ unless %w[parallel sequence scope].include?(node[:kind]) # a step, wait, timer or child can be undone, skipped, gated
512
+ if (c = node[:compensate])
513
+ out["compensate"] = { "activity" => c.activity,
514
+ "references" => JoobQ.collect_references(c.input.nil? ? output(node[:name]) : c.input) }
515
+ end
516
+ out["only_if"] = node[:only_if].expr unless node[:only_if].nil?
517
+ if (g = node[:gate])
518
+ gate_out = { "change" => g.change }
519
+ gate_out["min"] = g.min unless g.min.nil?
520
+ gate_out["max"] = g.max unless g.max.nil?
521
+ out["gate"] = gate_out
522
+ end
523
+ end
524
+ out["after"] = node[:after].dup unless node[:after].nil? || node[:after].empty?
525
+ out
526
+ end
527
+
528
+ private def blueprint_retry(r)
529
+ out = { "attempts" => r.attempts, "initial_ms" => r.initial_ms, "backoff" => r.backoff, "maximum_ms" => r.maximum_ms }
530
+ out["non_retryable"] = r.non_retryable.map(&:to_s) unless r.non_retryable.nil? || r.non_retryable.empty?
531
+ out
532
+ end
533
+
534
+ # The handlers of this workflow's steps, by the activity name the engine sends them under (`<workflow>/<step>`,
535
+ # compensations `<workflow>/<node>/compensate`, for waits, timers and children too; a child has no handler of its
536
+ # own: the engine runs the child workflow; update handlers `<workflow>/@update/<name>`, which receive the task's
537
+ # payload as is: the engine resolved their input). Each runs its step once: the engine conducts the run (it
538
+ # retries, compensates and waits). A step's input is resolved against the run's input (the task's payload) and the
539
+ # outputs of the steps that completed (the task's checkpoints).
540
+ def activities
541
+ found = {}
542
+ visit = lambda do |nodes|
543
+ nodes.each do |node|
544
+ case node[:kind]
545
+ when "sequence", "parallel", "scope"
546
+ visit.call(node[:children])
547
+ else
548
+ found[JoobQ.activity_task_name(@name, node[:name])] = step_activity(node) if node[:kind] == "step"
549
+ if (comp = node[:compensate]) # a step, wait, timer or child can be undone
550
+ found[JoobQ.activity_task_name(@name, node[:name], compensate: true)] = compensation_activity(node, comp)
551
+ end
552
+ end
553
+ end
554
+ end
555
+ visit.call(@steps)
556
+ @updates.each_value do |u|
557
+ found[JoobQ.update_task_name(@name, u.name)] = u.handler unless u.handler.nil?
558
+ end
559
+ found
560
+ end
561
+
562
+ private def step_activity(node)
563
+ lambda do |workflow_input, ctx|
564
+ node[:handler].call(JoobQ.resolve_references(node[:input_spec], workflow_input, ctx.task["checkpoints"] || {}), ctx)
565
+ end
566
+ end
567
+
568
+ private def compensation_activity(node, comp)
569
+ lambda do |workflow_input, ctx|
570
+ spec = comp.input.nil? ? output(node[:name]) : comp.input
571
+ comp.handler.call(JoobQ.resolve_references(spec, workflow_input, ctx.task["checkpoints"] || {}), ctx)
572
+ end
573
+ end
574
+
575
+ private
576
+
577
+ def mark_declared!(key)
578
+ @declared_steps[key.to_s] = true
579
+ end
580
+
581
+ def declared_keys
582
+ @declared_steps.keys
583
+ end
584
+
585
+ def enter_scope!
586
+ @in_scope = true
587
+ end
588
+
589
+ def check_gate!(step_name, gate)
590
+ return if gate.nil? || gate.is_a?(Gate)
591
+
592
+ raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': gate must be made with gate(change, min:, max:)"
593
+ end
594
+
595
+ # `only_if` must be a reference to the input or to an earlier step.
596
+ def check_condition!(step_name, only_if)
597
+ return if only_if.nil?
598
+ unless only_if.is_a?(DataRef)
599
+ raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': only_if must be a reference (input, output, ref)"
600
+ end
601
+
602
+ JoobQ.extract_step_dependencies(only_if).each do |dep|
603
+ unless @declared_steps.key?(dep)
604
+ raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}': only_if refers to unknown or downstream step '#{dep}'"
605
+ end
606
+ end
607
+ end
608
+
609
+ def validate_and_register!(step_name, input_spec, after = [])
610
+ raise ArgumentError, "Duplicate step '#{step_name}' in workflow '#{@name}'" if @declared_steps.key?(step_name)
611
+
612
+ after.each do |dep|
613
+ unless @declared_steps.key?(dep)
614
+ raise ArgumentError, "Step '#{step_name}' in workflow '#{@name}' waits (after:) for unknown or downstream step '#{dep}'"
615
+ end
616
+ end
617
+ JoobQ.extract_step_dependencies(input_spec).each do |dep|
618
+ unless @declared_steps.key?(dep)
619
+ raise ArgumentError, "Step '#{step_name}' references unknown or downstream step '#{dep}'"
620
+ end
621
+ end
622
+ mark_declared!(step_name)
623
+ end
624
+ end
625
+
626
+ # The activity the engine sends a step (or its compensation) of an SDK workflow under.
627
+ def self.activity_task_name(workflow_name, step, compensate: false)
628
+ compensate ? "#{workflow_name}/#{step}/compensate" : "#{workflow_name}/#{step}"
629
+ end
630
+
631
+ # The activity the engine sends an update handler of an SDK workflow under.
632
+ def self.update_task_name(workflow_name, update)
633
+ "#{workflow_name}/@update/#{update}"
634
+ end
635
+
636
+ # A workflow blueprint. `timeout_ms` is the whole run's deadline (none by default), enforced by the engine.
637
+ def self.workflow(name, version: 1, timeout_ms: nil, task_queue: "workflows", &block)
638
+ bp = WorkflowBlueprint.new(name, version: version, timeout_ms: timeout_ms, task_queue: task_queue)
639
+ bp.instance_eval(&block) if block
640
+ bp
641
+ end
642
+
643
+ class ExecutionContext
644
+ attr_reader :task, :worker_id, :client, :context, :lease_ttl_ms
645
+
646
+ def initialize(task, worker_id, client, lease_ttl_ms = 30_000, transport = nil)
647
+ @transport = transport
648
+ @task = task
649
+ @worker_id = worker_id
650
+ @client = client
651
+ @lease_ttl_ms = lease_ttl_ms
652
+ @context = (task["context"] || {}).dup
653
+ @last_beat_ms = nil
654
+ @beat_cancel = false
655
+ end
656
+
657
+ def attempt
658
+ (@task["attempt"] || 1).to_i
659
+ end
660
+
661
+ def dedupe_key
662
+ wf = @task["workflow_id"] || @task["id"]
663
+ st = @task["step_id"] || @task["name"]
664
+ "#{wf}:#{st}:#{attempt}"
665
+ end
666
+
667
+ # An engine step's heartbeat timeout (`task["heartbeat_ms"]`, nil for most tasks): call `heartbeat` more often.
668
+ def heartbeat_ms
669
+ @task["heartbeat_ms"]
670
+ end
671
+
672
+ # True when the run was cancelled. In a running worker it answers what the last lease renewal said without waiting
673
+ # for the gateway; the worker renews every lease on its own. For a step with `heartbeat_ms` it also
674
+ # tells the engine the step is alive (at most every third of `heartbeat_ms`): the worker's own renewals do not, so a
675
+ # handler that stops calling this is timed out.
676
+ def heartbeat(lease_ttl_ms: @lease_ttl_ms)
677
+ if (beat_ms = heartbeat_ms) && beat_ms.to_i.positive?
678
+ now = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
679
+ if @last_beat_ms.nil? || now - @last_beat_ms >= [100, beat_ms.to_i / 3].max
680
+ @last_beat_ms = now
681
+ begin
682
+ res = @client.heartbeat_task(@task["lease_token"], worker_id: @worker_id, lease_ttl_ms: lease_ttl_ms)
683
+ @beat_cancel = res.is_a?(Hash) && res["cancel_requested"] == true
684
+ rescue StandardError
685
+ nil # the next window's beat tries again; the engine's deadline decides
686
+ end
687
+ end
688
+ return true if @beat_cancel
689
+ end
690
+ return @transport.cancel_requested?(@task["lease_token"]) if @transport
691
+
692
+ res = @client.heartbeat_task(@task["lease_token"], worker_id: @worker_id, lease_ttl_ms: lease_ttl_ms)
693
+ res["cancel_requested"] == true
694
+ end
695
+
696
+ def enqueue(job_cls, payload = {}, context: {}, **opts)
697
+ @client.enqueue(job_cls, payload, context: @context.merge(context), **opts)
698
+ end
699
+
700
+ def enqueue_job(queue:, job_type:, payload: {}, context: {}, **opts)
701
+ @client.enqueue_job(queue: queue, job_type: job_type, payload: payload, context: @context.merge(context), **opts)
702
+ end
703
+
704
+ def start(workflow_def, input: nil, context: {}, **opts)
705
+ @client.start(workflow_def, input: input, context: @context.merge(context), **opts)
706
+ end
707
+
708
+ def start_workflow(name:, input: nil, context: {}, **opts)
709
+ @client.start_workflow(name: name, input: input, context: @context.merge(context), **opts)
710
+ end
711
+
712
+ def register_schedule(name, job_type, queue, **opts)
713
+ @client.register_schedule(name, job_type, queue, **opts)
714
+ end
715
+
716
+ def rate_limit(key, limit, period_ms)
717
+ @client.rate_limit(key, limit, period_ms)
718
+ end
719
+
720
+ # `Client#create_batch`; the members and callbacks carry this handler's context.
721
+ def create_batch(jobs, context: {}, **opts)
722
+ @client.create_batch(jobs, context: @context.merge(context), **opts)
723
+ end
724
+
725
+ def get_batch(id)
726
+ @client.get_batch(id)
727
+ end
728
+
729
+ def describe_workflow(id)
730
+ @client.describe_workflow(id)
731
+ end
732
+
733
+ def workflow_result(id, wait_ms: 0)
734
+ @client.workflow_result(id, wait_ms: wait_ms)
735
+ end
736
+
737
+ def workflow_history(id, after_seq: 0, limit: 200)
738
+ @client.workflow_history(id, after_seq: after_seq, limit: limit)
739
+ end
740
+
741
+ def query_workflow(id, name)
742
+ @client.query_workflow(id, name)
743
+ end
744
+
745
+ def signal_workflow(id:, name:, payload: nil)
746
+ @client.signal_workflow(id: id, name: name, payload: payload)
747
+ end
748
+
749
+ def update_workflow(id, name, input: nil, update_id: nil, wait_ms: nil)
750
+ @client.update_workflow(id, name, input: input, update_id: update_id, wait_ms: wait_ms)
751
+ end
752
+
753
+ def upsert_workflow_attributes(id, set: nil, remove: nil)
754
+ @client.upsert_workflow_attributes(id, set: set, remove: remove)
755
+ end
756
+ end
757
+
758
+ # The fields that report `error` to the gateway (`fail_task`, and a "fail" item of `ack_tasks`).
759
+ def self.failure(error, retryable: true)
760
+ frames = Array(error.backtrace).map do |line|
761
+ if line =~ /\A(.+):(\d+):in [`'](.+)'\z/
762
+ { file: Regexp.last_match(1), line: Regexp.last_match(2).to_i, function: Regexp.last_match(3) }
763
+ else
764
+ { file: line, line: 0, function: "unknown" }
765
+ end
766
+ end
767
+ message = error.message.to_s
768
+ message = error.class.name if message.empty?
769
+ {
770
+ error_class: error.class.name,
771
+ error_message: message[0, 4000],
772
+ retryable: retryable,
773
+ sdk_language: "ruby",
774
+ stack_frames: frames,
775
+ raw_stacktrace: Array(error.backtrace).join("\n")[0, 4000]
776
+ }
777
+ end
778
+
779
+ class Client
780
+ attr_reader :gateway_url, :token
781
+ # Seconds one gateway call may take. A worker lowers it (`Worker#initialize`): a call that hangs for longer than a
782
+ # lease would let every lease the worker holds expire while its tasks still run.
783
+ attr_accessor :request_timeout
784
+
785
+ def initialize(gateway_url:, token:, context_provider: nil)
786
+ @gateway_url = gateway_url.sub(%r{/+\z}, "")
787
+ @token = token
788
+ @request_timeout = 60
789
+ @context_provider = context_provider
790
+ @uri = URI.parse(@gateway_url)
791
+ @connection_key = :"joobq_http_#{object_id}" # one kept-alive connection per thread
792
+ end
793
+
794
+ # Enqueues many jobs with POST /sdk/jobs/batch (500 per call). Each item is `[JobClass, payload]`,
795
+ # `[JobClass, payload, { delay_ms:, jid:, unique_key: }]` or a hash of `enqueue_job` arguments. Answers one
796
+ # `{"jid", "queue", "status"}` per job, in order.
797
+ def enqueue_many(jobs, context: {})
798
+ bodies = jobs.map { |item| item_body(item, context) }
799
+ bodies.each_slice(500).flat_map { |slice| post("/api/v1/sdk/jobs/batch", { jobs: slice })["jobs"] || [] }
800
+ end
801
+
802
+ # Creates a Pro batch with POST /sdk/batches: 1 to 1000 member jobs (the shapes `enqueue_many` takes, without a
803
+ # `unique_key`) and callback jobs (the same shapes). `on_failure` is enqueued at the first dead member,
804
+ # `on_complete` once every member completed or was dead-lettered, `on_success` then too when none was dead; each
805
+ # once, with `{"batch" => {id, kind, total, succeeded, failed}}` merged into its payload. Answers
806
+ # `{"batch_id", "total", "jobs"}`; 402 pro_required without a Pro licence.
807
+ def create_batch(jobs, on_complete: nil, on_success: nil, on_failure: nil, description: nil, context: {})
808
+ body = { description: description, jobs: jobs.map { |item| batch_item_body(item, context) } }
809
+ { on_complete: on_complete, on_success: on_success, on_failure: on_failure }.each do |kind, callback|
810
+ body[kind] = batch_item_body(callback, context) unless callback.nil?
811
+ end
812
+ post("/api/v1/sdk/batches", body)
813
+ end
814
+
815
+ # A Pro batch's progress with GET /sdk/batches/:id: `{"batch_id", "description", "status", "total", "pending",
816
+ # "succeeded", "failed", "created_at", "finished_at"}`; `status` is pending, then success (no member dead) or
817
+ # complete.
818
+ def get_batch(id)
819
+ get("/api/v1/sdk/batches/#{segment(id)}")
820
+ end
821
+
822
+ # One member or callback of `create_batch`: the batch's `context` merged key by key under the job's own (the job's
823
+ # keys win).
824
+ def batch_item_body(item, context)
825
+ base = context || {}
826
+ if item.is_a?(Hash)
827
+ spec = item.transform_keys(&:to_sym)
828
+ item_body(spec.merge(context: base.merge(spec[:context] || {})), base)
829
+ else
830
+ job_cls, payload, opts = item
831
+ opts = (opts || {}).transform_keys(&:to_sym)
832
+ item_body([job_cls, payload, opts.merge(context: base.merge(opts[:context] || {}))], base)
833
+ end
834
+ end
835
+
836
+ # One job of `enqueue_many` or `create_batch`: a hash of `enqueue_job` arguments or `[JobClass, payload, opts]`.
837
+ def item_body(item, context)
838
+ if item.is_a?(Hash)
839
+ spec = item.transform_keys(&:to_sym)
840
+ job_body(context: context, **spec)
841
+ else
842
+ job_cls, payload, opts = item
843
+ opts = (opts || {}).transform_keys(&:to_sym)
844
+ job_body(queue: job_cls.queue, job_type: job_cls.job_name, payload: payload || {},
845
+ context: opts.fetch(:context, context), jid: opts[:jid], max_retries: job_cls.retries,
846
+ delay_ms: opts.fetch(:delay_ms, 0), run_at_ms: opts[:run_at_ms],
847
+ expires_ms: opts[:expires_ms], timeout_ms: opts[:timeout_ms],
848
+ encrypt_fields: job_cls.encrypted_fields,
849
+ unique_key: opts[:unique_key], unique_mode: opts.fetch(:unique_mode, "while_locked"),
850
+ unique_ttl_ms: job_cls.unique_ttl_ms)
851
+ end
852
+ end
853
+
854
+ # `enqueue` on a thread of its own: `value` on the returned thread waits for `{"jid", "queue", "status"}` (or raises
855
+ # the error). `enqueue` is the call that answers only once the job is stored.
856
+ def enqueue_async(job_cls, payload = {}, **opts)
857
+ Thread.new { enqueue(job_cls, payload, **opts) }.tap { |t| t.report_on_exception = false }
858
+ end
859
+
860
+ def enqueue(job_cls, payload = {}, context: {}, jid: nil, delay_ms: 0, run_at_ms: nil, expires_ms: nil, timeout_ms: nil, unique_key: nil, unique_mode: "while_locked", unique_ttl_ms: nil)
861
+ enqueue_job(
862
+ queue: job_cls.queue,
863
+ job_type: job_cls.job_name,
864
+ payload: payload,
865
+ context: context,
866
+ jid: jid,
867
+ max_retries: job_cls.retries,
868
+ delay_ms: delay_ms,
869
+ run_at_ms: run_at_ms,
870
+ expires_ms: expires_ms,
871
+ timeout_ms: timeout_ms,
872
+ encrypt_fields: job_cls.encrypted_fields,
873
+ unique_key: unique_key,
874
+ unique_mode: unique_mode,
875
+ unique_ttl_ms: unique_ttl_ms.nil? ? job_cls.unique_ttl_ms : unique_ttl_ms
876
+ )
877
+ end
878
+
879
+ def enqueue_job(**opts)
880
+ post("/api/v1/sdk/jobs", job_body(**opts))
881
+ end
882
+
883
+ def job_body(queue:, job_type:, payload: {}, context: {}, jid: nil, max_retries: 3, delay_ms: 0, run_at_ms: nil, expires_ms: nil, timeout_ms: nil, encrypt_fields: [], unique_key: nil, unique_mode: "while_locked", unique_ttl_ms: nil)
884
+ {
885
+ queue: queue,
886
+ job_type: job_type,
887
+ payload: payload,
888
+ context: resolve_context(context),
889
+ jid: jid,
890
+ max_retries: max_retries,
891
+ delay_ms: delay_ms,
892
+ run_at_ms: run_at_ms,
893
+ expires_ms: expires_ms,
894
+ timeout_ms: timeout_ms,
895
+ encrypt_fields: encrypt_fields,
896
+ unique_key: unique_key,
897
+ unique_mode: unique_mode,
898
+ unique_ttl_ms: unique_ttl_ms
899
+ }
900
+ end
901
+
902
+ def start(workflow_def, input: nil, context: {}, id: nil, search_attributes: nil, build: nil)
903
+ start_workflow(
904
+ name: workflow_def.name,
905
+ input: input,
906
+ context: context,
907
+ id: id,
908
+ search_attributes: search_attributes,
909
+ build: build
910
+ )
911
+ end
912
+
913
+ def start_workflow(name:, input: nil, context: {}, id: nil, search_attributes: nil, build: nil)
914
+ post("/api/v1/sdk/workflows", {
915
+ name: name,
916
+ input: input,
917
+ context: resolve_context(context),
918
+ id: id,
919
+ search_attributes: search_attributes,
920
+ build: build
921
+ })
922
+ end
923
+
924
+ # Registers a recurring schedule with POST /sdk/schedules. Pass exactly one of `cron:` or `every:`
925
+ # (`every` is "<count> <unit>", e.g. "3 seconds"). Answers `{"id", "name", "kind", "created"}`.
926
+ def register_schedule(name, job_type, queue, cron: nil, every: nil, timezone: "UTC", payload: nil, max_retries: 3, encrypt_fields: [], unique_key: nil)
927
+ post("/api/v1/sdk/schedules", {
928
+ name: name,
929
+ kind: "job",
930
+ cron: cron,
931
+ every: every,
932
+ timezone: timezone,
933
+ queue: queue,
934
+ job_type: job_type,
935
+ payload: payload || {},
936
+ max_retries: max_retries,
937
+ encrypt_fields: encrypt_fields,
938
+ unique_key: unique_key
939
+ })
940
+ end
941
+
942
+ # Consumes one unit of a Pro rate-limit window with POST /sdk/rate-limit. Answers
943
+ # `{"allowed" => Bool, "retry_after_ms" => Int}`.
944
+ def rate_limit(key, limit, period_ms)
945
+ post("/api/v1/sdk/rate-limit", { key: key, limit: limit, period_ms: period_ms })
946
+ end
947
+
948
+ def signal_workflow(id:, name:, payload: nil)
949
+ post("/api/v1/sdk/workflows/#{segment(id)}/signal", {
950
+ name: name,
951
+ payload: payload
952
+ })
953
+ end
954
+
955
+ # Sends an update to a running workflow's update handler `name` (Pro; 402 without a licence) with
956
+ # POST /sdk/workflows/:id/updates and waits up to `wait_ms` (100 to 30000, default 10000) for its outcome. Answers
957
+ # `{"update_id", "name", "status", "result", "error", "reason", "message"}`: `status` is completed, rejected
958
+ # (validate failed: `reason`, `message`), failed, or accepted (a 202) when the handler had not answered in time
959
+ # (send again with the same `update_id` to wait for the same update).
960
+ def update_workflow(id, name, input: nil, update_id: nil, wait_ms: nil)
961
+ body = { name: name, input: input }
962
+ body[:update_id] = update_id unless update_id.nil?
963
+ body[:wait_ms] = Integer(wait_ms) unless wait_ms.nil?
964
+ post("/api/v1/sdk/workflows/#{segment(id)}/updates", body)
965
+ end
966
+
967
+ # One run (Pro; 402 without a licence) with GET /sdk/workflows/:id. Answers `{"id", "name", "status", "closed",
968
+ # "run", "started_at", "closed_at", "continued_to", "output", "failure", "attributes", "queries"}`; payload fields
969
+ # with secret-looking names are masked, as in the console.
970
+ def describe_workflow(id)
971
+ get("/api/v1/sdk/workflows/#{segment(id)}")
972
+ end
973
+
974
+ # The run at the end of the workflow's continue-as-new chain (Pro; 402 without a licence) with
975
+ # GET /sdk/workflows/:id/result, waiting up to `wait_ms` (at most 25000) for it to close. Same shape as
976
+ # `describe_workflow`; `"closed" => false` means it was still open when the wait ended.
977
+ def workflow_result(id, wait_ms: 0)
978
+ get("/api/v1/sdk/workflows/#{segment(id)}/result?wait_ms=#{Integer(wait_ms)}")
979
+ end
980
+
981
+ # The run's events after `after_seq`, at most `limit`, oldest first (Pro; 402 without a licence), with
982
+ # GET /sdk/workflows/:id/history. Answers the array of events.
983
+ def workflow_history(id, after_seq: 0, limit: 200)
984
+ res = get("/api/v1/sdk/workflows/#{segment(id)}/history?after_seq=#{Integer(after_seq)}&limit=#{Integer(limit)}")
985
+ res["items"] || []
986
+ end
987
+
988
+ # Asks the run a read-only question (Pro; 402 without a licence) with POST /sdk/workflows/:id/query. Answers the
989
+ # query's result (any JSON value).
990
+ def query_workflow(id, name)
991
+ post("/api/v1/sdk/workflows/#{segment(id)}/query", { name: name })["result"]
992
+ end
993
+
994
+ # Sets and/or removes search attributes of the workflow's open run (Pro; 402 without a licence) with
995
+ # POST /sdk/workflows/:id/search-attributes. Answers `{"id", "attributes"}`.
996
+ def upsert_workflow_attributes(id, set: nil, remove: nil)
997
+ post("/api/v1/sdk/workflows/#{segment(id)}/search-attributes", { set: set, remove: remove })
998
+ end
999
+
1000
+ # Sends a workflow's blueprint; the gateway keeps each new content hash (`created`) and ignores a known one.
1001
+ def register_blueprint(workflow_def, worker_id: nil)
1002
+ post("/api/v1/sdk/blueprints", {
1003
+ name: workflow_def.name,
1004
+ sdk_language: "ruby",
1005
+ worker_id: worker_id,
1006
+ blueprint: workflow_def.to_blueprint
1007
+ })
1008
+ end
1009
+
1010
+ def register_worker(worker_id:, task_queues:, concurrency: 4, service_name: nil, team: nil)
1011
+ post("/api/v1/sdk/workers/register", {
1012
+ worker_id: worker_id,
1013
+ sdk_language: "ruby",
1014
+ sdk_version: SDK_VERSION,
1015
+ service_name: service_name,
1016
+ team: team || ENV["JOOBQ_TEAM"],
1017
+ host: Socket.gethostname,
1018
+ pid: Process.pid,
1019
+ task_queues: task_queues,
1020
+ concurrency: concurrency,
1021
+ delivery_mode: "poll"
1022
+ })
1023
+ end
1024
+
1025
+ def deregister_worker(worker_id:)
1026
+ post("/api/v1/sdk/workers/#{URI.encode_www_form_component(worker_id)}/deregister", {})
1027
+ end
1028
+
1029
+ def poll_tasks(worker_id:, task_queues:, max_tasks: 1, timeout_ms: 0, lease_ttl_ms: 30_000)
1030
+ res = post("/api/v1/sdk/tasks/poll", {
1031
+ worker_id: worker_id,
1032
+ task_queues: task_queues,
1033
+ max_tasks: max_tasks,
1034
+ timeout_ms: timeout_ms,
1035
+ lease_ttl_ms: lease_ttl_ms
1036
+ })
1037
+ res["tasks"] || []
1038
+ end
1039
+
1040
+ def heartbeat_task(lease_token, worker_id:, lease_ttl_ms: 30_000)
1041
+ post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/heartbeat", {
1042
+ worker_id: worker_id,
1043
+ lease_ttl_ms: lease_ttl_ms
1044
+ })
1045
+ end
1046
+
1047
+ def complete_task(lease_token, worker_id:, output: nil)
1048
+ post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/complete", {
1049
+ worker_id: worker_id,
1050
+ output: output
1051
+ })
1052
+ end
1053
+
1054
+ def fail_task(lease_token, worker_id:, error:, retryable: true)
1055
+ post("/api/v1/sdk/tasks/#{URI.encode_www_form_component(lease_token)}/fail",
1056
+ { worker_id: worker_id }.merge(JoobQ.failure(error, retryable: retryable)))
1057
+ end
1058
+
1059
+ # Settles up to 100 leases in one call (POST /sdk/tasks/ack): items are `{token:, outcome: "complete", output:}`
1060
+ # or `{token:, outcome: "fail", **JoobQ.failure(error)}`; each gets its own result.
1061
+ def ack_tasks(worker_id:, items:)
1062
+ post("/api/v1/sdk/tasks/ack", { worker_id: worker_id, items: items })
1063
+ end
1064
+
1065
+ # Renews many leases in one call (POST /sdk/tasks/heartbeat): `leases` are `{token:}`.
1066
+ def heartbeat_tasks(worker_id:, leases:, lease_ttl_ms: 30_000)
1067
+ post("/api/v1/sdk/tasks/heartbeat", { worker_id: worker_id, lease_ttl_ms: lease_ttl_ms, leases: leases })
1068
+ end
1069
+
1070
+ private
1071
+
1072
+ def resolve_context(explicit)
1073
+ ambient = @context_provider ? (@context_provider.call || {}) : {}
1074
+ ambient.merge(explicit || {})
1075
+ end
1076
+
1077
+ # This thread's kept-alive connection to the gateway (Net::HTTP honours the proxy environment variables).
1078
+ def connection
1079
+ http = Thread.current[@connection_key]
1080
+ return http if http&.started?
1081
+
1082
+ http = Net::HTTP.new(@uri.hostname, @uri.port)
1083
+ http.use_ssl = @uri.scheme == "https"
1084
+ http.open_timeout = 10
1085
+ http.read_timeout = @request_timeout
1086
+ http.write_timeout = @request_timeout
1087
+ http.keep_alive_timeout = 30
1088
+ http.start
1089
+ Thread.current[@connection_key] = http
1090
+ end
1091
+
1092
+ def drop_connection
1093
+ http = Thread.current[@connection_key]
1094
+ Thread.current[@connection_key] = nil
1095
+ http&.finish if http&.started?
1096
+ rescue IOError
1097
+ nil
1098
+ end
1099
+
1100
+ # A workflow id as one percent-encoded URL path segment.
1101
+ def segment(value)
1102
+ URI.encode_www_form_component(value.to_s).gsub("+", "%20")
1103
+ end
1104
+
1105
+ def post(path, body)
1106
+ req = Net::HTTP::Post.new("#{@uri.path}#{path}")
1107
+ req["Content-Type"] = "application/json"
1108
+ req.body = JSON.generate(body)
1109
+ send_request(req)
1110
+ end
1111
+
1112
+ def get(path)
1113
+ send_request(Net::HTTP::Get.new("#{@uri.path}#{path}"))
1114
+ end
1115
+
1116
+ def send_request(req)
1117
+ req["Authorization"] = "Bearer #{@token}"
1118
+ attempts = 0
1119
+ begin
1120
+ reused = !Thread.current[@connection_key].nil?
1121
+ res = connection.request(req)
1122
+ rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, IOError => e
1123
+ drop_connection
1124
+ attempts += 1
1125
+ # The gateway closed the kept-alive connection while it was idle: once more, on a new one.
1126
+ retry if reused && attempts == 1
1127
+ raise e
1128
+ rescue StandardError
1129
+ drop_connection
1130
+ raise
1131
+ end
1132
+ drop_connection if res["connection"].to_s.downcase == "close"
1133
+ raise GatewayError.new(res.code.to_i, res.body) unless res.code.to_i.between?(200, 299)
1134
+
1135
+ res.body.to_s.empty? ? {} : JSON.parse(res.body)
1136
+ end
1137
+ end
1138
+
1139
+ # Every gateway call of a running worker, on threads of its own, so a handler thread never waits for HTTP: a poller
1140
+ # keeps a small buffer of leased tasks (free slots plus `prefetch`), an acker sends the outcomes handlers queue
1141
+ # (POST /sdk/tasks/ack, up to 100 per call after at most `ack_linger_ms`, retried while the gateway is unavailable; a
1142
+ # task stays leased until its outcome is acknowledged; when `ack_buffer` outcomes wait, handlers wait), and a
1143
+ # heartbeater renews every lease the worker holds in one call (POST /sdk/tasks/heartbeat).
1144
+ class Transport
1145
+ attr_reader :ready, :lock, :changed
1146
+ attr_accessor :held
1147
+
1148
+ def initialize(ack_buffer)
1149
+ @ready = Queue.new
1150
+ @acks = SizedQueue.new(ack_buffer)
1151
+ @lock = Mutex.new
1152
+ @changed = ConditionVariable.new
1153
+ @held = 0 # buffered + running
1154
+ @leased = {}
1155
+ @cancelled = {}
1156
+ @beat_wake = Queue.new
1157
+ @ack_done = false
1158
+ @give_up = false
1159
+ @beat_stop = false
1160
+ end
1161
+
1162
+ attr_writer :ack_done, :give_up
1163
+
1164
+ # Ends the heartbeater at once instead of at its next renewal.
1165
+ def stop_heartbeat
1166
+ @beat_stop = true
1167
+ @beat_wake << true
1168
+ end
1169
+
1170
+ def give_up?
1171
+ @give_up
1172
+ end
1173
+
1174
+ def leased(tasks)
1175
+ @lock.synchronize do
1176
+ tasks.each { |t| @leased[t["lease_token"]] = true }
1177
+ @held += tasks.size
1178
+ end
1179
+ tasks.each { |t| @ready << t }
1180
+ end
1181
+
1182
+ def settle(item)
1183
+ loop do
1184
+ return if @acks.push(item, timeout: 0.5)
1185
+ return if @give_up # left to its lease
1186
+ end
1187
+ end
1188
+
1189
+ def cancel_requested?(token)
1190
+ @lock.synchronize { @cancelled[token] == true }
1191
+ end
1192
+
1193
+ def release(token)
1194
+ @lock.synchronize do
1195
+ @leased.delete(token)
1196
+ @cancelled.delete(token)
1197
+ end
1198
+ end
1199
+
1200
+ def ack(client, worker_id, linger)
1201
+ loop do
1202
+ first = @acks.pop(timeout: 0.1)
1203
+ if first.nil?
1204
+ return if @ack_done
1205
+
1206
+ next
1207
+ end
1208
+ batch = [first]
1209
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + linger
1210
+ while batch.size < 100
1211
+ left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
1212
+ more = left.positive? ? @acks.pop(timeout: left) : (@acks.pop(true) rescue nil)
1213
+ break if more.nil?
1214
+
1215
+ batch << more
1216
+ end
1217
+ send_acks(client, worker_id, batch)
1218
+ end
1219
+ end
1220
+
1221
+ def send_acks(client, worker_id, batch)
1222
+ delay = 0.25
1223
+ loop do
1224
+ begin
1225
+ client.ack_tasks(worker_id: worker_id, items: batch)
1226
+ break # a refused item (its lease is gone) is not sent again either
1227
+ rescue GatewayError => e
1228
+ if e.status == 413 && batch.size > 1 # too large for one call: in halves
1229
+ half = batch.size / 2
1230
+ send_acks(client, worker_id, batch[0, half])
1231
+ send_acks(client, worker_id, batch[half..])
1232
+ return
1233
+ end
1234
+ if e.status < 500 && e.status != 429
1235
+ warn "joobq: the gateway refused #{batch.size} outcomes: #{e.message}"
1236
+ break
1237
+ end
1238
+ rescue StandardError
1239
+ nil
1240
+ end
1241
+ break if @give_up # shutting down: the leases bring these tasks back
1242
+
1243
+ sleep delay
1244
+ delay = [delay * 2, 5.0].min
1245
+ end
1246
+ batch.each { |item| release(item[:token]) }
1247
+ end
1248
+
1249
+ def heartbeat(client, worker_id, lease_ttl_ms, &register)
1250
+ every = [lease_ttl_ms / 3000.0, 0.5].max
1251
+ now = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
1252
+ last_register = now.call
1253
+ until @beat_stop
1254
+ @beat_wake.pop(timeout: every)
1255
+ return if @beat_stop
1256
+
1257
+ tokens = @lock.synchronize { @leased.keys }
1258
+ tokens.each_slice(500) do |chunk|
1259
+ res = client.heartbeat_tasks(worker_id: worker_id, leases: chunk.map { |t| { token: t } }, lease_ttl_ms: lease_ttl_ms)
1260
+ @lock.synchronize do
1261
+ Array(res["leases"]).each do |lease|
1262
+ token = lease["lease_token"]
1263
+ if lease["error"]
1264
+ @leased.delete(token) # gone: its outcome will be refused, nothing to renew
1265
+ elsif @leased.key?(token)
1266
+ @cancelled[token] = lease["cancel_requested"] == true
1267
+ end
1268
+ end
1269
+ end
1270
+ rescue StandardError
1271
+ nil # renewed with the next heartbeat
1272
+ end
1273
+ if now.call - last_register > 10
1274
+ register.call
1275
+ last_register = now.call
1276
+ end
1277
+ end
1278
+ end
1279
+ end
1280
+
1281
+ # In-app worker: runs up to `concurrency` jobs and workflow steps at once on threads inside the booted application,
1282
+ # and on SIGTERM/SIGINT (or `stop`) drains the running tasks, sends their outcomes and deregisters. Its gateway calls
1283
+ # run beside the handlers (`Transport`), never in their way.
1284
+ class Worker
1285
+ attr_reader :task_queues
1286
+
1287
+ def initialize(client:, worker_id:, task_queues: [], concurrency: 4, service_name: nil, team: nil,
1288
+ lease_ttl_ms: 30_000, poll_timeout_ms: 5_000, drain_timeout: 30.0, prefetch: nil, ack_buffer: nil,
1289
+ ack_linger_ms: 5)
1290
+ @client = client
1291
+ @worker_id = worker_id
1292
+ @task_queues = Array(task_queues).dup
1293
+ @concurrency = [concurrency.to_i, 1].max
1294
+ @service_name = service_name
1295
+ @team = team
1296
+ @lease_ttl_ms = lease_ttl_ms
1297
+ @poll_timeout_ms = poll_timeout_ms
1298
+ @drain_timeout = drain_timeout
1299
+ # One hung call must not outlast the leases: a poll waits `poll_timeout_ms` at the gateway, and a sixth of a lease
1300
+ # on top keeps a renewal that hangs well inside the lease (10 s with the defaults).
1301
+ bound = (poll_timeout_ms / 1000.0) + (lease_ttl_ms / 6000.0)
1302
+ current = client.respond_to?(:request_timeout) ? client.request_timeout : nil
1303
+ client.request_timeout = bound if current.is_a?(Numeric) && current > bound
1304
+ # Leases held beyond the free slots, so a handler that finishes finds its next task at once.
1305
+ @prefetch = [prefetch.nil? ? [@concurrency / 2, 1].max : prefetch.to_i, 0].max
1306
+ # Outcomes queued for the acker before handlers wait for it.
1307
+ @ack_buffer = [ack_buffer.nil? ? 4 * @concurrency : ack_buffer.to_i, 1].max
1308
+ @ack_linger = [ack_linger_ms.to_f, 0].max / 1000.0
1309
+ @handlers = {}
1310
+ @middlewares = []
1311
+ @stopping = false
1312
+ @workflows = {}
1313
+ @blueprints_sent = {}
1314
+ @transport = nil
1315
+ end
1316
+
1317
+ def use(&block)
1318
+ @middlewares << block
1319
+ self
1320
+ end
1321
+
1322
+ def register_job(job_cls)
1323
+ @task_queues << job_cls.queue if job_cls.respond_to?(:queue) && !@task_queues.include?(job_cls.queue)
1324
+ @handlers[job_cls.job_name] = JoobQ.normalize_callable(job_cls)
1325
+ self
1326
+ end
1327
+
1328
+ def register_workflow(workflow_def)
1329
+ if workflow_def.respond_to?(:task_queue) && !@task_queues.include?(workflow_def.task_queue)
1330
+ @task_queues << workflow_def.task_queue
1331
+ end
1332
+ @workflows[workflow_def.name] = workflow_def
1333
+ # The engine conducts the workflow's runs and sends each step as its own task.
1334
+ @handlers.merge!(workflow_def.activities) if workflow_def.respond_to?(:activities)
1335
+ self
1336
+ end
1337
+
1338
+ def handle_job(job_type, &block)
1339
+ @handlers[job_type] = block
1340
+ self
1341
+ end
1342
+
1343
+ # Says once, on stderr, that the gateway refuses this worker (a wrong or rotated token, no Pro key): it goes on
1344
+ # retrying, and without this line it would sit idle with nothing in its output.
1345
+ def report_refusal(what, error)
1346
+ status = error.respond_to?(:status) ? error.status : nil
1347
+ return unless status.is_a?(Integer) && status >= 400 && status < 500 && ![408, 429].include?(status)
1348
+ return if @refused == status
1349
+
1350
+ @refused = status
1351
+ warn "joobq: the gateway refuses this worker (#{what}): #{error.message}. Retrying; check the SDK token and the server's licence."
1352
+ end
1353
+
1354
+ def process_task(task)
1355
+ token = task["lease_token"]
1356
+ handler = @handlers[task["name"]]
1357
+ unless handler
1358
+ settle(token, { outcome: "fail" }.merge(JoobQ.failure(NonRetryableError.new("No handler registered for #{task['name']}"), retryable: false)))
1359
+ return { "status" => "failed" }
1360
+ end
1361
+
1362
+ # Everything that can raise for this task is inside the begin: a task with no outcome keeps its lease renewed for
1363
+ # ever. Exception, not StandardError, so that a handler raising a ScriptError (NotImplementedError in a stub)
1364
+ # fails its task instead of killing its thread; signals still pass.
1365
+ begin
1366
+ ctx = ExecutionContext.new(task, @worker_id, @client, @lease_ttl_ms, @transport)
1367
+
1368
+ runner = @middlewares.reverse.inject(-> { handler.call(task["payload"], ctx) }) do |next_fn, mw|
1369
+ -> { mw.call(ctx, next_fn) }
1370
+ end
1371
+
1372
+ output = runner.call
1373
+ rescue SignalException
1374
+ raise
1375
+ rescue Exception => e # rubocop:disable Lint/RescueException
1376
+ retryable = !e.is_a?(NonRetryableError)
1377
+ settle(token, { outcome: "fail" }.merge(JoobQ.failure(e, retryable: retryable)))
1378
+ return { "status" => "failed" }
1379
+ end
1380
+ settle(token, { outcome: "complete", output: output })
1381
+ { "status" => "completed", "output" => output }
1382
+ end
1383
+
1384
+ # Stops polling; `start` returns once the running tasks have finished and their outcomes are acknowledged (or
1385
+ # `drain_timeout` passed).
1386
+ def stop
1387
+ @stopping = true
1388
+ end
1389
+
1390
+ # Runs until SIGTERM/SIGINT or `stop`, then drains the running tasks, sends their outcomes and deregisters.
1391
+ def start(install_signal_handlers: true)
1392
+ if install_signal_handlers
1393
+ %w[TERM INT].each { |sig| Signal.trap(sig) { @stopping = true } }
1394
+ end
1395
+ backoff = 0.25
1396
+ until @stopping
1397
+ begin
1398
+ register
1399
+ break
1400
+ rescue StandardError
1401
+ sleep(backoff)
1402
+ backoff = [backoff * 2, 5.0].min
1403
+ end
1404
+ end
1405
+ t = Transport.new(@ack_buffer)
1406
+ @transport = t
1407
+ threads = Array.new(@concurrency) do
1408
+ Thread.new do
1409
+ while (task = t.ready.pop)
1410
+ begin
1411
+ process_task(task)
1412
+ rescue StandardError => e
1413
+ warn "joobq: task #{task['id']} could not be settled: #{e.message}"
1414
+ ensure
1415
+ t.lock.synchronize do
1416
+ t.held -= 1
1417
+ t.changed.broadcast
1418
+ end
1419
+ end
1420
+ end
1421
+ end
1422
+ end
1423
+ acker = Thread.new { t.ack(@client, @worker_id, @ack_linger) }
1424
+ beat = Thread.new do
1425
+ t.heartbeat(@client, @worker_id, @lease_ttl_ms) do
1426
+ register unless @stopping
1427
+ rescue StandardError
1428
+ nil
1429
+ end
1430
+ end
1431
+
1432
+ capacity = @concurrency + @prefetch
1433
+ backoff = 0.25
1434
+ until @stopping
1435
+ room = t.lock.synchronize do
1436
+ t.changed.wait(t.lock, 0.5) while t.held >= capacity && !@stopping
1437
+ capacity - t.held
1438
+ end
1439
+ break if @stopping
1440
+
1441
+ begin
1442
+ tasks = @client.poll_tasks(worker_id: @worker_id, task_queues: @task_queues, max_tasks: [room, 100].min,
1443
+ timeout_ms: @poll_timeout_ms, lease_ttl_ms: @lease_ttl_ms)
1444
+ backoff = 0.25
1445
+ @refused = nil
1446
+ rescue StandardError => e
1447
+ report_refusal("poll for tasks", e)
1448
+ sleep(backoff + rand * 0.1)
1449
+ backoff = [backoff * 2, 5.0].min
1450
+ next
1451
+ end
1452
+ t.leased(tasks)
1453
+ end
1454
+
1455
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @drain_timeout
1456
+ left = -> { deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC) }
1457
+ t.lock.synchronize { t.changed.wait(t.lock, [left.call, 0.05].min) while t.held.positive? && left.call.positive? }
1458
+ threads.size.times { t.ready << nil }
1459
+ t.ack_done = true
1460
+ unless acker.join([left.call, 0.01].max)
1461
+ t.give_up = true
1462
+ acker.join(5)
1463
+ end
1464
+ t.give_up = true
1465
+ t.stop_heartbeat
1466
+ beat.join(5)
1467
+ @transport = nil
1468
+ begin
1469
+ @client.deregister_worker(worker_id: @worker_id)
1470
+ rescue StandardError
1471
+ nil
1472
+ end
1473
+ end
1474
+ alias run start
1475
+
1476
+ private
1477
+
1478
+ # Hands the outcome to the acker (waiting while its queue is full); outside `start` it is sent here.
1479
+ def settle(token, item)
1480
+ item = item.merge(token: token)
1481
+ if @transport
1482
+ @transport.settle(item)
1483
+ else
1484
+ report { @client.ack_tasks(worker_id: @worker_id, items: [item]) }
1485
+ end
1486
+ end
1487
+
1488
+ def register
1489
+ register_blueprints # first: a worker that shows up has its blueprints in
1490
+ @client.register_worker(worker_id: @worker_id, task_queues: @task_queues, concurrency: @concurrency,
1491
+ service_name: @service_name, team: @team)
1492
+ end
1493
+
1494
+ # Registers every workflow's blueprint once; one the gateway refuses (4xx) is logged and not sent again.
1495
+ def register_blueprints
1496
+ @workflows.each do |name, wf|
1497
+ next if @blueprints_sent[name]
1498
+
1499
+ begin
1500
+ @client.register_blueprint(wf, worker_id: @worker_id)
1501
+ rescue GatewayError => e
1502
+ raise if e.status >= 500
1503
+
1504
+ warn "joobq: blueprint of #{name} refused: #{e.message}"
1505
+ end
1506
+ @blueprints_sent[name] = true
1507
+ end
1508
+ end
1509
+
1510
+ # Sends an outcome, retrying while the gateway is unavailable; a lost outcome is left to the lease.
1511
+ def report
1512
+ delay = 0.25
1513
+ 6.times do
1514
+ yield
1515
+ return
1516
+ rescue GatewayError => e
1517
+ return if e.status < 500
1518
+
1519
+ sleep delay
1520
+ delay = [delay * 2, 5.0].min
1521
+ rescue StandardError
1522
+ sleep delay
1523
+ delay = [delay * 2, 5.0].min
1524
+ end
1525
+ end
1526
+ end
1527
+
1528
+ end