pgbus 0.19.4 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +97 -7
- data/Rakefile +6 -1
- data/app/controllers/pgbus/dashboard_controller.rb +2 -0
- data/app/controllers/pgbus/workflow_definitions_controller.rb +28 -0
- data/app/controllers/pgbus/workflows_controller.rb +197 -0
- data/app/frontend/pgbus/modules/charts.js +148 -4
- data/app/frontend/pgbus/modules/charts.min.js +1 -1
- data/app/frontend/pgbus/modules/charts.min.js.map +1 -1
- data/app/frontend/pgbus/style.css +1 -1
- data/app/helpers/pgbus/button_helper.rb +4 -2
- data/app/helpers/pgbus/workflow_insights_helper.rb +129 -0
- data/app/helpers/pgbus/workflows_helper.rb +165 -0
- data/app/models/pgbus/blocked_execution.rb +12 -0
- data/app/models/pgbus/uniqueness_key.rb +88 -3
- data/app/models/pgbus/workflow_entry.rb +172 -0
- data/app/models/pgbus/workflow_event.rb +43 -0
- data/app/models/pgbus/workflow_signal.rb +77 -0
- data/app/models/pgbus/workflow_step.rb +149 -0
- data/app/views/layouts/pgbus/application.html.erb +2 -0
- data/app/views/pgbus/dashboard/_stats_cards.html.erb +17 -0
- data/app/views/pgbus/workflows/_actions.html.erb +59 -0
- data/app/views/pgbus/workflows/_events_table.html.erb +24 -0
- data/app/views/pgbus/workflows/_filters.html.erb +29 -0
- data/app/views/pgbus/workflows/_signal_form.html.erb +31 -0
- data/app/views/pgbus/workflows/_signals_table.html.erb +40 -0
- data/app/views/pgbus/workflows/_status_filter.html.erb +12 -0
- data/app/views/pgbus/workflows/_steps_table.html.erb +79 -0
- data/app/views/pgbus/workflows/_timeline.html.erb +49 -0
- data/app/views/pgbus/workflows/_workflows_table.html.erb +59 -0
- data/app/views/pgbus/workflows/attention.html.erb +88 -0
- data/app/views/pgbus/workflows/definitions/_by_revision.html.erb +52 -0
- data/app/views/pgbus/workflows/definitions/_duration_table.html.erb +18 -0
- data/app/views/pgbus/workflows/definitions/_funnel.html.erb +49 -0
- data/app/views/pgbus/workflows/definitions/_pipeline.html.erb +16 -0
- data/app/views/pgbus/workflows/definitions/_step_card.html.erb +31 -0
- data/app/views/pgbus/workflows/definitions/_throughput_table.html.erb +32 -0
- data/app/views/pgbus/workflows/definitions/_window.html.erb +17 -0
- data/app/views/pgbus/workflows/definitions/index.html.erb +71 -0
- data/app/views/pgbus/workflows/definitions/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/definitions/show.html.erb +80 -0
- data/app/views/pgbus/workflows/index.html.erb +15 -0
- data/app/views/pgbus/workflows/not_found.html.erb +5 -0
- data/app/views/pgbus/workflows/show.html.erb +93 -0
- data/config/i18n-tasks.yml +6 -0
- data/config/locales/da.yml +445 -0
- data/config/locales/de.yml +445 -0
- data/config/locales/en.yml +445 -0
- data/config/locales/es.yml +445 -0
- data/config/locales/fi.yml +445 -0
- data/config/locales/fr.yml +445 -0
- data/config/locales/it.yml +445 -0
- data/config/locales/ja.yml +445 -0
- data/config/locales/nb.yml +445 -0
- data/config/locales/nl.yml +445 -0
- data/config/locales/pt.yml +445 -0
- data/config/locales/sv.yml +445 -0
- data/config/routes.rb +26 -0
- data/lib/generators/pgbus/add_workflow_signals_generator.rb +56 -0
- data/lib/generators/pgbus/add_workflows_generator.rb +54 -0
- data/lib/generators/pgbus/templates/add_workflow_signals.rb.erb +51 -0
- data/lib/generators/pgbus/templates/add_workflows.rb.erb +150 -0
- data/lib/generators/pgbus/templates/migration.rb.erb +127 -0
- data/lib/pgbus/active_job/adapter.rb +81 -3
- data/lib/pgbus/active_job/delivery.rb +63 -0
- data/lib/pgbus/active_job/executor.rb +20 -8
- data/lib/pgbus/concurrency/blocked_execution.rb +14 -6
- data/lib/pgbus/configuration/workflow_settings.rb +99 -0
- data/lib/pgbus/configuration.rb +64 -4
- data/lib/pgbus/dead_letter_header.rb +9 -1
- data/lib/pgbus/generators/migration_detector.rb +40 -0
- data/lib/pgbus/instrumentation.rb +33 -0
- data/lib/pgbus/integrations/appsignal/dashboard.json +157 -1
- data/lib/pgbus/integrations/appsignal/subscriber.rb +16 -1
- data/lib/pgbus/mcp/rack_app.rb +7 -2
- data/lib/pgbus/mcp/runner.rb +2 -1
- data/lib/pgbus/mcp/serialization.rb +25 -0
- data/lib/pgbus/mcp/server.rb +53 -10
- data/lib/pgbus/mcp/tools/deploy_tasks_tool.rb +1 -10
- data/lib/pgbus/mcp/tools/workflow_cancel_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_definitions_tool.rb +36 -0
- data/lib/pgbus/mcp/tools/workflow_detail_tool.rb +38 -0
- data/lib/pgbus/mcp/tools/workflow_resume_at_tool.rb +32 -0
- data/lib/pgbus/mcp/tools/workflow_resume_tool.rb +24 -0
- data/lib/pgbus/mcp/tools/workflow_run_now_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_signal_tool.rb +77 -0
- data/lib/pgbus/mcp/tools/workflow_skip_step_tool.rb +23 -0
- data/lib/pgbus/mcp/tools/workflow_stats_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_attention_tool.rb +42 -0
- data/lib/pgbus/mcp/tools/workflows_tool.rb +60 -0
- data/lib/pgbus/mcp/workflow_insights_payload.rb +15 -0
- data/lib/pgbus/mcp/workflow_payloads.rb +53 -0
- data/lib/pgbus/mcp/write_tool.rb +134 -0
- data/lib/pgbus/mcp.rb +15 -0
- data/lib/pgbus/metrics/backends/statsd.rb +6 -0
- data/lib/pgbus/metrics/subscriber.rb +14 -1
- data/lib/pgbus/metrics/workflow_metrics.rb +75 -0
- data/lib/pgbus/process/consumer.rb +2 -1
- data/lib/pgbus/process/dispatcher.rb +39 -3
- data/lib/pgbus/process/worker.rb +10 -1
- data/lib/pgbus/recurring/schedule.rb +1 -1
- data/lib/pgbus/streams/presence.rb +65 -36
- data/lib/pgbus/uniqueness.rb +23 -6
- data/lib/pgbus/version.rb +1 -1
- data/lib/pgbus/web/data_source/workflow_actions.rb +109 -0
- data/lib/pgbus/web/data_source/workflow_insights.rb +99 -0
- data/lib/pgbus/web/data_source/workflow_list.rb +291 -0
- data/lib/pgbus/web/data_source.rb +126 -15
- data/lib/pgbus/web/workflow_action_plan.rb +62 -0
- data/lib/pgbus/web/workflow_insights_queries.rb +250 -0
- data/lib/pgbus/web/workflow_insights_report.rb +171 -0
- data/lib/pgbus/web/workflow_payload.rb +36 -0
- data/lib/pgbus/web/workflow_query.rb +82 -0
- data/lib/pgbus/web/workflow_state.rb +89 -0
- data/lib/pgbus/web/workflow_timeline.rb +130 -0
- data/lib/pgbus/web/workflow_window.rb +51 -0
- data/lib/pgbus/workflow/batch_finished_job.rb +21 -0
- data/lib/pgbus/workflow/dsl.rb +105 -0
- data/lib/pgbus/workflow/error_capture.rb +63 -0
- data/lib/pgbus/workflow/exception_policy.rb +170 -0
- data/lib/pgbus/workflow/fan_out.rb +157 -0
- data/lib/pgbus/workflow/fan_out_definition.rb +41 -0
- data/lib/pgbus/workflow/flow_control.rb +94 -0
- data/lib/pgbus/workflow/instance.rb +183 -0
- data/lib/pgbus/workflow/outside_verbs.rb +138 -0
- data/lib/pgbus/workflow/payload.rb +53 -0
- data/lib/pgbus/workflow/registry.rb +53 -0
- data/lib/pgbus/workflow/removed_step.rb +14 -0
- data/lib/pgbus/workflow/runner.rb +218 -0
- data/lib/pgbus/workflow/signal_matcher.rb +53 -0
- data/lib/pgbus/workflow/signals.rb +132 -0
- data/lib/pgbus/workflow/states.rb +119 -0
- data/lib/pgbus/workflow/step_context.rb +58 -0
- data/lib/pgbus/workflow/step_definition.rb +133 -0
- data/lib/pgbus/workflow/step_execution.rb +242 -0
- data/lib/pgbus/workflow/step_job.rb +28 -0
- data/lib/pgbus/workflow/step_outcome.rb +151 -0
- data/lib/pgbus/workflow/steps.rb +103 -0
- data/lib/pgbus/workflow/sweep.rb +156 -0
- data/lib/pgbus/workflow/test_helpers.rb +152 -0
- data/lib/pgbus/workflow/wait_timeout.rb +94 -0
- data/lib/pgbus/workflow/waiting.rb +82 -0
- data/lib/pgbus/workflow.rb +183 -0
- data/sig/pgbus/configuration.rbs +23 -0
- metadata +92 -1
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Workflow
|
|
5
|
+
# Run workflow steps inline in a spec, ignoring their wait:.
|
|
6
|
+
#
|
|
7
|
+
# RSpec.configure { |config| config.include Pgbus::Workflow::TestHelpers }
|
|
8
|
+
#
|
|
9
|
+
# wf = SignupWorkflow.start!(user)
|
|
10
|
+
# perform_next_step(wf) # runs :welcome now
|
|
11
|
+
# run_step_inline(wf, :reminder) # runs :reminder, raising if another step is next
|
|
12
|
+
# speedrun_workflow(wf) # runs every step until the run stops being ready
|
|
13
|
+
# expect(wf.status).to eq("finished")
|
|
14
|
+
#
|
|
15
|
+
# deliver_signal(wf, :shipped, payload: { at: Time.current }) # signal!, then the step it wakes
|
|
16
|
+
# complete_fan_out(wf) # every member of the waiting fan_out's batch completes
|
|
17
|
+
# expire_wait(wf) # the waiting step's on_timeout applies now
|
|
18
|
+
#
|
|
19
|
+
# Each step runs through the real StepJob (claim, guards, policies, the
|
|
20
|
+
# audit trail) against the test database; the StepJob messages the
|
|
21
|
+
# model callback sends meanwhile go to whatever adapter the test uses
|
|
22
|
+
# and are not needed.
|
|
23
|
+
module TestHelpers
|
|
24
|
+
class Error < Workflow::Error; end
|
|
25
|
+
|
|
26
|
+
# Runs the run's scheduled step now. Returns the reloaded workflow.
|
|
27
|
+
def perform_next_step(workflow)
|
|
28
|
+
TestHelpers.ensure_ready!(workflow)
|
|
29
|
+
step = TestHelpers.scheduled_step(workflow) || raise(Error, "#{workflow.inspect} has no scheduled step")
|
|
30
|
+
TestHelpers.perform(step, workflow)
|
|
31
|
+
workflow.reload
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Runs the run's scheduled step when it is +step_name+; raises when the
|
|
35
|
+
# run is on another step.
|
|
36
|
+
def run_step_inline(workflow, step_name)
|
|
37
|
+
TestHelpers.ensure_ready!(workflow)
|
|
38
|
+
step = TestHelpers.scheduled_step(workflow)
|
|
39
|
+
unless step&.step_name == step_name.to_s
|
|
40
|
+
raise Error, "#{workflow.inspect} is not about to run #{step_name} (scheduled: #{step&.step_name || "nothing"})"
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
TestHelpers.perform(step, workflow)
|
|
44
|
+
workflow.reload
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Runs scheduled steps one after another until the run is no longer
|
|
48
|
+
# ready: it stops at paused, finished and cancelled, and raises
|
|
49
|
+
# Pgbus::Workflow::StepWaiting on waiting, naming what the step waits
|
|
50
|
+
# for (deliver_signal, complete_fan_out or expire_wait moves it on).
|
|
51
|
+
def speedrun_workflow(workflow, max_steps: 1_000)
|
|
52
|
+
max_steps.times do
|
|
53
|
+
case workflow.reload.status
|
|
54
|
+
when "ready" then perform_next_step(workflow)
|
|
55
|
+
when "waiting" then raise StepWaiting, "#{workflow.inspect} #{TestHelpers.waiting_for(workflow)}"
|
|
56
|
+
when "performing"
|
|
57
|
+
raise Error, "#{workflow.inspect} is performing: speedrun_workflow cannot move it on"
|
|
58
|
+
else return workflow
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
status = workflow.reload.status
|
|
62
|
+
raise StepWaiting, "#{workflow.inspect} #{TestHelpers.waiting_for(workflow)}" if status == "waiting"
|
|
63
|
+
return workflow unless %w[ready performing].include?(status)
|
|
64
|
+
|
|
65
|
+
raise Error, "#{workflow.inspect} was still #{workflow.status} after #{max_steps} steps"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# signal! the run, then run the step it scheduled (the waiting step it
|
|
69
|
+
# woke, or the step that claims a buffered signal). Returns the
|
|
70
|
+
# reloaded workflow.
|
|
71
|
+
# A redelivery (duplicate_delivery?) woke nothing, so it runs nothing.
|
|
72
|
+
def deliver_signal(workflow, name, payload: {}, idempotency_key: nil)
|
|
73
|
+
signal = workflow.signal!(name, payload: payload, idempotency_key: idempotency_key, actor: "test_helpers")
|
|
74
|
+
return workflow.reload if signal.duplicate_delivery?
|
|
75
|
+
|
|
76
|
+
perform_next_step(workflow)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Completes every member of the batch the run's waiting fan_out step
|
|
80
|
+
# waits on (Pgbus::Batch.job_completed, as the executor does), then
|
|
81
|
+
# runs the batch's BatchFinishedJob inline. The members do not run.
|
|
82
|
+
# Returns the reloaded workflow.
|
|
83
|
+
def complete_fan_out(workflow)
|
|
84
|
+
step = TestHelpers.waiting_step(workflow)
|
|
85
|
+
raise Error, "#{workflow.inspect} is not waiting on a fan_out batch" unless step&.batch_id
|
|
86
|
+
|
|
87
|
+
if Pgbus::Batch.executions_migrated?
|
|
88
|
+
Pgbus::BatchExecution.where(batch_id: step.batch_id).order(:id).pluck(:job_id).each do |job_id|
|
|
89
|
+
Pgbus::Batch.job_completed(step.batch_id, job_id: job_id)
|
|
90
|
+
end
|
|
91
|
+
else
|
|
92
|
+
Pgbus::Batch.find(step.batch_id)&.pending_jobs.to_i.times { Pgbus::Batch.job_completed(step.batch_id) }
|
|
93
|
+
end
|
|
94
|
+
BatchFinishedJob.new(step.id).perform_now
|
|
95
|
+
workflow.reload
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Applies the waiting step's on_timeout now, whatever its timeout_at.
|
|
99
|
+
# Returns the reloaded workflow.
|
|
100
|
+
def expire_wait(workflow)
|
|
101
|
+
step = TestHelpers.waiting_step(workflow)
|
|
102
|
+
raise Error, "#{workflow.inspect} has no waiting step with a wait_timeout:" unless step&.timeout_at
|
|
103
|
+
|
|
104
|
+
Runner.locked(workflow.workflow_id) do |record, notes|
|
|
105
|
+
WaitTimeout.fire!(record, WorkflowStep.find(step.id), notes: notes)
|
|
106
|
+
end
|
|
107
|
+
workflow.reload
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# A paused or finished run's scheduled step is not claimable: running
|
|
111
|
+
# it would silently do nothing (and pull its wait forward for resume!).
|
|
112
|
+
def self.ensure_ready!(workflow)
|
|
113
|
+
status = workflow.reload.status
|
|
114
|
+
return if status == "ready"
|
|
115
|
+
raise StepWaiting, "#{workflow.inspect} #{waiting_for(workflow)}" if status == "waiting"
|
|
116
|
+
|
|
117
|
+
hint = case status
|
|
118
|
+
when "paused" then "resume! it first"
|
|
119
|
+
when "finished", "cancelled" then "it is over; rerun! starts a new run"
|
|
120
|
+
when "performing" then "a step is still running"
|
|
121
|
+
else "nothing inline can move it on"
|
|
122
|
+
end
|
|
123
|
+
raise Error, "#{workflow.inspect} is #{status}, not ready: #{hint}"
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def self.waiting_step(workflow)
|
|
127
|
+
WorkflowStep.where(workflow_id: workflow.workflow_id, status: "waiting").order(:id).last
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# "is waiting for signal shipped (deliver_signal …)" / "… batch (complete_fan_out …)".
|
|
131
|
+
def self.waiting_for(workflow)
|
|
132
|
+
step = waiting_step(workflow)
|
|
133
|
+
return "is waiting" unless step
|
|
134
|
+
return "is waiting on step #{step.step_name}'s batch #{step.batch_id}: complete_fan_out moves it on" if step.batch_id
|
|
135
|
+
|
|
136
|
+
"is waiting on step #{step.step_name} for signal #{step.wait_for}: deliver_signal moves it on" \
|
|
137
|
+
"#{", expire_wait times it out" if step.timeout_at}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def self.scheduled_step(workflow)
|
|
141
|
+
WorkflowStep.where(workflow_id: workflow.workflow_id, status: "scheduled").order(:id).last
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# A step still waiting is pulled forward through run_now!, so the audit
|
|
145
|
+
# trail records the skipped wait (actor "test_helpers").
|
|
146
|
+
def self.perform(step, workflow)
|
|
147
|
+
workflow.run_now!(actor: "test_helpers", reason: "run inline by a test helper") if step.scheduled_at > Time.current
|
|
148
|
+
StepJob.new(step.id).perform_now
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Workflow
|
|
5
|
+
# wait_timeout: on a wait_for: step (issue #541). The dispatcher's
|
|
6
|
+
# workflow sweep finds waiting steps of waiting runs whose timeout_at
|
|
7
|
+
# passed and applies their on_timeout under the run's row lock, so a
|
|
8
|
+
# timeout fires within config.workflow_sweep_interval of its deadline:
|
|
9
|
+
#
|
|
10
|
+
# :pause! the step fails with WaitTimedOut and the run pauses
|
|
11
|
+
# :cancel! the run is cancelled
|
|
12
|
+
# :skip! the step is skipped and the run moves on
|
|
13
|
+
# :run the step is scheduled again; its body runs with
|
|
14
|
+
# received_signal nil and timed_out? true
|
|
15
|
+
# :notify only the event and the instrumentation (issue #565): the
|
|
16
|
+
# same attempt keeps waiting and timeout_at moves to its next
|
|
17
|
+
# deadline, one wait_timeout on (see next_deadline)
|
|
18
|
+
#
|
|
19
|
+
# A signal that arrives after the timeout fired stays pending, like any
|
|
20
|
+
# signal no step waits for, except under :notify, whose step still waits
|
|
21
|
+
# for it. A paused run's wait does not time out until it is resumed.
|
|
22
|
+
module WaitTimeout
|
|
23
|
+
module_function
|
|
24
|
+
|
|
25
|
+
# Drains every due wait, Sweep::BATCH_SIZE at a time (at most
|
|
26
|
+
# +max_batches+ of them per sweep, so one sweep cannot run unbounded).
|
|
27
|
+
def expire_due(now: Time.current, limit: Sweep::BATCH_SIZE, max_batches: 20)
|
|
28
|
+
total = 0
|
|
29
|
+
max_batches.times do
|
|
30
|
+
batch = WorkflowStep.wait_expired(now: now).order(:id).limit(limit).pluck(:id, :workflow_id)
|
|
31
|
+
total += expire_batch(batch, now: now)
|
|
32
|
+
break if batch.size < limit
|
|
33
|
+
end
|
|
34
|
+
total
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def expire_batch(batch, now:)
|
|
38
|
+
batch.count do |id, workflow_id|
|
|
39
|
+
Runner.locked(workflow_id) do |record, notes|
|
|
40
|
+
step = record && WorkflowStep.find_by(id: id)
|
|
41
|
+
next false unless record&.status == "waiting" && step&.status == "waiting" && step.timeout_at && step.timeout_at < now
|
|
42
|
+
|
|
43
|
+
fire!(record, step, notes: notes, now: now)
|
|
44
|
+
true
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Applies the step's on_timeout now (the sweep, and expire_wait in
|
|
50
|
+
# TestHelpers). Returns the action applied.
|
|
51
|
+
def fire!(record, step, notes:, now: Time.current)
|
|
52
|
+
definition = record.workflow_klass&.workflow_steps&.fetch(step.step_name)
|
|
53
|
+
action = definition&.on_timeout || :pause!
|
|
54
|
+
Runner.event!(record, "wait_timed_out", actor: Runner::SYSTEM_ACTOR, step: step,
|
|
55
|
+
details: { wait_for: step.wait_for, action: action.to_s,
|
|
56
|
+
timeout_at: step.timeout_at&.utc&.iso8601(3) }.compact)
|
|
57
|
+
notes.instrument("pgbus.workflow.wait_timed_out",
|
|
58
|
+
Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, action: action.to_s))
|
|
59
|
+
apply(record, step, action, notes: notes, now: now, wait_timeout: definition&.wait_timeout)
|
|
60
|
+
action
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def apply(record, step, action, notes:, now:, wait_timeout: nil)
|
|
64
|
+
return notify!(step, wait_timeout, now: now) if action == :notify
|
|
65
|
+
return Waiting.wake!(record, step, now: now, timeout_at: [step.timeout_at, now].compact.min) if action == :run
|
|
66
|
+
|
|
67
|
+
error = action == :pause! ? timed_out_error(step, now) : nil
|
|
68
|
+
Waiting.resolve!(record, step, FlowControl::Outcome.for(action), notes: notes, error: error)
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# The run and the step row stay as they are (no outcome, no StepJob);
|
|
72
|
+
# only timeout_at moves.
|
|
73
|
+
def notify!(step, wait_timeout, now:)
|
|
74
|
+
timeout_at = next_deadline(step.timeout_at, wait_timeout, now)
|
|
75
|
+
step.timeout_at = timeout_at if WorkflowStep.move_deadline!(step.id, timeout_at: timeout_at).positive?
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# The first deadline after +now+ on the wait's wait_timeout grid: one
|
|
79
|
+
# wait_timeout on when the sweep is within a period of the deadline
|
|
80
|
+
# (and for expire_wait ahead of it). A sweep that missed several
|
|
81
|
+
# deadlines (the dispatcher was down) reports them once: a deadline
|
|
82
|
+
# still behind now would be selected again by the next batch.
|
|
83
|
+
def next_deadline(timeout_at, wait_timeout, now)
|
|
84
|
+
missed = timeout_at > now ? 0 : ((now - timeout_at) / wait_timeout).floor
|
|
85
|
+
timeout_at + ((missed + 1) * wait_timeout)
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def timed_out_error(step, now)
|
|
89
|
+
WaitTimedOut.new("#{step.step_name} waited for #{step.wait_for} since #{step.waiting_since&.utc&.iso8601} " \
|
|
90
|
+
"and timed out at #{(step.timeout_at || now).utc.iso8601}")
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pgbus
|
|
4
|
+
class Workflow
|
|
5
|
+
# The waiting state wait_for: steps and fan_out steps share (issue #541),
|
|
6
|
+
# every move under the run's row lock:
|
|
7
|
+
#
|
|
8
|
+
# park! in_progress → waiting the gate found no signal, or the
|
|
9
|
+
# fan_out batch opened (run: park)
|
|
10
|
+
# wake! waiting → scheduled a signal it matches arrived, or the
|
|
11
|
+
# wait timed out with on_timeout: :run
|
|
12
|
+
# (run: wake); the model callback sends
|
|
13
|
+
# the StepJob once the write commits
|
|
14
|
+
# resolve! waiting → (an outcome) the batch finished, the wait timed
|
|
15
|
+
# out, or an operator skipped it: the
|
|
16
|
+
# outcome is applied by StepOutcome as
|
|
17
|
+
# for any finished attempt (run: resolve,
|
|
18
|
+
# then advance, finish, step_pause, ...)
|
|
19
|
+
#
|
|
20
|
+
# A run an outside pause! reached first stays paused (hold): the step row
|
|
21
|
+
# moves, nothing runs, and resume! picks it up.
|
|
22
|
+
module Waiting
|
|
23
|
+
module_function
|
|
24
|
+
|
|
25
|
+
# False when the attempt had already lost its in_progress row (stalled
|
|
26
|
+
# or cancelled while it ran).
|
|
27
|
+
# A fan_out's +output+ (the body's output(...)) waits on the row.
|
|
28
|
+
def park!(record, step, now:, batch_id: nil, wait_for: nil, timeout_at: nil, output: nil) # rubocop:disable Naming/PredicateMethod
|
|
29
|
+
attrs = { waiting_since: now, stall_at: nil, batch_id: batch_id, wait_for: wait_for, timeout_at: timeout_at }
|
|
30
|
+
attrs[:output] = output if output
|
|
31
|
+
return false if WorkflowStep.park!(step.id, **attrs).zero?
|
|
32
|
+
|
|
33
|
+
step.assign_attributes(status: "waiting", **attrs)
|
|
34
|
+
Runner.transition!(record, paused?(record) ? :hold : :park)
|
|
35
|
+
Runner.event!(record, "step_waiting", actor: Runner::SYSTEM_ACTOR, step: step,
|
|
36
|
+
details: { wait_for: wait_for, batch_id: batch_id,
|
|
37
|
+
timeout_at: timeout_at&.utc&.iso8601(3) }.compact)
|
|
38
|
+
true
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# A paused run's step is written without the model callback, so no
|
|
42
|
+
# StepJob goes out that its claim would only drop; resume! sends it.
|
|
43
|
+
def wake!(record, step, now:, **attrs)
|
|
44
|
+
if paused?(record)
|
|
45
|
+
step.update_columns(status: "scheduled", scheduled_at: now, **attrs)
|
|
46
|
+
Runner.transition!(record, :hold)
|
|
47
|
+
else
|
|
48
|
+
step.update!(status: "scheduled", scheduled_at: now, **attrs)
|
|
49
|
+
Runner.transition!(record, :wake)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Returns false when the row was no longer waiting.
|
|
54
|
+
def resolve!(record, step, outcome, notes:, error: nil)
|
|
55
|
+
return false if WorkflowStep.unpark!(step.id).zero?
|
|
56
|
+
|
|
57
|
+
step.assign_attributes(status: "in_progress")
|
|
58
|
+
Runner.transition!(record, :resolve) unless paused?(record)
|
|
59
|
+
output = step.respond_to?(:output) ? step.output : nil
|
|
60
|
+
StepOutcome.apply(record, step, outcome, notes: notes, error: error, output: output)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# start! (and rerun!) of a class with waiting steps needs their schema.
|
|
64
|
+
def check_schema!(klass)
|
|
65
|
+
definitions = klass.workflow_steps.definitions
|
|
66
|
+
fan_out = definitions.any? { |definition| definition.kind == "fan_out" }
|
|
67
|
+
return unless fan_out || definitions.any?(&:waits_for_signal?)
|
|
68
|
+
|
|
69
|
+
unless Sweep.waiting_ready?
|
|
70
|
+
raise WaitingStepsUnmigrated, "#{klass.name} has waiting steps (fan_out or wait_for:): run " \
|
|
71
|
+
"`rails generate pgbus:add_workflow_signals` and migrate first"
|
|
72
|
+
end
|
|
73
|
+
return unless fan_out && !Batch.callback_jobs_migrated?
|
|
74
|
+
|
|
75
|
+
raise BatchCallbacksUnmigrated, "#{klass.name} has a fan_out step, whose batch resumes the run through a " \
|
|
76
|
+
"callback job: run `rails generate pgbus:add_batch_callback_jobs` and migrate first"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def paused?(record) = record.status == "paused"
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "active_support"
|
|
4
|
+
require "active_support/core_ext/class/attribute"
|
|
5
|
+
require "active_support/core_ext/hash/indifferent_access"
|
|
6
|
+
|
|
7
|
+
module Pgbus
|
|
8
|
+
# A durable multi-step workflow (issue #540). Subclass it, declare steps,
|
|
9
|
+
# start a run:
|
|
10
|
+
#
|
|
11
|
+
# class SignupWorkflow < Pgbus::Workflow
|
|
12
|
+
# cancel_if { subject.nil? || subject.closed? }
|
|
13
|
+
# on_exception Net::OpenTimeout, action: :reattempt!, wait: 15.seconds, max_reattempts: 5
|
|
14
|
+
#
|
|
15
|
+
# step :welcome do
|
|
16
|
+
# WelcomeMailer.with(user: subject).welcome.deliver_later
|
|
17
|
+
# end
|
|
18
|
+
#
|
|
19
|
+
# step :reminder, wait: 2.days, skip_if: -> { subject.onboarded? } do
|
|
20
|
+
# ReminderMailer.with(user: subject).remind.deliver_later
|
|
21
|
+
# end
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# wf = SignupWorkflow.start!(user, plan: "pro")
|
|
25
|
+
# wf.pause!(actor: "alice", reason: "SMTP down") / wf.resume! / wf.cancel!
|
|
26
|
+
#
|
|
27
|
+
# Each step attempt is a pgbus_workflow_steps row and one
|
|
28
|
+
# Pgbus::Workflow::StepJob message, sent after the row commits. The step
|
|
29
|
+
# body runs on a fresh instance of the class, with +subject+ and
|
|
30
|
+
# +arguments+. Pgbus::WorkflowEntry is the run's row; this class is the
|
|
31
|
+
# DSL and the handle.
|
|
32
|
+
class Workflow
|
|
33
|
+
class Error < Pgbus::Error; end
|
|
34
|
+
|
|
35
|
+
# A second start! for a subject that already has an ongoing run of this
|
|
36
|
+
# class. #workflow is the run that holds the slot.
|
|
37
|
+
class AlreadyRunning < Error
|
|
38
|
+
attr_reader :workflow
|
|
39
|
+
|
|
40
|
+
def initialize(message = nil, workflow: nil)
|
|
41
|
+
super(message)
|
|
42
|
+
@workflow = workflow
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# A transition the States table does not allow from the workflow's state.
|
|
47
|
+
class InvalidTransition < Error; end
|
|
48
|
+
|
|
49
|
+
# resume!/resume_at! on a workflow that is not paused.
|
|
50
|
+
class NotPaused < InvalidTransition; end
|
|
51
|
+
|
|
52
|
+
# A row names a step the class does not define (and no removed_step
|
|
53
|
+
# gravestone covers it), or resume_at! names an unknown step.
|
|
54
|
+
class StepNotDefined < Error; end
|
|
55
|
+
|
|
56
|
+
# Recorded on a step whose worker died or that outlived its stall_at.
|
|
57
|
+
class StepStalled < Error; end
|
|
58
|
+
|
|
59
|
+
# Arguments or a step output over config.workflow_max_payload_size.
|
|
60
|
+
class PayloadTooLarge < Error; end
|
|
61
|
+
|
|
62
|
+
# A step-only verb or reader (reattempt!, finished!, output,
|
|
63
|
+
# step_idempotency_key, received_signal) called outside a step body.
|
|
64
|
+
class NotInStep < Error; end
|
|
65
|
+
|
|
66
|
+
# signal! on a finished or cancelled run, or by subject when the subject
|
|
67
|
+
# has no ongoing run of the class (missing: :raise).
|
|
68
|
+
class NotOngoing < InvalidTransition; end
|
|
69
|
+
|
|
70
|
+
# Recorded on a fan_out step whose batch had dead-lettered members.
|
|
71
|
+
class FanOutFailed < Error; end
|
|
72
|
+
|
|
73
|
+
# Recorded on a waiting step whose wait_timeout: passed with on_timeout: :pause!.
|
|
74
|
+
class WaitTimedOut < Error; end
|
|
75
|
+
|
|
76
|
+
# start! of a class with a fan_out step before the batch callback-job
|
|
77
|
+
# columns exist (rails generate pgbus:add_batch_callback_jobs).
|
|
78
|
+
class BatchCallbacksUnmigrated < Error; end
|
|
79
|
+
|
|
80
|
+
# start! of a class with a fan_out or wait_for: step before the
|
|
81
|
+
# waiting-step schema exists (rails generate pgbus:add_workflow_signals).
|
|
82
|
+
class WaitingStepsUnmigrated < Error; end
|
|
83
|
+
|
|
84
|
+
# speedrun_workflow reached a step that waits for a signal or a batch.
|
|
85
|
+
class StepWaiting < Error; end
|
|
86
|
+
|
|
87
|
+
extend DSL
|
|
88
|
+
include FlowControl
|
|
89
|
+
include Instance
|
|
90
|
+
|
|
91
|
+
# Class-level declarations: frozen values replaced whole by the DSL, set
|
|
92
|
+
# while the class body loads and only read afterwards.
|
|
93
|
+
# rubocop:disable ThreadSafety/ClassAndModuleAttributes -- written only by the class body at load, each a frozen value replaced whole
|
|
94
|
+
class_attribute :_workflow_steps, instance_accessor: false, instance_predicate: false, default: Steps::EMPTY
|
|
95
|
+
class_attribute :_cancel_conditions, instance_accessor: false, instance_predicate: false, default: [].freeze
|
|
96
|
+
class_attribute :_exception_policies, instance_accessor: false, instance_predicate: false, default: [].freeze
|
|
97
|
+
class_attribute :_step_job_options, instance_accessor: false, instance_predicate: false, default: {}.freeze
|
|
98
|
+
# rubocop:enable ThreadSafety/ClassAndModuleAttributes
|
|
99
|
+
|
|
100
|
+
class << self
|
|
101
|
+
def inherited(subclass)
|
|
102
|
+
super
|
|
103
|
+
Registry.register(subclass)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Every subclass that has a name, once (what the insights page lists).
|
|
107
|
+
def registered_classes = Registry.classes
|
|
108
|
+
|
|
109
|
+
# Start a run: the workflow row and its first step row, in one
|
|
110
|
+
# transaction; the first StepJob is sent once that commits.
|
|
111
|
+
#
|
|
112
|
+
# SignupWorkflow.start!(user, plan: "pro") # arguments["plan"]
|
|
113
|
+
# CheckoutWorkflow.start!(order, idempotency_key: "checkout-#{order.id}")
|
|
114
|
+
#
|
|
115
|
+
# +idempotency_key+ is reserved: with one, a second start! with the same
|
|
116
|
+
# key returns the first run (any status) with duplicate_start? true.
|
|
117
|
+
# Raises AlreadyRunning when the subject already has an ongoing run.
|
|
118
|
+
def start!(subject = nil, idempotency_key: nil, **arguments)
|
|
119
|
+
Runner.start(self, subject: subject, arguments: arguments, idempotency_key: idempotency_key)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The run with this workflow_id, as an instance of the class that
|
|
123
|
+
# started it; nil when there is none, or when that class is not this
|
|
124
|
+
# one or a subclass of it. Pgbus::Workflow.find finds any run (as a
|
|
125
|
+
# bare Pgbus::Workflow when its class no longer exists, so it can still
|
|
126
|
+
# be cancelled).
|
|
127
|
+
def find(workflow_id)
|
|
128
|
+
record = WorkflowEntry.find_by(workflow_id: workflow_id)
|
|
129
|
+
return nil unless record
|
|
130
|
+
|
|
131
|
+
klass = record.workflow_klass || Workflow
|
|
132
|
+
klass <= self ? klass.new(record) : nil
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Signal the ongoing run of this class for +subject+ (a webhook that
|
|
136
|
+
# knows the order, not the run):
|
|
137
|
+
#
|
|
138
|
+
# OrderWorkflow.signal!(order, :payment_confirmed, payload: { amount: 42 }, idempotency_key: event.id,
|
|
139
|
+
# missing: :ignore)
|
|
140
|
+
#
|
|
141
|
+
# The one-ongoing-per-subject index means at most one run matches. With
|
|
142
|
+
# none (never started, or already over), missing: :raise raises
|
|
143
|
+
# NotOngoing and missing: :ignore returns nil. A subject-less run is
|
|
144
|
+
# signalled through its handle: find(workflow_id).signal!.
|
|
145
|
+
def signal!(subject, name, payload: {}, idempotency_key: nil, missing: :raise, actor: nil, reason: nil)
|
|
146
|
+
raise ArgumentError, "missing: must be :raise or :ignore, got #{missing.inspect}" unless %i[raise ignore].include?(missing)
|
|
147
|
+
if subject.nil?
|
|
148
|
+
raise ArgumentError, "#{self.name}.signal! needs a subject; signal a subject-less run through its handle " \
|
|
149
|
+
"(#{self.name}.find(workflow_id).signal!)"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
attrs = Payload.subject_attributes(subject)
|
|
153
|
+
record = WorkflowEntry.ongoing.find_by(workflow_class: self.name, subject_type: attrs[:subject_type],
|
|
154
|
+
subject_id: attrs[:subject_id])
|
|
155
|
+
raise NotOngoing, "#{self.name} has no ongoing run for #{attrs[:subject_type]} #{attrs[:subject_id]}" unless record
|
|
156
|
+
|
|
157
|
+
new(record).signal!(name, payload: payload, idempotency_key: idempotency_key, actor: actor, reason: reason)
|
|
158
|
+
rescue NotOngoing
|
|
159
|
+
raise if missing == :raise
|
|
160
|
+
|
|
161
|
+
nil
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# How many ongoing runs still sit on each removed step:
|
|
165
|
+
# { "reminder" => 3 }. A gravestone can go once its count is zero.
|
|
166
|
+
def removed_steps_in_flight
|
|
167
|
+
names = _workflow_steps.removed_names
|
|
168
|
+
return {} if names.empty?
|
|
169
|
+
|
|
170
|
+
counts = WorkflowEntry.ongoing.where(workflow_class: name, current_step: names).group(:current_step).count
|
|
171
|
+
names.to_h { |step_name| [step_name, counts.fetch(step_name, 0)] }
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
attr_reader :record
|
|
176
|
+
|
|
177
|
+
def initialize(record, duplicate_start: false)
|
|
178
|
+
@record = record
|
|
179
|
+
@duplicate_start = duplicate_start
|
|
180
|
+
@step_context = nil
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
data/sig/pgbus/configuration.rbs
CHANGED
|
@@ -104,6 +104,8 @@ module Pgbus
|
|
|
104
104
|
def max_retries=: (untyped) -> untyped
|
|
105
105
|
def max_worker_lifetime: () -> untyped
|
|
106
106
|
def max_worker_lifetime=: (untyped) -> untyped
|
|
107
|
+
def mcp_write_tools: () -> bool
|
|
108
|
+
def mcp_write_tools=: (untyped) -> untyped
|
|
107
109
|
def metrics_backend: () -> untyped
|
|
108
110
|
def metrics_backend=: (untyped) -> untyped
|
|
109
111
|
def metrics_enabled: () -> bool
|
|
@@ -291,7 +293,10 @@ module Pgbus
|
|
|
291
293
|
def visibility_heartbeat_interval=: (untyped value) -> untyped
|
|
292
294
|
def visibility_timeout: () -> Numeric?
|
|
293
295
|
def visibility_timeout=: (untyped value) -> untyped
|
|
296
|
+
def warn_queue_prefix_underscores: () -> untyped
|
|
294
297
|
def warn_read_ahead_without_heartbeat: (untyped settings) -> untyped
|
|
298
|
+
def web_actor: () -> untyped
|
|
299
|
+
def web_actor=: (untyped) -> untyped
|
|
295
300
|
def web_auth: () -> untyped
|
|
296
301
|
def web_auth=: (untyped) -> untyped
|
|
297
302
|
def web_data_source: () -> untyped
|
|
@@ -320,6 +325,24 @@ module Pgbus
|
|
|
320
325
|
def worker_notify_wakeup?: () -> bool
|
|
321
326
|
def workers: () -> Array[untyped]
|
|
322
327
|
def workers=: (untyped value) -> untyped
|
|
328
|
+
def workflow_error_backtrace_lines: () -> Integer
|
|
329
|
+
def workflow_error_backtrace_lines=: (untyped) -> untyped
|
|
330
|
+
def workflow_max_payload_size: () -> Integer
|
|
331
|
+
def workflow_max_payload_size=: (untyped) -> untyped
|
|
332
|
+
def workflow_queue: () -> String
|
|
333
|
+
def workflow_queue=: (untyped value) -> untyped
|
|
334
|
+
def workflow_retention: () -> Numeric?
|
|
335
|
+
def workflow_retention=: (untyped value) -> untyped
|
|
336
|
+
def workflow_stall_action: () -> Symbol
|
|
337
|
+
def workflow_stall_action=: (untyped value) -> untyped
|
|
338
|
+
def workflow_stall_threshold: () -> Numeric?
|
|
339
|
+
def workflow_stall_threshold=: (untyped value) -> untyped
|
|
340
|
+
def workflow_subject_url: () -> untyped
|
|
341
|
+
def workflow_subject_url=: (untyped) -> untyped
|
|
342
|
+
def workflow_sweep_interval: () -> Numeric?
|
|
343
|
+
def workflow_sweep_interval=: (untyped value) -> untyped
|
|
344
|
+
def workflow_waiting_visibility_threshold: () -> Numeric?
|
|
345
|
+
def workflow_waiting_visibility_threshold=: (untyped value) -> untyped
|
|
323
346
|
def zombie_detection: () -> bool
|
|
324
347
|
def zombie_detection=: (untyped) -> untyped
|
|
325
348
|
end
|