solidflow 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +70 -0
- data/Rakefile +10 -0
- data/app/jobs/solidflow/jobs/run_execution_job.rb +21 -0
- data/app/jobs/solidflow/jobs/run_task_job.rb +107 -0
- data/app/jobs/solidflow/jobs/timer_sweep_job.rb +32 -0
- data/app/models/solidflow/application_record.rb +7 -0
- data/app/models/solidflow/event.rb +23 -0
- data/app/models/solidflow/execution.rb +36 -0
- data/app/models/solidflow/signal_message.rb +16 -0
- data/app/models/solidflow/timer.rb +19 -0
- data/bin/solidflow +7 -0
- data/db/migrate/001_create_solidflow_core_tables.rb +71 -0
- data/lib/solid_flow/cli.rb +68 -0
- data/lib/solid_flow/determinism.rb +39 -0
- data/lib/solid_flow/engine.rb +23 -0
- data/lib/solid_flow/errors.rb +52 -0
- data/lib/solid_flow/idempotency.rb +34 -0
- data/lib/solid_flow/instrumentation.rb +16 -0
- data/lib/solid_flow/registries/task_registry.rb +32 -0
- data/lib/solid_flow/registries/workflow_registry.rb +33 -0
- data/lib/solid_flow/replay.rb +258 -0
- data/lib/solid_flow/runner.rb +468 -0
- data/lib/solid_flow/serializers/oj.rb +24 -0
- data/lib/solid_flow/signals.rb +47 -0
- data/lib/solid_flow/stores/active_record.rb +461 -0
- data/lib/solid_flow/stores/base.rb +87 -0
- data/lib/solid_flow/task.rb +152 -0
- data/lib/solid_flow/testing.rb +28 -0
- data/lib/solid_flow/version.rb +5 -0
- data/lib/solid_flow/wait.rb +76 -0
- data/lib/solid_flow/workflow.rb +263 -0
- data/lib/solid_flow.rb +76 -0
- data/lib/solidflow.rb +3 -0
- metadata +213 -0
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidFlow
|
|
4
|
+
module Registries
|
|
5
|
+
class TaskRegistry
|
|
6
|
+
def initialize
|
|
7
|
+
@mutex = Mutex.new
|
|
8
|
+
@tasks = {}
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def register(name, klass)
|
|
12
|
+
@mutex.synchronize do
|
|
13
|
+
@tasks[name.to_sym] = klass
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def fetch(name)
|
|
18
|
+
@tasks.fetch(name.to_sym)
|
|
19
|
+
rescue KeyError
|
|
20
|
+
raise Errors::ConfigurationError, "Task #{name} is not registered"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def resolve(klass)
|
|
24
|
+
@tasks.key(klass)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def to_h
|
|
28
|
+
@tasks.dup
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidFlow
|
|
4
|
+
module Registries
|
|
5
|
+
class WorkflowRegistry
|
|
6
|
+
def initialize
|
|
7
|
+
@mutex = Mutex.new
|
|
8
|
+
@workflows = {}
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def register(name, klass)
|
|
12
|
+
@mutex.synchronize do
|
|
13
|
+
@workflows.delete_if { |_key, value| value == klass }
|
|
14
|
+
@workflows[name.to_s] = klass
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def fetch(name)
|
|
19
|
+
@workflows.fetch(name.to_s)
|
|
20
|
+
rescue KeyError
|
|
21
|
+
raise Errors::ConfigurationError, "Workflow #{name} is not registered"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def resolve(klass)
|
|
25
|
+
@workflows.key(klass)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def to_h
|
|
29
|
+
@workflows.dup
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module SolidFlow
|
|
4
|
+
class Replay
|
|
5
|
+
Event = Struct.new(:id, :type, :sequence, :payload, :recorded_at, keyword_init: true)
|
|
6
|
+
|
|
7
|
+
StepState = Struct.new(
|
|
8
|
+
:name,
|
|
9
|
+
:status,
|
|
10
|
+
:attempt,
|
|
11
|
+
:last_result,
|
|
12
|
+
:last_error,
|
|
13
|
+
:idempotency_key,
|
|
14
|
+
:wait_instructions,
|
|
15
|
+
keyword_init: true
|
|
16
|
+
) do
|
|
17
|
+
def waiting?
|
|
18
|
+
status == :waiting
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def completed?
|
|
22
|
+
status == :completed
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def failed?
|
|
26
|
+
status == :failed
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
TaskState = Struct.new(
|
|
31
|
+
:step,
|
|
32
|
+
:status,
|
|
33
|
+
:attempt,
|
|
34
|
+
:idempotency_key,
|
|
35
|
+
:last_result,
|
|
36
|
+
:last_error,
|
|
37
|
+
keyword_init: true
|
|
38
|
+
) do
|
|
39
|
+
def finished?
|
|
40
|
+
%i[completed failed].include?(status)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
TimerState = Struct.new(
|
|
45
|
+
:id,
|
|
46
|
+
:step,
|
|
47
|
+
:run_at,
|
|
48
|
+
:status,
|
|
49
|
+
:metadata,
|
|
50
|
+
keyword_init: true
|
|
51
|
+
) do
|
|
52
|
+
def pending?
|
|
53
|
+
status == :scheduled
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
AwaitingState = Struct.new(
|
|
58
|
+
:step,
|
|
59
|
+
:instructions,
|
|
60
|
+
keyword_init: true
|
|
61
|
+
) do
|
|
62
|
+
def timer_instructions
|
|
63
|
+
instructions.select { |inst| inst[:type].to_sym == :timer }
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def signal_instructions
|
|
67
|
+
instructions.select { |inst| inst[:type].to_sym == :signal }
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
ExecutionState = Struct.new(
|
|
72
|
+
:id,
|
|
73
|
+
:workflow,
|
|
74
|
+
:state,
|
|
75
|
+
:cursor_step,
|
|
76
|
+
:cursor_index,
|
|
77
|
+
:graph_signature,
|
|
78
|
+
:metadata,
|
|
79
|
+
keyword_init: true
|
|
80
|
+
) do
|
|
81
|
+
def finished?
|
|
82
|
+
%w[completed failed cancelled].include?(state)
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
State = Struct.new(
|
|
87
|
+
:execution_state,
|
|
88
|
+
:ctx,
|
|
89
|
+
:history,
|
|
90
|
+
:step_states,
|
|
91
|
+
:awaiting,
|
|
92
|
+
:task_states,
|
|
93
|
+
:timer_states,
|
|
94
|
+
:signal_buffer,
|
|
95
|
+
keyword_init: true
|
|
96
|
+
) do
|
|
97
|
+
def finished?
|
|
98
|
+
execution_state.finished?
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def current_step
|
|
102
|
+
execution_state.cursor_step
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def current_step_state
|
|
106
|
+
step_states[execution_state.cursor_step&.to_sym]
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
attr_reader :workflow_class, :events, :execution_record
|
|
111
|
+
|
|
112
|
+
def initialize(workflow_class:, events:, execution_record:)
|
|
113
|
+
@workflow_class = workflow_class
|
|
114
|
+
@events = Array(events)
|
|
115
|
+
@execution_record = execution_record
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def call
|
|
119
|
+
execution_state = ExecutionState.new(
|
|
120
|
+
id: execution_record.fetch(:id),
|
|
121
|
+
workflow: execution_record.fetch(:workflow),
|
|
122
|
+
state: execution_record.fetch(:state),
|
|
123
|
+
cursor_step: execution_record[:cursor_step]&.to_sym,
|
|
124
|
+
cursor_index: execution_record[:cursor_index] || 0,
|
|
125
|
+
graph_signature: execution_record[:graph_signature],
|
|
126
|
+
metadata: execution_record[:metadata] || {}
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
ctx = (execution_record[:ctx] || {}).with_indifferent_access
|
|
130
|
+
|
|
131
|
+
step_states = {}
|
|
132
|
+
workflow_class.steps.each do |step|
|
|
133
|
+
step_states[step.name] = StepState.new(
|
|
134
|
+
name: step.name,
|
|
135
|
+
status: :idle,
|
|
136
|
+
attempt: 0,
|
|
137
|
+
last_result: nil,
|
|
138
|
+
last_error: nil,
|
|
139
|
+
idempotency_key: nil,
|
|
140
|
+
wait_instructions: []
|
|
141
|
+
)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
task_states = {}
|
|
145
|
+
timer_states = {}
|
|
146
|
+
signal_buffer = Signals::Buffer.new
|
|
147
|
+
awaiting = nil
|
|
148
|
+
|
|
149
|
+
events.each do |event|
|
|
150
|
+
type = event.type.to_sym
|
|
151
|
+
raw_payload = event.payload || {}
|
|
152
|
+
payload = raw_payload.is_a?(Hash) ? raw_payload.deep_symbolize_keys : raw_payload
|
|
153
|
+
|
|
154
|
+
case type
|
|
155
|
+
when :workflow_started
|
|
156
|
+
ctx.merge!(payload[:input] || {})
|
|
157
|
+
when :step_waiting
|
|
158
|
+
step = payload.fetch(:step).to_sym
|
|
159
|
+
instructions = Array(payload[:instructions]).map do |inst|
|
|
160
|
+
inst.deep_symbolize_keys
|
|
161
|
+
end
|
|
162
|
+
awaiting = AwaitingState.new(step:, instructions:)
|
|
163
|
+
if step_states[step]
|
|
164
|
+
step_states[step].status = :waiting
|
|
165
|
+
step_states[step].wait_instructions = instructions
|
|
166
|
+
end
|
|
167
|
+
when :step_completed
|
|
168
|
+
step = payload.fetch(:step).to_sym
|
|
169
|
+
step_state = step_states[step]
|
|
170
|
+
if step_state
|
|
171
|
+
step_state.status = :completed
|
|
172
|
+
step_state.wait_instructions = []
|
|
173
|
+
step_state.last_result = payload[:result]
|
|
174
|
+
step_state.last_error = nil
|
|
175
|
+
step_state.attempt = payload[:attempt] if payload.key?(:attempt)
|
|
176
|
+
step_state.idempotency_key = payload[:idempotency_key]
|
|
177
|
+
end
|
|
178
|
+
ctx.merge!(payload[:ctx_snapshot] || {})
|
|
179
|
+
awaiting = nil if awaiting&.step == step
|
|
180
|
+
when :task_scheduled
|
|
181
|
+
step = payload.fetch(:step).to_sym
|
|
182
|
+
task_states[step] = TaskState.new(
|
|
183
|
+
step:,
|
|
184
|
+
status: :scheduled,
|
|
185
|
+
attempt: payload[:attempt] || 1,
|
|
186
|
+
idempotency_key: payload[:idempotency_key],
|
|
187
|
+
last_result: nil,
|
|
188
|
+
last_error: nil
|
|
189
|
+
)
|
|
190
|
+
if step_states[step]
|
|
191
|
+
step_states[step].status = :task_scheduled
|
|
192
|
+
step_states[step].attempt = payload[:attempt] || 1
|
|
193
|
+
step_states[step].idempotency_key = payload[:idempotency_key]
|
|
194
|
+
end
|
|
195
|
+
when :task_completed
|
|
196
|
+
step = payload.fetch(:step).to_sym
|
|
197
|
+
task_state = task_states[step] ||= TaskState.new(step:, status: :scheduled, attempt: 0, idempotency_key: nil, last_result: nil, last_error: nil)
|
|
198
|
+
task_state.status = :completed
|
|
199
|
+
task_state.attempt = payload[:attempt] || task_state.attempt
|
|
200
|
+
task_state.last_result = payload[:result]
|
|
201
|
+
task_state.last_error = nil
|
|
202
|
+
if step_states[step]
|
|
203
|
+
step_states[step].status = :completed
|
|
204
|
+
step_states[step].last_result = payload[:result]
|
|
205
|
+
step_states[step].last_error = nil
|
|
206
|
+
step_states[step].attempt = task_state.attempt
|
|
207
|
+
end
|
|
208
|
+
ctx.merge!(payload[:ctx_snapshot] || {})
|
|
209
|
+
when :task_failed
|
|
210
|
+
step = payload.fetch(:step).to_sym
|
|
211
|
+
task_state = task_states[step] ||= TaskState.new(step:, status: :scheduled, attempt: 0, idempotency_key: nil, last_result: nil, last_error: nil)
|
|
212
|
+
task_state.status = :failed
|
|
213
|
+
task_state.attempt = payload[:attempt] || task_state.attempt
|
|
214
|
+
task_state.last_error = payload[:error]
|
|
215
|
+
if step_states[step]
|
|
216
|
+
step_states[step].status = :failed
|
|
217
|
+
step_states[step].last_error = payload[:error]
|
|
218
|
+
step_states[step].attempt = task_state.attempt
|
|
219
|
+
end
|
|
220
|
+
when :timer_scheduled
|
|
221
|
+
timer_states[payload.fetch(:timer_id)] = TimerState.new(
|
|
222
|
+
id: payload.fetch(:timer_id),
|
|
223
|
+
step: payload[:step]&.to_sym,
|
|
224
|
+
run_at: payload[:run_at],
|
|
225
|
+
status: :scheduled,
|
|
226
|
+
metadata: payload[:metadata]
|
|
227
|
+
)
|
|
228
|
+
when :timer_fired
|
|
229
|
+
timer = timer_states[payload.fetch(:timer_id)]
|
|
230
|
+
timer.status = :fired if timer
|
|
231
|
+
when :signal_received
|
|
232
|
+
signal_buffer.push(Signals::Message.new(
|
|
233
|
+
name: payload.fetch(:signal).to_sym,
|
|
234
|
+
payload: payload[:payload],
|
|
235
|
+
metadata: payload[:metadata],
|
|
236
|
+
received_at: payload[:received_at],
|
|
237
|
+
consumed: false
|
|
238
|
+
))
|
|
239
|
+
when :signal_consumed
|
|
240
|
+
signal_buffer.consume(payload.fetch(:signal))
|
|
241
|
+
when :workflow_completed, :workflow_failed, :workflow_cancelled
|
|
242
|
+
execution_state.state = type.to_s.sub("workflow_", "")
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
State.new(
|
|
247
|
+
execution_state:,
|
|
248
|
+
ctx:,
|
|
249
|
+
history: events,
|
|
250
|
+
step_states:,
|
|
251
|
+
awaiting:,
|
|
252
|
+
task_states:,
|
|
253
|
+
timer_states:,
|
|
254
|
+
signal_buffer:
|
|
255
|
+
)
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
end
|