lepus-saga 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +28 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +237 -0
- data/LICENSE.txt +21 -0
- data/README.md +337 -0
- data/Rakefile +8 -0
- data/docs/brand/BRAND.md +67 -0
- data/docs/brand/apple-touch-icon.png +0 -0
- data/docs/brand/favicon-16.png +0 -0
- data/docs/brand/favicon-32.png +0 -0
- data/docs/brand/favicon.ico +0 -0
- data/docs/brand/favicon.svg +1 -0
- data/docs/brand/icon-512.png +0 -0
- data/docs/brand/lockup-lepus-mono.svg +1 -0
- data/docs/brand/lockup-lepus-on-dark.svg +1 -0
- data/docs/brand/lockup-lepus-on-light.svg +1 -0
- data/docs/brand/lockup-lepus-saga-mono.svg +1 -0
- data/docs/brand/lockup-lepus-saga-on-dark.svg +1 -0
- data/docs/brand/lockup-lepus-saga-on-light.svg +1 -0
- data/docs/brand/lockup-lepus-saga.svg +1 -0
- data/docs/brand/lockup-lepus.svg +1 -0
- data/docs/brand/mark-mono.svg +1 -0
- data/docs/brand/mark-on-dark.svg +1 -0
- data/docs/brand/mark-on-light.svg +1 -0
- data/docs/brand/mark-small-mono.svg +1 -0
- data/docs/brand/mark-small-on-dark.svg +1 -0
- data/docs/brand/mark-small-on-light.svg +1 -0
- data/docs/brand/mark-small.svg +1 -0
- data/docs/brand/mark.svg +1 -0
- data/lib/lepus/saga/configuration.rb +48 -0
- data/lib/lepus/saga/control.rb +27 -0
- data/lib/lepus/saga/definition.rb +123 -0
- data/lib/lepus/saga/execution_behaviour.rb +58 -0
- data/lib/lepus/saga/executor.rb +282 -0
- data/lib/lepus/saga/executor_consumer.rb +38 -0
- data/lib/lepus/saga/instrumentation.rb +41 -0
- data/lib/lepus/saga/log_subscriber.rb +42 -0
- data/lib/lepus/saga/queue_depth.rb +28 -0
- data/lib/lepus/saga/resume_stuck_job.rb +19 -0
- data/lib/lepus/saga/retry_policy.rb +19 -0
- data/lib/lepus/saga/rspec/engine_examples.rb +635 -0
- data/lib/lepus/saga/rspec/helpers.rb +22 -0
- data/lib/lepus/saga/rspec/query_examples.rb +378 -0
- data/lib/lepus/saga/rspec/store_examples.rb +514 -0
- data/lib/lepus/saga/rspec.rb +6 -0
- data/lib/lepus/saga/step.rb +9 -0
- data/lib/lepus/saga/step_attempt_behaviour.rb +35 -0
- data/lib/lepus/saga/step_producer.rb +37 -0
- data/lib/lepus/saga/stores/base.rb +305 -0
- data/lib/lepus/saga/stores/memory.rb +375 -0
- data/lib/lepus/saga/stores.rb +40 -0
- data/lib/lepus/saga/testing.rb +35 -0
- data/lib/lepus/saga/version.rb +7 -0
- data/lib/lepus/saga/web/api.rb +326 -0
- data/lib/lepus/saga/web/configuration.rb +49 -0
- data/lib/lepus/saga/web/guard.rb +38 -0
- data/lib/lepus/saga/web/serializer.rb +59 -0
- data/lib/lepus/saga/web/static.rb +108 -0
- data/lib/lepus/saga/web.rb +61 -0
- data/lib/lepus/saga.rb +212 -0
- data/lib/lepus-saga.rb +3 -0
- data/mise.toml +2 -0
- data/web/assets/app.css +913 -0
- data/web/assets/apple-touch-icon.png +0 -0
- data/web/assets/favicon-32.png +0 -0
- data/web/assets/favicon.svg +1 -0
- data/web/assets/fonts/overpass/OFL.txt +93 -0
- data/web/assets/fonts/overpass/overpass-latin-wght.woff2 +0 -0
- data/web/assets/js/api.js +50 -0
- data/web/assets/js/dom.js +84 -0
- data/web/assets/js/format.js +79 -0
- data/web/assets/js/live.js +74 -0
- data/web/assets/js/main.js +143 -0
- data/web/assets/js/map.js +639 -0
- data/web/assets/js/range.js +19 -0
- data/web/assets/js/sagas.js +18 -0
- data/web/assets/js/theme.js +8 -0
- data/web/assets/js/throughput.js +262 -0
- data/web/assets/js/views/execution.js +426 -0
- data/web/assets/js/views/executions.js +165 -0
- data/web/assets/js/views/overview.js +465 -0
- data/web/index.html +63 -0
- metadata +323 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Lepus
|
|
4
|
+
module Saga
|
|
5
|
+
module Stores
|
|
6
|
+
# The persistence contract the engine runs on. An adapter subclasses it,
|
|
7
|
+
# implements every method that raises NotImplementedError and returns
|
|
8
|
+
# its own records: an execution record includes
|
|
9
|
+
# Lepus::Saga::ExecutionBehaviour and a step attempt record includes
|
|
10
|
+
# Lepus::Saga::StepAttemptBehaviour. Both need readers and writers for
|
|
11
|
+
# their behaviour's ATTRIBUTES and `[](name)` returning the raw stored
|
|
12
|
+
# value.
|
|
13
|
+
#
|
|
14
|
+
# Concurrency model: an execution is only changed while its row is held
|
|
15
|
+
# by #with_lock or #with_locked_execution (an exclusive lock held until
|
|
16
|
+
# the block returns, like SELECT ... FOR UPDATE inside a transaction).
|
|
17
|
+
# The only writes made without it are #increment_resume_count, which
|
|
18
|
+
# must be atomic in storage, and #save_step_attempt on an attempt the
|
|
19
|
+
# caller opened itself.
|
|
20
|
+
#
|
|
21
|
+
# `require "lepus/saga/rspec"` and `it_behaves_like "a lepus saga store"`
|
|
22
|
+
# verify an adapter against this contract.
|
|
23
|
+
#
|
|
24
|
+
# The query methods after #step_succeeded? are optional: they feed
|
|
25
|
+
# read-only tools such as dashboards and are never called by the engine.
|
|
26
|
+
# An adapter that implements them returns true from #queries? and runs
|
|
27
|
+
# `it_behaves_like "a lepus saga query store"`.
|
|
28
|
+
class Base
|
|
29
|
+
QUERY_MAX_ROWS = 50_000
|
|
30
|
+
MAX_SERIES_BUCKETS = 1_000
|
|
31
|
+
GROUPABLE = %i[saga_name status partition current_step].freeze
|
|
32
|
+
SERIES_KEYS = ["started", *ExecutionBehaviour::TERMINAL_STATUSES].freeze
|
|
33
|
+
LATENCY_STATUSES = %w[succeeded failed].freeze
|
|
34
|
+
PERCENTILES = {p50: 50, p95: 95, p99: 99}.freeze
|
|
35
|
+
|
|
36
|
+
# Inserts an execution and returns its record. `attributes` holds
|
|
37
|
+
# saga_name, resource_type, resource_id, current_step, context,
|
|
38
|
+
# correlation_id, partition, started_at and last_progress_at. The store
|
|
39
|
+
# generates the id (a UUIDv7 string) and defaults status to "running",
|
|
40
|
+
# direction to "execute", epoch and resume_count to 0 and context to {}.
|
|
41
|
+
def create_execution(attributes)
|
|
42
|
+
raise NotImplementedError
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# The execution with this id, or nil. Never locks.
|
|
46
|
+
def find_execution(id)
|
|
47
|
+
raise NotImplementedError
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Persists every attribute of `execution` and returns it.
|
|
51
|
+
def save_execution(execution)
|
|
52
|
+
raise NotImplementedError
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Holds an exclusive lock on the execution's row for the duration of
|
|
56
|
+
# the block, after reloading `execution` in place from storage. Writes
|
|
57
|
+
# made in the block are atomic: they are kept when the block returns
|
|
58
|
+
# (including through `next`) and discarded when it raises. Another
|
|
59
|
+
# caller locking the same execution waits until the block finishes.
|
|
60
|
+
# Returns the block's value.
|
|
61
|
+
def with_lock(execution)
|
|
62
|
+
raise NotImplementedError
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Same guarantees as #with_lock, starting from an id: yields a freshly
|
|
66
|
+
# loaded, locked record, or nil when there is no such execution.
|
|
67
|
+
# Returns the block's value.
|
|
68
|
+
def with_locked_execution(id)
|
|
69
|
+
execution = find_execution(id)
|
|
70
|
+
return yield(nil) unless execution
|
|
71
|
+
|
|
72
|
+
with_lock(execution) { yield execution }
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Adds one to the stored resume_count in a single atomic write (never a
|
|
76
|
+
# read-modify-write of the record, so it cannot clobber or be clobbered
|
|
77
|
+
# by a concurrent locked update) and touches updated_at. Sets the
|
|
78
|
+
# record's resume_count and returns the stored value.
|
|
79
|
+
def increment_resume_count(execution)
|
|
80
|
+
raise NotImplementedError
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Executions in progress (running or compensating) whose
|
|
84
|
+
# last_progress_at is before `before`, oldest last_progress_at first.
|
|
85
|
+
# Returns an Enumerable; the caller stops iterating when done.
|
|
86
|
+
def stalled_executions(before:)
|
|
87
|
+
raise NotImplementedError
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Executions of a resource, most recently created first.
|
|
91
|
+
def executions_for_resource(resource_type, resource_id)
|
|
92
|
+
raise NotImplementedError
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Inserts a step attempt of `execution` and returns its record.
|
|
96
|
+
# `attributes` holds step_name, direction, epoch, attempt and
|
|
97
|
+
# started_at; status defaults to "running" and context_after to {}.
|
|
98
|
+
# (execution, step_name, direction, epoch, attempt) is unique: a second
|
|
99
|
+
# insert raises Lepus::Saga::DuplicateAttemptError, and raising it out
|
|
100
|
+
# of #with_lock discards the block's other writes.
|
|
101
|
+
def create_step_attempt(execution, attributes)
|
|
102
|
+
raise NotImplementedError
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Persists every attribute of `attempt` and returns it.
|
|
106
|
+
def save_step_attempt(attempt)
|
|
107
|
+
raise NotImplementedError
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
# Attempts of an execution ordered by started_at, then created_at.
|
|
111
|
+
def step_attempts(execution)
|
|
112
|
+
raise NotImplementedError
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Number of attempts of `execution` for this step, direction and epoch,
|
|
116
|
+
# whatever their status.
|
|
117
|
+
def count_step_attempts(execution, step_name:, direction:, epoch:)
|
|
118
|
+
raise NotImplementedError
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Whether a "running" attempt for this step, direction and epoch
|
|
122
|
+
# started at or after `started_since`.
|
|
123
|
+
def running_step_attempt?(execution, step_name:, direction:, epoch:, started_since:)
|
|
124
|
+
raise NotImplementedError
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# "running" attempts for this step, direction and epoch that started
|
|
128
|
+
# before `started_before`.
|
|
129
|
+
def stale_step_attempts(execution, step_name:, direction:, epoch:, started_before:)
|
|
130
|
+
raise NotImplementedError
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# Whether the step ever succeeded in the execute direction, in any
|
|
134
|
+
# epoch.
|
|
135
|
+
def step_succeeded?(execution, step_name)
|
|
136
|
+
raise NotImplementedError
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Whether the optional query methods below are implemented.
|
|
140
|
+
def queries? = false
|
|
141
|
+
|
|
142
|
+
# Executions matching the filters, newest first by id (UUIDv7, so by
|
|
143
|
+
# creation time), at most `limit` of them. `before_id` returns the
|
|
144
|
+
# page after the execution with that id. Every filter is optional:
|
|
145
|
+
# `saga_name`, `status` and `current_step` take a value or a list;
|
|
146
|
+
# `partition` matches the partition column, and executions without one
|
|
147
|
+
# whose definition runs in that partition; `resource` is a
|
|
148
|
+
# case-insensitive substring of resource_type or resource_id; `from`
|
|
149
|
+
# (inclusive) and `to` (exclusive) bound created_at, `progress_from`
|
|
150
|
+
# and `progress_to` bound last_progress_at the same way (in-progress
|
|
151
|
+
# executions with `progress_to: Time.now.utc - 15 * 60` are the
|
|
152
|
+
# stalled ones). `error_class` and `error_step` take a value or a list
|
|
153
|
+
# and keep executions with a failed attempt of that error class and at
|
|
154
|
+
# that step (either filter alone matches on its own attribute);
|
|
155
|
+
# `error_from` (inclusive) and `error_to` (exclusive) bound that
|
|
156
|
+
# failed attempt's started_at, the window #failure_summary counts.
|
|
157
|
+
def list_executions(saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
|
|
158
|
+
progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil, before_id: nil, limit: 50)
|
|
159
|
+
raise NotImplementedError
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Number of executions matching the #list_executions filters, as a
|
|
163
|
+
# Hash keyed by the value of `group_by` (one of GROUPABLE), or by an
|
|
164
|
+
# Array of values when `group_by` is an Array. Grouping by partition
|
|
165
|
+
# resolves executions without one to their definition's partition
|
|
166
|
+
# (nil for an unknown saga).
|
|
167
|
+
def count_executions(group_by:, saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
|
|
168
|
+
progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil)
|
|
169
|
+
raise NotImplementedError
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Executions started (by created_at) and finished in each terminal
|
|
173
|
+
# status (by finished_at) per `bucket` seconds between `from` and
|
|
174
|
+
# `to`. Returns `{rows: [[bucket_start, {"started" => n, "completed" =>
|
|
175
|
+
# n, ...}], ...], truncated:}` with every bucket of the window, bucket
|
|
176
|
+
# starts aligned to multiples of `bucket` since the epoch. At most
|
|
177
|
+
# `max_rows` of the newest timestamps of each kind are counted;
|
|
178
|
+
# `truncated` tells whether some were left out.
|
|
179
|
+
def execution_series(from:, to:, bucket:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
|
|
180
|
+
raise NotImplementedError
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Duration percentiles (nearest rank, in milliseconds) of the
|
|
184
|
+
# succeeded and failed attempts started between `from` and `to`, per
|
|
185
|
+
# saga, step and direction. Returns `{rows: [{saga_name:, step_name:,
|
|
186
|
+
# direction:, count:, p50:, p95:, p99:, max:}, ...], truncated:}`
|
|
187
|
+
# sorted by saga, step and direction, computed from at most `max_rows`
|
|
188
|
+
# of the newest attempts.
|
|
189
|
+
def step_latency(from:, to:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
|
|
190
|
+
raise NotImplementedError
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
# Failed attempts started between `from` and `to`, grouped by error
|
|
194
|
+
# class, saga and step, most frequent first (then most recently seen),
|
|
195
|
+
# at most `limit` groups. Returns `[{error_class:, saga_name:,
|
|
196
|
+
# step_name:, count:, execution_count:, last_seen_at:,
|
|
197
|
+
# sample_message:, sample_execution_id:}, ...]` where count is the
|
|
198
|
+
# failed attempts, execution_count the distinct executions they belong
|
|
199
|
+
# to and the sample the group's latest failure.
|
|
200
|
+
def failure_summary(from:, to:, saga_name: nil, limit: 20)
|
|
201
|
+
raise NotImplementedError
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# The distinct saga names in storage, sorted.
|
|
205
|
+
def saga_names
|
|
206
|
+
raise NotImplementedError
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def update_execution(execution, attributes)
|
|
210
|
+
assign(execution, attributes)
|
|
211
|
+
save_execution(execution)
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
def update_step_attempt(attempt, attributes)
|
|
215
|
+
assign(attempt, attributes)
|
|
216
|
+
save_step_attempt(attempt)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def finish_step_attempt(attempt, status, context_after: {}, error: nil)
|
|
220
|
+
update_step_attempt(attempt, attempt.finish_attributes(status, context_after: context_after, error: error))
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
private
|
|
224
|
+
|
|
225
|
+
def assign(record, attributes)
|
|
226
|
+
attributes.each { |name, value| record.public_send(:"#{name}=", value) }
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def filter_values(value)
|
|
230
|
+
values = Array(value).map(&:to_s)
|
|
231
|
+
values unless values.empty?
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def partition_saga_names(partition)
|
|
235
|
+
Lepus::Saga.definitions.filter_map { |name, definition| name if definition.partition == partition.to_s }
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def resolve_partition(partition, saga_name)
|
|
239
|
+
partition || Lepus::Saga.definitions[saga_name]&.partition
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
def group_keys(group_by)
|
|
243
|
+
keys = Array(group_by).map(&:to_sym)
|
|
244
|
+
unknown = keys - GROUPABLE
|
|
245
|
+
raise ArgumentError, "Cannot group executions by #{unknown.join(", ")}" if keys.empty? || unknown.any?
|
|
246
|
+
|
|
247
|
+
keys
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Sums `[attributes, count]` rows into the #count_executions Hash.
|
|
251
|
+
def tally_groups(rows, group_by)
|
|
252
|
+
keys = group_keys(group_by)
|
|
253
|
+
rows.each_with_object(Hash.new(0)) do |(attributes, count), counts|
|
|
254
|
+
values = keys.map do |key|
|
|
255
|
+
(key == :partition) ? resolve_partition(attributes[:partition], attributes[:saga_name]) : attributes[key]
|
|
256
|
+
end
|
|
257
|
+
counts[group_by.is_a?(Array) ? values : values.first] += count
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
# `rows` holds up to max_rows + 1 entries, newest first.
|
|
262
|
+
def take_rows(rows, max_rows)
|
|
263
|
+
[rows.first(max_rows), rows.size > max_rows]
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def series_bucket(from, to, bucket)
|
|
267
|
+
seconds = bucket.to_i
|
|
268
|
+
raise ArgumentError, "bucket must be at least one second" unless seconds.positive?
|
|
269
|
+
if series_size(from, to, seconds) > MAX_SERIES_BUCKETS
|
|
270
|
+
raise ArgumentError, "More than #{MAX_SERIES_BUCKETS} buckets between #{from} and #{to}"
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
seconds
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
def series_size(from, to, seconds)
|
|
277
|
+
((to.to_r - from.to_i.div(seconds) * seconds) / seconds).ceil
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
# Buckets `started` timestamps and `[finished_at, status]` pairs.
|
|
281
|
+
def build_series(from, to, seconds, started, finished)
|
|
282
|
+
first = from.to_i.div(seconds) * seconds
|
|
283
|
+
rows = Array.new(series_size(from, to, seconds)) do |index|
|
|
284
|
+
[Time.at(first + index * seconds).utc, SERIES_KEYS.to_h { |key| [key, 0] }]
|
|
285
|
+
end
|
|
286
|
+
started.each { |time| rows[(time.to_i - first).div(seconds)].last["started"] += 1 }
|
|
287
|
+
finished.each { |time, status| rows[(time.to_i - first).div(seconds)].last[status] += 1 }
|
|
288
|
+
rows
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Percentiles of `[saga_name, step_name, direction, duration_ms]` samples.
|
|
292
|
+
def latency_rows(samples)
|
|
293
|
+
samples.group_by { |sample| sample.first(3) }.sort_by(&:first).map do |(saga_name, step_name, direction), group|
|
|
294
|
+
durations = group.map { |sample| sample.last.to_f }.sort
|
|
295
|
+
{
|
|
296
|
+
saga_name: saga_name, step_name: step_name, direction: direction, count: durations.size,
|
|
297
|
+
**PERCENTILES.transform_values { |rank| durations[(Rational(rank, 100) * durations.size).ceil - 1] },
|
|
298
|
+
max: durations.last
|
|
299
|
+
}
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
end
|
|
304
|
+
end
|
|
305
|
+
end
|
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "monitor"
|
|
5
|
+
|
|
6
|
+
module Lepus
|
|
7
|
+
module Saga
|
|
8
|
+
module Stores
|
|
9
|
+
# Thread-safe in-process store for tests and single-process tools.
|
|
10
|
+
# Records are copies: changing one changes nothing until it is saved, and
|
|
11
|
+
# every read builds a fresh record. Locks are per execution and
|
|
12
|
+
# re-entrant; writes made under a lock are undone when the block raises.
|
|
13
|
+
# Readers see writes as soon as they happen (no isolation between
|
|
14
|
+
# concurrent transactions), so it is not a database.
|
|
15
|
+
class Memory < Base
|
|
16
|
+
class RecordNotFound < StandardError; end
|
|
17
|
+
|
|
18
|
+
ATTEMPT_KEY = %i[saga_execution_id step_name direction epoch attempt].freeze
|
|
19
|
+
JSON_ATTRIBUTES = %i[context context_after].freeze
|
|
20
|
+
|
|
21
|
+
class Record
|
|
22
|
+
class << self
|
|
23
|
+
attr_reader :attribute_names
|
|
24
|
+
|
|
25
|
+
def attributes(*names)
|
|
26
|
+
@attribute_names = names
|
|
27
|
+
include(Module.new { attr_accessor(*names) })
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def initialize(values = {})
|
|
32
|
+
load(values)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def [](name) = instance_variable_get(:"@#{name}")
|
|
36
|
+
|
|
37
|
+
def attributes
|
|
38
|
+
self.class.attribute_names.to_h { |name| [name, self[name]] }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def load(values)
|
|
42
|
+
values.each { |name, value| instance_variable_set(:"@#{name}", value) }
|
|
43
|
+
self
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def ==(other) = other.instance_of?(self.class) && !id.nil? && other.id == id
|
|
47
|
+
|
|
48
|
+
alias_method :eql?, :==
|
|
49
|
+
|
|
50
|
+
def hash = [self.class, id].hash
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
class Execution < Record
|
|
54
|
+
attributes(*ExecutionBehaviour::ATTRIBUTES)
|
|
55
|
+
|
|
56
|
+
include ExecutionBehaviour
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
class StepAttempt < Record
|
|
60
|
+
attributes(*StepAttemptBehaviour::ATTRIBUTES)
|
|
61
|
+
|
|
62
|
+
include StepAttemptBehaviour
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def initialize
|
|
66
|
+
super
|
|
67
|
+
@mutex = Mutex.new
|
|
68
|
+
@transaction_key = :"lepus_saga_memory_transaction_#{object_id}"
|
|
69
|
+
clear!
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def clear!
|
|
73
|
+
@mutex.synchronize do
|
|
74
|
+
@executions = {}
|
|
75
|
+
@step_attempts = {}
|
|
76
|
+
@attempt_ids = {}
|
|
77
|
+
@row_locks = Hash.new { |locks, id| locks[id] = Monitor.new }
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def create_execution(attributes)
|
|
82
|
+
now = Time.now.utc
|
|
83
|
+
row = {
|
|
84
|
+
status: "running", direction: "execute", epoch: 0, resume_count: 0,
|
|
85
|
+
context: {}, last_progress_at: now, **attributes.transform_keys(&:to_sym),
|
|
86
|
+
id: SecureRandom.uuid_v7, created_at: now, updated_at: now
|
|
87
|
+
}
|
|
88
|
+
Execution.new(insert(@executions, row))
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def find_execution(id)
|
|
92
|
+
read(@executions, id)&.then { |row| Execution.new(row) }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def save_execution(execution)
|
|
96
|
+
execution.updated_at = Time.now.utc
|
|
97
|
+
update(@executions, execution.attributes)
|
|
98
|
+
execution
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def with_lock(execution)
|
|
102
|
+
row_lock(execution.id).synchronize do
|
|
103
|
+
transaction do
|
|
104
|
+
execution.load(read(@executions, execution.id) || raise(RecordNotFound, "No execution #{execution.id}"))
|
|
105
|
+
yield execution
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def with_locked_execution(id)
|
|
111
|
+
row_lock(id).synchronize do
|
|
112
|
+
transaction { yield find_execution(id) }
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def increment_resume_count(execution)
|
|
117
|
+
row_lock(execution.id).synchronize do
|
|
118
|
+
execution.resume_count = @mutex.synchronize do
|
|
119
|
+
row = @executions.fetch(execution.id) { raise RecordNotFound, "No execution #{execution.id}" }
|
|
120
|
+
log_undo(@executions, execution.id)
|
|
121
|
+
@executions[execution.id] = row.merge(resume_count: row[:resume_count] + 1, updated_at: Time.now.utc)
|
|
122
|
+
row[:resume_count] + 1
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def stalled_executions(before:)
|
|
128
|
+
where(@executions, Execution) { |row| ExecutionBehaviour::IN_PROGRESS_STATUSES.include?(row[:status]) && row[:last_progress_at] < before }
|
|
129
|
+
.sort_by { |execution| [execution.last_progress_at, execution.id] }
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def executions_for_resource(resource_type, resource_id)
|
|
133
|
+
where(@executions, Execution) { |row| row[:resource_type] == resource_type && row[:resource_id] == resource_id }
|
|
134
|
+
.sort_by { |execution| [execution.created_at, execution.id] }.reverse
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def create_step_attempt(execution, attributes)
|
|
138
|
+
now = Time.now.utc
|
|
139
|
+
row = {
|
|
140
|
+
direction: "execute", epoch: 0, status: "running", context_after: {},
|
|
141
|
+
**attributes.transform_keys(&:to_sym),
|
|
142
|
+
id: SecureRandom.uuid_v7, saga_execution_id: execution.id, created_at: now, updated_at: now
|
|
143
|
+
}
|
|
144
|
+
StepAttempt.new(insert(@step_attempts, row, unique: ATTEMPT_KEY))
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def save_step_attempt(attempt)
|
|
148
|
+
attempt.updated_at = Time.now.utc
|
|
149
|
+
update(@step_attempts, attempt.attributes)
|
|
150
|
+
attempt
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def step_attempts(execution)
|
|
154
|
+
@mutex.synchronize { attempt_rows(execution.id) }.map { |row| StepAttempt.new(copy(row)) }
|
|
155
|
+
.sort_by { |attempt| [attempt.started_at, attempt.created_at, attempt.id] }
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def count_step_attempts(execution, step_name:, direction:, epoch:)
|
|
159
|
+
attempts_for_step(execution, step_name, direction, epoch).size
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def running_step_attempt?(execution, step_name:, direction:, epoch:, started_since:)
|
|
163
|
+
attempts_for_step(execution, step_name, direction, epoch)
|
|
164
|
+
.any? { |attempt| attempt.running? && attempt.started_at >= started_since }
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def stale_step_attempts(execution, step_name:, direction:, epoch:, started_before:)
|
|
168
|
+
attempts_for_step(execution, step_name, direction, epoch)
|
|
169
|
+
.select { |attempt| attempt.running? && attempt.started_at < started_before }
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
def step_succeeded?(execution, step_name)
|
|
173
|
+
step_attempts(execution).any? do |attempt|
|
|
174
|
+
attempt.step_name == step_name && attempt.direction == "execute" && attempt.succeeded?
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def queries? = true
|
|
179
|
+
|
|
180
|
+
def list_executions(before_id: nil, limit: 50, **filters)
|
|
181
|
+
executions = matching_executions(**filters)
|
|
182
|
+
executions = executions.select { |execution| execution.id < before_id } if before_id
|
|
183
|
+
executions.sort_by(&:id).reverse.first(limit)
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def count_executions(group_by:, **filters)
|
|
187
|
+
tally_groups(matching_executions(**filters).map { |execution| [execution.attributes, 1] }, group_by)
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
def execution_series(from:, to:, bucket:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
|
|
191
|
+
seconds = series_bucket(from, to, bucket)
|
|
192
|
+
window = from...to
|
|
193
|
+
executions = matching_executions(saga_name: saga_name)
|
|
194
|
+
started, started_truncated = take_rows(
|
|
195
|
+
executions.map(&:created_at).select { |time| window.cover?(time) }.sort.reverse, max_rows
|
|
196
|
+
)
|
|
197
|
+
finished, finished_truncated = take_rows(
|
|
198
|
+
executions.select { |execution| execution.terminal? && window.cover?(execution.finished_at) }
|
|
199
|
+
.map { |execution| [execution.finished_at, execution.status] }.sort_by(&:first).reverse,
|
|
200
|
+
max_rows
|
|
201
|
+
)
|
|
202
|
+
{rows: build_series(from, to, seconds, started, finished), truncated: started_truncated || finished_truncated}
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def step_latency(from:, to:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
|
|
206
|
+
attempts = attempts_with_saga(from, to, saga_name).select do |attempt, _|
|
|
207
|
+
LATENCY_STATUSES.include?(attempt.status) && attempt.duration_ms
|
|
208
|
+
end
|
|
209
|
+
samples, truncated = take_rows(
|
|
210
|
+
attempts.sort_by { |attempt, _| attempt.started_at }.reverse
|
|
211
|
+
.map { |attempt, saga| [saga, attempt.step_name, attempt.direction, attempt.duration_ms] },
|
|
212
|
+
max_rows
|
|
213
|
+
)
|
|
214
|
+
{rows: latency_rows(samples), truncated: truncated}
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def failure_summary(from:, to:, saga_name: nil, limit: 20)
|
|
218
|
+
attempts_with_saga(from, to, saga_name).select { |attempt, _| attempt.failed? }
|
|
219
|
+
.group_by { |attempt, saga| [attempt.error_class, saga, attempt.step_name] }
|
|
220
|
+
.map { |(error_class, saga, step_name), group| failure_group(error_class, saga, step_name, group.map(&:first)) }
|
|
221
|
+
.sort_by { |group| [-group[:count], -group[:last_seen_at].to_r] }
|
|
222
|
+
.first(limit)
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def saga_names
|
|
226
|
+
@mutex.synchronize { @executions.each_value.map { |row| row[:saga_name] } }.uniq.sort
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
private
|
|
230
|
+
|
|
231
|
+
def matching_executions(saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
|
|
232
|
+
progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil)
|
|
233
|
+
sagas = filter_values(saga_name)
|
|
234
|
+
statuses = filter_values(status)
|
|
235
|
+
steps = filter_values(current_step)
|
|
236
|
+
partition_sagas = partition_saga_names(partition) unless partition.to_s.strip.empty?
|
|
237
|
+
needle = resource.to_s.downcase
|
|
238
|
+
needle = nil if needle.strip.empty?
|
|
239
|
+
failing = failing_execution_ids(filter_values(error_class), filter_values(error_step), error_from, error_to)
|
|
240
|
+
where(@executions, Execution) do |row|
|
|
241
|
+
(sagas.nil? || sagas.include?(row[:saga_name])) &&
|
|
242
|
+
(statuses.nil? || statuses.include?(row[:status])) &&
|
|
243
|
+
(steps.nil? || steps.include?(row[:current_step])) &&
|
|
244
|
+
(from.nil? || row[:created_at] >= from) &&
|
|
245
|
+
(to.nil? || row[:created_at] < to) &&
|
|
246
|
+
(progress_from.nil? || row[:last_progress_at] >= progress_from) &&
|
|
247
|
+
(progress_to.nil? || row[:last_progress_at] < progress_to) &&
|
|
248
|
+
(failing.nil? || failing.include?(row[:id])) &&
|
|
249
|
+
(partition_sagas.nil? || in_partition?(row, partition.to_s, partition_sagas)) &&
|
|
250
|
+
(needle.nil? || row.values_at(:resource_type, :resource_id).any? { |value| value.to_s.downcase.include?(needle) })
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def failing_execution_ids(error_classes, steps, from, to)
|
|
255
|
+
return if [error_classes, steps, from, to].all?(&:nil?)
|
|
256
|
+
|
|
257
|
+
@mutex.synchronize do
|
|
258
|
+
@step_attempts.each_value.filter_map do |row|
|
|
259
|
+
next unless row[:status] == "failed"
|
|
260
|
+
next unless error_classes.nil? || error_classes.include?(row[:error_class])
|
|
261
|
+
next unless steps.nil? || steps.include?(row[:step_name])
|
|
262
|
+
next if from && row[:started_at] < from
|
|
263
|
+
next if to && row[:started_at] >= to
|
|
264
|
+
|
|
265
|
+
row[:saga_execution_id]
|
|
266
|
+
end.to_set
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def in_partition?(row, partition, saga_names)
|
|
271
|
+
row[:partition] ? row[:partition] == partition : saga_names.include?(row[:saga_name])
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# Attempts started in the window as `[attempt, saga_name]` pairs.
|
|
275
|
+
def attempts_with_saga(from, to, saga_name)
|
|
276
|
+
sagas = @mutex.synchronize { @executions.transform_values { |row| row[:saga_name] } }
|
|
277
|
+
wanted = filter_values(saga_name)
|
|
278
|
+
where(@step_attempts, StepAttempt) do |row|
|
|
279
|
+
(from...to).cover?(row[:started_at]) && (wanted.nil? || wanted.include?(sagas[row[:saga_execution_id]]))
|
|
280
|
+
end.map { |attempt| [attempt, sagas[attempt.saga_execution_id]] }
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
def failure_group(error_class, saga_name, step_name, attempts)
|
|
284
|
+
latest = attempts.max_by { |attempt| [attempt.finished_at, attempt.id] }
|
|
285
|
+
{
|
|
286
|
+
error_class: error_class, saga_name: saga_name, step_name: step_name, count: attempts.size,
|
|
287
|
+
execution_count: attempts.map(&:saga_execution_id).uniq.size, last_seen_at: latest.finished_at, sample_message: latest.error_message,
|
|
288
|
+
sample_execution_id: latest.saga_execution_id
|
|
289
|
+
}
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def attempts_for_step(execution, step_name, direction, epoch)
|
|
293
|
+
step_attempts(execution).select do |attempt|
|
|
294
|
+
attempt.step_name == step_name && attempt.direction == direction && attempt.epoch == epoch
|
|
295
|
+
end
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
def row_lock(id)
|
|
299
|
+
@mutex.synchronize { @row_locks[id] }
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
def transaction(&block)
|
|
303
|
+
undo_log ? yield : run_transaction(&block)
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def run_transaction
|
|
307
|
+
log = Thread.current[@transaction_key] = []
|
|
308
|
+
committed = false
|
|
309
|
+
result = yield
|
|
310
|
+
committed = true
|
|
311
|
+
result
|
|
312
|
+
ensure
|
|
313
|
+
Thread.current[@transaction_key] = nil
|
|
314
|
+
rollback(log) unless committed
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
def undo_log = Thread.current[@transaction_key]
|
|
318
|
+
|
|
319
|
+
def log_undo(table, id)
|
|
320
|
+
undo_log&.push([table, id, table[id]])
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def rollback(log)
|
|
324
|
+
@mutex.synchronize do
|
|
325
|
+
log.reverse_each do |table, id, previous|
|
|
326
|
+
previous ? table[id] = previous : table.delete(id)
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
end
|
|
330
|
+
|
|
331
|
+
def read(table, id)
|
|
332
|
+
@mutex.synchronize { table[id] }&.then { |row| copy(row) }
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def where(table, klass, &filter)
|
|
336
|
+
@mutex.synchronize { table.values.select(&filter) }.map { |row| klass.new(copy(row)) }
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
def insert(table, row, unique: nil)
|
|
340
|
+
row = copy(row)
|
|
341
|
+
@mutex.synchronize do
|
|
342
|
+
if unique && attempt_rows(row[:saga_execution_id]).any? { |other| other.values_at(*unique) == row.values_at(*unique) }
|
|
343
|
+
raise DuplicateAttemptError, "Duplicate #{row.slice(*unique).inspect}"
|
|
344
|
+
end
|
|
345
|
+
|
|
346
|
+
log_undo(table, row[:id])
|
|
347
|
+
table[row[:id]] = row
|
|
348
|
+
(@attempt_ids[row[:saga_execution_id]] ||= []) << row[:id] if table.equal?(@step_attempts)
|
|
349
|
+
end
|
|
350
|
+
copy(row)
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
def update(table, row)
|
|
354
|
+
row = copy(row)
|
|
355
|
+
@mutex.synchronize do
|
|
356
|
+
raise RecordNotFound, "No record #{row[:id]}" unless table.key?(row[:id])
|
|
357
|
+
|
|
358
|
+
log_undo(table, row[:id])
|
|
359
|
+
table[row[:id]] = row
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Attempt rows of an execution through the index kept on insert; ids of
|
|
364
|
+
# rows a rollback removed are skipped. Call it holding @mutex.
|
|
365
|
+
def attempt_rows(execution_id)
|
|
366
|
+
@attempt_ids.fetch(execution_id, []).filter_map { |id| @step_attempts[id] }
|
|
367
|
+
end
|
|
368
|
+
|
|
369
|
+
def copy(row)
|
|
370
|
+
row.to_h { |name, value| [name, JSON_ATTRIBUTES.include?(name) ? JSON.parse(JSON.generate(value)) : value] }
|
|
371
|
+
end
|
|
372
|
+
end
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
end
|