lepus-saga 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +28 -0
  3. data/Gemfile +5 -0
  4. data/Gemfile.lock +237 -0
  5. data/LICENSE.txt +21 -0
  6. data/README.md +337 -0
  7. data/Rakefile +8 -0
  8. data/docs/brand/BRAND.md +67 -0
  9. data/docs/brand/apple-touch-icon.png +0 -0
  10. data/docs/brand/favicon-16.png +0 -0
  11. data/docs/brand/favicon-32.png +0 -0
  12. data/docs/brand/favicon.ico +0 -0
  13. data/docs/brand/favicon.svg +1 -0
  14. data/docs/brand/icon-512.png +0 -0
  15. data/docs/brand/lockup-lepus-mono.svg +1 -0
  16. data/docs/brand/lockup-lepus-on-dark.svg +1 -0
  17. data/docs/brand/lockup-lepus-on-light.svg +1 -0
  18. data/docs/brand/lockup-lepus-saga-mono.svg +1 -0
  19. data/docs/brand/lockup-lepus-saga-on-dark.svg +1 -0
  20. data/docs/brand/lockup-lepus-saga-on-light.svg +1 -0
  21. data/docs/brand/lockup-lepus-saga.svg +1 -0
  22. data/docs/brand/lockup-lepus.svg +1 -0
  23. data/docs/brand/mark-mono.svg +1 -0
  24. data/docs/brand/mark-on-dark.svg +1 -0
  25. data/docs/brand/mark-on-light.svg +1 -0
  26. data/docs/brand/mark-small-mono.svg +1 -0
  27. data/docs/brand/mark-small-on-dark.svg +1 -0
  28. data/docs/brand/mark-small-on-light.svg +1 -0
  29. data/docs/brand/mark-small.svg +1 -0
  30. data/docs/brand/mark.svg +1 -0
  31. data/lib/lepus/saga/configuration.rb +48 -0
  32. data/lib/lepus/saga/control.rb +27 -0
  33. data/lib/lepus/saga/definition.rb +123 -0
  34. data/lib/lepus/saga/execution_behaviour.rb +58 -0
  35. data/lib/lepus/saga/executor.rb +282 -0
  36. data/lib/lepus/saga/executor_consumer.rb +38 -0
  37. data/lib/lepus/saga/instrumentation.rb +41 -0
  38. data/lib/lepus/saga/log_subscriber.rb +42 -0
  39. data/lib/lepus/saga/queue_depth.rb +28 -0
  40. data/lib/lepus/saga/resume_stuck_job.rb +19 -0
  41. data/lib/lepus/saga/retry_policy.rb +19 -0
  42. data/lib/lepus/saga/rspec/engine_examples.rb +635 -0
  43. data/lib/lepus/saga/rspec/helpers.rb +22 -0
  44. data/lib/lepus/saga/rspec/query_examples.rb +378 -0
  45. data/lib/lepus/saga/rspec/store_examples.rb +514 -0
  46. data/lib/lepus/saga/rspec.rb +6 -0
  47. data/lib/lepus/saga/step.rb +9 -0
  48. data/lib/lepus/saga/step_attempt_behaviour.rb +35 -0
  49. data/lib/lepus/saga/step_producer.rb +37 -0
  50. data/lib/lepus/saga/stores/base.rb +305 -0
  51. data/lib/lepus/saga/stores/memory.rb +375 -0
  52. data/lib/lepus/saga/stores.rb +40 -0
  53. data/lib/lepus/saga/testing.rb +35 -0
  54. data/lib/lepus/saga/version.rb +7 -0
  55. data/lib/lepus/saga/web/api.rb +326 -0
  56. data/lib/lepus/saga/web/configuration.rb +49 -0
  57. data/lib/lepus/saga/web/guard.rb +38 -0
  58. data/lib/lepus/saga/web/serializer.rb +59 -0
  59. data/lib/lepus/saga/web/static.rb +108 -0
  60. data/lib/lepus/saga/web.rb +61 -0
  61. data/lib/lepus/saga.rb +212 -0
  62. data/lib/lepus-saga.rb +3 -0
  63. data/mise.toml +2 -0
  64. data/web/assets/app.css +913 -0
  65. data/web/assets/apple-touch-icon.png +0 -0
  66. data/web/assets/favicon-32.png +0 -0
  67. data/web/assets/favicon.svg +1 -0
  68. data/web/assets/fonts/overpass/OFL.txt +93 -0
  69. data/web/assets/fonts/overpass/overpass-latin-wght.woff2 +0 -0
  70. data/web/assets/js/api.js +50 -0
  71. data/web/assets/js/dom.js +84 -0
  72. data/web/assets/js/format.js +79 -0
  73. data/web/assets/js/live.js +74 -0
  74. data/web/assets/js/main.js +143 -0
  75. data/web/assets/js/map.js +639 -0
  76. data/web/assets/js/range.js +19 -0
  77. data/web/assets/js/sagas.js +18 -0
  78. data/web/assets/js/theme.js +8 -0
  79. data/web/assets/js/throughput.js +262 -0
  80. data/web/assets/js/views/execution.js +426 -0
  81. data/web/assets/js/views/executions.js +165 -0
  82. data/web/assets/js/views/overview.js +465 -0
  83. data/web/index.html +63 -0
  84. metadata +323 -0
@@ -0,0 +1,305 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Lepus
4
+ module Saga
5
+ module Stores
6
+ # The persistence contract the engine runs on. An adapter subclasses it,
7
+ # implements every method that raises NotImplementedError and returns
8
+ # its own records: an execution record includes
9
+ # Lepus::Saga::ExecutionBehaviour and a step attempt record includes
10
+ # Lepus::Saga::StepAttemptBehaviour. Both need readers and writers for
11
+ # their behaviour's ATTRIBUTES and `[](name)` returning the raw stored
12
+ # value.
13
+ #
14
+ # Concurrency model: an execution is only changed while its row is held
15
+ # by #with_lock or #with_locked_execution (an exclusive lock held until
16
+ # the block returns, like SELECT ... FOR UPDATE inside a transaction).
17
+ # The only writes made without it are #increment_resume_count, which
18
+ # must be atomic in storage, and #save_step_attempt on an attempt the
19
+ # caller opened itself.
20
+ #
21
+ # `require "lepus/saga/rspec"` and `it_behaves_like "a lepus saga store"`
22
+ # verify an adapter against this contract.
23
+ #
24
+ # The query methods after #step_succeeded? are optional: they feed
25
+ # read-only tools such as dashboards and are never called by the engine.
26
+ # An adapter that implements them returns true from #queries? and runs
27
+ # `it_behaves_like "a lepus saga query store"`.
28
+ class Base
29
+ QUERY_MAX_ROWS = 50_000
30
+ MAX_SERIES_BUCKETS = 1_000
31
+ GROUPABLE = %i[saga_name status partition current_step].freeze
32
+ SERIES_KEYS = ["started", *ExecutionBehaviour::TERMINAL_STATUSES].freeze
33
+ LATENCY_STATUSES = %w[succeeded failed].freeze
34
+ PERCENTILES = {p50: 50, p95: 95, p99: 99}.freeze
35
+
36
+ # Inserts an execution and returns its record. `attributes` holds
37
+ # saga_name, resource_type, resource_id, current_step, context,
38
+ # correlation_id, partition, started_at and last_progress_at. The store
39
+ # generates the id (a UUIDv7 string) and defaults status to "running",
40
+ # direction to "execute", epoch and resume_count to 0 and context to {}.
41
+ def create_execution(attributes)
42
+ raise NotImplementedError
43
+ end
44
+
45
+ # The execution with this id, or nil. Never locks.
46
+ def find_execution(id)
47
+ raise NotImplementedError
48
+ end
49
+
50
+ # Persists every attribute of `execution` and returns it.
51
+ def save_execution(execution)
52
+ raise NotImplementedError
53
+ end
54
+
55
+ # Holds an exclusive lock on the execution's row for the duration of
56
+ # the block, after reloading `execution` in place from storage. Writes
57
+ # made in the block are atomic: they are kept when the block returns
58
+ # (including through `next`) and discarded when it raises. Another
59
+ # caller locking the same execution waits until the block finishes.
60
+ # Returns the block's value.
61
+ def with_lock(execution)
62
+ raise NotImplementedError
63
+ end
64
+
65
+ # Same guarantees as #with_lock, starting from an id: yields a freshly
66
+ # loaded, locked record, or nil when there is no such execution.
67
+ # Returns the block's value.
68
+ def with_locked_execution(id)
69
+ execution = find_execution(id)
70
+ return yield(nil) unless execution
71
+
72
+ with_lock(execution) { yield execution }
73
+ end
74
+
75
+ # Adds one to the stored resume_count in a single atomic write (never a
76
+ # read-modify-write of the record, so it cannot clobber or be clobbered
77
+ # by a concurrent locked update) and touches updated_at. Sets the
78
+ # record's resume_count and returns the stored value.
79
+ def increment_resume_count(execution)
80
+ raise NotImplementedError
81
+ end
82
+
83
+ # Executions in progress (running or compensating) whose
84
+ # last_progress_at is before `before`, oldest last_progress_at first.
85
+ # Returns an Enumerable; the caller stops iterating when done.
86
+ def stalled_executions(before:)
87
+ raise NotImplementedError
88
+ end
89
+
90
+ # Executions of a resource, most recently created first.
91
+ def executions_for_resource(resource_type, resource_id)
92
+ raise NotImplementedError
93
+ end
94
+
95
+ # Inserts a step attempt of `execution` and returns its record.
96
+ # `attributes` holds step_name, direction, epoch, attempt and
97
+ # started_at; status defaults to "running" and context_after to {}.
98
+ # (execution, step_name, direction, epoch, attempt) is unique: a second
99
+ # insert raises Lepus::Saga::DuplicateAttemptError, and raising it out
100
+ # of #with_lock discards the block's other writes.
101
+ def create_step_attempt(execution, attributes)
102
+ raise NotImplementedError
103
+ end
104
+
105
+ # Persists every attribute of `attempt` and returns it.
106
+ def save_step_attempt(attempt)
107
+ raise NotImplementedError
108
+ end
109
+
110
+ # Attempts of an execution ordered by started_at, then created_at.
111
+ def step_attempts(execution)
112
+ raise NotImplementedError
113
+ end
114
+
115
+ # Number of attempts of `execution` for this step, direction and epoch,
116
+ # whatever their status.
117
+ def count_step_attempts(execution, step_name:, direction:, epoch:)
118
+ raise NotImplementedError
119
+ end
120
+
121
+ # Whether a "running" attempt for this step, direction and epoch
122
+ # started at or after `started_since`.
123
+ def running_step_attempt?(execution, step_name:, direction:, epoch:, started_since:)
124
+ raise NotImplementedError
125
+ end
126
+
127
+ # "running" attempts for this step, direction and epoch that started
128
+ # before `started_before`.
129
+ def stale_step_attempts(execution, step_name:, direction:, epoch:, started_before:)
130
+ raise NotImplementedError
131
+ end
132
+
133
+ # Whether the step ever succeeded in the execute direction, in any
134
+ # epoch.
135
+ def step_succeeded?(execution, step_name)
136
+ raise NotImplementedError
137
+ end
138
+
139
+ # Whether the optional query methods below are implemented.
140
+ def queries? = false
141
+
142
+ # Executions matching the filters, newest first by id (UUIDv7, so by
143
+ # creation time), at most `limit` of them. `before_id` returns the
144
+ # page after the execution with that id. Every filter is optional:
145
+ # `saga_name`, `status` and `current_step` take a value or a list;
146
+ # `partition` matches the partition column, and executions without one
147
+ # whose definition runs in that partition; `resource` is a
148
+ # case-insensitive substring of resource_type or resource_id; `from`
149
+ # (inclusive) and `to` (exclusive) bound created_at, `progress_from`
150
+ # and `progress_to` bound last_progress_at the same way (in-progress
151
+ # executions with `progress_to: Time.now.utc - 15 * 60` are the
152
+ # stalled ones). `error_class` and `error_step` take a value or a list
153
+ # and keep executions with a failed attempt of that error class and at
154
+ # that step (either filter alone matches on its own attribute);
155
+ # `error_from` (inclusive) and `error_to` (exclusive) bound that
156
+ # failed attempt's started_at, the window #failure_summary counts.
157
+ def list_executions(saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
158
+ progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil, before_id: nil, limit: 50)
159
+ raise NotImplementedError
160
+ end
161
+
162
+ # Number of executions matching the #list_executions filters, as a
163
+ # Hash keyed by the value of `group_by` (one of GROUPABLE), or by an
164
+ # Array of values when `group_by` is an Array. Grouping by partition
165
+ # resolves executions without one to their definition's partition
166
+ # (nil for an unknown saga).
167
+ def count_executions(group_by:, saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
168
+ progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil)
169
+ raise NotImplementedError
170
+ end
171
+
172
+ # Executions started (by created_at) and finished in each terminal
173
+ # status (by finished_at) per `bucket` seconds between `from` and
174
+ # `to`. Returns `{rows: [[bucket_start, {"started" => n, "completed" =>
175
+ # n, ...}], ...], truncated:}` with every bucket of the window, bucket
176
+ # starts aligned to multiples of `bucket` since the epoch. At most
177
+ # `max_rows` of the newest timestamps of each kind are counted;
178
+ # `truncated` tells whether some were left out.
179
+ def execution_series(from:, to:, bucket:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
180
+ raise NotImplementedError
181
+ end
182
+
183
+ # Duration percentiles (nearest rank, in milliseconds) of the
184
+ # succeeded and failed attempts started between `from` and `to`, per
185
+ # saga, step and direction. Returns `{rows: [{saga_name:, step_name:,
186
+ # direction:, count:, p50:, p95:, p99:, max:}, ...], truncated:}`
187
+ # sorted by saga, step and direction, computed from at most `max_rows`
188
+ # of the newest attempts.
189
+ def step_latency(from:, to:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
190
+ raise NotImplementedError
191
+ end
192
+
193
+ # Failed attempts started between `from` and `to`, grouped by error
194
+ # class, saga and step, most frequent first (then most recently seen),
195
+ # at most `limit` groups. Returns `[{error_class:, saga_name:,
196
+ # step_name:, count:, execution_count:, last_seen_at:,
197
+ # sample_message:, sample_execution_id:}, ...]` where count is the
198
+ # failed attempts, execution_count the distinct executions they belong
199
+ # to and the sample the group's latest failure.
200
+ def failure_summary(from:, to:, saga_name: nil, limit: 20)
201
+ raise NotImplementedError
202
+ end
203
+
204
+ # The distinct saga names in storage, sorted.
205
+ def saga_names
206
+ raise NotImplementedError
207
+ end
208
+
209
+ def update_execution(execution, attributes)
210
+ assign(execution, attributes)
211
+ save_execution(execution)
212
+ end
213
+
214
+ def update_step_attempt(attempt, attributes)
215
+ assign(attempt, attributes)
216
+ save_step_attempt(attempt)
217
+ end
218
+
219
+ def finish_step_attempt(attempt, status, context_after: {}, error: nil)
220
+ update_step_attempt(attempt, attempt.finish_attributes(status, context_after: context_after, error: error))
221
+ end
222
+
223
+ private
224
+
225
+ def assign(record, attributes)
226
+ attributes.each { |name, value| record.public_send(:"#{name}=", value) }
227
+ end
228
+
229
+ def filter_values(value)
230
+ values = Array(value).map(&:to_s)
231
+ values unless values.empty?
232
+ end
233
+
234
+ def partition_saga_names(partition)
235
+ Lepus::Saga.definitions.filter_map { |name, definition| name if definition.partition == partition.to_s }
236
+ end
237
+
238
+ def resolve_partition(partition, saga_name)
239
+ partition || Lepus::Saga.definitions[saga_name]&.partition
240
+ end
241
+
242
+ def group_keys(group_by)
243
+ keys = Array(group_by).map(&:to_sym)
244
+ unknown = keys - GROUPABLE
245
+ raise ArgumentError, "Cannot group executions by #{unknown.join(", ")}" if keys.empty? || unknown.any?
246
+
247
+ keys
248
+ end
249
+
250
+ # Sums `[attributes, count]` rows into the #count_executions Hash.
251
+ def tally_groups(rows, group_by)
252
+ keys = group_keys(group_by)
253
+ rows.each_with_object(Hash.new(0)) do |(attributes, count), counts|
254
+ values = keys.map do |key|
255
+ (key == :partition) ? resolve_partition(attributes[:partition], attributes[:saga_name]) : attributes[key]
256
+ end
257
+ counts[group_by.is_a?(Array) ? values : values.first] += count
258
+ end
259
+ end
260
+
261
+ # `rows` holds up to max_rows + 1 entries, newest first.
262
+ def take_rows(rows, max_rows)
263
+ [rows.first(max_rows), rows.size > max_rows]
264
+ end
265
+
266
+ def series_bucket(from, to, bucket)
267
+ seconds = bucket.to_i
268
+ raise ArgumentError, "bucket must be at least one second" unless seconds.positive?
269
+ if series_size(from, to, seconds) > MAX_SERIES_BUCKETS
270
+ raise ArgumentError, "More than #{MAX_SERIES_BUCKETS} buckets between #{from} and #{to}"
271
+ end
272
+
273
+ seconds
274
+ end
275
+
276
+ def series_size(from, to, seconds)
277
+ ((to.to_r - from.to_i.div(seconds) * seconds) / seconds).ceil
278
+ end
279
+
280
+ # Buckets `started` timestamps and `[finished_at, status]` pairs.
281
+ def build_series(from, to, seconds, started, finished)
282
+ first = from.to_i.div(seconds) * seconds
283
+ rows = Array.new(series_size(from, to, seconds)) do |index|
284
+ [Time.at(first + index * seconds).utc, SERIES_KEYS.to_h { |key| [key, 0] }]
285
+ end
286
+ started.each { |time| rows[(time.to_i - first).div(seconds)].last["started"] += 1 }
287
+ finished.each { |time, status| rows[(time.to_i - first).div(seconds)].last[status] += 1 }
288
+ rows
289
+ end
290
+
291
+ # Percentiles of `[saga_name, step_name, direction, duration_ms]` samples.
292
+ def latency_rows(samples)
293
+ samples.group_by { |sample| sample.first(3) }.sort_by(&:first).map do |(saga_name, step_name, direction), group|
294
+ durations = group.map { |sample| sample.last.to_f }.sort
295
+ {
296
+ saga_name: saga_name, step_name: step_name, direction: direction, count: durations.size,
297
+ **PERCENTILES.transform_values { |rank| durations[(Rational(rank, 100) * durations.size).ceil - 1] },
298
+ max: durations.last
299
+ }
300
+ end
301
+ end
302
+ end
303
+ end
304
+ end
305
+ end
@@ -0,0 +1,375 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+ require "monitor"
5
+
6
+ module Lepus
7
+ module Saga
8
+ module Stores
9
+ # Thread-safe in-process store for tests and single-process tools.
10
+ # Records are copies: changing one changes nothing until it is saved, and
11
+ # every read builds a fresh record. Locks are per execution and
12
+ # re-entrant; writes made under a lock are undone when the block raises.
13
+ # Readers see writes as soon as they happen (no isolation between
14
+ # concurrent transactions), so it is not a database.
15
+ class Memory < Base
16
+ class RecordNotFound < StandardError; end
17
+
18
+ ATTEMPT_KEY = %i[saga_execution_id step_name direction epoch attempt].freeze
19
+ JSON_ATTRIBUTES = %i[context context_after].freeze
20
+
21
+ class Record
22
+ class << self
23
+ attr_reader :attribute_names
24
+
25
+ def attributes(*names)
26
+ @attribute_names = names
27
+ include(Module.new { attr_accessor(*names) })
28
+ end
29
+ end
30
+
31
+ def initialize(values = {})
32
+ load(values)
33
+ end
34
+
35
+ def [](name) = instance_variable_get(:"@#{name}")
36
+
37
+ def attributes
38
+ self.class.attribute_names.to_h { |name| [name, self[name]] }
39
+ end
40
+
41
+ def load(values)
42
+ values.each { |name, value| instance_variable_set(:"@#{name}", value) }
43
+ self
44
+ end
45
+
46
+ def ==(other) = other.instance_of?(self.class) && !id.nil? && other.id == id
47
+
48
+ alias_method :eql?, :==
49
+
50
+ def hash = [self.class, id].hash
51
+ end
52
+
53
+ class Execution < Record
54
+ attributes(*ExecutionBehaviour::ATTRIBUTES)
55
+
56
+ include ExecutionBehaviour
57
+ end
58
+
59
+ class StepAttempt < Record
60
+ attributes(*StepAttemptBehaviour::ATTRIBUTES)
61
+
62
+ include StepAttemptBehaviour
63
+ end
64
+
65
+ def initialize
66
+ super
67
+ @mutex = Mutex.new
68
+ @transaction_key = :"lepus_saga_memory_transaction_#{object_id}"
69
+ clear!
70
+ end
71
+
72
+ def clear!
73
+ @mutex.synchronize do
74
+ @executions = {}
75
+ @step_attempts = {}
76
+ @attempt_ids = {}
77
+ @row_locks = Hash.new { |locks, id| locks[id] = Monitor.new }
78
+ end
79
+ end
80
+
81
+ def create_execution(attributes)
82
+ now = Time.now.utc
83
+ row = {
84
+ status: "running", direction: "execute", epoch: 0, resume_count: 0,
85
+ context: {}, last_progress_at: now, **attributes.transform_keys(&:to_sym),
86
+ id: SecureRandom.uuid_v7, created_at: now, updated_at: now
87
+ }
88
+ Execution.new(insert(@executions, row))
89
+ end
90
+
91
+ def find_execution(id)
92
+ read(@executions, id)&.then { |row| Execution.new(row) }
93
+ end
94
+
95
+ def save_execution(execution)
96
+ execution.updated_at = Time.now.utc
97
+ update(@executions, execution.attributes)
98
+ execution
99
+ end
100
+
101
+ def with_lock(execution)
102
+ row_lock(execution.id).synchronize do
103
+ transaction do
104
+ execution.load(read(@executions, execution.id) || raise(RecordNotFound, "No execution #{execution.id}"))
105
+ yield execution
106
+ end
107
+ end
108
+ end
109
+
110
+ def with_locked_execution(id)
111
+ row_lock(id).synchronize do
112
+ transaction { yield find_execution(id) }
113
+ end
114
+ end
115
+
116
+ def increment_resume_count(execution)
117
+ row_lock(execution.id).synchronize do
118
+ execution.resume_count = @mutex.synchronize do
119
+ row = @executions.fetch(execution.id) { raise RecordNotFound, "No execution #{execution.id}" }
120
+ log_undo(@executions, execution.id)
121
+ @executions[execution.id] = row.merge(resume_count: row[:resume_count] + 1, updated_at: Time.now.utc)
122
+ row[:resume_count] + 1
123
+ end
124
+ end
125
+ end
126
+
127
+ def stalled_executions(before:)
128
+ where(@executions, Execution) { |row| ExecutionBehaviour::IN_PROGRESS_STATUSES.include?(row[:status]) && row[:last_progress_at] < before }
129
+ .sort_by { |execution| [execution.last_progress_at, execution.id] }
130
+ end
131
+
132
+ def executions_for_resource(resource_type, resource_id)
133
+ where(@executions, Execution) { |row| row[:resource_type] == resource_type && row[:resource_id] == resource_id }
134
+ .sort_by { |execution| [execution.created_at, execution.id] }.reverse
135
+ end
136
+
137
+ def create_step_attempt(execution, attributes)
138
+ now = Time.now.utc
139
+ row = {
140
+ direction: "execute", epoch: 0, status: "running", context_after: {},
141
+ **attributes.transform_keys(&:to_sym),
142
+ id: SecureRandom.uuid_v7, saga_execution_id: execution.id, created_at: now, updated_at: now
143
+ }
144
+ StepAttempt.new(insert(@step_attempts, row, unique: ATTEMPT_KEY))
145
+ end
146
+
147
+ def save_step_attempt(attempt)
148
+ attempt.updated_at = Time.now.utc
149
+ update(@step_attempts, attempt.attributes)
150
+ attempt
151
+ end
152
+
153
+ def step_attempts(execution)
154
+ @mutex.synchronize { attempt_rows(execution.id) }.map { |row| StepAttempt.new(copy(row)) }
155
+ .sort_by { |attempt| [attempt.started_at, attempt.created_at, attempt.id] }
156
+ end
157
+
158
+ def count_step_attempts(execution, step_name:, direction:, epoch:)
159
+ attempts_for_step(execution, step_name, direction, epoch).size
160
+ end
161
+
162
+ def running_step_attempt?(execution, step_name:, direction:, epoch:, started_since:)
163
+ attempts_for_step(execution, step_name, direction, epoch)
164
+ .any? { |attempt| attempt.running? && attempt.started_at >= started_since }
165
+ end
166
+
167
+ def stale_step_attempts(execution, step_name:, direction:, epoch:, started_before:)
168
+ attempts_for_step(execution, step_name, direction, epoch)
169
+ .select { |attempt| attempt.running? && attempt.started_at < started_before }
170
+ end
171
+
172
+ def step_succeeded?(execution, step_name)
173
+ step_attempts(execution).any? do |attempt|
174
+ attempt.step_name == step_name && attempt.direction == "execute" && attempt.succeeded?
175
+ end
176
+ end
177
+
178
+ def queries? = true
179
+
180
+ def list_executions(before_id: nil, limit: 50, **filters)
181
+ executions = matching_executions(**filters)
182
+ executions = executions.select { |execution| execution.id < before_id } if before_id
183
+ executions.sort_by(&:id).reverse.first(limit)
184
+ end
185
+
186
+ def count_executions(group_by:, **filters)
187
+ tally_groups(matching_executions(**filters).map { |execution| [execution.attributes, 1] }, group_by)
188
+ end
189
+
190
+ def execution_series(from:, to:, bucket:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
191
+ seconds = series_bucket(from, to, bucket)
192
+ window = from...to
193
+ executions = matching_executions(saga_name: saga_name)
194
+ started, started_truncated = take_rows(
195
+ executions.map(&:created_at).select { |time| window.cover?(time) }.sort.reverse, max_rows
196
+ )
197
+ finished, finished_truncated = take_rows(
198
+ executions.select { |execution| execution.terminal? && window.cover?(execution.finished_at) }
199
+ .map { |execution| [execution.finished_at, execution.status] }.sort_by(&:first).reverse,
200
+ max_rows
201
+ )
202
+ {rows: build_series(from, to, seconds, started, finished), truncated: started_truncated || finished_truncated}
203
+ end
204
+
205
+ def step_latency(from:, to:, saga_name: nil, max_rows: QUERY_MAX_ROWS)
206
+ attempts = attempts_with_saga(from, to, saga_name).select do |attempt, _|
207
+ LATENCY_STATUSES.include?(attempt.status) && attempt.duration_ms
208
+ end
209
+ samples, truncated = take_rows(
210
+ attempts.sort_by { |attempt, _| attempt.started_at }.reverse
211
+ .map { |attempt, saga| [saga, attempt.step_name, attempt.direction, attempt.duration_ms] },
212
+ max_rows
213
+ )
214
+ {rows: latency_rows(samples), truncated: truncated}
215
+ end
216
+
217
+ def failure_summary(from:, to:, saga_name: nil, limit: 20)
218
+ attempts_with_saga(from, to, saga_name).select { |attempt, _| attempt.failed? }
219
+ .group_by { |attempt, saga| [attempt.error_class, saga, attempt.step_name] }
220
+ .map { |(error_class, saga, step_name), group| failure_group(error_class, saga, step_name, group.map(&:first)) }
221
+ .sort_by { |group| [-group[:count], -group[:last_seen_at].to_r] }
222
+ .first(limit)
223
+ end
224
+
225
+ def saga_names
226
+ @mutex.synchronize { @executions.each_value.map { |row| row[:saga_name] } }.uniq.sort
227
+ end
228
+
229
+ private
230
+
231
+ def matching_executions(saga_name: nil, status: nil, current_step: nil, partition: nil, resource: nil, from: nil, to: nil,
232
+ progress_from: nil, progress_to: nil, error_class: nil, error_step: nil, error_from: nil, error_to: nil)
233
+ sagas = filter_values(saga_name)
234
+ statuses = filter_values(status)
235
+ steps = filter_values(current_step)
236
+ partition_sagas = partition_saga_names(partition) unless partition.to_s.strip.empty?
237
+ needle = resource.to_s.downcase
238
+ needle = nil if needle.strip.empty?
239
+ failing = failing_execution_ids(filter_values(error_class), filter_values(error_step), error_from, error_to)
240
+ where(@executions, Execution) do |row|
241
+ (sagas.nil? || sagas.include?(row[:saga_name])) &&
242
+ (statuses.nil? || statuses.include?(row[:status])) &&
243
+ (steps.nil? || steps.include?(row[:current_step])) &&
244
+ (from.nil? || row[:created_at] >= from) &&
245
+ (to.nil? || row[:created_at] < to) &&
246
+ (progress_from.nil? || row[:last_progress_at] >= progress_from) &&
247
+ (progress_to.nil? || row[:last_progress_at] < progress_to) &&
248
+ (failing.nil? || failing.include?(row[:id])) &&
249
+ (partition_sagas.nil? || in_partition?(row, partition.to_s, partition_sagas)) &&
250
+ (needle.nil? || row.values_at(:resource_type, :resource_id).any? { |value| value.to_s.downcase.include?(needle) })
251
+ end
252
+ end
253
+
254
+ def failing_execution_ids(error_classes, steps, from, to)
255
+ return if [error_classes, steps, from, to].all?(&:nil?)
256
+
257
+ @mutex.synchronize do
258
+ @step_attempts.each_value.filter_map do |row|
259
+ next unless row[:status] == "failed"
260
+ next unless error_classes.nil? || error_classes.include?(row[:error_class])
261
+ next unless steps.nil? || steps.include?(row[:step_name])
262
+ next if from && row[:started_at] < from
263
+ next if to && row[:started_at] >= to
264
+
265
+ row[:saga_execution_id]
266
+ end.to_set
267
+ end
268
+ end
269
+
270
+ def in_partition?(row, partition, saga_names)
271
+ row[:partition] ? row[:partition] == partition : saga_names.include?(row[:saga_name])
272
+ end
273
+
274
+ # Attempts started in the window as `[attempt, saga_name]` pairs.
275
+ def attempts_with_saga(from, to, saga_name)
276
+ sagas = @mutex.synchronize { @executions.transform_values { |row| row[:saga_name] } }
277
+ wanted = filter_values(saga_name)
278
+ where(@step_attempts, StepAttempt) do |row|
279
+ (from...to).cover?(row[:started_at]) && (wanted.nil? || wanted.include?(sagas[row[:saga_execution_id]]))
280
+ end.map { |attempt| [attempt, sagas[attempt.saga_execution_id]] }
281
+ end
282
+
283
+ def failure_group(error_class, saga_name, step_name, attempts)
284
+ latest = attempts.max_by { |attempt| [attempt.finished_at, attempt.id] }
285
+ {
286
+ error_class: error_class, saga_name: saga_name, step_name: step_name, count: attempts.size,
287
+ execution_count: attempts.map(&:saga_execution_id).uniq.size, last_seen_at: latest.finished_at, sample_message: latest.error_message,
288
+ sample_execution_id: latest.saga_execution_id
289
+ }
290
+ end
291
+
292
+ def attempts_for_step(execution, step_name, direction, epoch)
293
+ step_attempts(execution).select do |attempt|
294
+ attempt.step_name == step_name && attempt.direction == direction && attempt.epoch == epoch
295
+ end
296
+ end
297
+
298
+ def row_lock(id)
299
+ @mutex.synchronize { @row_locks[id] }
300
+ end
301
+
302
+ def transaction(&block)
303
+ undo_log ? yield : run_transaction(&block)
304
+ end
305
+
306
+ def run_transaction
307
+ log = Thread.current[@transaction_key] = []
308
+ committed = false
309
+ result = yield
310
+ committed = true
311
+ result
312
+ ensure
313
+ Thread.current[@transaction_key] = nil
314
+ rollback(log) unless committed
315
+ end
316
+
317
+ def undo_log = Thread.current[@transaction_key]
318
+
319
+ def log_undo(table, id)
320
+ undo_log&.push([table, id, table[id]])
321
+ end
322
+
323
+ def rollback(log)
324
+ @mutex.synchronize do
325
+ log.reverse_each do |table, id, previous|
326
+ previous ? table[id] = previous : table.delete(id)
327
+ end
328
+ end
329
+ end
330
+
331
+ def read(table, id)
332
+ @mutex.synchronize { table[id] }&.then { |row| copy(row) }
333
+ end
334
+
335
+ def where(table, klass, &filter)
336
+ @mutex.synchronize { table.values.select(&filter) }.map { |row| klass.new(copy(row)) }
337
+ end
338
+
339
+ def insert(table, row, unique: nil)
340
+ row = copy(row)
341
+ @mutex.synchronize do
342
+ if unique && attempt_rows(row[:saga_execution_id]).any? { |other| other.values_at(*unique) == row.values_at(*unique) }
343
+ raise DuplicateAttemptError, "Duplicate #{row.slice(*unique).inspect}"
344
+ end
345
+
346
+ log_undo(table, row[:id])
347
+ table[row[:id]] = row
348
+ (@attempt_ids[row[:saga_execution_id]] ||= []) << row[:id] if table.equal?(@step_attempts)
349
+ end
350
+ copy(row)
351
+ end
352
+
353
+ def update(table, row)
354
+ row = copy(row)
355
+ @mutex.synchronize do
356
+ raise RecordNotFound, "No record #{row[:id]}" unless table.key?(row[:id])
357
+
358
+ log_undo(table, row[:id])
359
+ table[row[:id]] = row
360
+ end
361
+ end
362
+
363
+ # Attempt rows of an execution through the index kept on insert; ids of
364
+ # rows a rollback removed are skipped. Call it holding @mutex.
365
+ def attempt_rows(execution_id)
366
+ @attempt_ids.fetch(execution_id, []).filter_map { |id| @step_attempts[id] }
367
+ end
368
+
369
+ def copy(row)
370
+ row.to_h { |name, value| [name, JSON_ATTRIBUTES.include?(name) ? JSON.parse(JSON.generate(value)) : value] }
371
+ end
372
+ end
373
+ end
374
+ end
375
+ end