task_progress 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +29 -0
  3. data/LICENSE +21 -0
  4. data/MANUAL.md +851 -0
  5. data/README.md +158 -0
  6. data/RECIPES.md +413 -0
  7. data/config/importmap.rb +17 -0
  8. data/exe/task_progress +51 -0
  9. data/js/local_tasks.d.ts +44 -0
  10. data/js/local_tasks.js +229 -0
  11. data/js/package.json +31 -0
  12. data/js/paths.d.ts +5 -0
  13. data/js/paths.js +7 -0
  14. data/js/react_task_tree.d.ts +49 -0
  15. data/js/react_task_tree.js +123 -0
  16. data/js/render_queue.d.ts +24 -0
  17. data/js/render_queue.js +79 -0
  18. data/js/row_view.d.ts +25 -0
  19. data/js/row_view.js +69 -0
  20. data/js/stimulus_controller.d.ts +10 -0
  21. data/js/stimulus_controller.js +137 -0
  22. data/js/task_progress_clock.d.ts +2 -0
  23. data/js/task_progress_clock.js +33 -0
  24. data/js/task_tree.css +51 -0
  25. data/js/task_tree.d.ts +111 -0
  26. data/js/task_tree.js +444 -0
  27. data/lib/task_progress/assertions.rb +49 -0
  28. data/lib/task_progress/batch.rb +23 -0
  29. data/lib/task_progress/branch.rb +74 -0
  30. data/lib/task_progress/buffered_output.rb +63 -0
  31. data/lib/task_progress/cancellations_app.rb +76 -0
  32. data/lib/task_progress/event.rb +46 -0
  33. data/lib/task_progress/execution_log_output.rb +183 -0
  34. data/lib/task_progress/fold.rb +18 -0
  35. data/lib/task_progress/html_renderer.rb +153 -0
  36. data/lib/task_progress/job_serializer.rb +92 -0
  37. data/lib/task_progress/journal.rb +35 -0
  38. data/lib/task_progress/log.rb +263 -0
  39. data/lib/task_progress/memory_cache.rb +31 -0
  40. data/lib/task_progress/memory_stream_store.rb +142 -0
  41. data/lib/task_progress/mount.rb +448 -0
  42. data/lib/task_progress/ndjson_output.rb +24 -0
  43. data/lib/task_progress/node.rb +52 -0
  44. data/lib/task_progress/poll.rb +87 -0
  45. data/lib/task_progress/publishing_output.rb +30 -0
  46. data/lib/task_progress/ractor_output.rb +23 -0
  47. data/lib/task_progress/rails_defaults.rb +46 -0
  48. data/lib/task_progress/railtie.rb +91 -0
  49. data/lib/task_progress/recording_output.rb +51 -0
  50. data/lib/task_progress/redis_stream_store.rb +131 -0
  51. data/lib/task_progress/squasher.rb +59 -0
  52. data/lib/task_progress/sse.rb +114 -0
  53. data/lib/task_progress/state.rb +319 -0
  54. data/lib/task_progress/stream_output.rb +222 -0
  55. data/lib/task_progress/stream_reader.rb +76 -0
  56. data/lib/task_progress/summary_output.rb +37 -0
  57. data/lib/task_progress/terminal_output.rb +241 -0
  58. data/lib/task_progress/token.rb +58 -0
  59. data/lib/task_progress/tracked_io.rb +68 -0
  60. data/lib/task_progress/transaction.rb +41 -0
  61. data/lib/task_progress/trees_app.rb +66 -0
  62. data/lib/task_progress/turbo_helper.rb +18 -0
  63. data/lib/task_progress/turbo_sse.rb +103 -0
  64. data/lib/task_progress/turbo_stream_output.rb +61 -0
  65. data/lib/task_progress/version.rb +7 -0
  66. data/lib/task_progress/wire.rb +211 -0
  67. data/lib/task_progress.rb +1036 -0
  68. data/sig/generated/task_progress/assertions.rbs +36 -0
  69. data/sig/generated/task_progress/batch.rbs +33 -0
  70. data/sig/generated/task_progress/branch.rbs +47 -0
  71. data/sig/generated/task_progress/buffered_output.rbs +38 -0
  72. data/sig/generated/task_progress/cancellations_app.rbs +41 -0
  73. data/sig/generated/task_progress/event.rbs +20 -0
  74. data/sig/generated/task_progress/execution_log_output.rbs +93 -0
  75. data/sig/generated/task_progress/fold.rbs +25 -0
  76. data/sig/generated/task_progress/html_renderer.rbs +71 -0
  77. data/sig/generated/task_progress/journal.rbs +25 -0
  78. data/sig/generated/task_progress/log.rbs +140 -0
  79. data/sig/generated/task_progress/memory_cache.rbs +28 -0
  80. data/sig/generated/task_progress/memory_stream_store.rbs +83 -0
  81. data/sig/generated/task_progress/mount.rbs +200 -0
  82. data/sig/generated/task_progress/ndjson_output.rbs +18 -0
  83. data/sig/generated/task_progress/node.rbs +66 -0
  84. data/sig/generated/task_progress/poll.rbs +50 -0
  85. data/sig/generated/task_progress/publishing_output.rbs +24 -0
  86. data/sig/generated/task_progress/ractor_output.rbs +17 -0
  87. data/sig/generated/task_progress/rails_defaults.rbs +33 -0
  88. data/sig/generated/task_progress/recording_output.rbs +41 -0
  89. data/sig/generated/task_progress/redis_stream_store.rbs +75 -0
  90. data/sig/generated/task_progress/squasher.rbs +20 -0
  91. data/sig/generated/task_progress/sse.rbs +73 -0
  92. data/sig/generated/task_progress/state.rbs +165 -0
  93. data/sig/generated/task_progress/stream_output.rbs +142 -0
  94. data/sig/generated/task_progress/stream_reader.rbs +46 -0
  95. data/sig/generated/task_progress/summary_output.rbs +19 -0
  96. data/sig/generated/task_progress/terminal_output.rbs +117 -0
  97. data/sig/generated/task_progress/token.rbs +41 -0
  98. data/sig/generated/task_progress/tracked_io.rbs +48 -0
  99. data/sig/generated/task_progress/transaction.rbs +33 -0
  100. data/sig/generated/task_progress/trees_app.rbs +40 -0
  101. data/sig/generated/task_progress/turbo_helper.rbs +14 -0
  102. data/sig/generated/task_progress/turbo_sse.rbs +69 -0
  103. data/sig/generated/task_progress/turbo_stream_output.rbs +51 -0
  104. data/sig/generated/task_progress/version.rbs +6 -0
  105. data/sig/generated/task_progress/wire.rbs +102 -0
  106. data/sig/generated/task_progress.rbs +553 -0
  107. metadata +178 -0
@@ -0,0 +1,319 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ class TaskProgress
5
+ # The tree as it stands after a run of events: what `apply` folds them into. It checks every
6
+ # event before it changes anything, so a bad one raises and leaves the state as it was. Not
7
+ # thread-safe on its own - the Log holds its lock around every apply
8
+ class State
9
+ # What an `open` or a `meta` may set
10
+ ATTRIBUTES = %i[name total weight expect indeterminate unit cancelable attempt].freeze #: Array[Symbol]
11
+ CHILD_IDS = 36**4 #: Integer - four base36 characters
12
+ # Siblings past this many are folded by `rows`...
13
+ FOLD_OVER = 12 #: Integer
14
+ # ...keeping this many of the done ones, the latest to finish, and of the ones waiting, the first
15
+ FOLD_KEEP = 3 #: Integer
16
+ ENDINGS = {done: :done, fail: :failed, cancel: :canceled}.freeze #: Hash[Symbol, Symbol]
17
+ # What a `put` carries - a Node, less what never goes over the wire
18
+ FIELDS = %i[name total count percent weight expect indeterminate unit cancelable attempt message status error stale
19
+ opened_at started_at finished_at].freeze #: Array[Symbol]
20
+ private_constant :CHILD_IDS, :ENDINGS
21
+
22
+ # @rbs @nodes: Hash[path, Node]
23
+
24
+ # A new node, opened at `at`, before its attributes are set
25
+ #: (Float) -> Node
26
+ def self.blank_node(at) = Node.new(name: "", count: 0, percent: 0, weight: 1.0, status: :pending, opened_at: at, children: [])
27
+
28
+ # Folding the squash of a state gives that state back, which is what makes a squashed log
29
+ # interchangeable with the one it came from
30
+ #: (Array[Event], ?at: Float) -> State
31
+ def self.fold(events, at: 0.0) = new.tap { |state| events.each { state.apply(it, at:) } }
32
+
33
+ # An empty tree - or one made of these nodes
34
+ #: (?Hash[path, Node]) -> void
35
+ def initialize(nodes = {})
36
+ @nodes = nodes
37
+ end
38
+
39
+ # The node at `path`, or nil
40
+ #: (path) -> Node?
41
+ def [](path) = @nodes[path]
42
+
43
+ # Whether there is a node at `path`
44
+ #: (path) -> bool
45
+ def key?(path) = @nodes.key?(path)
46
+
47
+ # Whether it has no root yet
48
+ #: () -> bool
49
+ def empty? = @nodes.empty?
50
+
51
+ # How many nodes it has
52
+ #: () -> Integer
53
+ def size = @nodes.size
54
+
55
+ # The paths of the nodes directly under `path`
56
+ #: (path) -> Array[path]
57
+ def children(path) = @nodes.fetch(path).children.map { [*path, it].freeze }
58
+
59
+ # A path for a new child of `path`. A child is named by a random id rather than its position,
60
+ # so a pruned task's path never turns up again on another one - not after a prune here, nor
61
+ # when a restarted remote replaces its tree. Ids only have to differ between siblings, since
62
+ # the path tells cousins apart, and a draw that hits a sibling is drawn again
63
+ #: (path, Random) -> path
64
+ def new_child(path, random) # :nodoc:
65
+ @nodes.fetch(path)
66
+ 100.times do
67
+ child = [*path, -random.rand(CHILD_IDS).to_s(36).rjust(4, "0")].freeze
68
+ return child unless @nodes.key?(child)
69
+ end
70
+ raise ArgumentError, "No id left for another task under #{path.inspect}"
71
+ end
72
+
73
+ # Pre-order, children in the order they opened - the order a renderer wants and the order
74
+ # puts replay in, so a reader rebuilding from a squash ends up with the same order
75
+ #: (?path) { (path, Node) -> void } -> void
76
+ #: (?path) -> Enumerator[[path, Node], void]
77
+ def each(path = [], &block)
78
+ return enum_for(:each, path) unless block
79
+ node = @nodes[path] or return
80
+ yield path, node
81
+ node.children.each { each([*path, it].freeze, &block) }
82
+ end
83
+
84
+ # What to draw, in the order of `each` - but where a task has more than `fold_over` children,
85
+ # the ones that are done (save the last `keep` to finish) are one Fold row ahead of the rest,
86
+ # and the ones still waiting (save the first `keep`) one Fold row after them. What is
87
+ # running, failed or canceled is always drawn. A Fold comes with its parent's path and
88
+ # `~done` or `~pending` after it, which no task id can be
89
+ #: (?path, ?fold_over: Integer, ?keep: Integer) { (path, Node | Fold) -> void } -> void
90
+ #: (?path, ?fold_over: Integer, ?keep: Integer) -> Enumerator[[path, Node | Fold], void]
91
+ def rows(path = [], fold_over: FOLD_OVER, keep: FOLD_KEEP, &block)
92
+ return enum_for(:rows, path, fold_over:, keep:) unless block
93
+ node = @nodes[path] or return
94
+ yield path, node
95
+ kids = node.children.map { [*path, it].freeze }
96
+ return kids.each { rows(it, fold_over:, keep:, &block) } if kids.length <= fold_over
97
+ done = kids.select { complete?(it) }
98
+ # The latest to finish, and of those that finished together - in one `done!` - the first
99
+ kept = done.each_with_index.max_by(keep) { |child, i| [@nodes.fetch(child).finished_at || 0.0, -i] }.map(&:first)
100
+ pending = kids.select { @nodes.fetch(it).status == :pending }
101
+ folded = Set.new(done - kept) | pending.drop(keep)
102
+ yield [*path, "~done"].freeze, Fold.new(status: :done, count: done.length - kept.length) if done.length > kept.length
103
+ kids.each { rows(it, fold_over:, keep:, &block) unless folded.include?(it) }
104
+ yield [*path, "~pending"].freeze, Fold.new(status: :pending, count: pending.length - keep) if pending.length > keep
105
+ end
106
+
107
+ # Folds one event in, or raises ArgumentError and leaves the state as it was
108
+ #: (Event, at: Float, ?journal: Journal?) -> self
109
+ def apply(event, at:, journal: nil) # :nodoc:
110
+ path, op, value = event.path, event.op, event.value
111
+ node = @nodes[path]
112
+ case op
113
+ when :open
114
+ raise ArgumentError, "A task already exists at #{path.inspect}" if node
115
+ parent = parent_of!(path)
116
+ journal&.touch(path, nil)
117
+ journal&.touch(up(path), parent) if parent
118
+ @nodes[path] = State.blank_node(at).tap { assign(it, value) }
119
+ adopt(parent, path) if parent
120
+ when :meta
121
+ node = existing!(path, node)
122
+ journal&.touch(path, node)
123
+ assign(node, value)
124
+ node.stale = value[:stale] if value.key?(:stale)
125
+ when :set
126
+ node = updatable!(path, node)
127
+ valid = (value.is_a?(Integer) || value.is_a?(Float)) && (0..100).cover?(value)
128
+ raise ArgumentError, "Percent must be between 0 and 100 inclusive, got #{value.inspect}" unless valid
129
+ start(path, at, journal)
130
+ node.percent = value
131
+ when :add
132
+ node = updatable!(path, node)
133
+ raise ArgumentError, "Count delta must be an Integer, got #{value.inspect}" unless value.is_a?(Integer)
134
+ start(path, at, journal)
135
+ node.count = [node.count + value, 0].max
136
+ when :note
137
+ node = updatable!(path, node)
138
+ start(path, at, journal)
139
+ node.message = value
140
+ when :done, :fail, :cancel
141
+ node = updatable!(path, node)
142
+ # Calling off what never began does not begin it
143
+ (op == :cancel) ? journal&.touch(path, node) : start(path, at, journal)
144
+ node.finished_at = at
145
+ node.status = ENDINGS.fetch(op)
146
+ node.error = value if op == :fail
147
+ node.message = value if op == :cancel && value
148
+ when :prune
149
+ return self unless node # so a squashed run may carry a prune for a node made within it
150
+ journal&.touch(path, node)
151
+ each(path).drop(1).each do |sub, subnode|
152
+ journal&.touch(sub, subnode)
153
+ @nodes.delete(sub)
154
+ end
155
+ node.children = [] #: Array[String]
156
+ when :say
157
+ existing!(path, node)
158
+ when :put
159
+ parent = parent_of!(path)
160
+ journal&.touch(path, node)
161
+ journal&.touch(up(path), parent) if parent && node.nil?
162
+ fresh = State.blank_node(at)
163
+ FIELDS.each { fresh[it] = value[it] if value.key?(it) }
164
+ fresh.status = fresh.status.to_sym
165
+ fresh.children = node ? node.children : []
166
+ fresh.sealed = node&.sealed
167
+ fresh.autofinish = node&.autofinish
168
+ @nodes[path] = fresh
169
+ adopt(parent, path) if parent && node.nil?
170
+ else
171
+ raise ArgumentError, "Unknown op #{op.inspect}"
172
+ end
173
+ self
174
+ end
175
+
176
+ # The tasks under `path` that have not ended, in pre-order - so reversed, every task comes
177
+ # before the one it is under. A mounted subtree is left out unless `mounts` - a remote that
178
+ # has not finished may yet, so a `done` from here would be a guess, but calling it off is not
179
+ #: (path, ?mounts: bool) -> Array[path]
180
+ def open_under(path, mounts: false)
181
+ open = [] #: Array[path]
182
+ sealed = nil #: path?
183
+ each(path).drop(1).each do |sub, node|
184
+ next if sealed && sub[0, sealed.length] == sealed
185
+ sealed = sub if node.sealed && !mounts
186
+ open << sub unless node.finished? || sealed == sub
187
+ end
188
+ open
189
+ end
190
+
191
+ # The state as a list of puts, one per node, in pre-order - or just the subtree at `path`
192
+ #: (?path) -> Array[Event]
193
+ def squash(path = []) = each(path).map { |path, node| Event.new(path:, op: :put, value: node.to_h) }
194
+
195
+ # Done, and everything under it ended - each task by a `done`, `fail` or `cancel` of its own.
196
+ # Children that are all through do not finish their parent, which may have more to do after
197
+ # them; a failed or canceled one does not hold back a parent that went on to finish regardless
198
+ #: (?path, ?Node?) -> bool
199
+ def complete?(path = [], node = @nodes[path])
200
+ return false unless node && node.status == :done
201
+ node.children.all? do |id|
202
+ child = @nodes.fetch([*path, id])
203
+ child.status == :failed || child.status == :canceled || complete?([*path, id], child)
204
+ end
205
+ end
206
+
207
+ # 0.0..1.0, unrounded. Children decide a parent's progress, by weight, and children it was
208
+ # told to expect but has not got yet count as zero. A leaf goes by its count when it has a
209
+ # total, and by its percent otherwise
210
+ #: (?path, ?Node?) -> Float
211
+ def fraction(path = [], node = @nodes[path])
212
+ return 0.0 unless node
213
+ return 1.0 if node.status == :done
214
+ if node.children.any? || missing(node).positive?
215
+ weights = missing(node).to_f
216
+ sum = 0.0
217
+ node.children.each do |id|
218
+ child_path = [*path, id]
219
+ child = @nodes.fetch(child_path)
220
+ weights += child.weight
221
+ sum += child.weight * fraction(child_path, child)
222
+ end
223
+ weights.zero? ? 0.0 : sum / weights
224
+ elsif (total = node.total)
225
+ total.zero? ? 0.0 : (node.count.to_f / total).clamp(0.0, 1.0)
226
+ elsif node.indeterminate
227
+ 0.0
228
+ else
229
+ node.percent / 100.0
230
+ end
231
+ end
232
+
233
+ # A whole percent, and 100 only once everything under it is done: 99.5 does not round up
234
+ # into a bar that reads as finished while work is still running
235
+ #: (?path) -> Integer
236
+ def percent(path = [])
237
+ return 100 if complete?(path)
238
+ [(fraction(path) * 100).round, 99].min
239
+ end
240
+
241
+ # Every node, as a `put` carries it
242
+ #: () -> Hash[path, Hash[Symbol, untyped]]
243
+ def to_h = @nodes.transform_values(&:to_h)
244
+
245
+ # A frozen copy, safe to read from any thread. Given an earlier snapshot of this state and the
246
+ # paths changed since, it copies only those, and takes the rest from the earlier one
247
+ #: (?State?, ?Enumerable[path]?) -> State
248
+ def snapshot(previous = nil, changed = nil)
249
+ return State.new(@nodes.to_h { |path, node| [path, frozen(node)] }.freeze).freeze unless previous && changed
250
+ nodes = previous.nodes.dup
251
+ changed.each { |path| (node = @nodes[path]) ? nodes[path] = frozen(node) : nodes.delete(path) }
252
+ State.new(nodes.freeze).freeze
253
+ end
254
+
255
+ # A copy that can change without changing this one
256
+ #: () -> State
257
+ def dup = State.new(@nodes.transform_values(&:deep_dup)) # :nodoc:
258
+
259
+ # Undoes what one transaction did
260
+ #: (Journal) -> void
261
+ def rollback(journal) = journal.rollback(@nodes) # :nodoc:
262
+
263
+ protected
264
+
265
+ # The nodes themselves, for an earlier snapshot to lend a new one what has not changed
266
+ #: () -> Hash[path, Node]
267
+ attr_reader :nodes #: Hash[path, Node]
268
+
269
+ private
270
+
271
+ #: (Node) -> Node
272
+ def frozen(node) = node.deep_dup.tap { it.children.freeze }.freeze
273
+
274
+ #: (Node) -> Integer
275
+ def missing(node) = [(node.expect || 0) - node.children.length, 0].max
276
+
277
+ #: (path) -> Node?
278
+ def parent_of!(path)
279
+ return nil if path.empty?
280
+ @nodes[up(path)] or raise ArgumentError, "No parent task at #{up(path).inspect} for #{path.inspect}"
281
+ end
282
+
283
+ #: (path, Node?) -> Node
284
+ def existing!(path, node)
285
+ node or raise ArgumentError, "No task at #{path.inspect}"
286
+ end
287
+
288
+ #: (path, Node?) -> Node
289
+ def updatable!(path, node)
290
+ node = existing!(path, node)
291
+ raise ArgumentError, "Cannot update task '#{node.name}' - already in #{node.status} state" if node.finished?
292
+ node
293
+ end
294
+
295
+ # Work started anywhere starts everything above it too
296
+ #: (path, Float, Journal?) -> void
297
+ def start(path, at, journal)
298
+ path.length.downto(0) do |length|
299
+ prefix = path[0, length] || []
300
+ node = @nodes.fetch(prefix)
301
+ journal&.touch(prefix, node)
302
+ node.status = :active if node.status == :pending
303
+ node.started_at ||= at
304
+ end
305
+ end
306
+
307
+ #: (Node, Hash[Symbol, untyped]) -> void
308
+ def assign(node, value)
309
+ ATTRIBUTES.each { node[it] = value[it] if value.key?(it) }
310
+ node.weight = node.weight.to_f
311
+ end
312
+
313
+ #: (path) -> path
314
+ def up(path) = path.take(path.length - 1)
315
+
316
+ #: (Node, path) -> void
317
+ def adopt(parent, path) = parent.children << path.fetch(-1)
318
+ end
319
+ end
@@ -0,0 +1,222 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "json"
5
+
6
+ class TaskProgress
7
+ # Writes a tree to a stream - a MemoryStreamStore, a RedisStreamStore - for readers anywhere to follow
8
+ # with StreamReader. What TaskProgress.handoff and TaskProgress.adopt write through. Under one
9
+ # key it keeps:
10
+ #
11
+ # key every batch, one entry each
12
+ # key:run the log id of the run that writes the tree now
13
+ # key:snapshot the tree squashed into one batch, and the id of the entry it is as of -
14
+ # where a reader starts, so the stream before it can be trimmed away
15
+ # key:beat there while the run is alive, and gone a few seconds after it is not
16
+ # key:control cancel requests, the other way - see Mount#follow_stream
17
+ #
18
+ # **One run at a time.** Each writer is a run, named by its log id, and claims `key:run` with its
19
+ # first batch. A run that a later one has taken over from - a job retried while the first
20
+ # attempt still limps on, two workers that adopted at once - may still append, and readers
21
+ # ignore what it appends; but every snapshot, trim, beat and expiry is written only while
22
+ # `key:run` still names the run that writes it, checked and written in one step. So a run that
23
+ # was replaced can never take the tree back.
24
+ #
25
+ # A snapshot is taken at the first batch, every `snapshot_every` seconds after, and when the
26
+ # root finishes, and each one trims the stream back to the one before it. Every key expires
27
+ # `retain` seconds after the last snapshot - a run that dies leaves nothing behind for good.
28
+ #
29
+ # With `heartbeat`, every so many seconds it marks itself alive and takes the cancel requests that
30
+ # came for it, on a thread of its own - from the first batch until the root finishes or it is
31
+ # closed. A request names the run it is for; a run also takes those for the run it took over
32
+ # from (`took_over`), so a cancel sent while a job still waited in the queue reaches whoever
33
+ # takes it up.
34
+ #
35
+ # A write that fails - Redis went away for a moment - is not lost: the next batch goes out as
36
+ # the whole tree, squashed, and readers start over from it
37
+ class StreamOutput
38
+ # The key it writes under
39
+ attr_reader :key #: String
40
+
41
+ # @rbs @stream: untyped
42
+ # @rbs @heartbeat: (Integer | Float)?
43
+ # @rbs @snapshot_every: Integer | Float
44
+ # @rbs @retain: Integer | Float
45
+ # @rbs @took_over: String?
46
+ # @rbs @run: String?
47
+ # @rbs @state: State
48
+ # @rbs @snapshot_id: String?
49
+ # @rbs @snapped_at: Float
50
+ # @rbs @beating: Thread?
51
+ # @rbs @stop: Thread::Queue
52
+ # @rbs @broken: bool
53
+ # @rbs @covered: Integer
54
+ # @rbs @replaced: bool
55
+ # @rbs @log: Log?
56
+
57
+ # The key of the squashed tree a reader starts from
58
+ #: (String) -> String
59
+ def self.snapshot_key(key) = "#{key}:snapshot"
60
+
61
+ # The key that is there while the writer is alive
62
+ #: (String) -> String
63
+ def self.beat_key(key) = "#{key}:beat"
64
+
65
+ # The key cancel requests are appended to
66
+ #: (String) -> String
67
+ def self.control_key(key) = "#{key}:control"
68
+
69
+ # Whether a cancel request is for one of these runs. A request names the run it was sent to -
70
+ # the one its sender saw writing the tree; one from a parent's control pipe names none
71
+ #: (String, *String?) -> bool
72
+ def self.for_run?(line, *runs)
73
+ named = JSON.parse(line)["run"] if line.start_with?("{")
74
+ named.nil? || runs.include?(named)
75
+ rescue JSON::ParserError
76
+ false
77
+ end
78
+
79
+ # The key that names the run writing the tree now
80
+ #: (String) -> String
81
+ def self.run_key(key) = "#{key}:run"
82
+
83
+ # A key for a new tree. The braces make it a Redis Cluster hash tag, so all five keys of a
84
+ # tree land on the same node
85
+ #: (String) -> String
86
+ def self.key_for(id) = "task_progress:{#{id}}"
87
+
88
+ # Writes to `stream` under `key`. `took_over` is the run it takes the tree over from
89
+ #: (untyped, String, ?heartbeat: (Integer | Float)?, ?snapshot_every: Integer | Float, ?retain: Integer | Float, ?took_over: String?) -> void
90
+ def initialize(stream, key, heartbeat: nil, snapshot_every: 2, retain: 86_400, took_over: nil)
91
+ @stream = stream
92
+ @key = key
93
+ @heartbeat = heartbeat
94
+ @snapshot_every = snapshot_every
95
+ @retain = retain
96
+ @took_over = took_over
97
+ @run = nil
98
+ @state = State.new
99
+ @snapshot_id = nil
100
+ @snapped_at = 0.0
101
+ @beating = nil
102
+ @stop = Thread::Queue.new
103
+ @broken = false
104
+ @covered = 0
105
+ @replaced = false
106
+ @log = nil
107
+ end
108
+
109
+ # Appends the batch, and takes a snapshot when one is due
110
+ #: (Batch, Log) -> void
111
+ def call(batch, log)
112
+ @log = log
113
+ return if @replaced
114
+ # A batch the last resync already carried
115
+ return if batch.seq <= @covered && !batch.reset?
116
+ batch = resync(log) if @broken
117
+ run = @run ||= claim(batch.log || @key)
118
+ id = @stream.append(@key, Wire.dump(batch))
119
+ @state = State.new if batch.reset?
120
+ batch.events.each { @state.apply(it, at: batch.at) }
121
+ finished = !!@state[[]]&.finished?
122
+ snapshot(id, batch, run) if batch.reset? || finished || now - @snapped_at >= @snapshot_every
123
+ if finished
124
+ finish(run)
125
+ elsif @heartbeat && !@beating && !@stop.closed?
126
+ @beating = Thread.new(log) { beat(it) }
127
+ end
128
+ rescue
129
+ @broken = true
130
+ raise
131
+ end
132
+
133
+ # Stops beating, and taking cancel requests - the work is over, one way or another. If the
134
+ # last write failed, the tree goes out whole once more, so readers do not stop short of the end
135
+ #: () -> void
136
+ def close
137
+ if @broken && (log = @log) && !@replaced
138
+ begin
139
+ call(log.squashed, log)
140
+ rescue => e
141
+ log.on_error.call(e, self)
142
+ end
143
+ end
144
+ stop
145
+ @beating&.join
146
+ end
147
+
148
+ # Whether a later run has taken the tree over, so this one writes no more
149
+ #: () -> bool
150
+ def replaced? = @replaced
151
+
152
+ private
153
+
154
+ #: () -> Float
155
+ def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
156
+
157
+ #: () -> void
158
+ def stop = @stop.close
159
+
160
+ #: (String) -> [String, String]
161
+ def guard(run) = [StreamOutput.run_key(@key), run]
162
+
163
+ # The run starts: from now on the tree is this run's to snapshot, trim and beat for
164
+ #: (String) -> String
165
+ def claim(run)
166
+ @stream.set(StreamOutput.run_key(@key), run, expires_in: @retain)
167
+ run
168
+ end
169
+
170
+ # The whole tree as one batch, in place of what failed to go out
171
+ #: (Log) -> Batch
172
+ def resync(log)
173
+ @broken = false
174
+ log.squashed.tap { @covered = it.seq }
175
+ end
176
+
177
+ #: (String, Batch, String) -> void
178
+ def snapshot(id, batch, run)
179
+ squash = Batch.new(log: batch.log, prev: 0, seq: batch.seq, at: batch.at, events: @state.squash.freeze)
180
+ written = @stream.set(StreamOutput.snapshot_key(@key), JSON.generate({id:, batch: Wire.to_h(squash)}), expires_in: @retain, guard: guard(run))
181
+ return replaced unless written
182
+ # A reader still on its way through the entries since the last snapshot gets to finish them
183
+ @stream.trim(@key, before: @snapshot_id, guard: guard(run)) if @snapshot_id
184
+ [@key, StreamOutput.control_key(@key), StreamOutput.run_key(@key)].each { @stream.expire(it, @retain, guard: guard(run)) }
185
+ @snapshot_id = id
186
+ @snapped_at = now
187
+ end
188
+
189
+ #: (String) -> void
190
+ def finish(run)
191
+ stop
192
+ @stream.delete(StreamOutput.beat_key(@key), guard: guard(run))
193
+ end
194
+
195
+ #: () -> void
196
+ def replaced
197
+ @replaced = true
198
+ stop
199
+ end
200
+
201
+ #: (Log) -> void
202
+ def beat(log)
203
+ root = TaskProgress.handle(log, [])
204
+ every = @heartbeat or return
205
+ run = @run or return
206
+ after = nil #: String?
207
+ until @stop.closed?
208
+ begin
209
+ alive = @stream.set(StreamOutput.beat_key(@key), run, expires_in: [every * 3, 1].max, guard: guard(run))
210
+ next replaced unless alive
211
+ @stream.read(StreamOutput.control_key(@key), after:).each do |id, line|
212
+ after = id
213
+ root.take_cancel_request(line) if StreamOutput.for_run?(line, run, @took_over)
214
+ end
215
+ rescue => e
216
+ log.on_error.call(e, self)
217
+ end
218
+ @stop.pop(timeout: every)
219
+ end
220
+ end
221
+ end
222
+ end
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ require "json"
5
+
6
+ class TaskProgress
7
+ # Reads a tree a StreamOutput writes: the snapshot first, as one batch with `prev` 0, then
8
+ # every batch after it as it comes. Each read picks up where the last one left off.
9
+ #
10
+ # It hands on what it reads, and judges none of it - Mount does that, see Mount#follow_stream.
11
+ # After a gap, `resync!` makes the next read start over from the latest snapshot
12
+ class StreamReader
13
+ # The key it reads
14
+ attr_reader :key #: String
15
+
16
+ # @rbs @stream: untyped
17
+ # @rbs @after: String?
18
+ # @rbs @started: bool
19
+
20
+ # Reads `stream` at `key`
21
+ #: (untyped, String) -> void
22
+ def initialize(stream, key)
23
+ @stream = stream
24
+ @key = key
25
+ @after = nil
26
+ @started = false
27
+ end
28
+
29
+ # The batches since the last read - starting from the snapshot. With a `timeout`, waits up
30
+ # to that many seconds for one when there are none yet
31
+ #: (?timeout: (Integer | Float)?) -> Array[Batch]
32
+ def read(timeout: nil)
33
+ batches = [] #: Array[Batch]
34
+ unless @started
35
+ @started = true
36
+ if (found = snapshot_entry)
37
+ @after, batch = found
38
+ batches << batch
39
+ end
40
+ end
41
+ @stream.read(@key, after: @after, timeout: (timeout if batches.empty?)).each do |id, line|
42
+ @after = id
43
+ batch = Wire.parse(line)
44
+ batches << batch if batch
45
+ end
46
+ batches
47
+ end
48
+
49
+ # The next read starts over, from the latest snapshot
50
+ #: () -> void
51
+ def resync!
52
+ @started = false
53
+ @after = nil
54
+ end
55
+
56
+ # The tree as of the latest snapshot, as one batch - or nil when nothing was ever written
57
+ # here, or it has expired
58
+ #: () -> Batch?
59
+ def snapshot = snapshot_entry&.last
60
+
61
+ # Whether the writer said lately that it is alive
62
+ #: () -> bool
63
+ def alive? = !@stream.get(StreamOutput.beat_key(@key)).nil?
64
+
65
+ private
66
+
67
+ #: () -> [String, Batch]?
68
+ def snapshot_entry
69
+ data = JSON.parse(@stream.get(StreamOutput.snapshot_key(@key)) || return)
70
+ batch = Wire.from_h(data["batch"]) if data.is_a?(Hash) && data["id"].is_a?(String)
71
+ [data["id"], batch] if batch
72
+ rescue JSON::ParserError
73
+ nil
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ class TaskProgress
5
+ # The tree boiled down to the root's percent and the latest thing said, for a progress
6
+ # column in a database or a single status line. A batch that changes neither is not passed on
7
+ class SummaryOutput
8
+ # @rbs @block: ^(percent: Integer, message: String?) -> void
9
+ # @rbs @percent: Integer
10
+
11
+ # Calls the block with `percent:` and `message:` whenever either changes
12
+ #: () { (percent: Integer, message: String?) -> void } -> void
13
+ def initialize(&block)
14
+ @block = block
15
+ @percent = 0
16
+ end
17
+
18
+ # Takes one batch, see Log#add_output
19
+ #: (Batch, Log) -> void
20
+ def call(batch, log)
21
+ snapshot = log.snapshot
22
+ message = nil #: String?
23
+ batch.events.each do |event|
24
+ case event.op
25
+ when :note, :say then message = event.value if event.value
26
+ when :done then message = "#{snapshot[event.path]&.name} complete"
27
+ when :fail then message = "#{snapshot[event.path]&.name} failed"
28
+ when :cancel then message = "#{snapshot[event.path]&.name} canceled"
29
+ end
30
+ end
31
+ percent = snapshot.percent
32
+ return if message.nil? && percent == @percent
33
+ @percent = percent
34
+ @block.call(percent:, message:)
35
+ end
36
+ end
37
+ end