task_progress 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +29 -0
  3. data/LICENSE +21 -0
  4. data/MANUAL.md +851 -0
  5. data/README.md +158 -0
  6. data/RECIPES.md +413 -0
  7. data/config/importmap.rb +17 -0
  8. data/exe/task_progress +51 -0
  9. data/js/local_tasks.d.ts +44 -0
  10. data/js/local_tasks.js +229 -0
  11. data/js/package.json +31 -0
  12. data/js/paths.d.ts +5 -0
  13. data/js/paths.js +7 -0
  14. data/js/react_task_tree.d.ts +49 -0
  15. data/js/react_task_tree.js +123 -0
  16. data/js/render_queue.d.ts +24 -0
  17. data/js/render_queue.js +79 -0
  18. data/js/row_view.d.ts +25 -0
  19. data/js/row_view.js +69 -0
  20. data/js/stimulus_controller.d.ts +10 -0
  21. data/js/stimulus_controller.js +137 -0
  22. data/js/task_progress_clock.d.ts +2 -0
  23. data/js/task_progress_clock.js +33 -0
  24. data/js/task_tree.css +51 -0
  25. data/js/task_tree.d.ts +111 -0
  26. data/js/task_tree.js +444 -0
  27. data/lib/task_progress/assertions.rb +49 -0
  28. data/lib/task_progress/batch.rb +23 -0
  29. data/lib/task_progress/branch.rb +74 -0
  30. data/lib/task_progress/buffered_output.rb +63 -0
  31. data/lib/task_progress/cancellations_app.rb +76 -0
  32. data/lib/task_progress/event.rb +46 -0
  33. data/lib/task_progress/execution_log_output.rb +183 -0
  34. data/lib/task_progress/fold.rb +18 -0
  35. data/lib/task_progress/html_renderer.rb +153 -0
  36. data/lib/task_progress/job_serializer.rb +92 -0
  37. data/lib/task_progress/journal.rb +35 -0
  38. data/lib/task_progress/log.rb +263 -0
  39. data/lib/task_progress/memory_cache.rb +31 -0
  40. data/lib/task_progress/memory_stream_store.rb +142 -0
  41. data/lib/task_progress/mount.rb +448 -0
  42. data/lib/task_progress/ndjson_output.rb +24 -0
  43. data/lib/task_progress/node.rb +52 -0
  44. data/lib/task_progress/poll.rb +87 -0
  45. data/lib/task_progress/publishing_output.rb +30 -0
  46. data/lib/task_progress/ractor_output.rb +23 -0
  47. data/lib/task_progress/rails_defaults.rb +46 -0
  48. data/lib/task_progress/railtie.rb +91 -0
  49. data/lib/task_progress/recording_output.rb +51 -0
  50. data/lib/task_progress/redis_stream_store.rb +131 -0
  51. data/lib/task_progress/squasher.rb +59 -0
  52. data/lib/task_progress/sse.rb +114 -0
  53. data/lib/task_progress/state.rb +319 -0
  54. data/lib/task_progress/stream_output.rb +222 -0
  55. data/lib/task_progress/stream_reader.rb +76 -0
  56. data/lib/task_progress/summary_output.rb +37 -0
  57. data/lib/task_progress/terminal_output.rb +241 -0
  58. data/lib/task_progress/token.rb +58 -0
  59. data/lib/task_progress/tracked_io.rb +68 -0
  60. data/lib/task_progress/transaction.rb +41 -0
  61. data/lib/task_progress/trees_app.rb +66 -0
  62. data/lib/task_progress/turbo_helper.rb +18 -0
  63. data/lib/task_progress/turbo_sse.rb +103 -0
  64. data/lib/task_progress/turbo_stream_output.rb +61 -0
  65. data/lib/task_progress/version.rb +7 -0
  66. data/lib/task_progress/wire.rb +211 -0
  67. data/lib/task_progress.rb +1036 -0
  68. data/sig/generated/task_progress/assertions.rbs +36 -0
  69. data/sig/generated/task_progress/batch.rbs +33 -0
  70. data/sig/generated/task_progress/branch.rbs +47 -0
  71. data/sig/generated/task_progress/buffered_output.rbs +38 -0
  72. data/sig/generated/task_progress/cancellations_app.rbs +41 -0
  73. data/sig/generated/task_progress/event.rbs +20 -0
  74. data/sig/generated/task_progress/execution_log_output.rbs +93 -0
  75. data/sig/generated/task_progress/fold.rbs +25 -0
  76. data/sig/generated/task_progress/html_renderer.rbs +71 -0
  77. data/sig/generated/task_progress/journal.rbs +25 -0
  78. data/sig/generated/task_progress/log.rbs +140 -0
  79. data/sig/generated/task_progress/memory_cache.rbs +28 -0
  80. data/sig/generated/task_progress/memory_stream_store.rbs +83 -0
  81. data/sig/generated/task_progress/mount.rbs +200 -0
  82. data/sig/generated/task_progress/ndjson_output.rbs +18 -0
  83. data/sig/generated/task_progress/node.rbs +66 -0
  84. data/sig/generated/task_progress/poll.rbs +50 -0
  85. data/sig/generated/task_progress/publishing_output.rbs +24 -0
  86. data/sig/generated/task_progress/ractor_output.rbs +17 -0
  87. data/sig/generated/task_progress/rails_defaults.rbs +33 -0
  88. data/sig/generated/task_progress/recording_output.rbs +41 -0
  89. data/sig/generated/task_progress/redis_stream_store.rbs +75 -0
  90. data/sig/generated/task_progress/squasher.rbs +20 -0
  91. data/sig/generated/task_progress/sse.rbs +73 -0
  92. data/sig/generated/task_progress/state.rbs +165 -0
  93. data/sig/generated/task_progress/stream_output.rbs +142 -0
  94. data/sig/generated/task_progress/stream_reader.rbs +46 -0
  95. data/sig/generated/task_progress/summary_output.rbs +19 -0
  96. data/sig/generated/task_progress/terminal_output.rbs +117 -0
  97. data/sig/generated/task_progress/token.rbs +41 -0
  98. data/sig/generated/task_progress/tracked_io.rbs +48 -0
  99. data/sig/generated/task_progress/transaction.rbs +33 -0
  100. data/sig/generated/task_progress/trees_app.rbs +40 -0
  101. data/sig/generated/task_progress/turbo_helper.rbs +14 -0
  102. data/sig/generated/task_progress/turbo_sse.rbs +69 -0
  103. data/sig/generated/task_progress/turbo_stream_output.rbs +51 -0
  104. data/sig/generated/task_progress/version.rbs +6 -0
  105. data/sig/generated/task_progress/wire.rbs +102 -0
  106. data/sig/generated/task_progress.rbs +553 -0
  107. metadata +178 -0
@@ -0,0 +1,1036 @@
1
+ # frozen_string_literal: true
2
+ # rbs_inline: enabled
3
+
4
+ # Hierarchical progress for task trees. A TaskProgress is a handle onto one node of a tree; the
5
+ # tree itself is a Log of change events, applied and numbered under one lock, which outputs
6
+ # receive in order - to draw a terminal, write to a pipe, stream to a browser, or feed a Mount
7
+ # that grafts the tree under a node of a tree in another process.
8
+ #
9
+ # TaskProgress.cli("Import") do |root|
10
+ # root.subtask("Download", unit: "bytes") { |t| t.track_io(io) ... }
11
+ # root.subtask("Rows", total: rows.size) { |t| rows.each { import(it); t.tick } }
12
+ # end
13
+ class TaskProgress
14
+ # @rbs!
15
+ # # A task's place in the tree: its ancestors' ids, then its own - `[]` is the root
16
+ # type path = Array[String]
17
+ #
18
+ # # Anything that takes the batches a log commits, in order - a lambda will do
19
+ # interface _Output
20
+ # def call: (Batch batch, Log log) -> void
21
+ # end
22
+ # type output = Object & _Output
23
+
24
+ PARTS = File.expand_path("task_progress", __dir__ || ".") #: String
25
+ private_constant :PARTS
26
+
27
+ # Each file loads when its constant is first used, along with what it needs from the standard
28
+ # library (json, openssl), so a program pays only for the parts it touches
29
+ autoload :VERSION, "#{PARTS}/version"
30
+ autoload :Event, "#{PARTS}/event"
31
+ autoload :OPS, "#{PARTS}/event"
32
+ autoload :Batch, "#{PARTS}/batch"
33
+ autoload :State, "#{PARTS}/state"
34
+ autoload :Node, "#{PARTS}/node"
35
+ autoload :Journal, "#{PARTS}/journal"
36
+ autoload :Wire, "#{PARTS}/wire"
37
+ autoload :Log, "#{PARTS}/log"
38
+ autoload :Transaction, "#{PARTS}/transaction"
39
+ autoload :Branch, "#{PARTS}/branch"
40
+ autoload :Mount, "#{PARTS}/mount"
41
+ autoload :Squasher, "#{PARTS}/squasher"
42
+ autoload :SummaryOutput, "#{PARTS}/summary_output"
43
+ autoload :NDJSONOutput, "#{PARTS}/ndjson_output"
44
+ autoload :PublishingOutput, "#{PARTS}/publishing_output"
45
+ autoload :BufferedOutput, "#{PARTS}/buffered_output"
46
+ autoload :RactorOutput, "#{PARTS}/ractor_output"
47
+ autoload :TerminalOutput, "#{PARTS}/terminal_output"
48
+ autoload :ExecutionLogOutput, "#{PARTS}/execution_log_output"
49
+ autoload :SSE, "#{PARTS}/sse"
50
+ autoload :Token, "#{PARTS}/token"
51
+ autoload :TreesApp, "#{PARTS}/trees_app"
52
+ autoload :TrackedIO, "#{PARTS}/tracked_io"
53
+ autoload :HTMLRenderer, "#{PARTS}/html_renderer"
54
+ autoload :TurboStreamOutput, "#{PARTS}/turbo_stream_output"
55
+ autoload :TurboSSE, "#{PARTS}/turbo_sse"
56
+ autoload :MemoryCache, "#{PARTS}/memory_cache"
57
+ autoload :CancellationsApp, "#{PARTS}/cancellations_app"
58
+ autoload :RailsDefaults, "#{PARTS}/rails_defaults"
59
+ autoload :MemoryStreamStore, "#{PARTS}/memory_stream_store"
60
+ autoload :RedisStreamStore, "#{PARTS}/redis_stream_store"
61
+ autoload :StreamOutput, "#{PARTS}/stream_output"
62
+ autoload :StreamReader, "#{PARTS}/stream_reader"
63
+ autoload :Poll, "#{PARTS}/poll"
64
+ autoload :Fold, "#{PARTS}/fold"
65
+ autoload :RecordingOutput, "#{PARTS}/recording_output"
66
+ autoload :Assertions, "#{PARTS}/assertions"
67
+ autoload :JobSerializer, "#{PARTS}/job_serializer"
68
+ autoload :TurboHelper, "#{PARTS}/turbo_helper"
69
+
70
+ # Ractors on Ruby 3.4 corrupt the VM's table of call sites: a `super` from Ruby into a C
71
+ # constructor (Data#initialize, say) while Ractors run brings the process down, sometimes
72
+ # minutes later and in unrelated code. Ruby 4.0 has it fixed, and the Ractor::Port API with it
73
+ RACTORS = RUBY_VERSION.split(".").first.to_i >= 4 #: bool
74
+
75
+ # Loads every part now rather than on first use - before forking workers, say, so they share
76
+ # the loaded code. TrackedIO is left out when the progressive_io gem is not there, and
77
+ # JobSerializer when ActiveJob is not
78
+ #: () -> singleton(TaskProgress)
79
+ def self.eager_load!
80
+ constants.each do |name|
81
+ const_get(name)
82
+ rescue LoadError
83
+ raise unless OPTIONAL.include?(name)
84
+ end
85
+ self
86
+ end
87
+
88
+ OPTIONAL = %i[TrackedIO JobSerializer].freeze #: Array[Symbol]
89
+ private_constant :OPTIONAL
90
+
91
+ # Raised at whoever works on a task that was called off, so the work stops at its next step.
92
+ # An Exception rather than a StandardError, like Interrupt: a `rescue => e` around one step
93
+ # should not swallow it and carry on with the next
94
+ class Canceled < Exception # rubocop:disable Lint/InheritException
95
+ # The task that was called off
96
+ attr_reader :task #: TaskProgress
97
+
98
+ # For the task that was called off, and what it is called
99
+ #: (TaskProgress, ?String) -> void
100
+ def initialize(task, name = task.name)
101
+ @task = task
102
+ super("Task '#{name}' was canceled")
103
+ end
104
+ end
105
+
106
+ # The log the task's tree lives in
107
+ attr_reader :log #: Log
108
+ # The task's place in the tree - `[]` for the root
109
+ attr_reader :path #: path
110
+
111
+ ROOT = [] #: path
112
+ ROOT.freeze
113
+ private_constant :ROOT
114
+
115
+ # A root reporting wherever suits a command-line program: a parent process, when it handed
116
+ # one down in TASK_PROGRESS_FD; a redrawn tree on a terminal; an ExecutionLog on CI or when
117
+ # piped. With a block, it runs the block as the root's work and returns its value
118
+ #: (?String, ?io: IO | StringIO, ?quiet: bool, **untyped) -> TaskProgress
119
+ #: [T] (?String, ?io: IO | StringIO, ?quiet: bool, **untyped) { (TaskProgress) -> T } -> T?
120
+ def self.cli(name = File.basename($PROGRAM_NAME), io: $stderr, quiet: false, **attributes, &block)
121
+ output = cli_output(io:, quiet:)
122
+ root = new(name, outputs: [output].compact, **attributes)
123
+ if (fd = ENV.delete("TASK_PROGRESS_CONTROL_FD"))
124
+ root.accept_cancel_requests(IO.new(Integer(fd), "r", autoclose: false).tap { it.close_on_exec = true })
125
+ end
126
+ return finish_at_exit(root, output) unless block
127
+ begin
128
+ root.run { block.call(it) }
129
+ ensure
130
+ close(output)
131
+ end
132
+ end
133
+
134
+ # The output `cli` and `bar` pick for where the program runs, or nil when `quiet`
135
+ #: (io: IO | StringIO, quiet: bool) -> output?
136
+ def self.cli_output(io:, quiet:) # :nodoc:
137
+ if (fd = ENV.delete("TASK_PROGRESS_FD"))
138
+ # Not handed down to a process this one starts, which would hold the pipe open past our exit
139
+ NDJSONOutput.new(IO.new(Integer(fd), "w", autoclose: false).tap { it.close_on_exec = true })
140
+ elsif quiet
141
+ nil
142
+ elsif io.tty? && !ENV["CI"] # some CI runners give a job a terminal, but nobody watches it redraw
143
+ TerminalOutput.new(io)
144
+ else
145
+ ExecutionLogOutput.new(io)
146
+ end
147
+ end
148
+
149
+ # A single bar, the way progress bars always worked: no block, and it finishes by itself once
150
+ # its count reaches `total`. Without a total it counts and spins. The last frame is drawn at
151
+ # exit, finished or not
152
+ #
153
+ # bar = TaskProgress.bar("Importing", total: rows.size)
154
+ # rows.each { import(it) && bar.tick }
155
+ #: (?String, ?total: Integer?, ?io: IO | StringIO, ?quiet: bool, **untyped) -> TaskProgress
156
+ def self.bar(name = File.basename($PROGRAM_NAME), total: nil, io: $stderr, quiet: false, **attributes)
157
+ output = cli_output(io:, quiet:)
158
+ root = new(name, outputs: [output].compact, total:, indeterminate: total.nil?, **attributes)
159
+ finish_at_exit(root, output)
160
+ end
161
+
162
+ # A root nobody runs a block for is finished when the program ends: done if it ends well,
163
+ # failed with the error if it is going down with one
164
+ #: (TaskProgress, output?) -> TaskProgress
165
+ def self.finish_at_exit(root, output) # :nodoc:
166
+ at_exit do
167
+ error = $!
168
+ unless root.finished?
169
+ (error.nil? || (error.is_a?(SystemExit) && error.success?)) ? root.done! : root.fail!(error)
170
+ end
171
+ ensure
172
+ close(output)
173
+ end
174
+ root
175
+ end
176
+
177
+ # Outputs that hold a thread or a screen - TerminalOutput, ExecutionLogOutput, BufferedOutput -
178
+ # can be closed, to draw their last and stop
179
+ #: (output?) -> void
180
+ def self.close(output) # :nodoc:
181
+ output.close if output.respond_to?(:close) # steep:ignore
182
+ end
183
+
184
+ # A root whose tree is drawn in pages with Turbo, from wherever it runs - a job, say. Every
185
+ # frame is the whole tree as HTML (see HTMLRenderer), sent as a Turbo Stream at most every
186
+ # `interval` seconds, and kept in `store` for a page to draw before the first one arrives.
187
+ # Cancel buttons post to `cancel_url`; the request waits in `store`, where this checks for it
188
+ # every `poll` seconds - so it works across processes too. With a block, the block is the
189
+ # root's work, as with `cli`.
190
+ #
191
+ # TaskProgress.broadcast(csv_import, "Importing") { |root| ... } # in a Rails job
192
+ # <%= task_progress_tree @csv_import %> # in its page
193
+ #
194
+ # In Rails, the target is a record, and the rest comes from RailsDefaults: Turbo::StreamsChannel,
195
+ # Rails.cache and the route the Railtie adds. Outside Rails the target is the element's id, and
196
+ # `to:` - called with each frame - is up to you
197
+ #: (untyped, ?String, ?to: (^(String) -> void)?, ?store: TurboStreamOutput::_Store?, ?tokens: Token?, ?cancel_url: String?, ?renderer: TurboStreamOutput::_Renderer?, ?interval: Integer | Float, ?poll: Integer | Float, **untyped) -> TaskProgress
198
+ #: [T] (untyped, ?String, ?to: (^(String) -> void)?, ?store: TurboStreamOutput::_Store?, ?tokens: Token?, ?cancel_url: String?, ?renderer: TurboStreamOutput::_Renderer?, ?interval: Integer | Float, ?poll: Integer | Float, **untyped) { (TaskProgress) -> T } -> T?
199
+ def self.broadcast(target, name = File.basename($PROGRAM_NAME), to: nil, store: nil, tokens: nil, cancel_url: nil, renderer: nil,
200
+ interval: 0.25, poll: 1, **attributes, &block)
201
+ defaults = RailsDefaults.for(target)
202
+ send = to || defaults[:to] or raise ArgumentError, "Give it to: - outside Rails with turbo-rails, there is nothing to send frames to"
203
+ store ||= defaults[:store]
204
+ tokens ||= defaults[:tokens]
205
+ cancel_url ||= defaults[:cancel_url]
206
+ key = RailsDefaults.key(target)
207
+ log = Log.new
208
+ # Tokens and requests are for this run alone: a page left open from the last run, or a click
209
+ # that came too late for it, cannot reach the next one for the same record
210
+ run = "#{key}/#{log.id}"
211
+ renderer ||= HTMLRenderer.new(id: RailsDefaults.id(target), cancel_url: (cancel_url if tokens), cancel_token: tokens&.issue(run, expires_in: 86_400))
212
+ output = BufferedOutput.new(TurboStreamOutput.new(renderer:, to: send, store:, key:), interval:)
213
+ root = new(name, outputs: [output], log:, **attributes)
214
+ watch_cancellations(root, store, run, poll) if store && tokens
215
+ return finish_at_exit(root, output) unless block
216
+ begin
217
+ root.run { block.call(it) }
218
+ ensure
219
+ close(output)
220
+ end
221
+ end
222
+
223
+ # Picks up the requests CancellationsApp leaves in the store, until the root has ended. A store
224
+ # that fails for a moment is tried again at the next poll
225
+ #: (TaskProgress, TurboStreamOutput::_Store, String, Integer | Float) -> Thread
226
+ def self.watch_cancellations(root, store, key, poll) # :nodoc:
227
+ Thread.new do
228
+ until root.finished?
229
+ sleep(poll)
230
+ begin
231
+ root.log.snapshot.each do |path, node|
232
+ next unless node.cancelable && !node.finished?
233
+ request = CancellationsApp.request_key(key, path)
234
+ reason = store.read(request) or next
235
+ store.delete(request)
236
+ root.request_cancel(path, reason.to_s)
237
+ end
238
+ rescue => e
239
+ root.log.on_error.call(e, store)
240
+ end
241
+ end
242
+ end
243
+ end
244
+
245
+ private_class_method :cli_output, :finish_at_exit, :close
246
+
247
+ @stream_store = nil #: untyped
248
+ @stream_store_lock = Mutex.new #: Thread::Mutex
249
+
250
+ # Where handed-off trees are kept, unless a call says otherwise - one connection, as
251
+ # `Rails.cache` is, and every tree in it a stream of its own under its own key.
252
+ #
253
+ # TaskProgress.stream_store = TaskProgress::RedisStreamStore.new(RedisClient.config(url: ENV["REDIS_URL"]).new_pool)
254
+ #
255
+ # Until it is set, a MemoryStreamStore: trees kept in this process alone, which is right for
256
+ # development and tests and quietly wrong anywhere a job runs in another process. So in
257
+ # production (RAILS_ENV, RACK_ENV or APP_ENV) it raises instead - set it to a MemoryStreamStore
258
+ # yourself if this process alone is what you mean
259
+ #: () -> untyped
260
+ def self.stream_store
261
+ @stream_store || @stream_store_lock.synchronize do
262
+ @stream_store ||= begin
263
+ if %w[RAILS_ENV RACK_ENV APP_ENV].any? { ENV[it] == "production" }
264
+ raise "TaskProgress.stream_store is not set. In production a handed-off tree has to be somewhere every " \
265
+ "process can reach it - set it to a TaskProgress::RedisStreamStore, or to a MemoryStreamStore " \
266
+ "if this process alone is what you mean"
267
+ end
268
+ MemoryStreamStore.new
269
+ end
270
+ end
271
+ end
272
+
273
+ # Sets where handed-off trees are kept, see `stream_store`
274
+ #: (untyped) -> void
275
+ def self.stream_store=(store)
276
+ @stream_store = store
277
+ end
278
+
279
+ @tokens = nil #: Token?
280
+ @trees_path = "/task_progress/trees" #: String
281
+
282
+ # Sets what signs the links `for_page` hands out. In Rails, the Railtie sets the app's own,
283
+ # made from its `secret_key_base`
284
+ #: (Token?) -> void
285
+ def self.tokens=(tokens)
286
+ @tokens = tokens
287
+ end
288
+
289
+ # Where TreesApp is mounted, for `for_page` to link to - `/task_progress/trees` by default
290
+ #: () -> String
291
+ def self.trees_path = @trees_path
292
+
293
+ # Sets where TreesApp is mounted
294
+ #: (String) -> void
295
+ def self.trees_path=(path)
296
+ @trees_path = path
297
+ end
298
+
299
+ # What signs the links `for_page` hands out - see `tokens=`
300
+ #: () -> Token
301
+ def self.tokens = @tokens || raise("Set TaskProgress.tokens = TaskProgress::Token.new(secret) - in Rails, the Railtie does")
302
+
303
+ # Everything a page needs to show a handed-off tree, as plain JSON-ready data - a prop for an
304
+ # Inertia or React page, a data attribute for Stimulus:
305
+ #
306
+ # render inertia: "Videos/Show", props: {progress: TaskProgress.for_page(video.progress_key)}
307
+ # # => {events: "/task_progress/trees/…/events", poll: "…/poll", cancel: "…/cancellations",
308
+ # # snapshot: {v: 1, …}}
309
+ #
310
+ # The links carry a Token good for `expires_in` seconds, not the tree's key: hand this out only
311
+ # where the page may see that tree. `snapshot` is the tree as it stands, so the page draws it
312
+ # before anything has come down the stream. TreesApp answers the links
313
+ #: (String | TaskProgress, ?expires_in: Integer, ?stream_store: untyped) -> Hash[Symbol, untyped]
314
+ def self.for_page(key, expires_in: 3600, stream_store: self.stream_store)
315
+ key = key.key || raise(ArgumentError, "#{key.inspect} was not handed off") if key.is_a?(TaskProgress)
316
+ base = "#{trees_path}/#{tokens.issue(key, expires_in:)}"
317
+ tree = fetch(key, stream_store:)
318
+ {events: "#{base}/events", poll: "#{base}/poll", cancel: "#{base}/cancellations", snapshot: Wire.to_h(tree.log.squashed)}
319
+ end
320
+
321
+ # A root to hand to whoever does the work - a job, a worker, another machine - which takes it
322
+ # up with `adopt`. Until then it waits, as "pending", where anyone can `follow` it: a page can
323
+ # show the job queued before any worker has picked it up.
324
+ #
325
+ # import = TaskProgress.handoff("Importing", cancelable: true)
326
+ # ImportJob.perform_later(upload, import) # ActiveJob takes the task itself
327
+ # Importer.perform_async(upload.id, import.key) # anything else, its key
328
+ #
329
+ # The tree lives in `stream_store`, under `key`. Once it is handed off, leave it to the worker: what
330
+ # this side writes after the worker has started is ignored
331
+ #: (String, ?stream_store: untyped, **untyped) -> TaskProgress
332
+ def self.handoff(name, stream_store: self.stream_store, retain: 86_400, **attributes)
333
+ key = StreamOutput.key_for(Log.new.id)
334
+ new(name, outputs: [StreamOutput.new(stream_store, key, retain:)], log: Log.new(key:), **attributes)
335
+ end
336
+
337
+ # Takes up a tree handed off with `handoff` - by its key, or the task itself - as a root of
338
+ # its own, with the name and attributes it was handed off with. It starts the tree over: a
339
+ # second worker to adopt it, a job retried, replaces what the last one wrote.
340
+ #
341
+ # Batches go out squashed, every `interval` seconds at most. Every `heartbeat` seconds it tells
342
+ # whoever follows that it is alive, and takes the cancel requests that came. With a block,
343
+ # the block is the root's work, see `run`, and it all stops when the block returns. Without
344
+ # one, call `log.close` once the work is over - ActiveJob does this by itself, see JobSerializer
345
+ #: (String | TaskProgress, ?stream_store: untyped, ?heartbeat: Integer | Float, ?interval: Integer | Float) -> TaskProgress
346
+ #: [T] (String | TaskProgress, ?stream_store: untyped, ?heartbeat: Integer | Float, ?interval: Integer | Float) { (TaskProgress) -> T } -> T?
347
+ def self.adopt(key, stream_store: self.stream_store, heartbeat: 1, interval: 0.1, retain: 86_400, &block)
348
+ key = key.key || raise(ArgumentError, "#{key.inspect} was not handed off") if key.is_a?(TaskProgress)
349
+ snapshot = StreamReader.new(stream_store, key).snapshot
350
+ handed = snapshot && State.fold(snapshot.events)[[]]
351
+ raise KeyError.new("Nothing was handed off at #{key}, or it has expired", key:) unless snapshot && handed
352
+ writer = StreamOutput.new(stream_store, key, heartbeat:, retain:, took_over: snapshot.log)
353
+ output = BufferedOutput.new(writer, interval:)
354
+ log = Log.new(key:)
355
+ log.on_close do
356
+ output.close
357
+ writer.close
358
+ end
359
+ root = new(handed.name, outputs: [output], log:, total: handed.total, expect: handed.expect,
360
+ indeterminate: !!handed.indeterminate, unit: handed.unit, cancelable: !!handed.cancelable)
361
+ root.send(:meta, attempt: handed.attempt.to_i + 1)
362
+ # Called off before anyone took it up - by the side that handed it off, or by a request that
363
+ # waited for a worker: it stays called off, and `run` will not start its work
364
+ root.cancel!(handed.message) if handed.status == :canceled
365
+ stream_store.read(StreamOutput.control_key(key)).each do |_, line|
366
+ root.take_cancel_request(line) if StreamOutput.for_run?(line, snapshot.log.to_s)
367
+ end
368
+ return root unless block
369
+ begin
370
+ root.run { block.call(it) }
371
+ ensure
372
+ log.close
373
+ end
374
+ end
375
+
376
+ # Draws the tree handed off at `key` where a command-line program draws one (see `cli`) until it
377
+ # finishes, and returns it. Ctrl-C stops watching, and leaves the work alone. What
378
+ # `task_progress watch` does
379
+ #: (String, ?stream_store: untyped, ?io: IO | StringIO) -> TaskProgress
380
+ def self.watch(key, stream_store: self.stream_store, io: $stderr)
381
+ output = cli_output(io:, quiet: false)
382
+ follow(key, stream_store:) do |tree|
383
+ tree.add_output(output)
384
+ sleep(0.1) until tree.finished?
385
+ tree
386
+ rescue Interrupt
387
+ tree
388
+ ensure
389
+ close(output)
390
+ end
391
+ end
392
+
393
+ # A tree that mirrors the one in `stream_store` at `key`, kept up to date on a thread of its own -
394
+ # for drawing it anywhere: a terminal, a parent's tree, an endpoint of your own. Requests to
395
+ # cancel its tasks go to whoever writes it; it goes stale when the writer goes quiet, and fails
396
+ # after `give_up_after` seconds of that, see Mount#follow_stream. With a block, it follows for
397
+ # as long as the block runs; without one, `log.close` stops it. SSE and Poll take the key
398
+ # itself, and follow for as long as they need to. A key nothing was handed off at is a KeyError
399
+ #: (String, ?stream_store: untyped, ?poll: Integer | Float, ?give_up_after: Integer | Float) -> TaskProgress
400
+ #: [T] (String, ?stream_store: untyped, ?poll: Integer | Float, ?give_up_after: Integer | Float) { (TaskProgress) -> T } -> T
401
+ def self.follow(key, stream_store: self.stream_store, poll: 1, give_up_after: 60)
402
+ root, mount = mirror(key, stream_store)
403
+ mount.follow_stream(stream_store, key, poll:, give_up_after:)
404
+ return root unless block_given?
405
+ begin
406
+ yield root
407
+ ensure
408
+ root.log.close
409
+ end
410
+ end
411
+
412
+ # The tree in `stream_store` at `key` as it is now, read once - for a page, or one poll, that needs
413
+ # no more than that. See `follow` for one kept up to date. A key nothing was handed off at -
414
+ # or whose tree has expired - is a KeyError, as with Hash#fetch
415
+ #: (String, ?stream_store: untyped) -> TaskProgress
416
+ def self.fetch(key, stream_store: self.stream_store)
417
+ root, mount = mirror(key, stream_store)
418
+ mount.follow_stream(stream_store, key, watch: false)
419
+ root
420
+ end
421
+
422
+ # Runs the block as the work of every root among `arguments` that a worker took up with `adopt`
423
+ # - done when it returns, failed when it raises - and closes them after. What JobSerializer
424
+ # wraps every ActiveJob in; for any other job runner, wrap `perform` in it the same way
425
+ #: [T] (Array[untyped]) { () -> T } -> T?
426
+ def self.working_on(arguments, &block)
427
+ roots = arguments.flat_map { it.is_a?(Hash) ? it.values : [it] }
428
+ .select { it.is_a?(TaskProgress) && it.key && !it.node.sealed } #: Array[TaskProgress]
429
+ begin
430
+ within(roots, &block)
431
+ ensure
432
+ roots.each { it.log.close }
433
+ end
434
+ end
435
+
436
+ # The block as the work of each root in turn, the first outermost
437
+ #: [T] (Array[TaskProgress]) { () -> T } -> T?
438
+ def self.within(roots, &block)
439
+ root = roots.first or return block.call
440
+ root.run { within(roots.drop(1), &block) }
441
+ end
442
+ private_class_method :within
443
+
444
+ # A root that is a mount and nothing else, named after `key` until the stream says otherwise
445
+ #: (String, untyped) -> [TaskProgress, Mount]
446
+ def self.mirror(key, stream_store)
447
+ StreamReader.new(stream_store, key).snapshot or raise KeyError.new("Nothing was handed off at #{key}, or it has expired", key:)
448
+ log = Log.new(key:)
449
+ root = new(key, log:)
450
+ mount = Mount.new(log, [], own: false)
451
+ log.transaction do |tx|
452
+ tx.journal.touch([], tx.state[[]])
453
+ tx.state[[]]&.sealed = mount
454
+ end
455
+ [root, mount]
456
+ end
457
+ private_class_method :mirror
458
+
459
+ # A size that is cheap to know, or nil - never `count`, which would run a lazy enumerator
460
+ #: (untyped) -> Integer?
461
+ def self.size_of(collection) = (collection.size if collection.respond_to?(:size)) # :nodoc:
462
+
463
+ # Where the browser client lives: `task_tree.js` (the fold, and `follow` for an SSE stream),
464
+ # `render_queue.js`, `row_view.js`, a Stimulus controller, a React component and
465
+ # `task_tree.css`. No build step - serve the directory as it is, under any prefix
466
+ #: () -> String
467
+ def self.js_path = File.expand_path("../js", __dir__ || ".")
468
+
469
+ # Copies the browser client into `dir`, for a bundler - Vite, esbuild - to take from there, and
470
+ # returns the names of the files it wrote. The copy is a package of its own, named
471
+ # `task_progress`: `npm install ./dir` puts it in `node_modules`, TypeScript declarations and all.
472
+ #
473
+ # `dir` is the gem's: whatever an earlier vendoring left there is replaced, so a file the gem
474
+ # no longer ships goes too. A directory with files of its own in it is left alone, and raises
475
+ #: (String) -> Array[String]
476
+ def self.vendor_js(dir)
477
+ require "fileutils"
478
+ require "json"
479
+ manifest = File.join(dir, "package.json")
480
+ ours = File.exist?(manifest) && JSON.parse(File.read(manifest))["name"] == "task_progress"
481
+ if Dir.exist?(dir) && !Dir.empty?(dir) && !ours
482
+ raise ArgumentError, "#{dir} has files of its own - vendor into an empty directory, or one vendored before"
483
+ end
484
+ FileUtils.rm_rf(dir)
485
+ FileUtils.mkdir_p(dir)
486
+ files = Dir[File.join(js_path, "*.{js,css,d.ts}")] + [File.join(js_path, "package.json")]
487
+ FileUtils.cp(files, dir)
488
+ files.map { File.basename(it) }.sort
489
+ rescue JSON::ParserError
490
+ raise ArgumentError, "#{dir} has a package.json of its own - vendor into an empty directory, or one vendored before"
491
+ end
492
+
493
+ # The version of the client vendored into `dir`, or nil when there is none
494
+ #: (String) -> String?
495
+ def self.vendored_js_version(dir)
496
+ require "json"
497
+ JSON.parse(File.read(File.join(dir, "package.json")))["version"] if File.exist?(File.join(dir, "package.json"))
498
+ rescue JSON::ParserError
499
+ nil
500
+ end
501
+
502
+ # A silent root, for a library's default
503
+ #: (?String) -> TaskProgress
504
+ def self.null(name = "progress") = new(name)
505
+
506
+ # A new tree, and its root. See `subtask` for `autofinish`
507
+ #: (?String, ?outputs: Array[output], ?log: Log, ?total: Integer?, ?expect: Integer?, ?indeterminate: bool, ?unit: String?, ?cancelable: bool, ?autofinish: bool) -> void
508
+ def initialize(name = "progress", outputs: [], log: Log.new, total: nil, expect: nil, indeterminate: false, unit: nil, cancelable: false, autofinish: true)
509
+ @log = log
510
+ @path = ROOT
511
+ outputs.each { @log.add_output(it) }
512
+ @log.transaction do |tx|
513
+ tx << Event.new(path: [], op: :open, value: TaskProgress.attributes(name:, total:, expect:, indeterminate:, unit:, cancelable:))
514
+ tx.state[[]]&.autofinish = autofinish
515
+ end
516
+ end
517
+
518
+ # The attributes an `open` carries, without what is at its default
519
+ #: (name: String, ?total: Integer?, ?weight: (Integer | Float)?, ?expect: Integer?, ?indeterminate: bool, ?unit: String?, ?cancelable: bool, ?attempt: Integer?) -> Hash[Symbol, untyped]
520
+ def self.attributes(name:, total: nil, weight: nil, expect: nil, indeterminate: false, unit: nil, cancelable: false, attempt: nil) # :nodoc:
521
+ {name: utf8(name), total:, weight:, expect:, indeterminate: (true if indeterminate), unit:, cancelable: (true if cancelable), attempt:}.compact
522
+ end
523
+
524
+ # A handle on the task at `path` in `log`, without opening anything
525
+ #: (Log, path) -> TaskProgress
526
+ def self.handle(log, path) = allocate.tap {
527
+ it.instance_variable_set(:@log, log)
528
+ it.instance_variable_set(:@path, path.frozen? ? path : path.dup.freeze)
529
+ } # :nodoc:
530
+
531
+ # The stream key a root from `handoff`, `adopt`, `follow` or `fetch` lives under, which is
532
+ # all it takes to `adopt` or `follow` it from anywhere. Nil for any other task
533
+ #: () -> String?
534
+ def key = (@log.key if @path.empty?)
535
+
536
+ # The task and everything under it as plain Hashes - its name, status and percent, and its
537
+ # count, total, message and error where it has them - for a test to compare, or a log line:
538
+ #
539
+ # {name: "Import", status: :done, percent: 100, children: [
540
+ # {name: "Rows", status: :failed, percent: 40, count: 4, total: 10, error: "RuntimeError: row 5"}]}
541
+ #: () -> Hash[Symbol, untyped]
542
+ def outline = TaskProgress.outline(@log.snapshot, @path)
543
+
544
+ # The node at `path` in `state` as an outline, see TaskProgress#outline
545
+ #: (State, path) -> Hash[Symbol, untyped]
546
+ def self.outline(state, path) # :nodoc:
547
+ node = state[path] or raise ArgumentError, "No task at #{path.inspect}"
548
+ error = node.error
549
+ {
550
+ name: node.name, status: node.status, percent: state.percent(path),
551
+ count: (node.count if node.total || node.count.positive?), total: node.total,
552
+ message: node.message, error: error && "#{error[:class]}: #{error[:message]}",
553
+ children: state.children(path).map { outline(state, it) }.then { it unless it.empty? }
554
+ }.compact
555
+ end
556
+
557
+ # Which run of the work this is, for a root taken up with `adopt`: 1, then 2 for a retry, and
558
+ # so on. Drawn with the task from 2 on. Nil for any other task
559
+ #: () -> Integer?
560
+ def attempt = node.attempt
561
+
562
+ # The task as the tree holds it now - frozen, and safe to read from any thread
563
+ #: () -> Node
564
+ def node = @log.snapshot[@path] || raise(ArgumentError, "No task at #{@path.inspect} - it was pruned")
565
+
566
+ # What it is called
567
+ #: () -> String
568
+ def name = node.name
569
+
570
+ # `:pending`, `:active`, `:done`, `:failed` or `:canceled`
571
+ #: () -> Symbol
572
+ def status = node.status
573
+
574
+ # A whole percent, 100 only once it and everything under it are done
575
+ #: () -> Integer
576
+ def percent = @log.snapshot.percent(@path)
577
+
578
+ # How far along, 0.0..1.0, unrounded
579
+ #: () -> Float
580
+ def fraction = @log.snapshot.fraction(@path)
581
+
582
+ # The latest thing it said, see `note`
583
+ #: () -> String?
584
+ def message = node.message
585
+
586
+ # What it failed with: `class`, `message`, `backtrace` and `cause`, see `fail!`
587
+ #: () -> Hash[Symbol, untyped]?
588
+ def error = node.error
589
+
590
+ # How many it has counted, see `advance`
591
+ #: () -> Integer
592
+ def count = node.count
593
+
594
+ # How many it counts to, or nil when it has no total
595
+ #: () -> Integer?
596
+ def total = node.total
597
+
598
+ # Done, and everything under it ended
599
+ #: () -> bool
600
+ def complete? = @log.snapshot.complete?(@path)
601
+
602
+ # Ended by `fail!`
603
+ #: () -> bool
604
+ def failed? = status == :failed
605
+
606
+ # Ended by `cancel!`, or by a task above it being canceled
607
+ #: () -> bool
608
+ def canceled? = status == :canceled
609
+
610
+ # Whether a reader may offer to call it off, and it has not ended yet
611
+ #: () -> bool
612
+ def cancelable? = !!node.cancelable && !finished?
613
+
614
+ # The task at `path` under this one - an Array of ids, or the ids joined with a dash, the way
615
+ # the browser client keys its rows (`x3k0-00af`) - or nil
616
+ #: (path | String) -> TaskProgress?
617
+ def find(path)
618
+ path = path.split("-") if path.is_a?(String)
619
+ full = [*@path, *path]
620
+ TaskProgress.handle(@log, full) if @log.snapshot.key?(full)
621
+ end
622
+
623
+ # Ended, by `done!`, `fail!` or `cancel!` - whether or not everything under it did
624
+ #: () -> bool
625
+ def finished? = node.finished?
626
+
627
+ # The task it is under, or nil for the root
628
+ #: () -> TaskProgress?
629
+ def parent = (TaskProgress.handle(@log, @path.take(@path.length - 1)) unless @path.empty?)
630
+
631
+ # The root of its tree
632
+ #: () -> TaskProgress
633
+ def root = TaskProgress.handle(@log, [])
634
+
635
+ # The tasks directly under it, in the order they opened
636
+ #: () -> Array[TaskProgress]
637
+ def subtasks = @log.snapshot.children(@path).map { TaskProgress.handle(@log, it) }
638
+
639
+ # Sends this task's subtree to an output, as a tree of its own rooted at `[]` - for the root
640
+ # that is the whole tree. See Log#add_output
641
+ #: (?output?, ?replay: bool) ?{ (Batch, Log) -> void } -> output
642
+ def add_output(output = nil, replay: true, &block) = @log.add_output(output, replay:, path: @path, &block)
643
+
644
+ # Stops sending to an output, as it was given to `add_output`
645
+ #: (output) -> void
646
+ def remove_output(output) = @log.remove_output(output)
647
+
648
+ # A child task. `total:` makes it counted (drive it with `advance`), `indeterminate: true`
649
+ # one with no measure of its own, and `weight:` how much of the parent it is. With a block
650
+ # it runs the block as the child's work, see `run`, and returns the block's value.
651
+ #
652
+ # A counted task is done once its count reaches its total. Give it `autofinish: false` when
653
+ # the total is a guess, and the work may go past it.
654
+ #
655
+ # `cancelable: true` tells whoever draws the tree that it may offer to call the task off - see
656
+ # `find` and `cancel!` for taking that request back
657
+ #: (String, ?total: Integer?, ?weight: (Integer | Float)?, ?expect: Integer?, ?indeterminate: bool, ?unit: String?, ?cancelable: bool, ?autofinish: bool) -> TaskProgress
658
+ #: [T] (String, ?total: Integer?, ?weight: (Integer | Float)?, ?expect: Integer?, ?indeterminate: bool, ?unit: String?, ?cancelable: bool, ?autofinish: bool) { (TaskProgress) -> T } -> T?
659
+ def subtask(name, total: nil, weight: nil, expect: nil, indeterminate: false, unit: nil, cancelable: false, autofinish: true, &block)
660
+ child = @log.transaction do |tx|
661
+ writable!(tx)
662
+ path = tx.new_child(@path)
663
+ tx << Event.new(path:, op: :open, value: TaskProgress.attributes(name:, total:, weight:, expect:, indeterminate:, unit:, cancelable:))
664
+ tx.state[path]&.autofinish = autofinish
665
+ TaskProgress.handle(@log, path)
666
+ end
667
+ block ? child.run { block.call(it) } : child
668
+ end
669
+
670
+ # Runs the block as this task's work: done when it returns, failed with the error when it
671
+ # raises - an Interrupt or a SignalException included - and the error raised on. A task
672
+ # canceled ends its work quietly: `run` returns nil. One canceled before it began does not
673
+ # begin
674
+ #: [T] () { (TaskProgress) -> T } -> T?
675
+ def run
676
+ # Called off before it began: there is no work to do
677
+ return nil if canceled?
678
+ result = yield self
679
+ done!
680
+ result
681
+ rescue Canceled => e
682
+ # This task called off - from another thread, say - stops its own work, and nothing above
683
+ # it notices. A task further up called off stops the work in between as well
684
+ return nil if e.task == self
685
+ raise
686
+ rescue Interrupt => e
687
+ cancel!(TaskProgress.interrupted_by(e)) # Ctrl-C calls the work off, it does not fail it
688
+ raise
689
+ rescue Exception => e # rubocop:disable Lint/RescueException
690
+ # Once called off, the work unwinding is no failure - an on_cancel that closed the connection
691
+ # the work was waiting on makes that call raise, and that is the cancel working
692
+ return nil if canceled?
693
+ fail!(e)
694
+ raise
695
+ end
696
+
697
+ # Moves the percent; 100 finishes the task
698
+ #: (Integer | Float, ?message: String?) -> self
699
+ def update(percent, message: nil)
700
+ @log.transaction do |tx|
701
+ writable!(tx)
702
+ tx << Event.new(path: @path, op: :note, value: TaskProgress.utf8(message)) if message
703
+ tx << Event.new(path: @path, op: (percent == 100) ? :done : :set, value: (percent unless percent == 100))
704
+ end
705
+ self
706
+ end
707
+
708
+ # Adds to the count - finishing the task, if that reaches its total
709
+ #: (?Integer, ?message: String?) -> self
710
+ def advance(by = 1, message: nil)
711
+ @log.transaction do |tx|
712
+ writable!(tx)
713
+ node = tx.state[@path] or next
714
+ total = node.total
715
+ next if node.autofinish && total && node.count >= total && node.status == :done # more work than the total said
716
+ tx << Event.new(path: @path, op: :add, value: by)
717
+ tx << Event.new(path: @path, op: :note, value: TaskProgress.utf8(message)) if message
718
+ tx << Event.new(path: @path, op: :done) if node.autofinish && total && node.count >= total
719
+ end
720
+ self
721
+ end
722
+
723
+ # Adds one to the count, see `advance`
724
+ #: (?message: String?) -> self
725
+ def tick(message: nil) = advance(1, message:)
726
+
727
+ # Goes through a collection a step per item. Given a name it opens a subtask for it, and
728
+ # otherwise counts on this task. The total comes from `size`, never `count` - so a lazy
729
+ # enumerator stays lazy, and one with no size just counts. Done when the loop ends, failed
730
+ # with the error if the block raises. Without a block it returns an Enumerator, which does
731
+ # the same as it is consumed
732
+ #
733
+ # root.iterate(files, "Files") { |file| copy(file) }
734
+ # root.iterate(rows, "Rows").map { transform(it) }
735
+ #: (Enumerable[untyped], ?String?, ?total: Integer?, ?step: Integer) ?{ (untyped) -> untyped } -> untyped
736
+ def iterate(collection, name = nil, total: TaskProgress.size_of(collection), step: 1, &block)
737
+ if name
738
+ task = subtask(name, total:, indeterminate: total.nil?)
739
+ else
740
+ task = self
741
+ meta(total:, indeterminate: false) if total && self.total.nil?
742
+ end
743
+ # Advancing costs a transaction, so it goes in steps of a thousandth of the total - what a
744
+ # bar can show anyway - and a loop over cheap items is not held up by its own progress. Each
745
+ # item counts `step`, so a step bigger than a thousandth advances every item
746
+ every = total ? [total / 1000, step].max : step
747
+ enumerator = Enumerator.new do |yielder|
748
+ pending = 0
749
+ task.run do
750
+ collection.each do |item|
751
+ yielder << item
752
+ next if (pending += step) < every
753
+ task.advance(pending)
754
+ pending = 0
755
+ end
756
+ ensure
757
+ task.advance(pending) if pending.positive? && !task.finished?
758
+ end
759
+ end
760
+ block ? enumerator.each(&block) : enumerator
761
+ end
762
+
763
+ # Prints a line above the tree, where it stays. It changes nothing in the tree itself, so a
764
+ # reader joining later never sees it - like anything said on a terminal
765
+ #: (_ToS) -> self
766
+ def puts(message)
767
+ @log.transaction { |tx| writable!(tx) && tx << Event.new(path: @path, op: :say, value: TaskProgress.utf8(message.to_s)) }
768
+ self
769
+ end
770
+
771
+ # The latest thing it says, shown on its row
772
+ #: (String?) -> self
773
+ def note(message)
774
+ @log.transaction { |tx| writable!(tx) && tx << Event.new(path: @path, op: :note, value: TaskProgress.utf8(message)) }
775
+ self
776
+ end
777
+
778
+ # Changes what it counts to
779
+ #: (Integer?) -> void
780
+ def total=(total)
781
+ meta(total:)
782
+ end
783
+
784
+ # Renames it
785
+ #: (String) -> void
786
+ def name=(name)
787
+ meta(name: TaskProgress.utf8(name))
788
+ end
789
+
790
+ # Finishes the task, and with it every task under it still open - each with a `done` of its
791
+ # own, deepest first, in the same batch. The work they were part of is over, and a reader
792
+ # should not be left waiting on a task nobody will ever finish. What was mounted is left to
793
+ # its Mount
794
+ #: (?String?) -> self
795
+ def done!(message = nil)
796
+ @log.transaction do |tx|
797
+ next unless writable!(tx, finishing: true)
798
+ tx << Event.new(path: @path, op: :note, value: TaskProgress.utf8(message)) if message
799
+ tx.state.open_under(@path).reverse_each { tx << Event.new(path: it, op: :done) }
800
+ tx << Event.new(path: @path, op: :done)
801
+ end
802
+ self
803
+ end
804
+
805
+ # Fails the task with an exception, a message, or nothing. `extra` goes along with the error:
806
+ # up to 8 of them, each true, false or an Integer, which is all a reader takes
807
+ #
808
+ # What is still open under it is canceled: it did not fail, it was cut short
809
+ #: (?(Exception | String)?, **bool | Integer) -> self
810
+ def fail!(error = nil, **extra)
811
+ kept = extra.length <= 8 && extra.each_value.all? { it == true || it == false || it.is_a?(Integer) }
812
+ unless kept
813
+ raise ArgumentError, "Up to 8 extras, each true, false or an Integer - nothing else survives the wire, so nothing else is kept here either"
814
+ end
815
+ @log.transaction do |tx|
816
+ next unless writable!(tx, finishing: true)
817
+ error = Wire.error_to_h(error || "#{tx.state[@path]&.name} failed").merge(extra)
818
+ tx.state.open_under(@path, mounts: true).reverse_each { tx << Event.new(path: it, op: :cancel) }
819
+ tx << Event.new(path: @path, op: :fail, value: error)
820
+ end
821
+ self
822
+ end
823
+
824
+ # Calls the task off, and every task still open under it, each with a `cancel` of its own -
825
+ # from any thread. Whoever works on one of them next gets a Canceled, which `run` takes as
826
+ # the end of that task's work. Canceling a finished task does nothing
827
+ #: (?String?) -> self
828
+ def cancel!(reason = nil)
829
+ @log.transaction { |tx| writable!(tx, finishing: true) && TaskProgress.cancel_in(tx, @path, reason) }
830
+ self
831
+ end
832
+
833
+ # Calls the block with the reason once this task is canceled, see Log#on_cancel. For work that
834
+ # blocks and so would not notice a cancel until it next touches its task:
835
+ #
836
+ # rows.on_cancel { http.finish }
837
+ #: () { (String?) -> void } -> self
838
+ def on_cancel(&block)
839
+ @log.on_cancel(@path, &block)
840
+ self
841
+ end
842
+
843
+ # Asks for the task at `path` under this one to be canceled - the way a request from outside
844
+ # (a browser, a parent process) comes in. Everything is decided in one transaction against the
845
+ # tree as it stands, so it cannot race the task finishing. Returns:
846
+ #
847
+ # :canceled it was, along with everything still open under it
848
+ # :forwarded it lives in a mounted child, which was asked; its cancel comes back up the stream
849
+ # :moot there is nothing left to cancel - no such task, or it has already ended
850
+ # :refused it is not cancelable
851
+ #
852
+ # A request that comes too late is moot, not an error: by the time it arrives the task may well
853
+ # have finished, and its process exited
854
+ #: (path | String, ?String?) -> Symbol
855
+ def request_cancel(path, reason = nil)
856
+ path = path.split("-") if path.is_a?(String)
857
+ full = [*@path, *path].freeze
858
+ outcome, mount, remote_path = @log.transaction do |tx|
859
+ node = tx.state[full]
860
+ next [:moot, nil, nil] if node.nil? || node.finished?
861
+ next [:refused, nil, nil] unless node.cancelable
862
+ mounted_at = (full.length - 1).downto(0).map { full.take(it) }.find { tx.state[it]&.sealed }
863
+ mount = tx.state[mounted_at]&.sealed if mounted_at
864
+ next [:forwarded, mount, full.drop(mounted_at.length)] if mount && mounted_at
865
+ TaskProgress.cancel_in(tx, full, reason)
866
+ [:canceled, nil, nil]
867
+ end #: [Symbol, Mount?, path?]
868
+ return mount.request_cancel(remote_path, reason) if mount && remote_path
869
+ outcome
870
+ end
871
+
872
+ # Reads cancel requests off an IO - one JSON line each, `{"cancel": [ids], "reason": "..."}` -
873
+ # on a thread of its own, and hands them to `request_cancel`. This is the far end of a Mount's
874
+ # control pipe; anything malformed is skipped. Returns the thread, which ends at EOF.
875
+ #
876
+ # The root itself, `[]`, is what the parent mounted, and only the parent asks for it: the
877
+ # mount was ended there, so the root is canceled whether it is cancelable or not
878
+ #: (IO) -> Thread
879
+ def accept_cancel_requests(io)
880
+ Thread.new do
881
+ Thread.current.report_on_exception = false
882
+ io.each_line("\n", Wire::TEXT_LIMIT) { take_cancel_request(it) }
883
+ end
884
+ end
885
+
886
+ # Takes one cancel request - a line as `accept_cancel_requests` reads them - and returns what
887
+ # came of it, see `request_cancel`; nil for a line that is not a request
888
+ #: (String) -> Symbol?
889
+ def take_cancel_request(line) # :nodoc:
890
+ require "json"
891
+ request = begin
892
+ JSON.parse(line)
893
+ rescue
894
+ return
895
+ end
896
+ return unless request.is_a?(Hash) && (path = request["cancel"]).is_a?(Array) && path.length <= Wire::MAX_DEPTH
897
+ return unless path.all? { it.is_a?(String) && it.match?(Wire::SEGMENT) }
898
+ return request_cancel(path, Wire.text(request["reason"])) unless path.empty?
899
+ cancel!(Wire.text(request["reason"]))
900
+ :canceled
901
+ rescue ArgumentError
902
+ nil
903
+ end
904
+
905
+ # The reason an Interrupt gives, if it gives one: Ctrl-C raises it saying "Interrupt", and a
906
+ # SIGINT from outside saying nothing at all
907
+ #: (Interrupt) -> String?
908
+ def self.interrupted_by(interrupt) = (interrupt.message unless ["", "Interrupt"].include?(interrupt.message)) # :nodoc:
909
+
910
+ # Every task called off carries the reason, so each of them can say why it stopped
911
+ #: (Transaction, path, String?) -> void
912
+ def self.cancel_in(tx, path, reason) # :nodoc:
913
+ reason = (utf8(reason.to_s) if reason)
914
+ tx.state.open_under(path, mounts: true).reverse_each { tx << Event.new(path: it, op: :cancel, value: reason) }
915
+ tx << Event.new(path:, op: :cancel, value: reason)
916
+ end
917
+
918
+ # Everything the block does to the tree lands as one batch, or none of it does
919
+ #: [T] () { (TaskProgress) -> T } -> T
920
+ def transaction
921
+ @log.transaction { yield self }
922
+ end
923
+
924
+ # A child whose subtree is fed from elsewhere - a subprocess, a Ractor, a stream. Only the
925
+ # Mount writes under it, see Mount
926
+ #: (String, ?weight: (Integer | Float)?, ?cancelable: bool) -> Mount
927
+ def mount(name, weight: nil, cancelable: false)
928
+ path = @log.transaction do |tx|
929
+ writable!(tx)
930
+ path = tx.new_child(@path)
931
+ tx << Event.new(path:, op: :open, value: TaskProgress.attributes(name:, weight:, cancelable:))
932
+ # The node holds its Mount, so a cancel request for a task under it can be passed on
933
+ mount = Mount.new(@log, path)
934
+ tx.state[path]&.sealed = mount
935
+ mount
936
+ end
937
+ end
938
+
939
+ # Wraps an IO so that reading it advances this task by the bytes read, see TrackedIO. The total
940
+ # is the IO's size unless given; with neither, the task counts bytes with no measure. With a
941
+ # block, the block runs with the wrapped IO as this task's work, see `run`. Needs the
942
+ # progressive_io gem, and an IO that can tell its position - for a pipe or a socket, call
943
+ # `advance` with each chunk instead
944
+ #: (untyped, ?total: Integer?, ?step: Integer?) -> TrackedIO
945
+ #: [T] (untyped, ?total: Integer?, ?step: Integer?) { (TrackedIO) -> T } -> T?
946
+ def track_io(io, total: nil, step: nil)
947
+ raise ArgumentError, "#{io.class} cannot tell its position, so reading it cannot be tracked - call advance with each chunk instead" unless positioned?(io)
948
+ total ||= io.size if io.respond_to?(:size)
949
+ meta(total:, unit: "bytes", indeterminate: total.nil?)
950
+ tracked = TrackedIO.new(io, self, total:, step:)
951
+ return tracked unless block_given?
952
+ run { yield(tracked).tap { tracked.flush_progress } } # steep:ignore
953
+ end
954
+
955
+ # The same task in the same tree
956
+ #: (untyped) -> bool
957
+ def ==(other) = other.is_a?(TaskProgress) && other.log.equal?(@log) && other.path == @path
958
+ alias_method :eql?, :==
959
+
960
+ # Equal handles hash alike, so a task works as a Hash key
961
+ #: () -> Integer
962
+ def hash = [@log.object_id, @path].hash
963
+
964
+ # Its path, name and status
965
+ #: () -> String
966
+ def inspect = "#<TaskProgress #{@path.inspect} #{@log.snapshot[@path]&.name.inspect} #{@log.snapshot[@path]&.status}>"
967
+
968
+ SCALE = [["TB", 1024**4], ["GB", 1024**3], ["MB", 1024**2], ["KB", 1024]].map(&:freeze).freeze #: Array[[String, Integer]]
969
+ private_constant :SCALE
970
+
971
+ # "4.2s", "3m07s", "2h15m"
972
+ #: (Integer | Float) -> String
973
+ def self.duration(seconds) # :nodoc:
974
+ if seconds < 60 then format("%.1fs", seconds)
975
+ elsif seconds < 3600 then format("%dm%02ds", seconds / 60, seconds % 60)
976
+ else format("%dh%02dm", seconds / 3600, seconds % 3600 / 60)
977
+ end
978
+ end
979
+
980
+ # "142 KB", "2.9 MB", "640 bytes" - a whole number drops its decimal
981
+ #: (Integer | Float) -> String
982
+ def self.human_bytes(n) # :nodoc:
983
+ n = n.to_i
984
+ found = SCALE.find { |_, size| n >= size } or return "#{n} bytes"
985
+ unit, size = found
986
+ value = (n.to_f / size).round(1)
987
+ number = (value % 1).zero? ? value.to_i.to_s : format("%.1f", value)
988
+ "#{number} #{unit}"
989
+ end
990
+
991
+ # Every message reaches the tree as valid UTF-8. Messages often name files, and a raw
992
+ # ZIP read hands those back tagged ASCII-8BIT, so a "Reading Alice’s.xhtml" would
993
+ # otherwise blow up whatever an output writes to (a database column, a UTF-8 log line)
994
+ #: (nil) -> nil
995
+ #: (untyped) -> String
996
+ def self.utf8(message) # :nodoc:
997
+ return if message.nil?
998
+ message = message.to_s
999
+ return message if message.encoding == Encoding::UTF_8 && message.valid_encoding?
1000
+ message.dup.force_encoding(Encoding::UTF_8).scrub("�")
1001
+ end
1002
+
1003
+ private
1004
+
1005
+ #: (**untyped) -> self
1006
+ def meta(**attributes)
1007
+ @log.transaction { |tx| writable!(tx) && tx << Event.new(path: @path, op: :meta, value: attributes) }
1008
+ self
1009
+ end
1010
+
1011
+ # Finishing a finished task is a no-op; anything else on one raises, as does writing under
1012
+ # a mount from here
1013
+ #: (Transaction, ?finishing: bool) -> bool
1014
+ def writable!(tx, finishing: false)
1015
+ node = tx.state[@path] or raise ArgumentError, "No task at #{@path.inspect}"
1016
+ # Under a mount only the Mount writes - but the mount's own task is the parent's to end
1017
+ if sealed?(tx.state) && !(finishing && node.sealed && !sealed?(tx.state, above: true))
1018
+ raise ArgumentError, "Task '#{node.name}' is mounted from elsewhere - write to it through its Mount"
1019
+ end
1020
+ raise Canceled.new(self, node.name) if node.status == :canceled && !finishing
1021
+ !(finishing && node.finished?)
1022
+ end
1023
+
1024
+ #: (State, ?above: bool) -> bool
1025
+ def sealed?(state, above: false) = (0..(above ? @path.length - 1 : @path.length)).any? { state[@path.take(it)]&.sealed }
1026
+
1027
+ #: (untyped) -> bool
1028
+ def positioned?(io)
1029
+ io.pos
1030
+ true
1031
+ rescue NoMethodError, Errno::ESPIPE, IOError
1032
+ false
1033
+ end
1034
+ end
1035
+
1036
+ require_relative "task_progress/railtie" if defined?(::Rails::Railtie)