task_progress 0.0.1

data/js/task_tree.js

@@ -0,0 +1,444 @@
+// The browser half of TaskProgress: folds batches into a tree the way TaskProgress::State
+// does, and works out percents by the same rules. Framework-free - the Stimulus and React
+// demos both sit on top of it.
+//
+//   followEventSourceAt(url, onChange)   a TaskProgress::SSE stream, drawn at most once a frame
+//   followPollingAt(url, onChange)       a TaskProgress::Poll endpoint, for where SSE does not get through
+//   followProgress(progress, onChange)   what TaskProgress.for_page hands a page - SSE, or polling if that fails
+//   batchFeed(onChange)              the same, for batches you get some other way
+//   followLocal(localTasks, onChange)    tasks running in this browser, on their own - uploads, say
+//   tree.graft(path, localTasks)         draws tasks running in this browser under a server task
+
+import { RenderQueue } from "./render_queue.js"
+import { pathKey as key } from "./paths.js"
+
+export { pathKey, pathFromKey } from "./paths.js"
+const parentKey = (path) => key(path.slice(0, -1))
+
+const blankNode = (path, at) => ({
+  path, name: "", total: null, count: 0, percent: 0, weight: 1, expect: null, indeterminate: false, cancelable: false, attempt: null,
+  unit: null, message: null, status: "pending", error: null, stale: false,
+  opened_at: at, started_at: null, finished_at: null, children: []
+})
+
+// The wire format's version, as TaskProgress::Wire::VERSION - a batch from a newer writer is
+// left alone rather than half understood
+export const WIRE_VERSION = 1
+
+// Siblings past FOLD_OVER are folded by TaskTree#rows, keeping FOLD_KEEP of the done and of the
+// waiting ones - as TaskProgress::State#rows does
+export const FOLD_OVER = 12
+export const FOLD_KEEP = 3
+
+const TIMES = ["opened_at", "started_at", "finished_at"]
+const shifted = (value, shift) => {
+  if (!shift) return value
+  const moved = { ...value }
+  for (const field of TIMES) if (moved[field] != null) moved[field] -= shift
+  return moved
+}
+
+// The least time between two long polls, in milliseconds
+const LONG_POLL_GAP = 250
+
+const FINISHED = new Set(["done", "failed", "canceled"])
+const ENDED_WITHOUT_DONE = new Set(["failed", "canceled"])
+const ATTRIBUTES = ["name", "total", "weight", "expect", "indeterminate", "unit", "cancelable", "attempt"]
+const FIELDS = [...ATTRIBUTES, "count", "percent", "message", "status", "error", "stale", "opened_at", "started_at", "finished_at"]
+
+export class TaskTree {
+  // `clock` gives seconds since the epoch, and only matters for elapsed times
+  constructor({ clock = () => Date.now() / 1000 } = {}) {
+    this.nodes = new Map()
+    this.log = null
+    this.seq = null
+    this.stale = false
+    this.clock = clock
+    this.offset = null
+    this.grafts = []
+    this.onLocalChange = null // set by a RenderQueue, to draw what a graft changes
+    this.local = null // the LocalTasks this whole tree mirrors, see followLocal
+  }
+
+  // Now, by the clock of whoever wrote the tree - so an elapsed time is never off by the skew
+  // between that machine and this one
+  now() { return this.clock() - (this.offset ?? 0) }
+
+  // Returns "applied", "duplicate" for a batch it already has, or "gap" when the batch does not follow the last one - the tree is then
+  // stale, and stays so until a batch with prev 0 (a fresh squash) replaces it. A batch in
+  // another version of the format is "unsupported", and changes nothing
+  apply(batch) {
+    if (batch.v !== WIRE_VERSION) return "unsupported"
+    if (batch.prev === 0) {
+      this.nodes = new Map()
+    } else if (batch.log === this.log && batch.seq <= this.seq) {
+      return "duplicate"
+    } else if (batch.log !== this.log || batch.prev !== this.seq) {
+      this.stale = true
+      return "gap"
+    }
+    for (const [path, op, value] of batch.ops) this.#applyOp(path, op, value, batch.at)
+    this.#measureOffset(batch)
+    this.log = batch.log
+    this.seq = batch.seq
+    this.stale = false
+    for (const graft of this.grafts) this.#tend(graft)
+    return "applied"
+  }
+
+  // Draws tasks running in this browser - a LocalTasks - as a child of the server task at
+  // `path`. Only this tree sees them. The graft is kept apart from what the server sends, so a
+  // reset or a reconnect, which starts the server's tree over, puts it back; and it waits for
+  // its task if that has not arrived yet. Returns a function that takes it off again.
+  //
+  // A server task with browser work still open under it is not complete - it reads 99% and
+  // is drawn as running even once the server has called it done. A server task that fails or
+  // is canceled cancels the browser work under it
+  graft(path, local) {
+    const graft = { anchor: [...path], local, id: local.log }
+    this.grafts.push(graft)
+    const unsubscribe = local.subscribe((batch) => this.#graftOps(graft, batch.ops, batch.at))
+    this.#tend(graft)
+    return () => {
+      unsubscribe()
+      this.grafts = this.grafts.filter((g) => g !== graft)
+      const top = [...graft.anchor, graft.id]
+      if (!this.get(top)) return
+      const prefix = key(top)
+      for (const k of [...this.nodes.keys()]) if (k === prefix || k.startsWith(prefix + "-")) this.nodes.delete(k)
+      const anchor = this.get(graft.anchor)
+      if (anchor) anchor.children = anchor.children.filter((id) => id !== graft.id)
+      this.onLocalChange?.([[top, "prune"]])
+    }
+  }
+
+  // Cancels a task running in this browser, by its path in this tree. Server tasks are not
+  // this tree's to end - see requestCancel
+  cancelLocal(path, reason) {
+    if (this.local) return this.local.at(path)?.cancel(reason)
+    const graft = this.grafts.find((g) => key([...g.anchor, g.id]) === key(path.slice(0, g.anchor.length + 1)))
+    graft?.local.at(path.slice(graft.anchor.length + 1))?.cancel(reason)
+  }
+
+  // Puts a graft back if the server's tree lost it, and cancels it if its server task ended badly
+  #tend(graft) {
+    const anchor = this.get(graft.anchor)
+    if (!anchor) return
+    if (!this.get([...graft.anchor, graft.id])) {
+      const squash = graft.local.squash()
+      this.#graftOps(graft, squash.ops, squash.at)
+    }
+    if (ENDED_WITHOUT_DONE.has(anchor.status) && !graft.local.root.finished) graft.local.root.cancel(anchor.message ?? `${anchor.name} ${anchor.status}`)
+  }
+
+  #graftOps(graft, ops, at) {
+    if (!this.get(graft.anchor)) return
+    const top = [...graft.anchor, graft.id]
+    // Lost to a reset or a prune since: the local tree already holds this change, so put it back whole
+    if (!this.get(top) && !(ops[0]?.[0].length === 0 && (ops[0][1] === "open" || ops[0][1] === "put"))) {
+      const squash = graft.local.squash()
+      return this.#graftOps(graft, squash.ops, squash.at)
+    }
+    // Times here are on this browser's clock; the tree keeps the server's, so they move over
+    const shift = this.offset ?? 0
+    const moved = ops.map(([path, op, value]) => [[...top, ...path], op, op === "put" ? shifted(value, shift) : value])
+    for (const [path, op, value] of moved) this.#applyOp(path, op, value, at - shift)
+    this.onLocalChange?.(moved)
+  }
+
+  get(path) { return this.nodes.get(key(path)) }
+
+  get root() { return this.nodes.get("") }
+
+  childrenOf(node) { return node.children.map((id) => this.nodes.get(key([...node.path, id]))) }
+
+  // Pre-order, children by position
+  *each(node = this.root) {
+    if (!node) return
+    yield node
+    for (const child of this.childrenOf(node)) yield* this.each(child)
+  }
+
+  // What to draw, in the order of `each` - but where a task has more than `foldOver` children, the
+  // ones that are done (save the last `keep` to finish) are one row ahead of the rest, and the
+  // ones still waiting (save the first `keep`) one row after them, see TaskProgress::State#rows.
+  // A folded row is { fold: true, path, status, count }, its path its parent's and `~done` or
+  // `~pending`, which no task id can be
+  *rows(node = this.root, { foldOver = FOLD_OVER, keep = FOLD_KEEP } = {}) {
+    if (!node) return
+    yield node
+    const kids = this.childrenOf(node)
+    const options = { foldOver, keep }
+    if (kids.length <= foldOver) {
+      for (const kid of kids) yield* this.rows(kid, options)
+      return
+    }
+    const done = kids.filter((kid) => this.complete(kid))
+    const kept = new Set([...done].sort((a, b) => (b.finished_at ?? 0) - (a.finished_at ?? 0)).slice(0, keep))
+    const pending = kids.filter((kid) => kid.status === "pending")
+    const folded = new Set([...done.filter((kid) => !kept.has(kid)), ...pending.slice(keep)])
+    if (done.length > kept.size) yield { fold: true, path: [...node.path, "~done"], status: "done", count: done.length - kept.size }
+    for (const kid of kids) if (!folded.has(kid)) yield* this.rows(kid, options)
+    if (pending.length > keep) yield { fold: true, path: [...node.path, "~pending"], status: "pending", count: pending.length - keep }
+  }
+
+  // Done, and everything under it ended - see TaskProgress::State#complete?
+  complete(node) {
+    if (node?.status !== "done") return false
+    return this.childrenOf(node).every((child) => ENDED_WITHOUT_DONE.has(child.status) || this.complete(child))
+  }
+
+  fraction(node) {
+    if (!node) return 0
+    if (node.status === "done") return 1
+    const missing = this.#missing(node)
+    if (node.children.length > 0 || missing > 0) {
+      let weights = missing
+      let sum = 0
+      for (const child of this.childrenOf(node)) {
+        weights += child.weight
+        sum += child.weight * this.fraction(child)
+      }
+      return weights === 0 ? 0 : sum / weights
+    }
+    if (node.total != null) return node.total === 0 ? 0 : Math.min(Math.max(node.count / node.total, 0), 1)
+    if (node.indeterminate) return 0
+    return node.percent / 100
+  }
+
+  // A whole percent that reaches 100 only once everything under the node is done. Math.round
+  // rounds .5 up like Ruby's round does for the positive numbers we have here
+  percentOf(node) {
+    if (this.complete(node)) return 100
+    return Math.min(Math.round(this.fraction(node) * 100), 99)
+  }
+
+  // Each batch says what the writer's clock read when it committed. The smallest gap to ours has
+  // the least time in transit in it, so that is the one kept
+  #measureOffset(batch) {
+    if (!batch.at) return
+    const offset = this.clock() - batch.at
+    this.offset = batch.log !== this.log || this.offset === null ? offset : Math.min(this.offset, offset)
+  }
+
+  #missing(node) { return Math.max((node.expect ?? 0) - node.children.length, 0) }
+
+  #start(path, at) {
+    for (let length = path.length; length >= 0; length--) {
+      const node = this.nodes.get(key(path.slice(0, length)))
+      if (node.status === "pending") node.status = "active"
+      node.started_at ??= at
+    }
+  }
+
+  #adopt(path) {
+    if (path.length === 0) return
+    const parent = this.nodes.get(parentKey(path))
+    const id = path[path.length - 1]
+    if (!parent.children.includes(id)) parent.children.push(id)
+  }
+
+  #applyOp(path, op, value, at) {
+    const node = this.nodes.get(key(path))
+    switch (op) {
+      case "open": {
+        const fresh = blankNode(path, at)
+        for (const field of ATTRIBUTES) if (value[field] !== undefined) fresh[field] = value[field]
+        this.nodes.set(key(path), fresh)
+        this.#adopt(path)
+        break
+      }
+      case "meta":
+        for (const field of [...ATTRIBUTES, "stale"]) if (value[field] !== undefined) node[field] = value[field]
+        break
+      case "set": this.#start(path, at); node.percent = value; break
+      case "add": this.#start(path, at); node.count = Math.max(node.count + value, 0); break
+      case "note": this.#start(path, at); node.message = value ?? null; break
+      case "say": break // a line for a log pane - nothing in the tree changes
+      case "done":
+      case "fail":
+        this.#start(path, at)
+        node.finished_at = at
+        node.status = op === "done" ? "done" : "failed"
+        if (op === "fail") node.error = value
+        break
+      case "cancel": // calling off what never began does not begin it
+        node.finished_at = at
+        node.status = "canceled"
+        if (value != null) node.message = value // an empty reason too, as State does
+        break
+      case "prune": {
+        if (!node) break
+        const prefix = key(path)
+        for (const k of [...this.nodes.keys()]) {
+          if (k !== prefix && (prefix === "" || k.startsWith(prefix + "-"))) this.nodes.delete(k)
+        }
+        node.children = []
+        break
+      }
+      case "put": {
+        const fresh = blankNode(path, at)
+        for (const field of FIELDS) if (value[field] !== undefined) fresh[field] = value[field]
+        fresh.children = node ? node.children : []
+        this.nodes.set(key(path), fresh)
+        if (!node) this.#adopt(path)
+        break
+      }
+    }
+  }
+}
+
+// Nothing more will come: the root has finished - or the tree is empty after a batch, its task
+// pruned away before anyone asked
+const ended = (tree) => tree.seq != null && (!tree.root || FINISHED.has(tree.root.status))
+
+// Folds batches pushed into it from anywhere - a WebSocket, ActionCable, a poller - and calls
+// onChange(tree, changes) at most once a frame with what they changed, see render_queue.js.
+// `push` returns what the tree made of the batch: on a "gap" it is for the caller to start
+// over, by fetching a fresh squash (a batch with prev 0)
+// `initial` is a batch to start from - the snapshot a page was rendered with, say - so the tree
+// is drawn before anything has come in
+export function batchFeed(onChange, { schedule, clock, initial } = {}) {
+  const tree = new TaskTree(clock ? { clock } : {})
+  const queue = new RenderQueue(tree, onChange, { schedule })
+  if (initial) queue.apply(initial)
+  return { tree, push: (batch) => queue.apply(batch), close: () => queue.close() }
+}
+
+// Follows a TaskProgress::SSE stream at `url`, see batchFeed. On a gap it reconnects, and
+// the server starts it over from a fresh squash. It stops once the root has finished
+//
+// `onUnreachable` is called, once, when nothing has come within `unreachableAfter` seconds of
+// connecting, or the connection failed before anything did - a proxy that buffers, a host that
+// cuts long responses. followProgress falls back to polling on it
+export function followEventSourceAt(url, onChange, { onUnreachable, unreachableAfter = 5, ...options } = {}) {
+  const following = batchFeed(onChange, options)
+  const { tree } = following
+  let source
+  let heard = false
+  let timer = null
+  const unreachable = () => {
+    if (heard || !onUnreachable) return
+    heard = true // once
+    clearTimeout(timer)
+    onUnreachable()
+  }
+  const connect = () => {
+    source = new EventSource(url)
+    source.addEventListener("batch", (event) => {
+      heard = true
+      clearTimeout(timer)
+      if (following.push(JSON.parse(event.data)) === "gap") {
+        source.close()
+        connect()
+      }
+      if (ended(tree)) source.close()
+    })
+    source.addEventListener("error", unreachable)
+  }
+  connect()
+  if (onUnreachable) timer = setTimeout(unreachable, unreachableAfter * 1000)
+  return { tree, close: () => { clearTimeout(timer); source.close(); following.close() } }
+}
+
+// Follows what TaskProgress.for_page hands a page - { events, poll, cancel, snapshot } - drawn
+// from its snapshot at once. `transport` is "sse", "poll", or "auto": SSE, falling back to
+// polling (every `poll` seconds, or long polling with `wait`) when the stream cannot be reached. `tree` is always the one being followed now
+// Draws tasks that run in this browser on their own, with no server tree around them - uploads,
+// say. Every row is local: its cancel button cancels the task here
+export function followLocal(local, onChange, options = {}) {
+  const feed = batchFeed(onChange, { ...options, initial: local.squash() })
+  feed.tree.local = local
+  const unsubscribe = local.subscribe((batch) => feed.push(batch))
+  return { tree: feed.tree, close: () => { unsubscribe(); feed.close() } }
+}
+
+export function followProgress(progress, onChange, { transport = "auto", poll = 1, wait = 0, ...options } = {}) {
+  const initial = progress.snapshot
+  const polling = (from) => followPollingAt(progress.poll, onChange, { poll, wait, ...options, initial: from })
+  if (transport === "poll" || !progress.events) return polling(initial)
+  let current = followEventSourceAt(progress.events, onChange, {
+    ...options,
+    initial,
+    onUnreachable: transport === "auto" ? () => {
+      current.close()
+      current = polling(initial)
+    } : undefined
+  })
+  return {
+    get tree() { return current.tree },
+    close: () => current.close()
+  }
+}
+
+// Polls a TaskProgress::Poll endpoint at `url`, see batchFeed - for where an EventSource does
+// not get through. Every answer is the whole tree, and its ETag goes back in If-None-Match, so a
+// tree that has not changed costs a 304. Polls every `poll` seconds; with `wait` (seconds too)
+// the server holds each poll until something changes, up to that long, and the next one goes
+// out at once - long polling. A poll that fails is tried again later and later, up to
+// half a minute apart. It stops once the root has finished
+export function followPollingAt(url, onChange, { poll: every = 1, wait = 0, ...options } = {}) {
+  const interval = every * 1000
+  const following = batchFeed(onChange, options)
+  const { tree } = following
+  const target = wait ? `${url}${url.includes("?") ? "&" : "?"}wait=${wait}` : url
+  let etag = null
+  let stopped = false
+  let timer = null
+  let controller = null
+  let delay = interval
+  const poll = async () => {
+    controller = new AbortController()
+    try {
+      const response = await fetch(target, { headers: etag ? { "if-none-match": etag } : {}, cache: "no-store", signal: controller.signal })
+      if (response.status === 200) {
+        const batch = await response.json()
+        etag = response.headers.get("etag")
+        following.push(batch)
+      } else if (response.status !== 304) {
+        throw new Error(`Polling ${url} got a ${response.status}`)
+      }
+      delay = interval
+    } catch {
+      if (stopped) return
+      delay = Math.min(Math.max(delay, interval) * 2, 30000)
+    }
+    if (stopped || ended(tree)) return
+    // Long polling goes straight back - but not faster than this, however busy the tree
+    timer = setTimeout(poll, wait && delay === interval ? LONG_POLL_GAP : delay)
+  }
+  poll()
+  return {
+    tree,
+    close: () => {
+      stopped = true
+      clearTimeout(timer)
+      controller?.abort()
+      following.close()
+    }
+  }
+}
+
+// Asks the server to cancel a task, by creating a cancellation: posts JSON with the task's path
+// (its pathKey) and the token the server issued for this tree. The server decides -
+// a task is only ever ended by its own tree - so nothing changes here until the `cancel` comes
+// back down the stream. A request that comes too late is not an error: the server takes it all
+// the same, and the row has already finished anyway
+//
+// On a Rails page it sends the page's CSRF token along, so `protect_from_forgery` lets it through
+export function requestCancel(url, path, { token } = {}) {
+  const csrf = globalThis.document?.querySelector('meta[name="csrf-token"]')?.content
+  return fetch(url, {
+    method: "POST",
+    headers: { "content-type": "application/json", ...(csrf ? { "x-csrf-token": csrf } : {}) },
+    body: JSON.stringify({ path: key(path), token })
+  })
+}
+
+export function humanBytes(n) {
+  const scale = [["TB", 1024 ** 4], ["GB", 1024 ** 3], ["MB", 1024 ** 2], ["KB", 1024]]
+  const found = scale.find(([, size]) => n >= size)
+  if (!found) return `${n} bytes`
+  const value = Math.round((n / found[1]) * 10) / 10
+  return `${Number.isInteger(value) ? value : value.toFixed(1)} ${found[0]}`
+}

data/lib/task_progress/assertions.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "pp"
+
+class TaskProgress
+  # Assertions for Minitest - and ActiveSupport::TestCase, which is Minitest underneath - about
+  # what work reported. Each failure shows the whole tree, see TaskProgress#outline.
+  #
+  #     class ImportTest < Minitest::Test
+  #       include TaskProgress::Assertions
+  #
+  #       def test_import
+  #         import = TaskProgress.new("Import")
+  #         Importer.new(progress: import).run
+  #         assert_task_complete import
+  #         assert_task_failed import.subtasks.first, with: /row 12/
+  #       end
+  #     end
+  module Assertions
+    # @rbs!
+    #   def assert: (untyped, ?String) -> void
+
+    # The task is done, and everything under it ended
+    #: (TaskProgress, ?String?) -> void
+    def assert_task_complete(task, message = nil)
+      assert task.complete?, message || "Expected '#{task.name}' to be complete, but it is #{task.status}#{TaskProgress::Assertions.tree(task)}"
+    end
+
+    # The task failed - with an error whose message matches `with`, if given
+    #: (TaskProgress, ?with: (String | Regexp)?, ?message: String?) -> void
+    def assert_task_failed(task, with: nil, message: nil)
+      error = task.error
+      matched = with.nil? || (with.is_a?(Regexp) ? with.match?(error&.dig(:message).to_s) : error&.dig(:message) == with)
+      assert task.failed? && matched,
+        message || "Expected '#{task.name}' to have failed#{" with #{with.inspect}" if with}, but it is #{task.status}#{" with #{error[:message].inspect}" if error}#{TaskProgress::Assertions.tree(task)}"
+    end
+
+    # The task was canceled
+    #: (TaskProgress, ?String?) -> void
+    def assert_task_canceled(task, message = nil)
+      assert task.canceled?, message || "Expected '#{task.name}' to have been canceled, but it is #{task.status}#{TaskProgress::Assertions.tree(task)}"
+    end
+
+    # The tree, as a failure message ends with it
+    #: (TaskProgress) -> String
+    def self.tree(task) = "\n\n#{PP.pp(task.root.outline, +"", 100)}"
+  end
+end

data/lib/task_progress/batch.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+class TaskProgress
+  # The unit of commit, see below
+  Batch = Data.define(
+    :log, #: String?
+    :prev, #: Integer
+    :seq, #: Integer
+    :at, #: Float
+    :events #: Array[Event]
+  )
+
+  # The unit of commit: everything one transaction changed, in order. `seq` counts batches,
+  # not events, and `prev` names the batch it follows - so a reader that sees a `prev` other
+  # than the `seq` it last saw knows it missed something. `prev` 0 means "from the start",
+  # which is also how a squashed replay announces itself
+  class Batch
+    # Whether it starts over from nothing, as a squash does
+    #: () -> bool
+    def reset? = prev.zero?
+  end
+end

data/lib/task_progress/branch.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+class TaskProgress
+  # An output for one subtree, which it sees as a tree of its own: the branch's node becomes the
+  # root, `[*branch, "x3k0"]` becomes `["x3k0"]`, and batches that do not touch the branch are left
+  # out. That is a Mount run backwards - a Mount fed from a Branch grafts the subtree back whole.
+  #
+  # Since batches get left out, a branch numbers its own: `log` is the log's id with the branch
+  # path appended, and its `prev`/`seq` follow on from each other just like the log's do.
+  #
+  # Pruning an ancestor takes the branch's node away. Usually for good, since a pruned path is
+  # never given out again - and the branch then sends one last reset with nothing in it, and
+  # falls silent. But a Mount starting over (a remote that restarted, a stream resynced) prunes
+  # its subtree and puts the same tasks back in the same batch: the branch's node comes back,
+  # and the branch starts over with it.
+  #
+  # Its first batch always starts from nothing, `prev` 0: one for a node that comes into being
+  # after the branch was made carries the node's `open`
+  class Branch # :nodoc:
+    # The output the branch feeds
+    attr_reader :output #: output
+    # The branch's task, in the whole tree
+    attr_reader :path #: path
+    # The log id its batches carry: the log's, with the path appended
+    attr_reader :id #: String
+
+    # @rbs @seq: Integer
+    # @rbs @gone: bool
+    # @rbs @started: bool
+
+    # The log id of the branch at `path`
+    #: (String, path) -> String
+    def self.id(log_id, path) = [log_id, *path].join("-")
+
+    # For an output, the branch at `path` of the log `log_id`, numbering on from `seq`
+    #: (output, path, String, Integer, ?started: bool) -> void
+    def initialize(output, path, log_id, seq, started: true)
+      @output = output
+      @path = path
+      @id = Branch.id(log_id, path)
+      @seq = seq
+      @gone = false
+      @started = started
+    end
+
+    # Passes on what of the batch is in the branch, moved to its root
+    #: (Batch, Log) -> void
+    def call(batch, log)
+      return if @gone
+      reset = batch.reset? || !@started
+      pruned = false
+      events = [] #: Array[Event]
+      batch.events.each do |event|
+        if event.op == :prune && event.path.length < @path.length && @path[0, event.path.length] == event.path
+          pruned = reset = true
+          events.clear
+        elsif event.path[0, @path.length] == @path
+          # Put back after the prune - the branch starts over from it
+          pruned = false if event.path.length == @path.length && (event.op == :put || event.op == :open)
+          events << Event.new(path: event.path.drop(@path.length).freeze, op: event.op, value: event.value)
+        end
+      end
+      if pruned
+        @gone = true
+        events.clear
+      end
+      return if events.empty? && !reset
+      @started = true
+      @seq += 1
+      @output.call(Batch.new(log: @id, prev: reset ? 0 : @seq - 1, seq: @seq, at: batch.at, events: events.freeze), log)
+    end
+  end
+end

data/lib/task_progress/buffered_output.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+class TaskProgress
+  # Puts a thread between the log and a slow output. The first batch goes through at once,
+  # and whatever piles up while the output works (or during `interval` after it) goes out
+  # squashed into one - so a database write a few times a second keeps up with any number of
+  # workers, and never holds one of them up
+  class BufferedOutput
+    # @rbs @output: output
+    # @rbs @interval: Integer | Float
+    # @rbs @queue: Thread::Queue[Batch]
+    # @rbs @log: Log?
+    # @rbs @squasher: Squasher
+    # @rbs @thread: Thread
+
+    # For the slow output, sending to it at most every `interval` seconds
+    #: (output, ?interval: Integer | Float) -> void
+    def initialize(output, interval: 0.25)
+      @output = output
+      @interval = interval
+      @queue = Thread::Queue.new
+      @log = nil
+      @squasher = Squasher.new
+      @thread = Thread.new { run }
+    end
+
+    # Takes one batch, to deliver from its own thread
+    #: (Batch, Log) -> void
+    def call(batch, log)
+      @log = log
+      @queue << batch
+    rescue ClosedQueueError
+      nil
+    end
+
+    # Delivers what is still queued, then stops
+    #: () -> void
+    def close
+      @queue.close
+      @thread.join
+    end
+
+    private
+
+    #: () -> void
+    def run
+      while (batch = @queue.pop)
+        batches = [batch]
+        while !@queue.empty? && (more = @queue.pop)
+          batches << more
+        end
+        log = @log or next
+        begin
+          @output.call(@squasher.squash(batches), log)
+        rescue => e
+          log.on_error.call(e, @output)
+        end
+        sleep(@interval) unless @queue.closed?
+      end
+    end
+  end
+end