@50arms/session-telemetry 0.0.0-stage → 0.1.66

package/dist/dispatch-usage.d.ts

@@ -0,0 +1,321 @@
+/**
+ * DX-2963 — a running dispatch's usage, worker side.
+ *
+ * The worker no longer sums anything. It feeds every transcript line into
+ * the ONE shared counter (`UsageLedgerCounter`, per-field maximum per reply)
+ * and ships the entries that are new or changed since the last acknowledged
+ * write to the server (`applyStats`), which derives every total from the
+ * ledger and hands them back. The worker's own view of the totals is what the
+ * server last returned — plus, for the token ceiling only, a local estimate of
+ * what it has read but the server has not acknowledged yet.
+ *
+ * Totals are taken from the server ONLY at process start (the first write,
+ * sent with no entries if there are none) and from each write's response —
+ * never seeded by hand (the reattach hand seeds this replaces).
+ *
+ * DB-free: runs in the DB-less worker. The sink decides the transport
+ * (HTTP on a registered worker, getPool-direct on the dashboard).
+ */
+import { UsageLedgerCounter } from "./usage-ledger.js";
+import type { CursorPastEofInfo, EntryMeta } from "./session-log-watcher.js";
+import { type ActivityChange, type ActivityMessage, type ActivityProblemLink, type SourceCursors, type StatsPayload, type StatsResponse, type UsageTotals } from "./stats-wire.js";
+/** Where stats writes go — both dispatch-row sinks implement it. */
+export interface StatsSink {
+    applyStats(dispatchId: string, payload: StatsPayload): Promise<StatsResponse>;
+}
+export declare function zeroUsageTotals(): UsageTotals;
+export type FlushOutcome = "ok" | "failed" | "refused";
+/**
+ * DX-3962 - which transcript owner a collector is built for. It decides whether
+ * replies are attributed to cards, and it has no default: every construction names
+ * its path.
+ *
+ *   - `"worker"` - a dispatched worker. NO claim tracker, and every entry's
+ *     `issueId` is `null` whatever the transcript contains (a worker picks its card
+ *     up server-side, its transcript has no pickup, and its dispatch row already
+ *     carries the card). The server refuses a non-null `issueId` from a non-operator
+ *     dispatch by refusing the WHOLE batch, which would stall the worker's stats,
+ *     token ceiling and spend alert on every resend - so a worker is structurally
+ *     unable to send one.
+ *   - `"operator-bridge"` - the plan event relay of an operator session
+ *     (`platform-mcp`'s `session-stats.ts`), the only path that attributes. The value keeps its old name on purpose: it is
+ *     this package's published API (the MCP package passes it), so renaming it is a new telemetry release, not a text fix.
+ */
+export type DispatchUsageMode = "worker" | "operator-bridge";
+export declare class DispatchUsage {
+    readonly counter: UsageLedgerCounter;
+    /** DX-3962 - `null` in `"worker"` mode: the worker can never create a claim, so it can never send an `issueId`. */
+    private readonly claims;
+    /**
+     * DX-3962 - per source, the claim state as of its `acked` offset: what the next
+     * write reports with `acked`, promoted together with it. The state the tracker
+     * holds NOW belongs to `delivered`, which runs ahead.
+     */
+    private readonly ackedClaims;
+    private serverTotals;
+    private readonly acked;
+    private healthy;
+    private refused;
+    private started;
+    private readonly queue;
+    constructor(mode: DispatchUsageMode);
+    /** How to read the watcher's live per-source cursor state. */
+    private readCursorState;
+    /**
+     * Per source, the `delivered` offset as of the last ACKNOWLEDGED write — which
+     * is precisely what `acked` means. Seeded from the cursors this process
+     * resumed from, so a restart that writes before reading anything re-reports
+     * its own resume point instead of regressing it.
+     */
+    private readonly ackedCursor;
+    /**
+     * DX-3972 - per source, the `acked` the server holds for this dispatch: the `acked` of the last ACKNOWLEDGED
+     * report that carried the source. It is NOT {@link DispatchUsage.ackedCursor}: that is what the NEXT report
+     * claims (one write behind `delivered`), this is what the LAST one claimed. The terminal catch-up
+     * ({@link DispatchUsage.flushFinal}) is needed exactly while this lags the watcher's `delivered`.
+     */
+    private readonly storedAcked;
+    /**
+     * DX-2954 (AC 34324) — the cursors this process STARTED from, reported
+     * verbatim for every source the watcher has not opened yet until one write
+     * carrying them has landed. See {@link DispatchUsage.buildCursorReports}.
+     */
+    private startCursors;
+    private startCursorsPersisted;
+    /**
+     * DX-2954 — set the moment the first flush begins. `attachCursors` after that
+     * point would set the starting cursors AFTER a write that carried none, and
+     * `startCursorsPersisted` (set by any landed write) would then keep them from
+     * ever being carried — the exact AC 34324 loss, silently. So it fails loud.
+     */
+    private flushBegan;
+    /** Past-EOF resets to report once, then forget. */
+    private pastEof;
+    /**
+     * DX-2954 — replies this process read as REPLAYS. The server already counted
+     * them for THIS dispatch (they are below its stored `acked`), so they must not
+     * enter the unacknowledged excess the token-ceiling estimate adds on top of the
+     * server totals.
+     */
+    private readonly replayedMessageIds;
+    /** Running-set changes to report on the next write. */
+    private pendingActivities;
+    /** DX-5333 - sub-agent messages to report on the next write, one per `(activityId, messageKey)`. */
+    private pendingMessages;
+    /** DX-5333 - card problems sub-agents created, to report on the next write, one per `(activityId, problemRefNum)`. */
+    private pendingProblemLinks;
+    /** The session transcript path — the ONE path field the wire carries. */
+    private transcriptPath;
+    private transcriptPathSent;
+    /**
+     * DX-2954 — wire the cursor half up at attach.
+     *
+     * `startCursors` are the cursors this process RESUMED from
+     * (`resolveStartCursors`); their `acked` values seed this collector's own
+     * acknowledged floor. `readCursorState` reads the watcher's live offsets at
+     * each write.
+     */
+    attachCursors(readCursorState: () => Record<string, {
+        startOffset: number;
+        delivered: number;
+    }>, startCursors: SourceCursors): void;
+    /**
+     * DX-2954 — a stored cursor was past its file's end; report the reset on the
+     * next write so the server records a `transcript_cursor_past_eof` event and
+     * permits the one sanctioned backwards move of `acked`.
+     */
+    recordPastEof(info: CursorPastEofInfo): void;
+    /**
+     * DX-2954 — queue running-set changes for the next write.
+     *
+     * Kept COLLAPSED (one entry per `(kind, activityId)`) rather than appended
+     * raw, so the queue is bounded by how many activities the dispatch has, not by
+     * how much they moved since the last write. A producer reports an activity's
+     * full current state every time anything about it changes, and a chatty
+     * sub-agent advances its `lastActivityAt` continuously — appending those would
+     * grow without bound, eventually exceed `ACTIVITY_BATCH_CAP` and take the
+     * WHOLE stats write down (not just its activity part) to describe a handful of
+     * activities.
+     *
+     * `collapseActivityChanges` applies the server's own merge rule, so collapsing
+     * here reaches the identical stored row; it is not a lossy summarisation.
+     */
+    recordActivities(changes: readonly ActivityChange[]): void;
+    /**
+     * DX-5333 - queue sub-agent messages for the next write. One entry per `(activityId, messageKey)`: the server's insert is
+     * idempotent on that key, so a message read twice (a collector that re-reads a file) is sent once.
+     */
+    recordMessages(messages: readonly ActivityMessage[]): void;
+    /** DX-5333 - queue the card problems a sub-agent created for the next write (idempotent on the pair, like {@link DispatchUsage.recordMessages}). */
+    recordProblemLinks(links: readonly ActivityProblemLink[]): void;
+    /**
+     * DX-2954 — the session's own transcript path, recorded on the dispatch's
+     * `jsonl_path`. Sent once, with the first stats write that carries it; the
+     * server also keeps only the first path it is given, so a repeat is harmless.
+     */
+    setTranscriptPath(path: string): void;
+    /**
+     * DX-2954 — this write's cursor reports, plus the `delivered` value to promote
+     * to `acked` per source once the write is acknowledged.
+     *
+     * **`acked` deliberately LAGS `delivered` by one write.** A write reports the
+     * position it has read to (`delivered`) and the position whose entries a
+     * PREVIOUS write already had acknowledged (`acked`); only when THIS write
+     * succeeds does its own `delivered` become the next write's `acked`.
+     *
+     * The lag is not slack — it is what makes the pair safe. A reply's ledger entry
+     * can be RAISED after it was first reported (its usage grows as the reply
+     * streams), and a delivered line need not produce an entry at all (a
+     * `tool_result`), so "the server committed my last write" is NOT the same as
+     * "every line I have read is finally accounted for". Reporting `acked` one
+     * write behind is the conservative claim that is always true.
+     *
+     * What it costs: a restart re-reads the last write's window. That is exactly
+     * the window `EntryMeta.replay` covers — the ledger re-credits nothing (its
+     * excess rule) and every side-effecting subscriber skips it — so the cost is
+     * one window of re-read, never a double count or a duplicate side effect.
+     *
+     * ## The terminal write closes the lag (DX-3972)
+     *
+     * A restart is a crash, and a crash's window is the cost above. A dispatch that
+     * ENDS is not one: its watcher is drained and stopped, so every line it read has been
+     * fed to this collector and nothing will raise a reply any more. The terminal
+     * {@link DispatchUsage.flushFinal} therefore reports `acked` equal to `delivered` for
+     * every source it has opened (see {@link DispatchUsage.catchUpReports}), and a
+     * `--resume` of the session starts at the end of what this dispatch read instead of
+     * replaying the whole file. Without it a dispatch with ONE line-bearing write ended at
+     * `{acked: 0, delivered: EOF}`, and every resume of the session re-read the whole
+     * transcript and re-shipped it into its own `dispatch_logs`.
+     *
+     * ## The process-start write carries the STARTING cursors (AC 34324)
+     *
+     * `attachMonitoringStack` deliberately does NOT await `watcher.start()` —
+     * file discovery polls for up to 180 s and must not block the spawn — and the
+     * process-start stats write fires immediately after it. At that moment the
+     * watcher has opened no file, so its live `cursorState()` is empty.
+     *
+     * So every source in the cursors this process STARTED from that the watcher
+     * has not opened yet is reported VERBATIM — the exact `{acked, delivered}`
+     * pair it resumed from — until a write that carries them has landed. That is
+     * what makes "every attach persists its starting cursors" true on the FIRST
+     * write rather than one stats tick (15-30 s) later: a reattach of this row
+     * inside its first interval reads them back (start-order rule 1) and starts
+     * exactly where this process did, instead of finding no cursor, resolving
+     * rule 4 and skipping every line written since spawn. A fresh session's
+     * starting cursor is explicit too (`resolveStartCursors` rule 3 returns the
+     * main source at 0), and a `--resume`'s is its predecessor's (rule 2).
+     *
+     * Verbatim is the only honest claim for a source this process has not read:
+     * a higher `acked` would let a restart skip the replay window it has not
+     * re-credited, a lower one is refused as a regression. For the same reason
+     * the start write does not PROMOTE such a source's acknowledged floor — it
+     * delivered nothing there — so it stays out of `deliveredAtSend`.
+     *
+     * The one case with nothing to carry is rule 4 (a continuing dispatch with no
+     * cursor for ANY source): its start is "each existing file's end at
+     * discovery", which does not exist until discovery runs. Its cursors reach
+     * the server on the first write after discovery. Rule 4 is the pre-card
+     * deploy transition only — every dispatch since this card stores cursors on
+     * its first write — so that bounded window is the one the card accepts.
+     */
+    private buildCursorReports;
+    /**
+     * DX-2954 — whether the cursor half has anything new to say. An IDLE dispatch
+     * (no new lines, no hook events) must still cost zero writes per tick, which
+     * is why a write is not forced merely because cursors EXIST.
+     */
+    private cursorsMoved;
+    /**
+     * DX-3972 - the TERMINAL write's reports: the tick's reports with every source this process has read claiming its
+     * whole `delivered` as `acked`, and the claim state AS OF that offset (the tracker's current state belongs to
+     * `delivered`, so it describes the new `acked` exactly, never a later position). Sources the watcher has not
+     * opened (a start cursor reported verbatim) are untouched: this process read nothing there to claim.
+     */
+    private catchUpReports;
+    /**
+     * DX-3972 - whether the terminal write has anything to say: the server's `acked` still lags what the catch-up now
+     * claims. `ackedCursor` (the NEXT tick's claim) cannot tell, because a dispatch whose last write reported
+     * `delivered` with an older `acked` has `ackedCursor === delivered` and still lags; {@link DispatchUsage.storedAcked}
+     * is what the server holds. A source never reported at all has no entry, so it counts as lagging.
+     */
+    private storedAckedLags;
+    /**
+     * Feed one raw transcript line of the usage feed set (assistant, user, attachment, system), in file order.
+     * DX-3964 - the usage collector is fed from the watcher's collector-only `onUsageLine` subscription, never from
+     * `onEntry`: `convertJsonlEntry` drops human prompts, attachments and system lines, and widening it would hand
+     * every other `onEntry` subscriber a new volume of entries.
+     *
+     * `meta.replay` (DX-2954) marks a line re-read after a restart. The line STILL goes to the counter - the
+     * ledger is idempotent and dropping it would lose usage a restart is meant to recover - but a replayed
+     * assistant reply's id is remembered so the local ceiling estimate does not count it twice (see
+     * {@link DispatchUsage.unacknowledged}).
+     */
+    observeRawLine(raw: Record<string, unknown>, meta: EntryMeta): void;
+    /**
+     * Ship every changed entry in capped batches, one after another. The first
+     * call always writes (even with nothing to send) so the process takes the
+     * server totals at start. A failed batch re-queues itself and every later
+     * batch; a refusal (the dispatch went terminal) stops all later writes.
+     *
+     * DX-2954 — the write also carries the cursor half. A write happens when there
+     * are entries, OR the cursors moved, OR there are activity changes, OR it is
+     * the process-start write; an idle dispatch still writes nothing. Cursors,
+     * activity changes and the transcript path ride the LAST batch only, so a
+     * multi-batch backlog does not repeat them and the cursor can never be
+     * acknowledged ahead of the entries it covers.
+     */
+    flush(sink: StatsSink, dispatchId: string): Promise<FlushOutcome>;
+    /**
+     * DX-3972 - the TERMINAL write: {@link DispatchUsage.flush} that also catches the stored `acked` up to
+     * `delivered` for every source (see {@link DispatchUsage.catchUpReports}). Call it once, when the dispatch ends,
+     * after the watcher is drained and stopped (that is the whole precondition: every line it read has reached this
+     * collector, so `delivered` is covered by the entries this very write carries), and BEFORE the row goes terminal
+     * (the server refuses a terminal dispatch's stats, which would drop the catch-up silently). A write that does not
+     * land promotes nothing, exactly as in a tick; a dispatch already caught up sends no write.
+     *
+     * It does NOT wait for the dispatch's log shipper: the spawn path flushes it before this call, the reattach path
+     * after (`reattach.ts` `_cleanup`), so on a reattach `acked` can claim a line whose `dispatch_logs` row is still in
+     * the shipper's buffer. The shipper is best-effort today and a resume's shipper skips replays either way, so
+     * nothing is lost that was not already; acking only after the log line is acknowledged is DX-2956's change.
+     */
+    flushFinal(sink: StatsSink, dispatchId: string): Promise<FlushOutcome>;
+    private write;
+    /** The dispatch's totals as the server last returned them. */
+    totals(): UsageTotals;
+    /** Window tokens (input + output + cache writes) of the server totals. */
+    serverWindowTokens(): number;
+    /** Whether the most recent stats write succeeded. */
+    statsHealthy(): boolean;
+    /**
+     * The per-field excess of each local reply's maximum over what the server
+     * last acknowledged for it.
+     *
+     * DX-2954 — REPLAYED replies are excluded. The pre-DX-2954 comment here
+     * asserted that "every id this process observed was read at or after its
+     * starting cursor … so lines re-read on a restart never enter the estimate",
+     * which stopped being true the moment replays became a deliberate input: a
+     * replayed reply has no entry in `this.acked` at process start, so its FULL
+     * usage would be added on top of `serverTotals` that already contain it. That
+     * feeds a fabricated total to the token ceiling (`agent-status.ts` uses this
+     * estimate precisely when a write did NOT land) and to the context-overflow
+     * decision — either can stop a healthy dispatch.
+     */
+    private unacknowledged;
+    /**
+     * The dispatch's usage as far as this process knows it: the server totals
+     * plus the per-field excess this process has read but the server has not
+     * acknowledged. A replay block adds nothing (its excess over the
+     * acknowledged total is zero). For a job with no dispatch row (nothing is
+     * ever acknowledged) this is exactly the local counter's total.
+     */
+    estimate(): {
+        tokensIn: number;
+        tokensOut: number;
+        cacheRead: number;
+        cacheWrite: number;
+    };
+    /** Token-ceiling input: window tokens (input + output + cache writes) of {@link estimate}. */
+    windowTokensEstimate(): number;
+    /** All-field total of {@link estimate}. */
+    totalTokensEstimate(): number;
+}