@sema-agent/server 1.323.0 → 2.0.1

package/dist/orchestration/workflow-notify-journal.d.ts

@@ -1,24 +1,51 @@
 import type { WorkflowRun, WorkflowRunStore, WorkflowRunStatus, WorkflowRunSummary } from "@sema-agent/core";
 import type { FleetEventBus } from "../fleet/fleet-bus.js";
+/** A workflow completion notification's terminal payload — the bounded, redacted shape core's notifier seam
+ *  carries, re-derivable from a durable {@link import("@sema-agent/core").WorkflowRun} on the recovery path. */
 export interface WorkflowCompletionPayload {
     runId: string;
     status: "completed" | "failed";
+    /** Redacted + length-bounded — never host-internal paths/tokens (parity with core's `boundedSummary`). */
     summary: string;
     sourceTaskId?: string;
     principal?: string;
+    /** core 1.208: the ORIGINATING session id, closed over by the Runner at RunWorkflow mount time —
+     *  the lookup-free routing key for the completion inbox. Present on the LIVE notify path; ABSENT on the
+     *  crash-recovery path (the durable `WorkflowRun` does not carry it — relayed to core to thread it there too,
+     *  parity with `sourceTaskId`'s 1.187 story), where the delivery falls back to the `sourceTaskId` run-row lookup. */
     originatingSessionId?: string;
 }
+/** The downstream delivery the journal guards (the deployment's real completion route — log+meter today, a full
+ *  re-invoke into the originator's stream/inbox tomorrow). Idempotent on `runId` by contract; the journal still
+ *  de-dups so it is invoked AT MOST once per run in steady state and at most one extra time across a crash. */
 export type WorkflowCompletionDelivery = (p: WorkflowCompletionPayload) => Promise<void> | void;
+/** One journal entry: the durable to-do record for a single run's completion notify. `acked` flips true ONLY
+ *  after delivery succeeds (deliver-then-ack — a crash mid-delivery re-delivers on recovery). */
 export interface WorkflowNotifyJournalEntry {
     runId: string;
+    /** The run's scope (= creating principal) — needed to re-fetch / owner-attribute on recovery. */
     scope: string;
+    /** Whether the completion has been DELIVERED to the downstream sink (and may be GC'd). */
     acked: boolean;
+    /** The originating task id, threaded to the delivery (parity with core's notifier input). */
     sourceTaskId?: string;
+    /** The originating principal, threaded to the delivery. */
     principal?: string;
+    /** When the run was journaled (epoch ms) — for retention + observability. */
     createdAt: number;
+    /** When the entry was acked (epoch ms) — set with `acked`. */
     ackedAt?: number;
 }
+/**
+ * The pluggable durable seam for the notify journal. Intentionally tiny: record a started run, mark it acked,
+ * and enumerate the un-acked backlog the recovery sweep walks. `listPending` is the one method the
+ * `WorkflowRunStore` cannot provide (it has no cross-scope enumeration) and is the whole reason the journal is a
+ * separate store. A backend (the File default, or a follow-on TiDB/PG) implements this; the gate logic in
+ * {@link WorkflowNotifyGate} is backend-agnostic.
+ */
 export interface WorkflowNotifyJournalStore {
+    /** Record a STARTED run as pending-notify. Idempotent on `runId`: a second `record` for the same run is a
+     *  no-op (it must NOT reset an already-`acked` entry back to pending — that would re-deliver forever). */
     record(entry: {
         runId: string;
         scope: string;
@@ -26,11 +53,27 @@ export interface WorkflowNotifyJournalStore {
         principal?: string;
         createdAt: number;
     }): Promise<void>;
+    /** Mark a run's completion DELIVERED. Idempotent: acking an already-acked or missing run is a no-op. */
     ack(runId: string, ackedAt: number): Promise<void>;
+    /** Every NOT-`acked` entry — the recovery backlog. Order is unspecified (the sweep handles each independently). */
     listPending(): Promise<WorkflowNotifyJournalEntry[]>;
+    /** Read one entry (`null` when absent) — lets the gate check `acked` before a live delivery (the dedup). */
     get(runId: string): Promise<WorkflowNotifyJournalEntry | null>;
 }
 export declare function recoverySummary(status: WorkflowRunStatus, error?: string): string;
+/**
+ * The AT-LEAST-ONCE completion-notify gate. Wrap the deployment's real delivery once at boot, then:
+ *  - {@link buildNotifier} hands core a {@link import("@sema-agent/core").WorkflowCompletionNotifier} whose `notify`
+ *    runs the LIVE deliver-then-ack path (the in-process at-most-once notify, now journaled into at-least-once).
+ *  - {@link onWorkflowStart} journals a run the moment it starts (so a crash BEFORE its terminal notify still
+ *    leaves a pending entry the recovery sweep finds). Call it from wherever the runId becomes known.
+ *  - {@link recover} runs ONCE at boot: re-derives every pending run's terminal state from the
+ *    {@link WorkflowRunStore} and re-delivers + acks the ones that are terminal.
+ *
+ * The gate is the SINGLE writer of `acked`, and dedups every delivery against the journal, so the wrapped
+ * delivery sink is invoked at most once per run in steady state (and at most one extra time across a crash —
+ * the at-least-once / idempotent contract).
+ */
 export declare class WorkflowNotifyGate {
     private readonly journal;
     private readonly runStore;
@@ -38,21 +81,60 @@ export declare class WorkflowNotifyGate {
     private readonly opts;
     constructor(journal: WorkflowNotifyJournalStore, runStore: WorkflowRunStore, deliver: WorkflowCompletionDelivery, opts?: {
         now?: () => number;
+        /** Surfaced on a failure (so a transient downstream/journal error is observable, NOT swallowed silently —
+         *  it stays pending + retries on the next recovery sweep). Never throws back into core's notify. `record` =
+         *  the start-time journal write failed; `deliver` = the downstream delivery threw; `recover` = a per-entry
+         *  recovery-sweep error. */
         onError?: (stage: "record" | "deliver" | "recover", runId: string, err: unknown) => void;
     });
     private now;
+    /**
+     * Journal a STARTED run as pending-notify. Call this with the synchronous `runId` from `startWorkflow` /
+     * `run_workflow` — BEFORE the workflow can reach terminal — so a crash mid-run still leaves a recoverable
+     * to-do. Best-effort + idempotent: a journal throw is reported, never propagated (it must not break the
+     * workflow start; worst case the recovery sweep won't know about this run — degraded, not corrupting).
+     */
     onWorkflowStart(input: {
         runId: string;
         scope: string;
         sourceTaskId?: string;
         principal?: string;
     }): Promise<void>;
+    /**
+     * The core-facing notifier. core calls this exactly once per run on terminal (its own at-most-once, runId-dedup
+     * `fire`). We turn it into at-least-once: ensure the run is journaled (the start hook may have been skipped),
+     * then deliver-then-ack. Idempotent: if the run is ALREADY acked (a recovery sweep beat us, or a duplicate
+     * core fire), skip — at most one delivery.
+     */
     buildNotifier(): {
         notify: (input: WorkflowCompletionPayload & {
             scope?: string;
         }) => Promise<void>;
     };
+    /**
+     * Deliver a run's completion exactly once (idempotent on the journal). Ensures the entry exists (record is a
+     * no-op if it already does), SKIPS if already acked, otherwise delivers THEN acks (a crash between deliver and
+     * ack re-delivers on recovery — at-least-once). A delivery throw is reported and re-thrown to the LIVE caller
+     * only as a swallowed report (core's `fire` already `.catch`es), leaving the entry pending for the next sweep.
+     */
     private deliverOnce;
+    /**
+     * RECOVERY sweep — run at boot AND PERIODICALLY (wired into the service reaper), BEFORE/while serving traffic.
+     * For every pending (not-acked) journal entry, re-fetch the run from the durable {@link WorkflowRunStore}:
+     *  - TERMINAL (completed/failed): its in-process notify was lost to a crash (or it finished while we were
+     *    down) → re-deliver + ack (idempotent on the receiver).
+     *  - `running` but STALE (run.startedAt older than `orphanGraceMs`): a crash-orphaned run — core never resumes
+     *    a prior `running` row (resume mints a NEW runId) and never reaps a `running` row, so NOTHING will ever
+     *    flip it terminal. Finalize-as-ABANDONED: deliver a `failed` completion (so the originator learns the
+     *    workflow died) + ack, instead of leaking the entry forever. This closes the exact crash topology SVC-1
+     *    exists for (a replica SIGKILLed mid-run).
+     *  - `running` and FRESH (within the grace window): genuinely in flight → leave pending (the owning process
+     *    delivers its terminal notify, or the next sweep catches it once it goes terminal or stale).
+     *  - MISSING (reaped / never persisted): ack-as-abandoned so the journal doesn't chase a ghost forever.
+     * `orphanGraceMs` MUST exceed the max expected workflow runtime (the run store has no cross-replica liveness
+     * signal, so age is the only orphan proxy). Returns a tally. A per-entry throw is isolated so one bad entry
+     * can't abort the sweep.
+     */
     recover(opts?: {
         orphanGraceMs?: number;
     }): Promise<{
@@ -62,13 +144,28 @@ export declare class WorkflowNotifyGate {
         abandoned: number;
     }>;
 }
+/**
+ * Default zero-dependency, crash-safe {@link WorkflowNotifyJournalStore}. An append-only JSONL ledger replayed
+ * at boot (last-writer-wins per `runId`), the same durable posture as core's `FileWorkflowRunStore` — fsync'd
+ * appends, atomic single-line records. Single-instance / TOC-local: a cross-replica TiDB/PG port is a follow-on
+ * (the recovery sweep + gate are backend-agnostic).
+ *
+ * The ledger only ever grows by one line per `record`/`ack`; an acked entry's line is not deleted (the replay
+ * folds it), so retention is a future compaction concern — the backlog `listPending` walks is bounded by
+ * IN-FLIGHT + un-swept runs, not history.
+ */
 export declare class FileWorkflowNotifyJournalStore implements WorkflowNotifyJournalStore {
     private readonly fsyncEnabled;
     private readonly ledgerPath;
     private readonly entries;
     private fd;
     constructor(root: string, fsyncEnabled?: boolean);
+    /** Replay the ledger into the authoritative map (last-writer-wins by line order). Tolerates a torn final line
+     *  (a crash mid-append) by dropping an unparsable trailing record — the same crash-safety FileWorkflowRunStore
+     *  relies on (a partial append is simply not yet committed state). */
     private replay;
+    /** Commit one entry: append the JSON line (fsync) THEN flip the in-memory map (crash-safe ordering — the
+     *  durable record lands before the in-memory state the caller observes). */
     private commit;
     record(input: {
         runId: string;
@@ -80,9 +177,15 @@ export declare class FileWorkflowNotifyJournalStore implements WorkflowNotifyJou
     ack(runId: string, ackedAt: number): Promise<void>;
     listPending(): Promise<WorkflowNotifyJournalEntry[]>;
     get(runId: string): Promise<WorkflowNotifyJournalEntry | null>;
+    /** Release the append handle (best-effort) — called on shutdown. */
     close(): void;
+    /** Test/inspection: total entries (acked + pending). */
     get size(): number;
 }
+/** In-process {@link WorkflowNotifyJournalStore} — tests / single-instance ephemeral. Does NOT survive a
+ *  restart (so it provides NO crash-recovery on its own); use {@link FileWorkflowNotifyJournalStore} or a
+ *  durable backend for the real at-least-once guarantee. Useful to unit-test the gate's live + idempotency
+ *  paths without touching disk, and to simulate a crash by handing a FRESH store to a new gate. */
 export declare class InMemoryWorkflowNotifyJournalStore implements WorkflowNotifyJournalStore {
     private readonly entries;
     record(input: {
@@ -97,11 +200,39 @@ export declare class InMemoryWorkflowNotifyJournalStore implements WorkflowNotif
     get(runId: string): Promise<WorkflowNotifyJournalEntry | null>;
     get size(): number;
 }
+/**
+ * A {@link WorkflowRunStore} DECORATOR that journals a run the instant core persists it at START. core's
+ * `startWorkflow` `put`s the run row (`status: "running"`) synchronously before the body runs, carrying both the
+ * `runId` and the `scope` — the one synchronous start-time hook the service gets WITHOUT a new core seam (core
+ * exposes no `onWorkflowStart`). Wrapping the run store here observes that `put`, journals the started run as
+ * pending-notify, then delegates to the wrapped store. Every other method is a pure pass-through (the run store
+ * stays core's source of truth for `/workflows`). This is the [[core-service-boundary]]-clean wiring: reuse the
+ * core store seam, observe writes through it, never fork a parallel run model.
+ *
+ * Journaling on `put` (not on notify) is what makes recovery cover a crash BEFORE terminal: the entry exists the
+ * moment the run starts, so the boot sweep finds it even if the process died mid-run.
+ */
 export declare class JournalingWorkflowRunStore implements WorkflowRunStore {
     private readonly inner;
+    /** [1262] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory
+     *  backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI
+     *  state and must not be gated on the durable-notify machinery (the regression that emptied clay's
+     *  workflow panel on memory-backend deployments). */
     private readonly gate?;
+    /** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation
+     *  point as the notify journal). Optional — undefined ⇒ no fleet publish. */
     private readonly fleetBus?;
-    constructor(inner: WorkflowRunStore, gate?: Pick<WorkflowNotifyGate, "onWorkflowStart"> | undefined, fleetBus?: FleetEventBus | undefined);
+    constructor(inner: WorkflowRunStore, 
+    /** [1262] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory
+     *  backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI
+     *  state and must not be gated on the durable-notify machinery (the regression that emptied clay's
+     *  workflow panel on memory-backend deployments). */
+    gate?: Pick<WorkflowNotifyGate, "onWorkflowStart"> | undefined, 
+    /** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation
+     *  point as the notify journal). Optional — undefined ⇒ no fleet publish. */
+    fleetBus?: FleetEventBus | undefined);
+    /** Derive + publish the MF-Fleet workflow row from a WorkflowRun (doneCount/totalCount from agents, tokens from
+     *  stats); a terminal workflow LEAVES the fleet (publish terminal then remove), a running one stays/updates. */
     private publishFleet;
     put(id: string, run: WorkflowRun): Promise<void>;
     update(id: string, scope: string, run: WorkflowRun, expect?: {

package/dist/orchestration/workflow-notify-journal.js

@@ -1,7 +1,52 @@
+/**
+ * SVC-1 (core contract) — the AT-LEAST-ONCE workflow completion-notify TRUST GATE.
+ *
+ * ## Why this exists
+ * core's `run_workflow` tool already does the BACKGROUND-RUN half of SVC-1: `startWorkflow` returns a `runId`
+ * synchronously (the caller never blocks), the workflow runs to terminal in the background, and on terminal the
+ * tool fires a single in-process completion notify through the {@link import("@sema-agent/core").WorkflowCompletionNotifier}
+ * seam (see core `run-workflow-tool.js`). That notify is **at-MOST-once**: it lives only in the process that
+ * started the run. If the replica CRASHES between the workflow reaching terminal and the receiver acking the
+ * notification — or if it crashes before the run even finishes — the notification is LOST. Nothing re-derives it.
+ *
+ * SVC-1 makes the completion notify **at-LEAST-once**, the half that is service's (design boundary
+ * [[core-service-boundary]] — the durable wiring + the deployment notify route is TOB, not core):
+ *  1. **terminal → notify** (steady state): every started run is journaled as `pending`; when its in-process
+ *     notify fires, we record the terminal status durably, DELIVER to the wrapped notifier, and mark the journal
+ *     entry `acked`. Delivery before ack ⇒ a crash mid-delivery re-delivers on recovery (never a silent drop).
+ *  2. **crash-recovery → re-derive + notify** (startup): on boot we read the journal for every NOT-`acked` entry
+ *     and RE-FETCH the run from the durable {@link import("@sema-agent/core").WorkflowRunStore}. A run that is now
+ *     TERMINAL (its in-process notify was lost to the crash, or it finished while we were down) is RE-DELIVERED
+ *     and acked. A run still `running` is left pending — the live process owns its terminal notify, and the next
+ *     recovery sweep (or that process's own delivery) will catch it. A run MISSING from the store (reaped /
+ *     never persisted) is acked-as-abandoned so the journal does not grow unbounded chasing a ghost.
+ *  3. **idempotent**: the journal de-dups by `runId` (a delivered+acked run is never re-delivered), so the
+ *     wrapped notifier — and the downstream receiver — may safely be invoked twice for the same run across a
+ *     crash boundary (the SVC-1 contract: the receiver is idempotent on `runId`). We dedup on OUR side too, so a
+ *     double recovery sweep, or a recovery racing the live notify, delivers at most one extra time, not N.
+ *
+ * ## Durability model
+ * The journal is itself a {@link WorkflowNotifyJournalStore} — a tiny append-only `(runId → entry)` durable map,
+ * deliberately SEPARATE from the `WorkflowRunStore`: the run store is core-owned (its schema/CAS is core's), and
+ * critically it has **no cross-scope enumeration** (`listByScope` needs a scope; there is no "list all runs" /
+ * "list all scopes"), so it cannot by itself answer "which runs might still owe a notify?" after a restart. The
+ * journal records `(runId, scope)` for exactly that: a bounded, service-owned to-do list of notifications, scoped
+ * to the deployment, that the recovery sweep walks. It is the SINGLE source of truth for "has this run's
+ * completion been delivered?".
+ *
+ * The default {@link FileWorkflowNotifyJournalStore} is zero-dependency + crash-safe (atomic-append ledger, same
+ * posture as core's `FileWorkflowRunStore`) so the no-DB / TOC-local path is durable out of the box. A durable
+ * cross-replica backend (TiDB/PG) is a thin port of the same interface (a follow-on; the recovery sweep + the
+ * gate logic here are backend-agnostic).
+ */
 import { mkdirSync, readFileSync, existsSync, openSync, writeSync, fsyncSync, closeSync } from "node:fs";
 import { join } from "node:path";
 import { deriveAgentDisplayStatus } from "@sema-agent/core";
 import { redactSecrets } from "../trace/redact.js";
+/** Build a bounded + redacted completion summary from a durable run's terminal state (the recovery path — the
+ *  live path uses core's own `boundedSummary`). We never have the raw result here (the run store keeps the run,
+ *  not the originator's payload), so we synthesize a conservative one: status + the run's already-bounded error.
+ *  Mirrors core `failureSummary`/`redactSummary` posture — strip URLs/tokens/paths defensively. */
 const SUMMARY_MAX = 500;
 export function recoverySummary(status, error) {
     if (status === "completed")
@@ -12,9 +57,24 @@ export function recoverySummary(status, error) {
         .replace(/\b(?:Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+/gi, "[redacted-token]")
         .replace(/\b(?:api[_-]?key|secret|token|password|authorization)\b\s*[:=]\s*"?[^\s"',}]+/gi, "[redacted-secret]")
         .replace(/(?:\/[A-Za-z0-9_.-]+){2,}/g, "[redacted-path]")
+        // Windows/UNC paths too (SVC-1 review): `C:\dir\file` + `\\host\share\…` would otherwise surface VERBATIM — a
+        // host-path/identity leak the forward-slash rule above misses. `\\{1,2}` covers the UNC double-leading-backslash.
         .replace(/(?:[A-Za-z]:)?(?:\\{1,2}[A-Za-z0-9_.$-]+){2,}/g, "[redacted-path]");
     return redacted.length > SUMMARY_MAX ? `${redacted.slice(0, SUMMARY_MAX)}…[+${redacted.length - SUMMARY_MAX} chars]` : redacted;
 }
+/**
+ * The AT-LEAST-ONCE completion-notify gate. Wrap the deployment's real delivery once at boot, then:
+ *  - {@link buildNotifier} hands core a {@link import("@sema-agent/core").WorkflowCompletionNotifier} whose `notify`
+ *    runs the LIVE deliver-then-ack path (the in-process at-most-once notify, now journaled into at-least-once).
+ *  - {@link onWorkflowStart} journals a run the moment it starts (so a crash BEFORE its terminal notify still
+ *    leaves a pending entry the recovery sweep finds). Call it from wherever the runId becomes known.
+ *  - {@link recover} runs ONCE at boot: re-derives every pending run's terminal state from the
+ *    {@link WorkflowRunStore} and re-delivers + acks the ones that are terminal.
+ *
+ * The gate is the SINGLE writer of `acked`, and dedups every delivery against the journal, so the wrapped
+ * delivery sink is invoked at most once per run in steady state (and at most one extra time across a crash —
+ * the at-least-once / idempotent contract).
+ */
 export class WorkflowNotifyGate {
     journal;
     runStore;
@@ -29,6 +89,12 @@ export class WorkflowNotifyGate {
     now() {
         return this.opts.now ? this.opts.now() : Date.now();
     }
+    /**
+     * Journal a STARTED run as pending-notify. Call this with the synchronous `runId` from `startWorkflow` /
+     * `run_workflow` — BEFORE the workflow can reach terminal — so a crash mid-run still leaves a recoverable
+     * to-do. Best-effort + idempotent: a journal throw is reported, never propagated (it must not break the
+     * workflow start; worst case the recovery sweep won't know about this run — degraded, not corrupting).
+     */
     async onWorkflowStart(input) {
         try {
             await this.journal.record({
@@ -40,9 +106,15 @@ export class WorkflowNotifyGate {
             });
         }
         catch (err) {
-            this.opts.onError?.("record", input.runId, err);
+            this.opts.onError?.("record", input.runId, err); // start-time JOURNAL write failed (not a delivery)
         }
     }
+    /**
+     * The core-facing notifier. core calls this exactly once per run on terminal (its own at-most-once, runId-dedup
+     * `fire`). We turn it into at-least-once: ensure the run is journaled (the start hook may have been skipped),
+     * then deliver-then-ack. Idempotent: if the run is ALREADY acked (a recovery sweep beat us, or a duplicate
+     * core fire), skip — at most one delivery.
+     */
     buildNotifier() {
         return {
             notify: async (input) => {
@@ -52,16 +124,24 @@ export class WorkflowNotifyGate {
                     summary: input.summary,
                     ...(input.sourceTaskId ? { sourceTaskId: input.sourceTaskId } : {}),
                     ...(input.principal ? { principal: input.principal } : {}),
-                    ...(input.originatingSessionId ? { originatingSessionId: input.originatingSessionId } : {}),
+                    ...(input.originatingSessionId ? { originatingSessionId: input.originatingSessionId } : {}), // core 1.208: lookup-free session routing
                 }, input.scope ?? input.principal ?? "default");
             },
         };
     }
+    /**
+     * Deliver a run's completion exactly once (idempotent on the journal). Ensures the entry exists (record is a
+     * no-op if it already does), SKIPS if already acked, otherwise delivers THEN acks (a crash between deliver and
+     * ack re-delivers on recovery — at-least-once). A delivery throw is reported and re-thrown to the LIVE caller
+     * only as a swallowed report (core's `fire` already `.catch`es), leaving the entry pending for the next sweep.
+     */
     async deliverOnce(p, scope) {
         const existing = await this.journal.get(p.runId);
         if (existing?.acked)
-            return;
+            return; // already delivered — idempotent skip
         if (!existing) {
+            // The start hook didn't journal it (e.g. a run started before this gate existed, or the hook was skipped).
+            // Record it now so the deliver-then-ack ordering still holds and a crash mid-delivery is recoverable.
             await this.journal.record({
                 runId: p.runId,
                 scope,
@@ -74,13 +154,31 @@ export class WorkflowNotifyGate {
             await this.deliver(p);
         }
         catch (err) {
+            // Stays PENDING (not acked) → the next recovery sweep re-delivers. Report, never throw back into core.
             this.opts.onError?.("deliver", p.runId, err);
             return;
         }
         await this.journal.ack(p.runId, this.now());
     }
+    /**
+     * RECOVERY sweep — run at boot AND PERIODICALLY (wired into the service reaper), BEFORE/while serving traffic.
+     * For every pending (not-acked) journal entry, re-fetch the run from the durable {@link WorkflowRunStore}:
+     *  - TERMINAL (completed/failed): its in-process notify was lost to a crash (or it finished while we were
+     *    down) → re-deliver + ack (idempotent on the receiver).
+     *  - `running` but STALE (run.startedAt older than `orphanGraceMs`): a crash-orphaned run — core never resumes
+     *    a prior `running` row (resume mints a NEW runId) and never reaps a `running` row, so NOTHING will ever
+     *    flip it terminal. Finalize-as-ABANDONED: deliver a `failed` completion (so the originator learns the
+     *    workflow died) + ack, instead of leaking the entry forever. This closes the exact crash topology SVC-1
+     *    exists for (a replica SIGKILLed mid-run).
+     *  - `running` and FRESH (within the grace window): genuinely in flight → leave pending (the owning process
+     *    delivers its terminal notify, or the next sweep catches it once it goes terminal or stale).
+     *  - MISSING (reaped / never persisted): ack-as-abandoned so the journal doesn't chase a ghost forever.
+     * `orphanGraceMs` MUST exceed the max expected workflow runtime (the run store has no cross-replica liveness
+     * signal, so age is the only orphan proxy). Returns a tally. A per-entry throw is isolated so one bad entry
+     * can't abort the sweep.
+     */
     async recover(opts = {}) {
-        const orphanGraceMs = opts.orphanGraceMs ?? 24 * 60 * 60 * 1000;
+        const orphanGraceMs = opts.orphanGraceMs ?? 24 * 60 * 60 * 1000; // 24h default — well beyond any normal workflow
         const pending = await this.journal.listPending();
         let redelivered = 0;
         let stillRunning = 0;
@@ -89,19 +187,31 @@ export class WorkflowNotifyGate {
             try {
                 const run = await this.runStore.get(entry.runId);
                 if (!run) {
+                    // No durable run — reaped or never persisted. Don't keep it pending forever.
                     await this.journal.ack(entry.runId, this.now());
                     abandoned++;
                     continue;
                 }
+                // principal fallback: the start-hook (JournalingWorkflowRunStore.put) records only runId+scope — scope IS
+                // the creating principal, so it's the sound owner attribution when entry.principal is absent (M6: a
+                // recovered notify must still carry an owner).
                 const principal = entry.principal ?? entry.scope;
+                // #3 G (core 1.187): sourceTaskId used to be "genuinely unrecoverable" here (the durable WorkflowRun didn't
+                // carry it). core now threads it onto WorkflowRun.sourceTaskId (startWorkflow stores it / RunWorkflow passes the
+                // host taskId), so fall back to run's copy when the journal entry lost it — the recovered notify can finally
+                // attribute the originating host task.
                 const sourceTaskId = entry.sourceTaskId ?? run.sourceTaskId;
+                // core 1.210: the durable WorkflowRun now records `originatingSessionId` (same story as
+                // sourceTaskId's 1.187) — thread it onto the RECOVERED payload so crash-recovery routes lookup-free,
+                // symmetric with the live notify (resolveCompletionRoute reads it first).
                 const originatingSessionId = run.originatingSessionId;
                 if (run.status === "running") {
                     const startedAt = run.startedAt || entry.createdAt;
                     if (this.now() - startedAt <= orphanGraceMs) {
                         stillRunning++;
-                        continue;
+                        continue; // fresh → genuinely in flight; the owner (or a later sweep) delivers terminal
                     }
+                    // Stale `running` ⇒ crash-orphan (nothing will ever flip it terminal). Finalize as abandoned/failed.
                     await this.deliverOnce({
                         runId: entry.runId,
                         status: "failed",
@@ -114,6 +224,7 @@ export class WorkflowNotifyGate {
                         abandoned++;
                     continue;
                 }
+                // Terminal but not acked ⇒ the crash dropped its notify. Re-derive the bounded summary + re-deliver.
                 await this.deliverOnce({
                     runId: entry.runId,
                     status: run.status,
@@ -122,6 +233,7 @@ export class WorkflowNotifyGate {
                     ...(principal ? { principal } : {}),
                     ...(originatingSessionId ? { originatingSessionId } : {}),
                 }, entry.scope);
+                // deliverOnce only acks on a clean delivery; count a redelivery whether or not it stuck (it logs on fail).
                 const after = await this.journal.get(entry.runId);
                 if (after?.acked)
                     redelivered++;
@@ -133,6 +245,16 @@ export class WorkflowNotifyGate {
         return { scanned: pending.length, redelivered, stillRunning, abandoned };
     }
 }
+/**
+ * Default zero-dependency, crash-safe {@link WorkflowNotifyJournalStore}. An append-only JSONL ledger replayed
+ * at boot (last-writer-wins per `runId`), the same durable posture as core's `FileWorkflowRunStore` — fsync'd
+ * appends, atomic single-line records. Single-instance / TOC-local: a cross-replica TiDB/PG port is a follow-on
+ * (the recovery sweep + gate are backend-agnostic).
+ *
+ * The ledger only ever grows by one line per `record`/`ack`; an acked entry's line is not deleted (the replay
+ * folds it), so retention is a future compaction concern — the backlog `listPending` walks is bounded by
+ * IN-FLIGHT + un-swept runs, not history.
+ */
 export class FileWorkflowNotifyJournalStore {
     fsyncEnabled;
     ledgerPath;
@@ -141,10 +263,15 @@ export class FileWorkflowNotifyJournalStore {
     constructor(root, fsyncEnabled = true) {
         this.fsyncEnabled = fsyncEnabled;
         const dir = join(root, "workflow-notify");
+        // 0o700 dir + 0o600 ledger: the journal persists tenant identity (scope/principal/sourceTaskId) — it must not
+        // be group/world-readable (parity with the durable-secret posture; a default-umask file would leak identity).
         mkdirSync(dir, { recursive: true, mode: 0o700 });
         this.ledgerPath = join(dir, "journal.jsonl");
         this.replay();
     }
+    /** Replay the ledger into the authoritative map (last-writer-wins by line order). Tolerates a torn final line
+     *  (a crash mid-append) by dropping an unparsable trailing record — the same crash-safety FileWorkflowRunStore
+     *  relies on (a partial append is simply not yet committed state). */
     replay() {
         if (!existsSync(this.ledgerPath))
             return;
@@ -157,15 +284,17 @@ export class FileWorkflowNotifyJournalStore {
                 ev = JSON.parse(line);
             }
             catch {
-                continue;
+                continue; // torn/partial line — skip (uncommitted)
             }
             if (ev && typeof ev.runId === "string")
                 this.entries.set(ev.runId, ev);
         }
     }
+    /** Commit one entry: append the JSON line (fsync) THEN flip the in-memory map (crash-safe ordering — the
+     *  durable record lands before the in-memory state the caller observes). */
     commit(entry) {
         if (this.fd === undefined)
-            this.fd = openSync(this.ledgerPath, "a", 0o600);
+            this.fd = openSync(this.ledgerPath, "a", 0o600); // owner-only (identity data)
         writeSync(this.fd, `${JSON.stringify(entry)}\n`);
         if (this.fsyncEnabled)
             fsyncSync(this.fd);
@@ -173,6 +302,7 @@ export class FileWorkflowNotifyJournalStore {
     }
     async record(input) {
         const existing = this.entries.get(input.runId);
+        // Idempotent: never reset an existing entry (esp. an acked one → it would re-deliver forever).
         if (existing)
             return;
         this.commit({
@@ -187,7 +317,7 @@ export class FileWorkflowNotifyJournalStore {
     async ack(runId, ackedAt) {
         const existing = this.entries.get(runId);
         if (!existing || existing.acked)
-            return;
+            return; // idempotent
         this.commit({ ...existing, acked: true, ackedAt });
     }
     async listPending() {
@@ -196,25 +326,32 @@ export class FileWorkflowNotifyJournalStore {
     async get(runId) {
         return this.entries.get(runId) ?? null;
     }
+    /** Release the append handle (best-effort) — called on shutdown. */
     close() {
         if (this.fd !== undefined) {
             try {
                 closeSync(this.fd);
             }
             catch {
+                /* best-effort */
             }
             this.fd = undefined;
         }
     }
+    /** Test/inspection: total entries (acked + pending). */
     get size() {
         return this.entries.size;
     }
 }
+/** In-process {@link WorkflowNotifyJournalStore} — tests / single-instance ephemeral. Does NOT survive a
+ *  restart (so it provides NO crash-recovery on its own); use {@link FileWorkflowNotifyJournalStore} or a
+ *  durable backend for the real at-least-once guarantee. Useful to unit-test the gate's live + idempotency
+ *  paths without touching disk, and to simulate a crash by handing a FRESH store to a new gate. */
 export class InMemoryWorkflowNotifyJournalStore {
     entries = new Map();
     async record(input) {
         if (this.entries.has(input.runId))
-            return;
+            return; // idempotent
         this.entries.set(input.runId, {
             runId: input.runId,
             scope: input.scope,
@@ -240,28 +377,58 @@ export class InMemoryWorkflowNotifyJournalStore {
         return this.entries.size;
     }
 }
+/**
+ * A {@link WorkflowRunStore} DECORATOR that journals a run the instant core persists it at START. core's
+ * `startWorkflow` `put`s the run row (`status: "running"`) synchronously before the body runs, carrying both the
+ * `runId` and the `scope` — the one synchronous start-time hook the service gets WITHOUT a new core seam (core
+ * exposes no `onWorkflowStart`). Wrapping the run store here observes that `put`, journals the started run as
+ * pending-notify, then delegates to the wrapped store. Every other method is a pure pass-through (the run store
+ * stays core's source of truth for `/workflows`). This is the [[core-service-boundary]]-clean wiring: reuse the
+ * core store seam, observe writes through it, never fork a parallel run model.
+ *
+ * Journaling on `put` (not on notify) is what makes recovery cover a crash BEFORE terminal: the entry exists the
+ * moment the run starts, so the boot sweep finds it even if the process died mid-run.
+ */
 export class JournalingWorkflowRunStore {
     inner;
     gate;
     fleetBus;
-    constructor(inner, gate, fleetBus) {
+    constructor(inner, 
+    /** [1262] optional since the fleet decoupling: undefined ⇒ no start-time journal (ephemeral memory
+     *  backend has nothing to recover), the FLEET half still publishes — the panel is replica-local UI
+     *  state and must not be gated on the durable-notify machinery (the regression that emptied clay's
+     *  workflow panel on memory-backend deployments). */
+    gate, 
+    /** MF-Fleet (data contract): publish the workflow's fleet row on every put/update (the same write-observation
+     *  point as the notify journal). Optional — undefined ⇒ no fleet publish. */
+    fleetBus) {
         this.inner = inner;
         this.gate = gate;
         this.fleetBus = fleetBus;
     }
+    /** Derive + publish the MF-Fleet workflow row from a WorkflowRun (doneCount/totalCount from agents, tokens from
+     *  stats); a terminal workflow LEAVES the fleet (publish terminal then remove), a running one stays/updates. */
     publishFleet(id, run) {
         if (!this.fleetBus)
             return;
         const agents = run.agents ?? [];
         const done = agents.filter((a) => a.status === "completed" || a.status === "failed").length;
         const failed = agents.filter((a) => a.status === "failed").length;
+        // cli [1726] 二①:CC 规模告警的分母是 **started**(已启动),而 `totalCount`(= agents.length)是**计划总数**
+        // (含排队中)。判别口径走 core 导出的 `deriveAgentDisplayStatus` —— 一个 agent 已在 `run.agents` 里但
+        // `startedAt` 未置就是排队中。**不自己写 `startedAt !== undefined`**:那是 core 的显示契约,两端各写一遍
+        // 必然漂移(本仓刚在 coarse 门镜像上吃过同款),用它的函数则 core 一改、两端同时跟随。
         const started = agents.filter((a) => deriveAgentDisplayStatus(a, run.status) !== "queued").length;
         const terminal = run.status === "completed" || run.status === "failed";
         this.fleetBus.publishWorkflow({
             id,
+            // [WF2-A parity] redact the workflow label surfaces for parity with the run + subagent-child names (fleet-bus.ts):
+            // a tool-launched (LLM-authored) workflow's meta.name/description is task-controlled and could carry a secret shape.
             name: redactSecrets(run.name ?? "Dynamic workflow"),
             ...(run.description ? { description: redactSecrets(run.description) } : {}),
             scope: run.scope,
+            // codex-6 F2:sessionId 必须随行——streamFleet 对无 sessionId 的行按「同 principal 全会话可见」
+            // 兜底,漏发=A 会话的 workflow 名/进度/token 泄进 B 会话的 ?session= 过滤流。
             ...(run.originatingSessionId ? { sessionId: run.originatingSessionId } : {}),
             status: run.status,
             doneCount: done,
@@ -269,23 +436,30 @@ export class JournalingWorkflowRunStore {
             failedCount: failed,
             startedCount: started,
             tokens: (run.stats?.tokens ?? 0) + (run.stats?.nested?.tokens ?? 0),
+            // [1294]:跑动中也带时长(1.232 只在 endedAt 后带——clay 验收轮实锚面板恒显 0s)。终态用
+            // endedAt 定格,活跑用 now-startedAt(每次 put/update 观察点刷新,壳侧读帧即当前时长)。
             elapsedMs: (run.endedAt ?? Date.now()) - run.startedAt,
         });
         if (terminal)
-            this.fleetBus.removeWorkflow(id);
+            this.fleetBus.removeWorkflow(id); // terminal workflow leaves the active fleet (the shell saw the final frame)
     }
     async put(id, run) {
+        // Journal FIRST (best-effort, swallowed inside onWorkflowStart) so a started run is always recoverable, THEN
+        // persist. Order matters: if persist throws, the journal entry still lets recovery re-check the (absent) run
+        // and ack-as-abandoned — no leak, no lost notify owed.
         await this.gate?.onWorkflowStart({
             runId: id,
             scope: run.scope,
+            // core threads sourceTaskId/principal to the NOTIFIER, not onto the run row, so they're unavailable here;
+            // the notifier path (deliverOnce) fills them in when it records/delivers. Recovery still has runId+scope.
         });
         await this.inner.put(id, run);
-        this.publishFleet(id, run);
+        this.publishFleet(id, run); // MF-Fleet: the run appeared (running) / its row state changed
     }
     async update(id, scope, run, expect) {
         const ok = await this.inner.update(id, scope, run, expect);
         if (ok)
-            this.publishFleet(id, run);
+            this.publishFleet(id, run); // MF-Fleet: only on a successful CAS (the persisted state is what the fleet shows)
         return ok;
     }
     get(id) {