@nanobpm/nano-workforce 0.174.0 → 0.175.0

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## [0.175.0](https://github.com/nanobpm/nano-workforce/compare/v0.174.1...v0.175.0) (2026-09-02)
+
+### Features
+
+* **agentic:** engine/poller-owned relay transcript completion (retire the presence heuristic) ([#708](https://github.com/nanobpm/nano-workforce/issues/708)) ([74f42f6](https://github.com/nanobpm/nano-workforce/commit/74f42f6fa3f1b07a6608ccf50979693eecb8ead4)), closes [689/#690](https://github.com/689/nano-workforce/issues/690) [#661](https://github.com/nanobpm/nano-workforce/issues/661) [#691](https://github.com/nanobpm/nano-workforce/issues/691) [#691](https://github.com/nanobpm/nano-workforce/issues/691) [#691](https://github.com/nanobpm/nano-workforce/issues/691) [#unlink](https://github.com/nanobpm/nano-workforce/issues/unlink)
+
+## [0.174.1](https://github.com/nanobpm/nano-workforce/compare/v0.174.0...v0.174.1) (2026-09-02)
+
+### Bug Fixes
+
+* **reconcile:** fold vanished engine instances to orphaned, unwedging inflight runs ([#706](https://github.com/nanobpm/nano-workforce/issues/706)) ([68461da](https://github.com/nanobpm/nano-workforce/commit/68461dae90fba5ab6caaa8eef595e63f3995dc88)), closes [#2](https://github.com/nanobpm/nano-workforce/issues/2) [#630](https://github.com/nanobpm/nano-workforce/issues/630) [#627](https://github.com/nanobpm/nano-workforce/issues/627)
+
 ## [0.174.0](https://github.com/nanobpm/nano-workforce/compare/v0.173.0...v0.174.0) (2026-09-02)
 
 ### Features

package/app/agentic/families/relay.family.test.ts

@@ -956,6 +956,198 @@ test("#689 mid-job reconnect: with no isInstanceLive wired, a producer disconnec
   service.teardown();
 });
 
+test("#691 engine-owned disconnect: a mid-job reconnect whose engine job is still parked keeps the correlation live", async () => {
+  const registry = new ConnectionRegistry();
+  const correlation = new CorrelationRegistry();
+  const byConnection = new Map([["conn-old", "worker-E"]]);
+  // The engine is the completion authority (#691): while the harness holds the lease the job stays
+  // parked (resolver returns an element-instance key), so a producer disconnect must NOT complete it.
+  let parked = true;
+  const resolveElementInstance = (jobKey: string) =>
+    Promise.resolve(parked && jobKey === "9001" ? "ei-9001" : undefined);
+  // isInstanceLive is deliberately NOT wired (static () => false): the engine job-state — not the
+  // presence heuristic — must be what spares the reconnect, proving presence is no longer the authority.
+  const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
+    resolveElementInstance,
+  });
+  const p = connect("conn-old", registry);
+  hub.handler?.(produce(jobStream("9001"), 1, "booting agent"), p.conn);
+  assertEquals(correlation.jobKeysFor("worker-E"), ["9001"], "the job links on first produce");
+
+  // The producer WS blips: the old connection drops (no new one yet — presence would report the
+  // instance NOT live). A frame drives #reconcile → drops the dead producer + kicks the engine reconcile.
+  registry.remove("conn-old");
+  const other = connect("cons", registry);
+  hub.handler?.(grant(0), other.conn);
+  await tick(); // let the fire-and-forget engine reconcile settle
+  assertEquals(correlation.jobKeysFor("worker-E"), ["9001"], "a still-parked job survives the disconnect");
+  assertEquals(service.transcriptOf(jobStream("9001")), undefined, "the still-active stream is NOT archived");
+  assertEquals(service.liveFallback(jobStream("9001"))?.ring !== undefined, true, "the transcript is still live");
+
+  // The worker resumes producing on a NEW connection over the same job → re-attributed, still one job.
+  byConnection.set("conn-new", "worker-E");
+  const p2 = connect("conn-new", registry);
+  hub.handler?.(produce(jobStream("9001"), 1, "resumed output"), p2.conn);
+  assertEquals(correlation.jobKeysFor("worker-E"), ["9001"], "the resumed producer stays linked to the same job");
+
+  // The job genuinely ends: the engine park vanishes. The periodic backstop pass completes + archives it.
+  parked = false;
+  await service.reconcileEngineCorrelations();
+  assertEquals(correlation.jobKeysFor("worker-E"), [], "the ended job is released");
+  assertEquals(service.transcriptOf(jobStream("9001"))?.status, "completed", "and its transcript is archived");
+  service.teardown();
+});
+
+test("#691 engine-owned disconnect: a true exit whose engine job is gone completes + archives the stream", async () => {
+  const registry = new ConnectionRegistry();
+  const correlation = new CorrelationRegistry();
+  const byConnection = new Map([["prod", "worker-X"]]);
+  // The engine reports the job GONE at disconnect time (a clean completion whose terminal lifecycle
+  // event was missed, or an unclean exit): the disconnect-driven engine reconcile completes it.
+  const resolveElementInstance = () => Promise.resolve(undefined);
+  const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
+    resolveElementInstance,
+  });
+  const p = connect("prod", registry);
+  hub.handler?.(produce(jobStream("kX"), 1, "x"), p.conn);
+  assertEquals(correlation.jobKeysFor("worker-X"), ["kX"]);
+
+  registry.remove("prod");
+  const other = connect("cons", registry);
+  hub.handler?.(grant(0), other.conn);
+  await tick(); // the fire-and-forget engine reconcile resolves "gone" → completes
+  assertEquals(correlation.jobKeysFor("worker-X"), [], "a truly-ended job is released on disconnect");
+  assertEquals(service.transcriptOf(jobStream("kX"))?.status, "completed", "and its transcript is archived");
+  service.teardown();
+});
+
+test("#691 engine-owned disconnect: an UNLINKED job stream (register/produce race) whose engine job is still parked is NOT archived on disconnect", async () => {
+  const registry = new ConnectionRegistry();
+  const correlation = new CorrelationRegistry();
+  // The connection→instance map is deliberately EMPTY at first produce: the producer's presence
+  // instance is not yet resolvable (the documented register/produce race), so #link leaves the job
+  // stream UNLINKED (`state.linked` stays false) even though it IS a job stream and a resolver is
+  // wired. The engine — not `state.linked` — must own the disconnect completion decision, otherwise
+  // a mid-job reconnect that raced the link would wrongly fall through to the presence fallback and
+  // archive a still-parked job.
+  const byConnection = new Map<string, string>();
+  let parked = true;
+  const resolveElementInstance = (jobKey: string) =>
+    Promise.resolve(parked && jobKey === "7742" ? "ei-7742" : undefined);
+  const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
+    resolveElementInstance,
+  });
+  const p = connect("conn-old", registry);
+  hub.handler?.(produce(jobStream("7742"), 1, "booting agent"), p.conn);
+  assertEquals(correlation.count(), 0, "the stream did NOT link — the instance was not resolvable at produce time");
+
+  // The old producer connection blips before a later produce could retry the link. A frame drives
+  // #reconcile → it must defer to the engine (job still parked) rather than archive on presence.
+  registry.remove("conn-old");
+  const other = connect("cons", registry);
+  hub.handler?.(grant(0), other.conn);
+  await tick(); // let the fire-and-forget engine reconcile settle
+  assertEquals(
+    service.transcriptOf(jobStream("7742")),
+    undefined,
+    "an unlinked-but-still-parked job is NOT archived on disconnect",
+  );
+  assertEquals(service.liveFallback(jobStream("7742"))?.ring !== undefined, true, "its transcript is still live");
+
+  // The worker resumes on a NEW connection; the instance now resolves → the late link finally lands.
+  byConnection.set("conn-new", "worker-R");
+  const p2 = connect("conn-new", registry);
+  hub.handler?.(produce(jobStream("7742"), 1, "resumed output"), p2.conn);
+  assertEquals(correlation.jobKeysFor("worker-R"), ["7742"], "the late link succeeds on reconnect once the instance resolves");
+
+  // The job genuinely ends: the engine park vanishes → the backstop pass completes + archives it.
+  parked = false;
+  await service.reconcileEngineCorrelations();
+  assertEquals(correlation.jobKeysFor("worker-R"), [], "the ended job is released");
+  assertEquals(service.transcriptOf(jobStream("7742"))?.status, "completed", "and its transcript is archived");
+  service.teardown();
+});
+
+test("#708 periodic backstop covers an UNLINKED job stream: an unlinked-but-parked disconnect that never reconnects is completed once the engine park ends", async () => {
+  const registry = new ConnectionRegistry();
+  const correlation = new CorrelationRegistry();
+  // Same register/produce race as #691: the connection→instance map is empty, so #link leaves the
+  // job stream UNLINKED. The producer then drops while the engine job is still parked, so the
+  // disconnect path keeps the stream live and clears `state.producer` (so #reconcile does not
+  // re-trigger). Critically the worker NEVER reconnects — no later `produce` ever links it. The
+  // periodic backstop is now the ONLY actor that can retire it, so it must include unlinked job
+  // streams in its reconcile snapshot; otherwise the stream leaks live forever once the engine job
+  // becomes terminal.
+  const byConnection = new Map<string, string>();
+  let parked = true;
+  const resolveElementInstance = (jobKey: string) =>
+    Promise.resolve(parked && jobKey === "7708" ? "ei-7708" : undefined);
+  const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
+    resolveElementInstance,
+  });
+  const p = connect("conn-old", registry);
+  hub.handler?.(produce(jobStream("7708"), 1, "booting agent"), p.conn);
+  assertEquals(correlation.count(), 0, "the stream did NOT link — the instance was not resolvable at produce time");
+
+  // Producer blips; the disconnect engine-reconcile keeps the still-parked stream live and clears
+  // the producer so #reconcile will not revisit it on later frames.
+  registry.remove("conn-old");
+  const other = connect("cons", registry);
+  hub.handler?.(grant(0), other.conn);
+  await tick();
+  assertEquals(
+    service.transcriptOf(jobStream("7708")),
+    undefined,
+    "an unlinked-but-still-parked job is NOT archived on disconnect",
+  );
+
+  // The engine job genuinely ends. With NO reconnect to link it, only the periodic backstop can
+  // retire it — and it must, even though the stream is unlinked.
+  parked = false;
+  await service.reconcileEngineCorrelations();
+  assertEquals(
+    service.transcriptOf(jobStream("7708"))?.status,
+    "completed",
+    "the periodic backstop completes an unlinked job stream once the engine park disappears",
+  );
+  assertEquals(service.liveFallback(jobStream("7708")), undefined, "and its live ring is retired");
+  service.teardown();
+});
+
+test("#691 engine-owned disconnect: a transient engine read at disconnect never falsely completes; the backstop retries", async () => {
+  const registry = new ConnectionRegistry();
+  const correlation = new CorrelationRegistry();
+  const byConnection = new Map([["prod", "worker-T"]]);
+  // The disconnect-time engine read throws (unavailable). A transient fault must be treated as
+  // "unknown — keep it linked", NEVER as "job gone": the stream stays live until a later pass resolves.
+  let failing = true;
+  let gone = false;
+  const resolveElementInstance = () => {
+    if (failing) return Promise.reject(new Error("engine unavailable"));
+    return Promise.resolve(gone ? undefined : "ei-T");
+  };
+  const { service, hub } = mkCorrelatedService(registry, memoryDb(), correlation, byConnection, {
+    resolveElementInstance,
+  });
+  const p = connect("prod", registry);
+  hub.handler?.(produce(jobStream("kT"), 1, "x"), p.conn);
+
+  registry.remove("prod");
+  const other = connect("cons", registry);
+  hub.handler?.(grant(0), other.conn);
+  await tick(); // the engine read rejects → the stream must stay linked, not complete
+  assertEquals(correlation.jobKeysFor("worker-T"), ["kT"], "a transient engine failure leaves the job linked");
+  assertEquals(service.transcriptOf(jobStream("kT")), undefined, "and its transcript is NOT archived");
+
+  // The engine recovers and now reports the job gone: the periodic backstop pass completes it.
+  failing = false;
+  gone = true;
+  await service.reconcileEngineCorrelations();
+  assertEquals(correlation.jobKeysFor("worker-T"), [], "the backstop pass releases the ended job");
+  assertEquals(service.transcriptOf(jobStream("kT"))?.status, "completed", "and archives its transcript");
+  service.teardown();
+});
+
 test("H6 correlation write-side: non-job streams are never linked; a link retries until the instance resolves", () => {
   const registry = new ConnectionRegistry();
   const correlation = new CorrelationRegistry();

package/app/agentic/families/relay.family.ts

@@ -254,15 +254,18 @@ export interface RelayTranscriptServiceOptions {
   readonly instanceForConnection?: (connectionId: string) => string | undefined;
   /**
    * Whether a worker instance is still live on any hub connection (#689). Wired to the presence
-   * registry's {@link PresenceRegistry.isInstanceLive}. The disconnect-driven reconcile uses it to
-   * spare a still-active job's stream when its worker merely RECONNECTED mid-job (old producer
-   * connection dropped, a new one re-registered under the same instance): completing then would
-   * archive a live job's transcript and release its correlation, wedging the cockpit (the reconnected
-   * worker's produce frames hit a terminal `completed` stream and are ignored) while the harness
-   * keeps the engine lease. When wired to presence, a true worker-exit still completes: presence
-   * has dropped the instance, so this returns false. Omitted (`() => false`, the static default) →
-   * the prior always-complete-on-disconnect behaviour — every disconnect completes, reconnect
-   * included (the seam never consults presence, so its value does not vary).
+   * registry's {@link PresenceRegistry.isInstanceLive}. Since #691 the disconnect-driven reconcile
+   * defers to the engine job-state (the poller-owned completion authority) whenever an engine view
+   * ({@link resolveElementInstance}) is wired, so this presence signal is consulted ONLY as the
+   * engine-less fallback: on a host with no engine read-model (or a non-job stream — no jobKey, so
+   * nothing for the engine to reconcile against; an unlinked *job* stream still has a jobKey and DOES
+   * reconcile against the engine since #691) it spares a still-active job's stream when its worker merely
+   * RECONNECTED mid-job (old producer connection dropped, a new one re-registered under the same
+   * instance) — completing then would archive a live job's transcript and release its correlation,
+   * wedging the cockpit (the reconnected worker's produce frames hit a terminal `completed` stream and
+   * are ignored). A true worker-exit still completes (presence has dropped the instance → false).
+   * Omitted (`() => false`, the static default) → the prior always-complete-on-disconnect behaviour in
+   * that fallback path.
    */
   readonly isInstanceLive?: (instance: string) => boolean;
   /**
@@ -725,27 +728,56 @@ export class RelayTranscriptService {
   }
 
   /**
-   * Flush + complete every ephemeral stream whose producer connection is no longer live (the S1
-   * registry dropped it on close or liveness timeout), and release its job correlation (H6, #149).
-   * Lazy, like the relay hub's own dead-subscriber prune: it runs on each inbound frame, and shutdown
-   * covers the quiescent tail via {@link teardown}. The correlation release is store-independent (it
-   * runs even for an unpersisted relay), so a dropped worker's `jobKeys` always clear.
+   * Reconcile every ephemeral stream whose producer connection is no longer live (the S1 registry
+   * dropped it on close or liveness timeout). For a job stream with an engine view wired this
+   * defers the completion decision to the engine job-state (#691) — the poller-owned authority —
+   * regardless of whether the stream is linked yet (an unlinked job stream can still be mid-reconnect
+   * on the register/produce race), so a mid-job reconnect never archives a still-active job; for an
+   * engine-less host (or a non-job stream with no engine job to reconcile against) it flushes +
+   * completes the ephemeral stream and releases its correlation (H6, #149), with
+   * presence liveness (#689/#690) sparing a still-live instance as the only fallback signal. Lazy, like
+   * the relay hub's own dead-subscriber prune: it runs on each inbound frame, and shutdown covers the
+   * quiescent tail via {@link teardown}. The correlation release is store-independent (it runs even for
+   * an unpersisted relay), so a dropped worker's `jobKeys` always clear.
    */
   #reconcile(): void {
     for (const [stream, state] of this.#streams) {
       if (state.producer !== undefined && !this.#registry.has(state.producer)) {
-        // Producer connection gone. Normally that means the job it was relaying ended — release the
-        // correlation and flush+complete its ephemeral transcript. BUT a worker that merely
-        // RECONNECTED mid-job (#689) also loses its old producer connection while its job keeps
-        // running (the harness holds the engine lease and extends it). Presence re-registers the SAME
-        // instance under the new connection, so the instance stays live even though this specific
-        // producer connection is gone. Completing then would archive a still-active job's transcript
-        // to `historical` and drop its correlation — and because a completed stream is terminal
-        // (`#observe` ignores later frames), the reconnected worker could NEVER re-correlate: the
-        // cockpit shows the worker idle with a frozen transcript while the job is genuinely running.
-        // So spare the stream while its instance is still live; the next `produce` re-attributes the
-        // live connection (via `#observe`), a NEW job supersedes it (via `#link`), or — once the
-        // worker truly exits — presence drops the instance and a later reconcile completes it.
+        // Producer connection gone. Normally that means the job it was relaying ended — but a worker
+        // that merely RECONNECTED mid-job (#689) also loses its old producer connection while its job
+        // keeps running (the harness holds the engine lease and extends it). Inferring job-end from the
+        // relay-connection layer (the old presence heuristic, #690) conflates two independent liveness
+        // signals (engine lease vs. WS connection); the authoritative one is the engine job-state the
+        // poller already reconciles (#691).
+        if (this.#resolveElementInstance !== undefined && jobKeyOfStream(stream) !== undefined) {
+          // Engine/poller-owned completion (#691): drop the dead producer so `#reconcile` does not
+          // re-trigger on every subsequent frame, then reconcile THIS stream against the engine's view
+          // of its job (fire-and-forget — the sync frame handler must not await an engine read). A
+          // reconnected worker's job is still parked → kept live (the reconnect's next `produce`
+          // re-attributes the live connection via `#observe`); a genuine exit's job is gone →
+          // completed + archived. The periodic {@link reconcileEngineCorrelations} pass is the backstop
+          // if this read faults transiently. Presence liveness is no longer consulted here — the
+          // engine job-state is the sole completion authority whenever an engine view is wired.
+          //
+          // NB: this deliberately does NOT require `state.linked`. A job stream can still be UNLINKED
+          // during the documented register/produce race (the producer's presence instance was not yet
+          // resolvable at `produce` time, so `#link` deferred). Gating the engine path on `linked`
+          // would fall an unlinked-but-engine-wired job through to the presence fallback below and
+          // archive a still-parked job mid-reconnect — the very heuristic #691 removes. The engine is
+          // resolvable by jobKey alone (as at link time), so defer to it regardless of `linked`;
+          // `#unlink` inside `completeStream` stays a no-op for a stream that never linked.
+          state.producer = undefined;
+          void this.#reconcileStreamAgainstEngine(stream).catch((err: unknown) => {
+            this.#log.warn("agentic relay disconnect engine-reconcile failed", { stream, err: String(err) });
+          });
+          continue;
+        }
+        // Engine-less fallback (no engine view wired — e.g. an engine-less host, or a non-job stream
+        // with no engine job to reconcile against): presence-liveness spares a still-live
+        // instance's stream (#689/#690) and a truly-gone instance completes. Presence remains a
+        // completion signal ONLY in this degraded path where there is no engine job-state to consult —
+        // never the sole authority on the engine-wired path above. (A job stream with an engine view
+        // wired always took the engine path above, linked or not — it never reaches here.)
         if (state.instance !== undefined && this.#isInstanceLive(state.instance)) continue;
         // Producer connection gone → the job it was relaying ended: release its correlation.
         this.#unlink(stream, state);
@@ -756,58 +788,98 @@ export class RelayTranscriptService {
   }
 
   /**
-   * Defensive engine-reconcile safety net (#661): release any linked jobKey whose engine JOB park is
-   * no longer live. The precise, fast release is the terminal `lifecycle` event
+   * Defensive engine-reconcile safety net (#661): release any job stream — linked OR unlinked (#708) —
+   * whose engine JOB park is no longer live. The precise, fast release is the terminal `lifecycle` event
    * ({@link #observeTerminalLifecycle}), but an UNCLEAN worker exit (crash/kill) can skip that event —
    * and because the worker's relay connection is persistent across jobs, the disconnect release never
    * fires either, so the finished job would linger as a phantom active job on the worker's supply row.
    * This periodic pass asks the engine read model (the same {@link ElementInstanceResolver} the link
-   * path uses at #544) whether each linked job is still parked; a job the engine no longer parks
+   * path uses at #544) whether each job is still parked; a job the engine no longer parks
    * (resolver returns `undefined`) is released and its transcript completed. Bounds staleness regardless
    * of whether the worker emitted a clean terminal event — and also covers a worker that emitted nothing
-   * and merely went quiet.
+   * and merely went quiet. It covers UNLINKED job streams too (#708): the disconnect path (#691)
+   * can leave a job stream unlinked-but-still-parked with no producer, and jobKey alone resolves
+   * terminality, so this backstop is the only actor that can retire one whose worker never reconnects.
    *
    * Advisory and best-effort: a no-op with no resolver wired, and a resolver THROW / REJECTION for a
-   * given job is treated as "unknown — keep it linked" (never a false release of a genuinely active
-   * job). The linked set is snapshotted before any await so a concurrent completion (a terminal
+   * given job is treated as "unknown — keep it" (never a false release of a genuinely active
+   * job). The job-stream set is snapshotted before any await so a concurrent completion (a terminal
    * lifecycle event landing mid-pass) cannot corrupt iteration, and each release re-checks the current
    * stream state so a job already released between snapshot and resolution is not double-completed.
    */
   async reconcileEngineCorrelations(): Promise<void> {
-    const resolve = this.#resolveElementInstance;
-    if (resolve === undefined) return;
-    const linked: { stream: string; jobKey: string; processInstanceKey?: string }[] = [];
+    if (this.#resolveElementInstance === undefined) return;
+    // Snapshot the job-stream ids BEFORE any await so a concurrent completion (a terminal lifecycle
+    // event, a supersede, or a disconnect-triggered engine reconcile landing mid-pass) cannot corrupt
+    // iteration; the per-stream helper re-reads live state, so a job released between snapshot and
+    // resolution is not double-completed.
+    const streams: string[] = [];
     for (const [stream, state] of this.#streams) {
-      if (!state.linked || state.completed) continue;
-      const jobKey = jobKeyOfStream(stream);
-      if (jobKey === undefined) continue;
-      const processInstanceKey = this.#correlation()?.resolve?.(jobKey)?.processInstanceKey;
-      linked.push({ stream, jobKey, processInstanceKey });
+      if (state.completed) continue;
+      if (jobKeyOfStream(stream) === undefined) continue;
+      // Include UNLINKED job streams too (#708). The disconnect path (#691) routes an unlinked
+      // job stream through the engine reconcile and clears `state.producer`, so if the engine
+      // reports "still parked" at disconnect (or the read faults transiently) and the worker never
+      // reconnects to `produce` a link, this periodic pass is the ONLY actor left that can retire
+      // it. jobKey alone resolves terminality (as at link time), so gating this snapshot on
+      // `state.linked` would leak such a stream live forever once its engine job becomes terminal.
+      streams.push(stream);
     }
-    for (const { stream, jobKey, processInstanceKey } of linked) {
-      let activeKey: string | undefined;
-      try {
-        activeKey = await resolve(jobKey, processInstanceKey);
-      } catch (err) {
-        // A transient engine read failure must NOT be read as "job gone" — leave the job linked; a
-        // later pass (or the terminal lifecycle event) releases it. Advisory, never a false release.
-        this.#log.warn("agentic relay engine-reconcile read failed — leaving correlation linked", {
-          stream,
-          jobKey,
-          err: String(err),
-        });
-        continue;
-      }
-      // A live JOB park (a resolved element-instance key) means the job is genuinely active — keep it.
-      if (activeKey !== undefined) continue;
-      // The engine no longer parks this job → it ended (possibly via an unclean exit that skipped the
-      // terminal lifecycle event). Re-check the current state — a concurrent completion may already
-      // have released it — then release its correlation and flush its transcript.
-      const current = this.#streams.get(stream);
-      if (current === undefined || current.completed || !current.linked) continue;
-      this.#log.info("agentic relay engine-reconcile released a stale correlation", { stream, jobKey });
-      this.completeStream(stream);
+    for (const stream of streams) await this.#reconcileStreamAgainstEngine(stream);
+  }
+
+  /**
+   * Reconcile ONE `job:<jobKey>` stream against the engine's view of its job (the poller-owned
+   * completion authority, #691). Asks the engine read model (the {@link ElementInstanceResolver} the
+   * #544 link path uses) whether the job is still parked: still parked → genuinely active, kept live;
+   * gone → ended (a clean completion whose terminal lifecycle event was missed, or an unclean exit) →
+   * released + its transcript flushed/archived. This is the single canonical engine-reconcile step,
+   * shared by BOTH the periodic safety-net pass ({@link reconcileEngineCorrelations}) and the
+   * disconnect-driven `#reconcile` — so a producer disconnect no longer completes a stream by inferring
+   * job-end from relay-connection/presence liveness (#689/#690), but by the authoritative engine
+   * job-state. A no-op with no resolver wired, and a resolver THROW / REJECTION is treated as
+   * "unknown — keep it" so a transient engine read never falsely releases a genuinely active
+   * job (a later pass, or the terminal lifecycle event, releases it). Re-reads live state before
+   * completing so a job released between the read and the completion is not double-completed.
+   *
+   * Does NOT require `state.linked`: the disconnect path (#691) also routes an UNLINKED job stream
+   * here (a job stream whose link deferred on the register/produce race), so the engine — not the
+   * link flag — owns its completion. The engine resolves by jobKey alone (as at link time); an
+   * unlinked stream the engine says is gone is still flushed/archived, and `#unlink` inside
+   * `completeStream` stays a no-op for it.
+   */
+  async #reconcileStreamAgainstEngine(stream: string): Promise<void> {
+    const resolve = this.#resolveElementInstance;
+    if (resolve === undefined) return;
+    const before = this.#streams.get(stream);
+    if (before === undefined || before.completed) return;
+    const jobKey = jobKeyOfStream(stream);
+    if (jobKey === undefined) return;
+    const processInstanceKey = this.#correlation()?.resolve?.(jobKey)?.processInstanceKey;
+    let activeKey: string | undefined;
+    try {
+      activeKey = await resolve(jobKey, processInstanceKey);
+    } catch (err) {
+      // A transient engine read failure must NOT be read as "job gone" — keep the stream live; a
+      // later pass (or the terminal lifecycle event) releases it. Advisory, never a false release.
+      this.#log.warn("agentic relay engine-reconcile read failed — keeping stream live", {
+        stream,
+        jobKey,
+        err: String(err),
+      });
+      return;
     }
+    // A live JOB park (a resolved element-instance key) means the job is genuinely active — keep it.
+    // This is what spares a mid-job RECONNECT: the harness still holds the engine lease, so the job is
+    // still parked even though the old producer connection dropped.
+    if (activeKey !== undefined) return;
+    // The engine no longer parks this job → it ended (possibly via an unclean exit that skipped the
+    // terminal lifecycle event). Re-check the current state — a concurrent completion may already
+    // have released it — then release its correlation and flush its transcript.
+    const current = this.#streams.get(stream);
+    if (current === undefined || current.completed) return;
+    this.#log.info("agentic relay engine-reconcile released a stale correlation", { stream, jobKey });
+    this.completeStream(stream);
   }
 
   /**
@@ -886,10 +958,11 @@ export function createRelayFamily(options: {
         // are read per call, so this works regardless of family mount order (relay may mount before
         // presence/correlation). Absent registries → no linking, still advisory-correct.
         instanceForConnection: (connectionId) => currentPresenceRegistry()?.instanceForConnection(connectionId),
-        // #689: is the producing worker instance still live on ANY connection? Gates the
-        // disconnect-driven reconcile so a mid-job RECONNECT (old producer connection dropped, the
-        // same instance re-registered) does not archive a still-active job's stream and wedge its
-        // correlation. Read per call for the same mount-order independence as the resolvers above.
+        // #689/#691: is the producing worker instance still live on ANY connection? Since #691 the
+        // disconnect-driven reconcile decides completion by the engine job-state (below) whenever the
+        // element-instance resolver is wired; this presence signal is the engine-LESS fallback that
+        // spares a mid-job RECONNECT's still-active stream where no engine view is available. Read per
+        // call for the same mount-order independence as the resolvers around it.
         isInstanceLive: (instance) => currentPresenceRegistry()?.isInstanceLive(instance) ?? false,
         // #485: resolve a completed job's worker attribution (presence identity/host) from the live
         // presence registry, read per call for the same mount-order independence. Absent → attribution

package/app/reconcile.test.ts

@@ -1,4 +1,4 @@
-// Red/green coverage for the app-side engine-reset reconciliation surface (issue #622).
+// Red/green coverage for the app-side engine-reset reconciliation surface (issues #622 and #630).
 //
 // The core scenario the incident (Magikcraft/nano-bpm#1065) demanded a supported remedy for: the
 // engine is reset and its incarnation epoch REGRESSES, while `app.db` still projects engine-backed
@@ -13,21 +13,55 @@ import { test } from "node:test";
 import { assertEquals } from "#test-assert";
 import { freshData } from "../test/reconcileDb.ts";
 import {
+  DEFAULT_VANISHED_GRACE_MS,
   ORPHANED_STATUS,
   parseEngineEpoch,
   RECONCILE_ORPHAN_REASON,
+  RECONCILE_VANISHED_REASON,
   reconcileEngineBackedWork,
+  reconcileVanishedInstances,
+  runEngineReconcile,
 } from "./reconcile.ts";
 
 const AT = () => new Date("2026-02-02T00:00:00.000Z");
 
-function seedFeatureRun(raw: DatabaseSync, key: string, status: string, processKey: string | null): void {
+/** The canonical `_urban_instance_state` DDL (urban's framework projection, `_urban_`-prefixed so it
+ *  is provisioned by the runtime — NOT our migrations). Mirrors `InstanceStateStore`'s schema so the
+ *  vanished-instance reconcile is exercised against exactly the table it reads in production. */
+function ensureInstanceState(raw: DatabaseSync): void {
+  raw.exec(
+    `CREATE TABLE IF NOT EXISTS _urban_instance_state (
+       process_instance_key TEXT NOT NULL,
+       state                TEXT NOT NULL,
+       waiting_on_human     INTEGER NOT NULL DEFAULT 0,
+       updated_at           TEXT NOT NULL,
+       PRIMARY KEY (process_instance_key)
+     );`,
+  );
+}
+
+function seedInstanceState(raw: DatabaseSync, processKey: string, state: string): void {
+  raw
+    .prepare(
+      `INSERT INTO _urban_instance_state (process_instance_key, state, waiting_on_human, updated_at)
+       VALUES (?, ?, 0, '2026-01-15')`,
+    )
+    .run(processKey, state);
+}
+
+function seedFeatureRun(
+  raw: DatabaseSync,
+  key: string,
+  status: string,
+  processKey: string | null,
+  updatedAt = "2026-01-01",
+): void {
   raw
     .prepare(
       `INSERT INTO feature_runs (feature_key, repo, issue_number, issue_url, base_branch, status, process_key, created_at, updated_at)
-       VALUES (?, 'o/r', 1, 'https://x', 'main', ?, ?, '2026-01-01', '2026-01-01')`,
+       VALUES (?, 'o/r', 1, 'https://x', 'main', ?, ?, '2026-01-01', ?)`,
     )
-    .run(key, status, processKey);
+    .run(key, status, processKey, updatedAt);
 }
 
 function seedDeliveryGraphRun(raw: DatabaseSync, runKey: string, status: string, processKey: string | null): void {
@@ -210,3 +244,199 @@ test("RED→GREEN: orphaning stamps updated_at so the transition timestamp isn't
   assertEquals(dg.status, ORPHANED_STATUS);
   assertEquals(dg.updated_at, at);
 });
+
+// --- Vanished-instance reconciliation (issue #630) --------------------------------------------
+// The "instance absent/unknown" gap, DISTINCT from the epoch-regression reset above: when an engine
+// instance VANISHES from the read model (`_urban_instance_state` row pruned/never re-created after a
+// clean reset), the derived terminal edge has no `TERMINATED` row to match, so the run freezes at its
+// last worker-owned status (`escalated`) and wedges Active forever. `reconcileVanishedInstances`
+// drives those orphaned-in-truth rows to `orphaned` WITH PROVENANCE — gated on a grace window so a
+// still-starting run (not yet projected) is spared.
+
+test("RED→GREEN: a vanished instance (no _urban_instance_state row, past grace) is orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // The pre-reset orphan from the incident: escalated, keyed on a HIGH pre-reset process_key whose
+  // instance is absent from the current read model. Its updated_at is ~32 days before AT() (past grace).
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+  // A live sibling: still ACTIVE in the projection — must be left untouched.
+  seedFeatureRun(raw, "o/r#live", "running", "200");
+  seedInstanceState(raw, "200", "ACTIVE");
+
+  // RED (pre-fix): the orphan reads `escalated` (Active) indefinitely — no terminal edge fires.
+  const before = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
+    status: string;
+  };
+  assertEquals(before.status, "escalated");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.reason, "instance-vanished");
+  assertEquals(res.orphanedCount, 1);
+
+  const orphan = raw
+    .prepare("SELECT status, updated_at FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'")
+    .get() as { status: string; updated_at: string };
+  assertEquals(orphan.status, ORPHANED_STATUS);
+  // The transition refreshes updated_at like every other status transition (not left stale).
+  assertEquals(orphan.updated_at, AT().toISOString());
+
+  // The live instance (ACTIVE row present) is never touched.
+  const live = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#live'").get() as { status: string };
+  assertEquals(live.status, "running");
+
+  const prov = raw
+    .prepare("SELECT * FROM reconcile_provenance WHERE source_table='feature_runs'")
+    .get() as Record<string, unknown>;
+  assertEquals(prov.to_status, ORPHANED_STATUS);
+  assertEquals(prov.from_status, "escalated");
+  assertEquals(prov.reason, RECONCILE_VANISHED_REASON);
+  assertEquals(prov.key_value, "71506");
+  assertEquals(prov.run_id, "van-1");
+  assertEquals(prov.observed_epoch, null);
+
+  const run = raw.prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id='van-1'").get() as {
+    reason: string;
+    orphaned_count: number;
+  };
+  assertEquals(run.reason, "instance-vanished");
+  assertEquals(run.orphaned_count, 1);
+});
+
+test("a still-starting run within the grace window is NOT prematurely folded", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // Dispatched moments ago — its process_key is set but the reconciler has not yet projected the
+  // instance into _urban_instance_state. updated_at is 30s before AT(), inside the grace window.
+  const justNow = new Date(AT().getTime() - 30_000).toISOString();
+  seedFeatureRun(raw, "o/r#starting", "running", "999", justNow);
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#starting'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "running");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+  // A generous grace window is the point — the default comfortably exceeds a poll cycle.
+  assertEquals(DEFAULT_VANISHED_GRACE_MS >= 60_000, true);
+});
+
+test("RED→GREEN: a row whose updated_at is null/unparseable is spared, never orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A tracked table's `updated_at` can be nullable (e.g. delivery_units.updated_at,
+  // db/migrations/088_delivery_units.sql) or carry an unparseable value. Its instance is absent from
+  // the read model, so without a usable age we cannot tell a genuinely-vanished row from a live one.
+  // RED (pre-fix): withinGrace treated an unestablishable age as "old enough" and folded the row.
+  // GREEN: we err toward sparing — an ageless row is treated as within grace and left untouched.
+  seedFeatureRun(raw, "o/r#ageless", "running", "888", "not-a-timestamp");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#ageless'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "running");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("terminal history, keyless rows, and rows with a live instance are never folded as vanished", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "term#1", "merged", "88"); // terminal — not in activeStatuses
+  seedFeatureRun(raw, "nokeed#1", "running", null); // active but never dispatched (no engine key)
+  seedFeatureRun(raw, "live#1", "escalated", "89"); // active, but its instance is still present
+  seedInstanceState(raw, "89", "ACTIVE");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const statuses = raw.prepare("SELECT feature_key, status FROM feature_runs ORDER BY feature_key").all() as {
+    feature_key: string;
+    status: string;
+  }[];
+  assertEquals(statuses.find((r) => r.feature_key === "term#1")?.status, "merged");
+  assertEquals(statuses.find((r) => r.feature_key === "nokeed#1")?.status, "running");
+  assertEquals(statuses.find((r) => r.feature_key === "live#1")?.status, "escalated");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("no-op when the _urban_instance_state projection is absent (never orphan on its absence)", async () => {
+  const { data, raw } = freshData();
+  // NOTE: no ensureInstanceState — the framework projection has not been provisioned.
+  seedFeatureRun(raw, "o/r#1", "escalated", "71506");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.reason, "no-op");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
+  assertEquals(row.status, "escalated");
+  const run = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id='van-1'").get() as { reason: string };
+  assertEquals(run.reason, "no-op");
+});
+
+test("idempotent: a second vanished pass is a no-op (the orphaned row left activeStatuses)", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "o/r#1", "escalated", "71506");
+
+  const first = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+  assertEquals(first.orphanedCount, 1);
+
+  const second = await reconcileVanishedInstances(data, { now: AT, runId: "van-2" });
+  assertEquals(second.reason, "no-op");
+  assertEquals(second.orphanedCount, 0);
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 1);
+});
+
+// --- Merged seam: runEngineReconcile (both passes, one result) --------------------------------
+// The operator/startup seam merges the epoch-regression and vanished-instance passes into ONE
+// result. This guards the merged behavior the two per-pass suites above don't reach: run-id
+// correlation (the vanished pass's provenance must be locatable from the returned `runId`) and
+// `reason` selection when the epoch pass is `engine-unreachable` yet vanished instances are orphaned.
+
+test("runEngineReconcile: engine-unreachable epoch pass still folds vanished instances, with a correlatable run id", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A vanished orphan (escalated, past grace, instance absent from the read model).
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+
+  // The engine is unreachable — the epoch probe fails, so the epoch pass reports `engine-unreachable`
+  // and orphans nothing; the vanished pass must still act.
+  const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
+  const res = await runEngineReconcile(
+    data,
+    { restAddress: "http://engine.invalid" },
+    { now: AT, fetchImpl },
+  );
+
+  // The vanished pass acted even though the epoch pass could not reach the engine.
+  assertEquals(res.reason, "instance-vanished");
+  assertEquals(res.orphanedCount, 1);
+  const orphan = raw
+    .prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'")
+    .get() as { status: string };
+  assertEquals(orphan.status, ORPHANED_STATUS);
+
+  // The vanished pass's provenance is stamped with the DERIVED, correlatable id `<runId>-vanished`
+  // (the boot path omits opts.runId, so a bare random UUID would be non-locatable from the result).
+  const prov = raw
+    .prepare("SELECT run_id FROM reconcile_provenance WHERE source_table='feature_runs'")
+    .get() as { run_id: string };
+  assertEquals(prov.run_id, `${res.runId}-vanished`);
+
+  // Both passes recorded their own reconcile_runs row under correlatable ids.
+  const epochRun = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id=?").get(res.runId) as
+    | { reason: string }
+    | undefined;
+  assertEquals(epochRun?.reason, "engine-unreachable");
+  const vanishedRun = raw
+    .prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id=?")
+    .get(`${res.runId}-vanished`) as { reason: string; orphaned_count: number } | undefined;
+  assertEquals(vanishedRun?.reason, "instance-vanished");
+  assertEquals(vanishedRun?.orphaned_count, 1);
+});

package/app/reconcile.ts

@@ -29,12 +29,24 @@
 //     engine is a no-op too: reconcile NEVER orphans when it could not confirm a reset (a 401/5xx or
 //     a network error yields `reachable:false`, not a false "engine missing").
 //
+// A second, complementary pass covers the "instance absent/unknown" gap (issue #630) the epoch signal
+// alone cannot: an inflight run whose engine instance has VANISHED from the read model — its
+// `keyField` (process_key) has NO `_urban_instance_state` row at all (engine clean-reset, cluster
+// rebuild, or read-model pruning removed it), so the derived tracking edge has no `TERMINATED` row to
+// match and the run freezes at its last worker-owned status, wedging Active forever with no
+// reconciliation path (the observed pre-reset orphan `Magikcraft/nano-bpm#1051`, process_key 71506).
+// `reconcileVanishedInstances` drives every such row — active, dispatched, its key absent from
+// `_urban_instance_state`, and PAST A GRACE WINDOW (so a still-starting run not yet projected is
+// spared) — to the same `orphaned` terminal, with a DISTINCT provenance reason so an operator can
+// tell a vanished-instance orphan apart from an epoch-regression one. `runEngineReconcile` runs BOTH
+// passes, so startup and the operator command converge both failure modes in one call.
+//
 // The provenance is app-owned (not urban's `_urban_write_provenance`, which is a domain-free
 // insert-join sidecar written only inside a job): reconcile runs at boot / over HTTP, outside any
 // job, and needs to record the REASON + epoch + run id — which the app-owned `reconcile_provenance`
 // table carries, and the existing `app.db` backup convention makes the whole mutation reversible.
 
-import type { DataLayer, GatewayDataSource as DataSource } from "@nanobpm/urban";
+import type { DataLayer, GatewayDataSource as DataSource, InstanceTracking } from "@nanobpm/urban";
 import type { TopologyProbe } from "./enginePreflight.ts";
 import { activeStatusesFor, baseStatusFieldFor, engineBackedBindings, keyFieldFor } from "./instanceTracking.ts";
 
@@ -49,6 +61,28 @@ export const ORPHANED_STATUS = "orphaned";
  *  recorded incarnation epoch regressed (the #1065 signature). */
 export const RECONCILE_ORPHAN_REASON = "engine-reset/epoch-regression";
 
+/** The provenance reason stamped when a row is orphaned because its engine instance VANISHED from the
+ *  read model — the run's `keyField` (process_key) has NO `_urban_instance_state` row at all, so the
+ *  instance is absent/unknown in engine truth (engine clean-reset, cluster rebuild, or read-model
+ *  pruning removed the instance-state row entirely — issue #630). Deliberately DISTINCT from
+ *  {@link RECONCILE_ORPHAN_REASON} so an operator can tell an epoch-regression orphan apart from a
+ *  vanished-instance orphan, even though both land on the same `orphaned` terminal. */
+export const RECONCILE_VANISHED_REASON = "engine-instance/vanished";
+
+/** The default grace window (ms) a dispatched-but-instance-less row is spared before it is considered
+ *  vanished. A run dispatched moments ago (its `process_key` set) has not yet been polled into
+ *  `_urban_instance_state` by the instanceTracking reconciler (`pollMs` 5s + engine search latency),
+ *  so it transiently looks "vanished". This window (comfortably larger than a poll cycle) keeps a
+ *  legitimately-still-starting run from being folded to terminal prematurely (issue #630 AC #2). */
+export const DEFAULT_VANISHED_GRACE_MS = 5 * 60_000;
+
+/** The framework's canonical per-instance engine-lifecycle projection table (urban's
+ *  `_urban_instance_state`, keyed by `process_instance_key`). The vanished-instance reconcile joins
+ *  each engine-backed row's `keyField` against it: a run whose key has NO row here has no backing
+ *  instance in engine truth. `_urban_` prefixed (framework bookkeeping) so it is provisioned by the
+ *  runtime, not our migrations — the reconcile guards on its existence before acting. */
+const INSTANCE_STATE_TABLE = "_urban_instance_state";
+
 /** The single-row epoch ledger + its append-only run/provenance sidecars (migration 092). */
 const INCARNATION_TABLE = "engine_incarnation";
 const RUNS_TABLE = "reconcile_runs";
@@ -71,7 +105,12 @@ export interface EngineEpochObservation {
 }
 
 /** Why a reconcile pass acted (or did not). */
-export type ReconcileReason = "epoch-regression" | "seed-epoch" | "no-op" | "engine-unreachable";
+export type ReconcileReason =
+  | "epoch-regression"
+  | "seed-epoch"
+  | "no-op"
+  | "engine-unreachable"
+  | "instance-vanished";
 
 /** One orphaned engine-backed row. */
 export interface OrphanedRow {
@@ -107,6 +146,14 @@ export interface ReconcileOptions {
   log?: ReconcileLog;
 }
 
+/** Options for the vanished-instance reconcile pass ({@link reconcileVanishedInstances}). */
+export interface VanishedReconcileOptions extends ReconcileOptions {
+  /** How long (ms) a dispatched-but-instance-less row is spared before it is folded to terminal, so a
+   *  still-starting run (not yet projected into `_urban_instance_state`) is not orphaned prematurely.
+   *  Defaults to {@link DEFAULT_VANISHED_GRACE_MS}. */
+  graceMs?: number;
+}
+
 /** Read the incarnation epoch out of a `/v2/topology` body — `nano.incarnation` (or its `epoch`
  *  alias), coerced from a number or a numeric string. Any other shape (absent, non-numeric) yields
  *  null: "the engine exposes no epoch". A null is a no-op ONLY when no epoch was previously recorded;
@@ -175,6 +222,78 @@ async function persistEpoch(src: DataSource, epoch: number, at: string): Promise
   );
 }
 
+/** The resolved schema + tracking selectors for one engine-backed binding, or `null` when the binding
+ *  carries no `activeStatuses` selector (it cannot classify "in-flight", so it is skipped). */
+interface BindingShape {
+  table: string;
+  active: readonly string[];
+  statusField: string;
+  keyField: string;
+  pkCol: string;
+  hasUpdatedAt: boolean;
+}
+
+/** The shape of a row selected for possible orphaning. */
+interface OrphanCandidate {
+  __pk: unknown;
+  __key: unknown;
+  __status: unknown;
+  __updated?: unknown;
+}
+
+/** Resolve a binding's tracking selectors + physical schema, or `null` to skip a selector-less
+ *  binding (activeStatusesFor would throw; we tolerate it here). */
+async function resolveShape(src: DataSource, binding: InstanceTracking): Promise<BindingShape | null> {
+  if (!binding.activeStatuses?.length) return null;
+  const table = binding.table;
+  const { pkCol, hasUpdatedAt } = await tableShape(src, table);
+  return {
+    table,
+    active: activeStatusesFor(table),
+    statusField: baseStatusFieldFor(table),
+    keyField: keyFieldFor(table),
+    pkCol,
+    hasUpdatedAt,
+  };
+}
+
+/** Drive ONE candidate row to `orphaned` with provenance, GUARDED: the UPDATE re-asserts the exact
+ *  status read AND a populated key (plus any `extraGuardSql`, e.g. the still-vanished re-check), so a
+ *  writer that flipped the row to a newer terminal status (or an instance that reappeared) between the
+ *  SELECT and this UPDATE wins the race — we never clobber that history back to `orphaned`. Only a row
+ *  we actually transitioned (`res.changed > 0`) gets provenance and is returned. `updated_at` is
+ *  stamped (when the table has one) so the transition refreshes the row's timestamp like every other
+ *  status transition in the codebase. Runs inside the caller's transaction. */
+async function orphanRow(
+  src: DataSource,
+  shape: BindingShape,
+  row: OrphanCandidate,
+  reason: string,
+  observedEpoch: number | null,
+  runId: string,
+  at: string,
+  extraGuardSql = "",
+): Promise<OrphanedRow | null> {
+  const { table, pkCol, statusField, keyField, hasUpdatedAt } = shape;
+  const pk = String(row.__pk);
+  const key = row.__key == null ? null : String(row.__key);
+  const fromStatus = String(row.__status);
+  const res = await src.exec(
+    `UPDATE ${q(table)} SET ${q(statusField)} = ?` +
+      (hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} = ?` : "") +
+      ` WHERE ${q(pkCol)} = ? AND ${q(statusField)} = ? AND ${q(keyField)} IS NOT NULL${extraGuardSql}`,
+    hasUpdatedAt ? [ORPHANED_STATUS, at, row.__pk, fromStatus] : [ORPHANED_STATUS, row.__pk, fromStatus],
+  );
+  if (res.changed <= 0) return null;
+  await src.exec(
+    `INSERT INTO ${PROVENANCE_TABLE} ` +
+      `(run_id, source_table, pk_value, key_value, from_status, to_status, reason, observed_epoch, at) ` +
+      `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [runId, table, pk, key, fromStatus, ORPHANED_STATUS, reason, observedEpoch, at],
+  );
+  return { table, pk, key, fromStatus };
+}
+
 /** Orphan every NON-terminal, engine-backed row across all instanceTracking bindings, recording one
  *  `reconcile_provenance` row per transition. Runs inside the caller's transaction. */
 async function orphanEngineBackedRows(
@@ -185,52 +304,87 @@ async function orphanEngineBackedRows(
 ): Promise<OrphanedRow[]> {
   const orphaned: OrphanedRow[] = [];
   for (const binding of engineBackedBindings()) {
-    const table = binding.table;
-    // A binding with no active-status selector cannot classify "in-flight" — skip it rather than
-    // guess (activeStatusesFor would throw; we tolerate a selector-less binding).
-    if (!binding.activeStatuses?.length) continue;
-    const active = activeStatusesFor(table);
-    const statusField = baseStatusFieldFor(table);
-    const keyField = keyFieldFor(table);
-    const { pkCol, hasUpdatedAt } = await tableShape(src, table);
-    const placeholders = active.map(() => "?").join(", ");
-    const rows = await src.query<{ __pk: unknown; __key: unknown; __status: unknown }>(
-      `SELECT ${q(pkCol)} AS __pk, ${q(keyField)} AS __key, ${q(statusField)} AS __status ` +
-        `FROM ${q(table)} WHERE ${q(statusField)} IN (${placeholders}) AND ${q(keyField)} IS NOT NULL`,
-      [...active],
+    const shape = await resolveShape(src, binding);
+    if (!shape) continue;
+    const placeholders = shape.active.map(() => "?").join(", ");
+    const rows = await src.query<OrphanCandidate>(
+      `SELECT ${q(shape.pkCol)} AS __pk, ${q(shape.keyField)} AS __key, ${q(shape.statusField)} AS __status ` +
+        `FROM ${q(shape.table)} WHERE ${q(shape.statusField)} IN (${placeholders}) AND ${q(shape.keyField)} IS NOT NULL`,
+      [...shape.active],
     );
     for (const row of rows) {
-      const pk = String(row.__pk);
-      const key = row.__key == null ? null : String(row.__key);
-      const fromStatus = String(row.__status);
-      // GUARDED update: re-assert the exact status we read AND a populated key, so a writer that
-      // flipped the row to a newer terminal status (or cleared its key) between the SELECT above and
-      // this UPDATE wins the race — we never clobber that terminal history back to `orphaned`. Only a
-      // row we actually transitioned (`res.changed > 0`) gets provenance and is counted. We also stamp
-      // `updated_at` (when the table has one) so the transition to `orphaned` refreshes the row's
-      // timestamp the same way every other status transition in the codebase does — leaving it stale
-      // would misrepresent the orphaning moment to the UI/audits.
-      const res = await src.exec(
-        `UPDATE ${q(table)} SET ${q(statusField)} = ?` +
-          (hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} = ?` : "") +
-          ` WHERE ${q(pkCol)} = ? AND ${q(statusField)} = ? AND ${q(keyField)} IS NOT NULL`,
-        hasUpdatedAt
-          ? [ORPHANED_STATUS, at, row.__pk, fromStatus]
-          : [ORPHANED_STATUS, row.__pk, fromStatus],
-      );
-      if (res.changed <= 0) continue;
-      await src.exec(
-        `INSERT INTO ${PROVENANCE_TABLE} ` +
-          `(run_id, source_table, pk_value, key_value, from_status, to_status, reason, observed_epoch, at) ` +
-          `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-        [runId, table, pk, key, fromStatus, ORPHANED_STATUS, RECONCILE_ORPHAN_REASON, observedEpoch, at],
-      );
-      orphaned.push({ table, pk, key, fromStatus });
+      const o = await orphanRow(src, shape, row, RECONCILE_ORPHAN_REASON, observedEpoch, runId, at);
+      if (o) orphaned.push(o);
     }
   }
   return orphaned;
 }
 
+/** Whether a row's `updated_at` is younger than the grace window — i.e. it was (re)dispatched too
+ *  recently to have been projected into `_urban_instance_state` yet, so it must NOT be folded. A
+ *  null/unparseable timestamp is treated as "within grace" (spared): when we cannot establish a row's
+ *  age we must NOT orphan it — a nullable `updated_at` (e.g. `delivery_units.updated_at`,
+ *  db/migrations/088_delivery_units.sql) would otherwise fold a live row. Erring toward sparing at
+ *  worst leaves a genuinely-vanished ageless row for a later pass once it carries a usable timestamp;
+ *  erring the other way wedges/destroys a live run, so we choose the safe default. */
+function withinGrace(updated: unknown, nowMs: number, graceMs: number): boolean {
+  if (updated == null) return true;
+  const t = Date.parse(String(updated));
+  if (!Number.isFinite(t)) return true;
+  return nowMs - t < graceMs;
+}
+
+/** Orphan every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
+ *  `_urban_instance_state` row — the instance is absent/unknown in engine truth (vanished, issue
+ *  #630) — and whose last transition is older than the grace window. Records one
+ *  `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). Runs inside
+ *  the caller's transaction. */
+async function orphanVanishedRows(
+  src: DataSource,
+  runId: string,
+  at: string,
+  nowMs: number,
+  graceMs: number,
+): Promise<OrphanedRow[]> {
+  const orphaned: OrphanedRow[] = [];
+  for (const binding of engineBackedBindings()) {
+    const shape = await resolveShape(src, binding);
+    if (!shape) continue;
+    const placeholders = shape.active.map(() => "?").join(", ");
+    const updatedSel = shape.hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} AS __updated` : "";
+    // Active, dispatched (key populated) rows whose engine instance key has NO matching
+    // `_urban_instance_state` row — absent/unknown in engine truth.
+    const rows = await src.query<OrphanCandidate>(
+      `SELECT ${q(shape.pkCol)} AS __pk, ${q(shape.keyField)} AS __key, ${q(shape.statusField)} AS __status${updatedSel} ` +
+        `FROM ${q(shape.table)} b WHERE ${q(shape.statusField)} IN (${placeholders}) AND ${q(shape.keyField)} IS NOT NULL ` +
+        `AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s WHERE s.process_instance_key = b.${q(shape.keyField)})`,
+      [...shape.active],
+    );
+    // Re-assert "still no instance-state row" in the guarded UPDATE too, so an instance that reappears
+    // (the poller records it) between the SELECT above and the UPDATE wins the race.
+    const stillVanishedGuard =
+      ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
+      `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
+    for (const row of rows) {
+      if (shape.hasUpdatedAt && withinGrace(row.__updated, nowMs, graceMs)) continue;
+      const o = await orphanRow(src, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
+      if (o) orphaned.push(o);
+    }
+  }
+  return orphaned;
+}
+
+/** Whether the framework `_urban_instance_state` projection exists in this source. When it does NOT,
+ *  the vanished-instance pass is a hard no-op: without the projection every dispatched row would look
+ *  "vanished", so we must never orphan on its absence. */
+async function instanceStateTableExists(src: DataSource): Promise<boolean> {
+  const rows = await src.query<{ n: number }>(
+    `SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
+    [INSTANCE_STATE_TABLE],
+  );
+  return rows.length > 0 && Number(rows[0].n) > 0;
+}
+
 /**
  * Reconcile the app's engine-backed projections against one epoch observation. Pure of I/O beyond the
  * data layer (the topology probe is {@link probeEngineEpoch}, injected as `observation`), so the
@@ -300,6 +454,75 @@ export async function reconcileEngineBackedWork(
   return { runId, reason: result.reason, observedEpoch, recordedEpoch, orphanedCount: result.orphaned.length, orphaned: result.orphaned };
 }
 
+/**
+ * Reconcile engine-backed inflight work against the framework's canonical per-instance projection
+ * (`_urban_instance_state`) — the "instance absent/unknown" gap (issue #630), DISTINCT from the
+ * epoch-regression reset the {@link reconcileEngineBackedWork} pass handles.
+ *
+ * When an engine instance VANISHES from the read model — engine clean-reset, cluster rebuild, or
+ * read-model pruning removes the `_urban_instance_state` row entirely — there is no `TERMINATED` row
+ * for the derived tracking edge to match, so the run freezes at its last worker-owned status
+ * (`escalated`/`awaiting_operator`) and wedges the Active list forever with no reconciliation path.
+ * "The instance backing this run no longer exists in engine truth" is a terminal condition: this pass
+ * drives every such row (active, dispatched, its `keyField` absent from `_urban_instance_state`, and
+ * past the grace window) to the defined `orphaned` terminal WITH PROVENANCE.
+ *
+ * Safety:
+ *   • GRACE WINDOW — a just-dispatched run has not yet been polled into `_urban_instance_state`; only
+ *     rows whose last transition is older than `graceMs` are folded, so a still-starting run is never
+ *     prematurely orphaned (AC #2).
+ *   • PROJECTION-PRESENT — if `_urban_instance_state` does not exist (the runtime has not provisioned
+ *     it), every dispatched row would look vanished, so the pass is a hard no-op.
+ *   • GUARDED — the same status-re-assert as the epoch pass, plus a still-vanished re-check, so a
+ *     concurrent terminal write or a reappearing instance wins the race.
+ *   • This pass reads the app's OWN last-known projection, not a live probe, so it acts correctly on a
+ *     genuinely-vanished instance regardless of transient engine reachability (a live instance keeps
+ *     its persisted ACTIVE row across a restart, so it is never mistaken for vanished).
+ */
+export async function reconcileVanishedInstances(
+  data: DataLayer,
+  opts: VanishedReconcileOptions = {},
+): Promise<ReconcileResult> {
+  const src = data.open(opts.sourceName);
+  const clock = opts.now?.() ?? new Date();
+  const at = clock.toISOString();
+  const nowMs = clock.getTime();
+  const runId = opts.runId ?? crypto.randomUUID();
+  const graceMs = opts.graceMs ?? DEFAULT_VANISHED_GRACE_MS;
+
+  // Without the framework projection we cannot tell a vanished instance from a live one — every
+  // dispatched row would look vanished. NO-OP rather than orphan live work.
+  if (!(await instanceStateTableExists(src))) {
+    await recordRun(src, { runId, at, observedEpoch: null, recordedEpoch: null, reason: "no-op", orphanedCount: 0 });
+    opts.log?.info(`reconcile(vanished): instance-state projection absent — no-op [run ${runId}].`);
+    return { runId, reason: "no-op", observedEpoch: null, recordedEpoch: null, orphanedCount: 0, orphaned: [] };
+  }
+
+  const orphaned = await src.tx(async (t) => {
+    const rows = await orphanVanishedRows(t, runId, at, nowMs, graceMs);
+    const reason: ReconcileReason = rows.length > 0 ? "instance-vanished" : "no-op";
+    await recordRun(t, { runId, at, observedEpoch: null, recordedEpoch: null, reason, orphanedCount: rows.length });
+    return rows;
+  });
+
+  if (orphaned.length > 0) {
+    opts.log?.warn(
+      `reconcile(vanished): orphaned ${orphaned.length} inflight row(s) whose engine instance ` +
+        `vanished from the read model [run ${runId}].`,
+    );
+  } else {
+    opts.log?.info(`reconcile(vanished): no vanished instances — no-op [run ${runId}].`);
+  }
+  return {
+    runId,
+    reason: orphaned.length > 0 ? "instance-vanished" : "no-op",
+    observedEpoch: null,
+    recordedEpoch: null,
+    orphanedCount: orphaned.length,
+    orphaned,
+  };
+}
+
 async function recordRun(
   src: DataSource,
   run: { runId: string; at: string; observedEpoch: number | null; recordedEpoch: number | null; reason: ReconcileReason; orphanedCount: number },
@@ -312,15 +535,47 @@ async function recordRun(
 }
 
 /** Probe the engine's incarnation epoch, then reconcile — the wiring both startup (main.ts) and the
- *  `reconcileEngineState` operator command share, so the two paths can never diverge. */
+ *  `reconcileEngineState` operator command share, so the two paths can never diverge. Runs BOTH the
+ *  epoch-regression pass (engine reset/rewind) and the vanished-instance pass (an inflight run whose
+ *  instance is absent/unknown in `_urban_instance_state` — issue #630), returning ONE merged result.
+ *  Reason precedence: `epoch-regression` always wins; otherwise, if the vanished pass orphaned any
+ *  rows the reason is `instance-vanished` (even when the epoch pass reported a non-regression state
+ *  such as `engine-unreachable`); otherwise the epoch pass's reason stands. `orphanedCount` /
+ *  `orphaned` cover both passes. The two passes never double-fold a row: once the epoch pass orphans a
+ *  row it leaves `activeStatuses`, so the vanished pass no longer selects it. */
 export async function runEngineReconcile(
   data: DataLayer,
   engineRest: { restAddress: string; token?: string },
-  opts: ReconcileOptions & { fetchImpl?: typeof fetch } = {},
+  opts: VanishedReconcileOptions & { fetchImpl?: typeof fetch } = {},
 ): Promise<ReconcileResult> {
   const observation = await probeEngineEpoch(engineRest.restAddress, {
     token: engineRest.token,
     fetchImpl: opts.fetchImpl,
   });
-  return reconcileEngineBackedWork(data, observation, opts);
+  const epoch = await reconcileEngineBackedWork(data, observation, opts);
+  // A distinct run id so the vanished pass's `reconcile_runs`/provenance rows never collide with the
+  // epoch pass's (run_id is a PRIMARY KEY). DERIVE it from the epoch pass's resolved run id — which is
+  // also the merged result's `runId` — so it is `<runId>-vanished` on EVERY path, including the boot
+  // path where `opts.runId` is omitted (a bare random UUID here would be non-correlatable to the
+  // returned `runId`). Operators can always locate the vanished pass's provenance from the reported id.
+  const vanished = await reconcileVanishedInstances(data, {
+    ...opts,
+    runId: `${epoch.runId}-vanished`,
+  });
+
+  const orphaned = [...epoch.orphaned, ...vanished.orphaned];
+  const reason: ReconcileReason =
+    epoch.reason === "epoch-regression"
+      ? "epoch-regression"
+      : vanished.orphanedCount > 0
+        ? "instance-vanished"
+        : epoch.reason;
+  return {
+    runId: epoch.runId,
+    reason,
+    observedEpoch: epoch.observedEpoch,
+    recordedEpoch: epoch.recordedEpoch,
+    orphanedCount: orphaned.length,
+    orphaned,
+  };
 }

package/main.ts

@@ -103,12 +103,14 @@ if (httpServer instanceof Server) {
   app.log.warn("agentic channel not mounted: app.httpServer is not a node:http Server on this host");
 }
 
-// Engine-reset reconciliation (issue #622). On boot, compare the engine's incarnation epoch against
-// the last-seen value; on a REGRESSION (the engine was reset/restored/rewound and re-minted its keys,
-// Magikcraft/nano-bpm#1065) drive every dangling engine-backed inflight row to the defined `orphaned`
-// terminal WITH PROVENANCE — BEFORE the pollers below start projecting off stale, dead instances.
-// Guarded: an unreachable engine is a no-op (it never orphans live work), and any failure degrades to
-// a warn so reconcile can never block boot.
+// Engine-reset reconciliation (issues #622, #630). On boot, compare the engine's incarnation epoch
+// against the last-seen value; on a REGRESSION (the engine was reset/restored/rewound and re-minted
+// its keys, Magikcraft/nano-bpm#1065) drive every dangling engine-backed inflight row to the defined
+// `orphaned` terminal WITH PROVENANCE. A second pass also folds any run whose engine instance has
+// VANISHED from the read model (no `_urban_instance_state` row, past a grace window — issue #630),
+// which the epoch signal alone can't catch — BEFORE the pollers below start projecting off stale,
+// dead instances. Guarded: an unreachable engine / an absent projection is a no-op (never orphans
+// live work), and any failure degrades to a warn so reconcile can never block boot.
 if (app.data) {
   try {
     const reconciled = await runEngineReconcile(

package/openapi.yaml

@@ -1042,7 +1042,10 @@ components:
           type: integer
     ReconcileReport:
       type: object
-      description: The result of one engine-reset reconciliation pass (issue #622).
+      description: "The merged result of the engine-reconcile invocation — an aggregate of both the
+        epoch-regression (engine reset/rewind) pass and the vanished-instance pass (issues #622 and
+        #630). `runId` is the epoch pass's run id; the vanished pass records its own provenance under
+        the derived id `<runId>-vanished`."
       additionalProperties: false
       required:
         - runId
@@ -1054,15 +1057,19 @@ components:
       properties:
         runId:
           type: string
-          description: The reconcile run id every orphaned transition's provenance is stamped with.
+          description: The reconcile run id the epoch pass's orphaned-transition provenance is stamped
+            with. The vanished-instance pass's provenance is stamped with the derived, deterministic id
+            `<runId>-vanished`, so operators can locate the provenance rows for either pass from this id.
         reason:
           type: string
-          description: Why this pass acted (or did not).
+          description: Why this pass acted (or did not). `instance-vanished` — a run whose engine
+            instance is absent/unknown in the read model was orphaned (issue #630).
           enum:
             - epoch-regression
             - seed-epoch
             - no-op
             - engine-unreachable
+            - instance-vanished
         observedEpoch:
           type: integer
           nullable: true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nanobpm/nano-workforce",
-  "version": "0.174.0",
+  "version": "0.175.0",
   "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
   "type": "module",
   "main": "main.ts",