@nanobpm/nano-workforce 0.174.0 → 0.174.1

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## [0.174.1](https://github.com/nanobpm/nano-workforce/compare/v0.174.0...v0.174.1) (2026-09-02)
+
+### Bug Fixes
+
+* **reconcile:** fold vanished engine instances to orphaned, unwedging inflight runs ([#706](https://github.com/nanobpm/nano-workforce/issues/706)) ([68461da](https://github.com/nanobpm/nano-workforce/commit/68461dae90fba5ab6caaa8eef595e63f3995dc88)), closes [#2](https://github.com/nanobpm/nano-workforce/issues/2) [#630](https://github.com/nanobpm/nano-workforce/issues/630) [#627](https://github.com/nanobpm/nano-workforce/issues/627)
+
 ## [0.174.0](https://github.com/nanobpm/nano-workforce/compare/v0.173.0...v0.174.0) (2026-09-02)
 
 ### Features

package/app/reconcile.test.ts

@@ -1,4 +1,4 @@
-// Red/green coverage for the app-side engine-reset reconciliation surface (issue #622).
+// Red/green coverage for the app-side engine-reset reconciliation surface (issues #622 and #630).
 //
 // The core scenario the incident (Magikcraft/nano-bpm#1065) demanded a supported remedy for: the
 // engine is reset and its incarnation epoch REGRESSES, while `app.db` still projects engine-backed
@@ -13,21 +13,55 @@ import { test } from "node:test";
 import { assertEquals } from "#test-assert";
 import { freshData } from "../test/reconcileDb.ts";
 import {
+  DEFAULT_VANISHED_GRACE_MS,
   ORPHANED_STATUS,
   parseEngineEpoch,
   RECONCILE_ORPHAN_REASON,
+  RECONCILE_VANISHED_REASON,
   reconcileEngineBackedWork,
+  reconcileVanishedInstances,
+  runEngineReconcile,
 } from "./reconcile.ts";
 
 const AT = () => new Date("2026-02-02T00:00:00.000Z");
 
-function seedFeatureRun(raw: DatabaseSync, key: string, status: string, processKey: string | null): void {
+/** The canonical `_urban_instance_state` DDL (urban's framework projection, `_urban_`-prefixed so it
+ *  is provisioned by the runtime — NOT our migrations). Mirrors `InstanceStateStore`'s schema so the
+ *  vanished-instance reconcile is exercised against exactly the table it reads in production. */
+function ensureInstanceState(raw: DatabaseSync): void {
+  raw.exec(
+    `CREATE TABLE IF NOT EXISTS _urban_instance_state (
+       process_instance_key TEXT NOT NULL,
+       state                TEXT NOT NULL,
+       waiting_on_human     INTEGER NOT NULL DEFAULT 0,
+       updated_at           TEXT NOT NULL,
+       PRIMARY KEY (process_instance_key)
+     );`,
+  );
+}
+
+function seedInstanceState(raw: DatabaseSync, processKey: string, state: string): void {
+  raw
+    .prepare(
+      `INSERT INTO _urban_instance_state (process_instance_key, state, waiting_on_human, updated_at)
+       VALUES (?, ?, 0, '2026-01-15')`,
+    )
+    .run(processKey, state);
+}
+
+function seedFeatureRun(
+  raw: DatabaseSync,
+  key: string,
+  status: string,
+  processKey: string | null,
+  updatedAt = "2026-01-01",
+): void {
   raw
     .prepare(
       `INSERT INTO feature_runs (feature_key, repo, issue_number, issue_url, base_branch, status, process_key, created_at, updated_at)
-       VALUES (?, 'o/r', 1, 'https://x', 'main', ?, ?, '2026-01-01', '2026-01-01')`,
+       VALUES (?, 'o/r', 1, 'https://x', 'main', ?, ?, '2026-01-01', ?)`,
     )
-    .run(key, status, processKey);
+    .run(key, status, processKey, updatedAt);
 }
 
 function seedDeliveryGraphRun(raw: DatabaseSync, runKey: string, status: string, processKey: string | null): void {
@@ -210,3 +244,199 @@ test("RED→GREEN: orphaning stamps updated_at so the transition timestamp isn't
   assertEquals(dg.status, ORPHANED_STATUS);
   assertEquals(dg.updated_at, at);
 });
+
+// --- Vanished-instance reconciliation (issue #630) --------------------------------------------
+// The "instance absent/unknown" gap, DISTINCT from the epoch-regression reset above: when an engine
+// instance VANISHES from the read model (`_urban_instance_state` row pruned/never re-created after a
+// clean reset), the derived terminal edge has no `TERMINATED` row to match, so the run freezes at its
+// last worker-owned status (`escalated`) and wedges Active forever. `reconcileVanishedInstances`
+// drives those orphaned-in-truth rows to `orphaned` WITH PROVENANCE — gated on a grace window so a
+// still-starting run (not yet projected) is spared.
+
+test("RED→GREEN: a vanished instance (no _urban_instance_state row, past grace) is orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // The pre-reset orphan from the incident: escalated, keyed on a HIGH pre-reset process_key whose
+  // instance is absent from the current read model. Its updated_at is ~32 days before AT() (past grace).
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+  // A live sibling: still ACTIVE in the projection — must be left untouched.
+  seedFeatureRun(raw, "o/r#live", "running", "200");
+  seedInstanceState(raw, "200", "ACTIVE");
+
+  // RED (pre-fix): the orphan reads `escalated` (Active) indefinitely — no terminal edge fires.
+  const before = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
+    status: string;
+  };
+  assertEquals(before.status, "escalated");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.reason, "instance-vanished");
+  assertEquals(res.orphanedCount, 1);
+
+  const orphan = raw
+    .prepare("SELECT status, updated_at FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'")
+    .get() as { status: string; updated_at: string };
+  assertEquals(orphan.status, ORPHANED_STATUS);
+  // The transition refreshes updated_at like every other status transition (not left stale).
+  assertEquals(orphan.updated_at, AT().toISOString());
+
+  // The live instance (ACTIVE row present) is never touched.
+  const live = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#live'").get() as { status: string };
+  assertEquals(live.status, "running");
+
+  const prov = raw
+    .prepare("SELECT * FROM reconcile_provenance WHERE source_table='feature_runs'")
+    .get() as Record<string, unknown>;
+  assertEquals(prov.to_status, ORPHANED_STATUS);
+  assertEquals(prov.from_status, "escalated");
+  assertEquals(prov.reason, RECONCILE_VANISHED_REASON);
+  assertEquals(prov.key_value, "71506");
+  assertEquals(prov.run_id, "van-1");
+  assertEquals(prov.observed_epoch, null);
+
+  const run = raw.prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id='van-1'").get() as {
+    reason: string;
+    orphaned_count: number;
+  };
+  assertEquals(run.reason, "instance-vanished");
+  assertEquals(run.orphaned_count, 1);
+});
+
+test("a still-starting run within the grace window is NOT prematurely folded", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // Dispatched moments ago — its process_key is set but the reconciler has not yet projected the
+  // instance into _urban_instance_state. updated_at is 30s before AT(), inside the grace window.
+  const justNow = new Date(AT().getTime() - 30_000).toISOString();
+  seedFeatureRun(raw, "o/r#starting", "running", "999", justNow);
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#starting'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "running");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+  // A generous grace window is the point — the default comfortably exceeds a poll cycle.
+  assertEquals(DEFAULT_VANISHED_GRACE_MS >= 60_000, true);
+});
+
+test("RED→GREEN: a row whose updated_at is null/unparseable is spared, never orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A tracked table's `updated_at` can be nullable (e.g. delivery_units.updated_at,
+  // db/migrations/088_delivery_units.sql) or carry an unparseable value. Its instance is absent from
+  // the read model, so without a usable age we cannot tell a genuinely-vanished row from a live one.
+  // RED (pre-fix): withinGrace treated an unestablishable age as "old enough" and folded the row.
+  // GREEN: we err toward sparing — an ageless row is treated as within grace and left untouched.
+  seedFeatureRun(raw, "o/r#ageless", "running", "888", "not-a-timestamp");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#ageless'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "running");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("terminal history, keyless rows, and rows with a live instance are never folded as vanished", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "term#1", "merged", "88"); // terminal — not in activeStatuses
+  seedFeatureRun(raw, "nokeed#1", "running", null); // active but never dispatched (no engine key)
+  seedFeatureRun(raw, "live#1", "escalated", "89"); // active, but its instance is still present
+  seedInstanceState(raw, "89", "ACTIVE");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.orphanedCount, 0);
+  const statuses = raw.prepare("SELECT feature_key, status FROM feature_runs ORDER BY feature_key").all() as {
+    feature_key: string;
+    status: string;
+  }[];
+  assertEquals(statuses.find((r) => r.feature_key === "term#1")?.status, "merged");
+  assertEquals(statuses.find((r) => r.feature_key === "nokeed#1")?.status, "running");
+  assertEquals(statuses.find((r) => r.feature_key === "live#1")?.status, "escalated");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("no-op when the _urban_instance_state projection is absent (never orphan on its absence)", async () => {
+  const { data, raw } = freshData();
+  // NOTE: no ensureInstanceState — the framework projection has not been provisioned.
+  seedFeatureRun(raw, "o/r#1", "escalated", "71506");
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+
+  assertEquals(res.reason, "no-op");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
+  assertEquals(row.status, "escalated");
+  const run = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id='van-1'").get() as { reason: string };
+  assertEquals(run.reason, "no-op");
+});
+
+test("idempotent: a second vanished pass is a no-op (the orphaned row left activeStatuses)", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "o/r#1", "escalated", "71506");
+
+  const first = await reconcileVanishedInstances(data, { now: AT, runId: "van-1" });
+  assertEquals(first.orphanedCount, 1);
+
+  const second = await reconcileVanishedInstances(data, { now: AT, runId: "van-2" });
+  assertEquals(second.reason, "no-op");
+  assertEquals(second.orphanedCount, 0);
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 1);
+});
+
+// --- Merged seam: runEngineReconcile (both passes, one result) --------------------------------
+// The operator/startup seam merges the epoch-regression and vanished-instance passes into ONE
+// result. This guards the merged behavior the two per-pass suites above don't reach: run-id
+// correlation (the vanished pass's provenance must be locatable from the returned `runId`) and
+// `reason` selection when the epoch pass is `engine-unreachable` yet vanished instances are orphaned.
+
+test("runEngineReconcile: engine-unreachable epoch pass still folds vanished instances, with a correlatable run id", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A vanished orphan (escalated, past grace, instance absent from the read model).
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+
+  // The engine is unreachable — the epoch probe fails, so the epoch pass reports `engine-unreachable`
+  // and orphans nothing; the vanished pass must still act.
+  const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
+  const res = await runEngineReconcile(
+    data,
+    { restAddress: "http://engine.invalid" },
+    { now: AT, fetchImpl },
+  );
+
+  // The vanished pass acted even though the epoch pass could not reach the engine.
+  assertEquals(res.reason, "instance-vanished");
+  assertEquals(res.orphanedCount, 1);
+  const orphan = raw
+    .prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'")
+    .get() as { status: string };
+  assertEquals(orphan.status, ORPHANED_STATUS);
+
+  // The vanished pass's provenance is stamped with the DERIVED, correlatable id `<runId>-vanished`
+  // (the boot path omits opts.runId, so a bare random UUID would be non-locatable from the result).
+  const prov = raw
+    .prepare("SELECT run_id FROM reconcile_provenance WHERE source_table='feature_runs'")
+    .get() as { run_id: string };
+  assertEquals(prov.run_id, `${res.runId}-vanished`);
+
+  // Both passes recorded their own reconcile_runs row under correlatable ids.
+  const epochRun = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id=?").get(res.runId) as
+    | { reason: string }
+    | undefined;
+  assertEquals(epochRun?.reason, "engine-unreachable");
+  const vanishedRun = raw
+    .prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id=?")
+    .get(`${res.runId}-vanished`) as { reason: string; orphaned_count: number } | undefined;
+  assertEquals(vanishedRun?.reason, "instance-vanished");
+  assertEquals(vanishedRun?.orphaned_count, 1);
+});

package/app/reconcile.ts

@@ -29,12 +29,24 @@
 //     engine is a no-op too: reconcile NEVER orphans when it could not confirm a reset (a 401/5xx or
 //     a network error yields `reachable:false`, not a false "engine missing").
 //
+// A second, complementary pass covers the "instance absent/unknown" gap (issue #630) the epoch signal
+// alone cannot: an inflight run whose engine instance has VANISHED from the read model — its
+// `keyField` (process_key) has NO `_urban_instance_state` row at all (engine clean-reset, cluster
+// rebuild, or read-model pruning removed it), so the derived tracking edge has no `TERMINATED` row to
+// match and the run freezes at its last worker-owned status, wedging Active forever with no
+// reconciliation path (the observed pre-reset orphan `Magikcraft/nano-bpm#1051`, process_key 71506).
+// `reconcileVanishedInstances` drives every such row — active, dispatched, its key absent from
+// `_urban_instance_state`, and PAST A GRACE WINDOW (so a still-starting run not yet projected is
+// spared) — to the same `orphaned` terminal, with a DISTINCT provenance reason so an operator can
+// tell a vanished-instance orphan apart from an epoch-regression one. `runEngineReconcile` runs BOTH
+// passes, so startup and the operator command converge both failure modes in one call.
+//
 // The provenance is app-owned (not urban's `_urban_write_provenance`, which is a domain-free
 // insert-join sidecar written only inside a job): reconcile runs at boot / over HTTP, outside any
 // job, and needs to record the REASON + epoch + run id — which the app-owned `reconcile_provenance`
 // table carries, and the existing `app.db` backup convention makes the whole mutation reversible.
 
-import type { DataLayer, GatewayDataSource as DataSource } from "@nanobpm/urban";
+import type { DataLayer, GatewayDataSource as DataSource, InstanceTracking } from "@nanobpm/urban";
 import type { TopologyProbe } from "./enginePreflight.ts";
 import { activeStatusesFor, baseStatusFieldFor, engineBackedBindings, keyFieldFor } from "./instanceTracking.ts";
 
@@ -49,6 +61,28 @@ export const ORPHANED_STATUS = "orphaned";
  *  recorded incarnation epoch regressed (the #1065 signature). */
 export const RECONCILE_ORPHAN_REASON = "engine-reset/epoch-regression";
 
+/** The provenance reason stamped when a row is orphaned because its engine instance VANISHED from the
+ *  read model — the run's `keyField` (process_key) has NO `_urban_instance_state` row at all, so the
+ *  instance is absent/unknown in engine truth (engine clean-reset, cluster rebuild, or read-model
+ *  pruning removed the instance-state row entirely — issue #630). Deliberately DISTINCT from
+ *  {@link RECONCILE_ORPHAN_REASON} so an operator can tell an epoch-regression orphan apart from a
+ *  vanished-instance orphan, even though both land on the same `orphaned` terminal. */
+export const RECONCILE_VANISHED_REASON = "engine-instance/vanished";
+
+/** The default grace window (ms) a dispatched-but-instance-less row is spared before it is considered
+ *  vanished. A run dispatched moments ago (its `process_key` set) has not yet been polled into
+ *  `_urban_instance_state` by the instanceTracking reconciler (`pollMs` 5s + engine search latency),
+ *  so it transiently looks "vanished". This window (comfortably larger than a poll cycle) keeps a
+ *  legitimately-still-starting run from being folded to terminal prematurely (issue #630 AC #2). */
+export const DEFAULT_VANISHED_GRACE_MS = 5 * 60_000;
+
+/** The framework's canonical per-instance engine-lifecycle projection table (urban's
+ *  `_urban_instance_state`, keyed by `process_instance_key`). The vanished-instance reconcile joins
+ *  each engine-backed row's `keyField` against it: a run whose key has NO row here has no backing
+ *  instance in engine truth. `_urban_` prefixed (framework bookkeeping) so it is provisioned by the
+ *  runtime, not our migrations — the reconcile guards on its existence before acting. */
+const INSTANCE_STATE_TABLE = "_urban_instance_state";
+
 /** The single-row epoch ledger + its append-only run/provenance sidecars (migration 092). */
 const INCARNATION_TABLE = "engine_incarnation";
 const RUNS_TABLE = "reconcile_runs";
@@ -71,7 +105,12 @@ export interface EngineEpochObservation {
 }
 
 /** Why a reconcile pass acted (or did not). */
-export type ReconcileReason = "epoch-regression" | "seed-epoch" | "no-op" | "engine-unreachable";
+export type ReconcileReason =
+  | "epoch-regression"
+  | "seed-epoch"
+  | "no-op"
+  | "engine-unreachable"
+  | "instance-vanished";
 
 /** One orphaned engine-backed row. */
 export interface OrphanedRow {
@@ -107,6 +146,14 @@ export interface ReconcileOptions {
   log?: ReconcileLog;
 }
 
+/** Options for the vanished-instance reconcile pass ({@link reconcileVanishedInstances}). */
+export interface VanishedReconcileOptions extends ReconcileOptions {
+  /** How long (ms) a dispatched-but-instance-less row is spared before it is folded to terminal, so a
+   *  still-starting run (not yet projected into `_urban_instance_state`) is not orphaned prematurely.
+   *  Defaults to {@link DEFAULT_VANISHED_GRACE_MS}. */
+  graceMs?: number;
+}
+
 /** Read the incarnation epoch out of a `/v2/topology` body — `nano.incarnation` (or its `epoch`
  *  alias), coerced from a number or a numeric string. Any other shape (absent, non-numeric) yields
  *  null: "the engine exposes no epoch". A null is a no-op ONLY when no epoch was previously recorded;
@@ -175,6 +222,78 @@ async function persistEpoch(src: DataSource, epoch: number, at: string): Promise
   );
 }
 
+/** The resolved schema + tracking selectors for one engine-backed binding, or `null` when the binding
+ *  carries no `activeStatuses` selector (it cannot classify "in-flight", so it is skipped). */
+interface BindingShape {
+  table: string;
+  active: readonly string[];
+  statusField: string;
+  keyField: string;
+  pkCol: string;
+  hasUpdatedAt: boolean;
+}
+
+/** The shape of a row selected for possible orphaning. */
+interface OrphanCandidate {
+  __pk: unknown;
+  __key: unknown;
+  __status: unknown;
+  __updated?: unknown;
+}
+
+/** Resolve a binding's tracking selectors + physical schema, or `null` to skip a selector-less
+ *  binding (activeStatusesFor would throw; we tolerate it here). */
+async function resolveShape(src: DataSource, binding: InstanceTracking): Promise<BindingShape | null> {
+  if (!binding.activeStatuses?.length) return null;
+  const table = binding.table;
+  const { pkCol, hasUpdatedAt } = await tableShape(src, table);
+  return {
+    table,
+    active: activeStatusesFor(table),
+    statusField: baseStatusFieldFor(table),
+    keyField: keyFieldFor(table),
+    pkCol,
+    hasUpdatedAt,
+  };
+}
+
+/** Drive ONE candidate row to `orphaned` with provenance, GUARDED: the UPDATE re-asserts the exact
+ *  status read AND a populated key (plus any `extraGuardSql`, e.g. the still-vanished re-check), so a
+ *  writer that flipped the row to a newer terminal status (or an instance that reappeared) between the
+ *  SELECT and this UPDATE wins the race — we never clobber that history back to `orphaned`. Only a row
+ *  we actually transitioned (`res.changed > 0`) gets provenance and is returned. `updated_at` is
+ *  stamped (when the table has one) so the transition refreshes the row's timestamp like every other
+ *  status transition in the codebase. Runs inside the caller's transaction. */
+async function orphanRow(
+  src: DataSource,
+  shape: BindingShape,
+  row: OrphanCandidate,
+  reason: string,
+  observedEpoch: number | null,
+  runId: string,
+  at: string,
+  extraGuardSql = "",
+): Promise<OrphanedRow | null> {
+  const { table, pkCol, statusField, keyField, hasUpdatedAt } = shape;
+  const pk = String(row.__pk);
+  const key = row.__key == null ? null : String(row.__key);
+  const fromStatus = String(row.__status);
+  const res = await src.exec(
+    `UPDATE ${q(table)} SET ${q(statusField)} = ?` +
+      (hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} = ?` : "") +
+      ` WHERE ${q(pkCol)} = ? AND ${q(statusField)} = ? AND ${q(keyField)} IS NOT NULL${extraGuardSql}`,
+    hasUpdatedAt ? [ORPHANED_STATUS, at, row.__pk, fromStatus] : [ORPHANED_STATUS, row.__pk, fromStatus],
+  );
+  if (res.changed <= 0) return null;
+  await src.exec(
+    `INSERT INTO ${PROVENANCE_TABLE} ` +
+      `(run_id, source_table, pk_value, key_value, from_status, to_status, reason, observed_epoch, at) ` +
+      `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+    [runId, table, pk, key, fromStatus, ORPHANED_STATUS, reason, observedEpoch, at],
+  );
+  return { table, pk, key, fromStatus };
+}
+
 /** Orphan every NON-terminal, engine-backed row across all instanceTracking bindings, recording one
  *  `reconcile_provenance` row per transition. Runs inside the caller's transaction. */
 async function orphanEngineBackedRows(
@@ -185,52 +304,87 @@ async function orphanEngineBackedRows(
 ): Promise<OrphanedRow[]> {
   const orphaned: OrphanedRow[] = [];
   for (const binding of engineBackedBindings()) {
-    const table = binding.table;
-    // A binding with no active-status selector cannot classify "in-flight" — skip it rather than
-    // guess (activeStatusesFor would throw; we tolerate a selector-less binding).
-    if (!binding.activeStatuses?.length) continue;
-    const active = activeStatusesFor(table);
-    const statusField = baseStatusFieldFor(table);
-    const keyField = keyFieldFor(table);
-    const { pkCol, hasUpdatedAt } = await tableShape(src, table);
-    const placeholders = active.map(() => "?").join(", ");
-    const rows = await src.query<{ __pk: unknown; __key: unknown; __status: unknown }>(
-      `SELECT ${q(pkCol)} AS __pk, ${q(keyField)} AS __key, ${q(statusField)} AS __status ` +
-        `FROM ${q(table)} WHERE ${q(statusField)} IN (${placeholders}) AND ${q(keyField)} IS NOT NULL`,
-      [...active],
+    const shape = await resolveShape(src, binding);
+    if (!shape) continue;
+    const placeholders = shape.active.map(() => "?").join(", ");
+    const rows = await src.query<OrphanCandidate>(
+      `SELECT ${q(shape.pkCol)} AS __pk, ${q(shape.keyField)} AS __key, ${q(shape.statusField)} AS __status ` +
+        `FROM ${q(shape.table)} WHERE ${q(shape.statusField)} IN (${placeholders}) AND ${q(shape.keyField)} IS NOT NULL`,
+      [...shape.active],
     );
     for (const row of rows) {
-      const pk = String(row.__pk);
-      const key = row.__key == null ? null : String(row.__key);
-      const fromStatus = String(row.__status);
-      // GUARDED update: re-assert the exact status we read AND a populated key, so a writer that
-      // flipped the row to a newer terminal status (or cleared its key) between the SELECT above and
-      // this UPDATE wins the race — we never clobber that terminal history back to `orphaned`. Only a
-      // row we actually transitioned (`res.changed > 0`) gets provenance and is counted. We also stamp
-      // `updated_at` (when the table has one) so the transition to `orphaned` refreshes the row's
-      // timestamp the same way every other status transition in the codebase does — leaving it stale
-      // would misrepresent the orphaning moment to the UI/audits.
-      const res = await src.exec(
-        `UPDATE ${q(table)} SET ${q(statusField)} = ?` +
-          (hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} = ?` : "") +
-          ` WHERE ${q(pkCol)} = ? AND ${q(statusField)} = ? AND ${q(keyField)} IS NOT NULL`,
-        hasUpdatedAt
-          ? [ORPHANED_STATUS, at, row.__pk, fromStatus]
-          : [ORPHANED_STATUS, row.__pk, fromStatus],
-      );
-      if (res.changed <= 0) continue;
-      await src.exec(
-        `INSERT INTO ${PROVENANCE_TABLE} ` +
-          `(run_id, source_table, pk_value, key_value, from_status, to_status, reason, observed_epoch, at) ` +
-          `VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-        [runId, table, pk, key, fromStatus, ORPHANED_STATUS, RECONCILE_ORPHAN_REASON, observedEpoch, at],
-      );
-      orphaned.push({ table, pk, key, fromStatus });
+      const o = await orphanRow(src, shape, row, RECONCILE_ORPHAN_REASON, observedEpoch, runId, at);
+      if (o) orphaned.push(o);
     }
   }
   return orphaned;
 }
 
+/** Whether a row's `updated_at` is younger than the grace window — i.e. it was (re)dispatched too
+ *  recently to have been projected into `_urban_instance_state` yet, so it must NOT be folded. A
+ *  null/unparseable timestamp is treated as "within grace" (spared): when we cannot establish a row's
+ *  age we must NOT orphan it — a nullable `updated_at` (e.g. `delivery_units.updated_at`,
+ *  db/migrations/088_delivery_units.sql) would otherwise fold a live row. Erring toward sparing at
+ *  worst leaves a genuinely-vanished ageless row for a later pass once it carries a usable timestamp;
+ *  erring the other way wedges/destroys a live run, so we choose the safe default. */
+function withinGrace(updated: unknown, nowMs: number, graceMs: number): boolean {
+  if (updated == null) return true;
+  const t = Date.parse(String(updated));
+  if (!Number.isFinite(t)) return true;
+  return nowMs - t < graceMs;
+}
+
+/** Orphan every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
+ *  `_urban_instance_state` row — the instance is absent/unknown in engine truth (vanished, issue
+ *  #630) — and whose last transition is older than the grace window. Records one
+ *  `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). Runs inside
+ *  the caller's transaction. */
+async function orphanVanishedRows(
+  src: DataSource,
+  runId: string,
+  at: string,
+  nowMs: number,
+  graceMs: number,
+): Promise<OrphanedRow[]> {
+  const orphaned: OrphanedRow[] = [];
+  for (const binding of engineBackedBindings()) {
+    const shape = await resolveShape(src, binding);
+    if (!shape) continue;
+    const placeholders = shape.active.map(() => "?").join(", ");
+    const updatedSel = shape.hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} AS __updated` : "";
+    // Active, dispatched (key populated) rows whose engine instance key has NO matching
+    // `_urban_instance_state` row — absent/unknown in engine truth.
+    const rows = await src.query<OrphanCandidate>(
+      `SELECT ${q(shape.pkCol)} AS __pk, ${q(shape.keyField)} AS __key, ${q(shape.statusField)} AS __status${updatedSel} ` +
+        `FROM ${q(shape.table)} b WHERE ${q(shape.statusField)} IN (${placeholders}) AND ${q(shape.keyField)} IS NOT NULL ` +
+        `AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s WHERE s.process_instance_key = b.${q(shape.keyField)})`,
+      [...shape.active],
+    );
+    // Re-assert "still no instance-state row" in the guarded UPDATE too, so an instance that reappears
+    // (the poller records it) between the SELECT above and the UPDATE wins the race.
+    const stillVanishedGuard =
+      ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
+      `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
+    for (const row of rows) {
+      if (shape.hasUpdatedAt && withinGrace(row.__updated, nowMs, graceMs)) continue;
+      const o = await orphanRow(src, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
+      if (o) orphaned.push(o);
+    }
+  }
+  return orphaned;
+}
+
+/** Whether the framework `_urban_instance_state` projection exists in this source. When it does NOT,
+ *  the vanished-instance pass is a hard no-op: without the projection every dispatched row would look
+ *  "vanished", so we must never orphan on its absence. */
+async function instanceStateTableExists(src: DataSource): Promise<boolean> {
+  const rows = await src.query<{ n: number }>(
+    `SELECT COUNT(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
+    [INSTANCE_STATE_TABLE],
+  );
+  return rows.length > 0 && Number(rows[0].n) > 0;
+}
+
 /**
  * Reconcile the app's engine-backed projections against one epoch observation. Pure of I/O beyond the
  * data layer (the topology probe is {@link probeEngineEpoch}, injected as `observation`), so the
@@ -300,6 +454,75 @@ export async function reconcileEngineBackedWork(
   return { runId, reason: result.reason, observedEpoch, recordedEpoch, orphanedCount: result.orphaned.length, orphaned: result.orphaned };
 }
 
+/**
+ * Reconcile engine-backed inflight work against the framework's canonical per-instance projection
+ * (`_urban_instance_state`) — the "instance absent/unknown" gap (issue #630), DISTINCT from the
+ * epoch-regression reset the {@link reconcileEngineBackedWork} pass handles.
+ *
+ * When an engine instance VANISHES from the read model — engine clean-reset, cluster rebuild, or
+ * read-model pruning removes the `_urban_instance_state` row entirely — there is no `TERMINATED` row
+ * for the derived tracking edge to match, so the run freezes at its last worker-owned status
+ * (`escalated`/`awaiting_operator`) and wedges the Active list forever with no reconciliation path.
+ * "The instance backing this run no longer exists in engine truth" is a terminal condition: this pass
+ * drives every such row (active, dispatched, its `keyField` absent from `_urban_instance_state`, and
+ * past the grace window) to the defined `orphaned` terminal WITH PROVENANCE.
+ *
+ * Safety:
+ *   • GRACE WINDOW — a just-dispatched run has not yet been polled into `_urban_instance_state`; only
+ *     rows whose last transition is older than `graceMs` are folded, so a still-starting run is never
+ *     prematurely orphaned (AC #2).
+ *   • PROJECTION-PRESENT — if `_urban_instance_state` does not exist (the runtime has not provisioned
+ *     it), every dispatched row would look vanished, so the pass is a hard no-op.
+ *   • GUARDED — the same status-re-assert as the epoch pass, plus a still-vanished re-check, so a
+ *     concurrent terminal write or a reappearing instance wins the race.
+ *   • This pass reads the app's OWN last-known projection, not a live probe, so it acts correctly on a
+ *     genuinely-vanished instance regardless of transient engine reachability (a live instance keeps
+ *     its persisted ACTIVE row across a restart, so it is never mistaken for vanished).
+ */
+export async function reconcileVanishedInstances(
+  data: DataLayer,
+  opts: VanishedReconcileOptions = {},
+): Promise<ReconcileResult> {
+  const src = data.open(opts.sourceName);
+  const clock = opts.now?.() ?? new Date();
+  const at = clock.toISOString();
+  const nowMs = clock.getTime();
+  const runId = opts.runId ?? crypto.randomUUID();
+  const graceMs = opts.graceMs ?? DEFAULT_VANISHED_GRACE_MS;
+
+  // Without the framework projection we cannot tell a vanished instance from a live one — every
+  // dispatched row would look vanished. NO-OP rather than orphan live work.
+  if (!(await instanceStateTableExists(src))) {
+    await recordRun(src, { runId, at, observedEpoch: null, recordedEpoch: null, reason: "no-op", orphanedCount: 0 });
+    opts.log?.info(`reconcile(vanished): instance-state projection absent — no-op [run ${runId}].`);
+    return { runId, reason: "no-op", observedEpoch: null, recordedEpoch: null, orphanedCount: 0, orphaned: [] };
+  }
+
+  const orphaned = await src.tx(async (t) => {
+    const rows = await orphanVanishedRows(t, runId, at, nowMs, graceMs);
+    const reason: ReconcileReason = rows.length > 0 ? "instance-vanished" : "no-op";
+    await recordRun(t, { runId, at, observedEpoch: null, recordedEpoch: null, reason, orphanedCount: rows.length });
+    return rows;
+  });
+
+  if (orphaned.length > 0) {
+    opts.log?.warn(
+      `reconcile(vanished): orphaned ${orphaned.length} inflight row(s) whose engine instance ` +
+        `vanished from the read model [run ${runId}].`,
+    );
+  } else {
+    opts.log?.info(`reconcile(vanished): no vanished instances — no-op [run ${runId}].`);
+  }
+  return {
+    runId,
+    reason: orphaned.length > 0 ? "instance-vanished" : "no-op",
+    observedEpoch: null,
+    recordedEpoch: null,
+    orphanedCount: orphaned.length,
+    orphaned,
+  };
+}
+
 async function recordRun(
   src: DataSource,
   run: { runId: string; at: string; observedEpoch: number | null; recordedEpoch: number | null; reason: ReconcileReason; orphanedCount: number },
@@ -312,15 +535,47 @@ async function recordRun(
 }
 
 /** Probe the engine's incarnation epoch, then reconcile — the wiring both startup (main.ts) and the
- *  `reconcileEngineState` operator command share, so the two paths can never diverge. */
+ *  `reconcileEngineState` operator command share, so the two paths can never diverge. Runs BOTH the
+ *  epoch-regression pass (engine reset/rewind) and the vanished-instance pass (an inflight run whose
+ *  instance is absent/unknown in `_urban_instance_state` — issue #630), returning ONE merged result.
+ *  Reason precedence: `epoch-regression` always wins; otherwise, if the vanished pass orphaned any
+ *  rows the reason is `instance-vanished` (even when the epoch pass reported a non-regression state
+ *  such as `engine-unreachable`); otherwise the epoch pass's reason stands. `orphanedCount` /
+ *  `orphaned` cover both passes. The two passes never double-fold a row: once the epoch pass orphans a
+ *  row it leaves `activeStatuses`, so the vanished pass no longer selects it. */
 export async function runEngineReconcile(
   data: DataLayer,
   engineRest: { restAddress: string; token?: string },
-  opts: ReconcileOptions & { fetchImpl?: typeof fetch } = {},
+  opts: VanishedReconcileOptions & { fetchImpl?: typeof fetch } = {},
 ): Promise<ReconcileResult> {
   const observation = await probeEngineEpoch(engineRest.restAddress, {
     token: engineRest.token,
     fetchImpl: opts.fetchImpl,
   });
-  return reconcileEngineBackedWork(data, observation, opts);
+  const epoch = await reconcileEngineBackedWork(data, observation, opts);
+  // A distinct run id so the vanished pass's `reconcile_runs`/provenance rows never collide with the
+  // epoch pass's (run_id is a PRIMARY KEY). DERIVE it from the epoch pass's resolved run id — which is
+  // also the merged result's `runId` — so it is `<runId>-vanished` on EVERY path, including the boot
+  // path where `opts.runId` is omitted (a bare random UUID here would be non-correlatable to the
+  // returned `runId`). Operators can always locate the vanished pass's provenance from the reported id.
+  const vanished = await reconcileVanishedInstances(data, {
+    ...opts,
+    runId: `${epoch.runId}-vanished`,
+  });
+
+  const orphaned = [...epoch.orphaned, ...vanished.orphaned];
+  const reason: ReconcileReason =
+    epoch.reason === "epoch-regression"
+      ? "epoch-regression"
+      : vanished.orphanedCount > 0
+        ? "instance-vanished"
+        : epoch.reason;
+  return {
+    runId: epoch.runId,
+    reason,
+    observedEpoch: epoch.observedEpoch,
+    recordedEpoch: epoch.recordedEpoch,
+    orphanedCount: orphaned.length,
+    orphaned,
+  };
 }

package/main.ts

@@ -103,12 +103,14 @@ if (httpServer instanceof Server) {
   app.log.warn("agentic channel not mounted: app.httpServer is not a node:http Server on this host");
 }
 
-// Engine-reset reconciliation (issue #622). On boot, compare the engine's incarnation epoch against
-// the last-seen value; on a REGRESSION (the engine was reset/restored/rewound and re-minted its keys,
-// Magikcraft/nano-bpm#1065) drive every dangling engine-backed inflight row to the defined `orphaned`
-// terminal WITH PROVENANCE — BEFORE the pollers below start projecting off stale, dead instances.
-// Guarded: an unreachable engine is a no-op (it never orphans live work), and any failure degrades to
-// a warn so reconcile can never block boot.
+// Engine-reset reconciliation (issues #622, #630). On boot, compare the engine's incarnation epoch
+// against the last-seen value; on a REGRESSION (the engine was reset/restored/rewound and re-minted
+// its keys, Magikcraft/nano-bpm#1065) drive every dangling engine-backed inflight row to the defined
+// `orphaned` terminal WITH PROVENANCE. A second pass also folds any run whose engine instance has
+// VANISHED from the read model (no `_urban_instance_state` row, past a grace window — issue #630),
+// which the epoch signal alone can't catch — BEFORE the pollers below start projecting off stale,
+// dead instances. Guarded: an unreachable engine / an absent projection is a no-op (never orphans
+// live work), and any failure degrades to a warn so reconcile can never block boot.
 if (app.data) {
   try {
     const reconciled = await runEngineReconcile(

package/openapi.yaml

@@ -1042,7 +1042,10 @@ components:
           type: integer
     ReconcileReport:
       type: object
-      description: The result of one engine-reset reconciliation pass (issue #622).
+      description: "The merged result of the engine-reconcile invocation — an aggregate of both the
+        epoch-regression (engine reset/rewind) pass and the vanished-instance pass (issues #622 and
+        #630). `runId` is the epoch pass's run id; the vanished pass records its own provenance under
+        the derived id `<runId>-vanished`."
       additionalProperties: false
       required:
         - runId
@@ -1054,15 +1057,19 @@ components:
       properties:
         runId:
           type: string
-          description: The reconcile run id every orphaned transition's provenance is stamped with.
+          description: The reconcile run id the epoch pass's orphaned-transition provenance is stamped
+            with. The vanished-instance pass's provenance is stamped with the derived, deterministic id
+            `<runId>-vanished`, so operators can locate the provenance rows for either pass from this id.
         reason:
           type: string
-          description: Why this pass acted (or did not).
+          description: Why this pass acted (or did not). `instance-vanished` — a run whose engine
+            instance is absent/unknown in the read model was orphaned (issue #630).
           enum:
             - epoch-regression
             - seed-epoch
             - no-op
             - engine-unreachable
+            - instance-vanished
         observedEpoch:
           type: integer
           nullable: true

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nanobpm/nano-workforce",
-  "version": "0.174.0",
+  "version": "0.174.1",
   "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
   "type": "module",
   "main": "main.ts",