@nanobpm/nano-workforce 0.183.0 → 0.183.2

package/app/reconcile.test.ts

@@ -10,10 +10,12 @@
 // `makeGateway`), so the tables/columns/indexes reconcile reads and writes are the shipping schema.
 import { DatabaseSync } from "node:sqlite";
 import { test } from "node:test";
-import { assertEquals } from "#test-assert";
+import type { DataLayer, GatewayDataSource as DataSource } from "@nanobpm/urban";
+import { assertEquals, assertNotEquals } from "#test-assert";
 import { freshData } from "../test/reconcileDb.ts";
 import {
   DEFAULT_VANISHED_GRACE_MS,
+  makeEngineActiveProbe,
   ORPHANED_STATUS,
   parseEngineEpoch,
   RECONCILE_ORPHAN_REASON,
@@ -393,28 +395,115 @@ test("idempotent: a second vanished pass is a no-op (the orphaned row left activ
   assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 1);
 });
 
+// --- Engine-truth cross-check before orphaning a "vanished" row (issue #736) -------------------
+// The `_urban_instance_state` projection is an app-side read model that can lag / be pruned / be
+// rebuilt while the instance is still ACTIVE on the engine. "No projection row" is therefore NOT
+// "instance vanished": on merlin (whose engine exposes no incarnation epoch, so the robust epoch
+// detector is disabled) this false-orphaned 3 concurrently-LIVE instances in one pass. The vanished
+// pass now cross-checks ENGINE TRUTH via `engineActive` before folding — an ACTIVE instance is spared.
+
+test("RED→GREEN #736: an engine-ACTIVE instance with no _urban_instance_state row (past grace) is NOT orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // The merlin repro: an inflight run past grace whose projection row is absent (lagging/pruned) but
+  // whose instance the engine still reports ACTIVE. RED (pre-fix): folded to `orphaned`. GREEN: spared.
+  seedFeatureRun(raw, "nanobpm/nano-workforce#336", "running", "11625");
+  // Engine truth says ACTIVE for this key.
+  const engineActive = async (key: string) => (key === "11625" ? true : false);
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
+
+  assertEquals(res.reason, "no-op");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#336'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "running");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("#736: an engine-CONFIRMED-gone instance (engineActive=false) IS still orphaned", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+  // Engine answered and the instance is absent/terminated — genuinely gone in engine truth.
+  const engineActive = async () => false;
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
+
+  assertEquals(res.reason, "instance-vanished");
+  assertEquals(res.orphanedCount, 1);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, ORPHANED_STATUS);
+});
+
+test("#736: an instance whose engine truth is UNKNOWN (engineActive=null) is spared — never orphan unconfirmed", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "o/r#unknown", "escalated", "71506");
+  // The engine truth could not be established (unreachable / non-2xx / malformed) — we must NOT orphan.
+  const engineActive = async () => null;
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
+
+  assertEquals(res.reason, "no-op");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#unknown'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "escalated");
+});
+
 // --- Merged seam: runEngineReconcile (both passes, one result) --------------------------------
 // The operator/startup seam merges the epoch-regression and vanished-instance passes into ONE
 // result. This guards the merged behavior the two per-pass suites above don't reach: run-id
-// correlation (the vanished pass's provenance must be locatable from the returned `runId`) and
-// `reason` selection when the epoch pass is `engine-unreachable` yet vanished instances are orphaned.
+// correlation (the vanished pass's provenance must be locatable from the returned `runId`), the
+// engine-truth cross-check the seam wires from the live engine (#736), and `reason` selection.
+
+/** A `/v2` fetch stub: `/topology` answers `topologyBody` (200), and `/process-instances/search`
+ *  answers with `searchItems` (200) — the engine-truth cross-check the vanished pass runs (#736). */
+function engineFetch(
+  topologyBody: unknown,
+  searchItems: { processInstanceKey?: string | number; state?: string }[],
+): typeof fetch {
+  return (async (url: string, init?: { method?: string }) => {
+    const u = String(url);
+    if (u.endsWith("/topology")) return new Response(JSON.stringify(topologyBody), { status: 200 });
+    if (u.endsWith("/process-instances/search") && init?.method === "POST") {
+      return new Response(JSON.stringify({ items: searchItems }), { status: 200 });
+    }
+    return new Response("not found", { status: 404 });
+  }) as unknown as typeof fetch;
+}
+
+test("runEngineReconcile #736: a reachable engine reporting the instance ACTIVE spares it (no false-orphan)", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // No projection row, past grace — but the engine (reachable, no epoch, like merlin) reports ACTIVE.
+  seedFeatureRun(raw, "nanobpm/nano-workforce#731", "escalated", "11644");
+  const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, [{ processInstanceKey: "11644", state: "ACTIVE" }]);
+
+  const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#731'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "escalated");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
 
-test("runEngineReconcile: engine-unreachable epoch pass still folds vanished instances, with a correlatable run id", async () => {
+test("runEngineReconcile #736: a reachable engine that no longer knows the instance folds it, with a correlatable run id", async () => {
   const { data, raw } = freshData();
   ensureInstanceState(raw);
-  // A vanished orphan (escalated, past grace, instance absent from the read model).
+  // A genuinely-vanished orphan (escalated, past grace); the engine answers but the instance is absent.
   seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+  const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, []);
 
-  // The engine is unreachable — the epoch probe fails, so the epoch pass reports `engine-unreachable`
-  // and orphans nothing; the vanished pass must still act.
-  const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
-  const res = await runEngineReconcile(
-    data,
-    { restAddress: "http://engine.invalid" },
-    { now: AT, fetchImpl },
-  );
+  const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
 
-  // The vanished pass acted even though the epoch pass could not reach the engine.
   assertEquals(res.reason, "instance-vanished");
   assertEquals(res.orphanedCount, 1);
   const orphan = raw
@@ -429,14 +518,189 @@ test("runEngineReconcile: engine-unreachable epoch pass still folds vanished ins
     .get() as { run_id: string };
   assertEquals(prov.run_id, `${res.runId}-vanished`);
 
-  // Both passes recorded their own reconcile_runs row under correlatable ids.
-  const epochRun = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id=?").get(res.runId) as
-    | { reason: string }
-    | undefined;
-  assertEquals(epochRun?.reason, "engine-unreachable");
   const vanishedRun = raw
     .prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id=?")
     .get(`${res.runId}-vanished`) as { reason: string; orphaned_count: number } | undefined;
   assertEquals(vanishedRun?.reason, "instance-vanished");
   assertEquals(vanishedRun?.orphaned_count, 1);
 });
+
+test("runEngineReconcile #736: an UNREACHABLE engine spares vanished candidates (truth unconfirmed → never orphan)", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A candidate that looks vanished (escalated, past grace, no projection row) — but with the engine
+  // unreachable we cannot confirm it is gone, so it MUST be spared (issue #736): the old projection-only
+  // behavior would have orphaned it, potentially false-orphaning a live instance mid-outage.
+  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
+
+  const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
+  const res = await runEngineReconcile(data, { restAddress: "http://engine.invalid" }, { now: AT, fetchImpl });
+
+  // The epoch pass could not reach the engine, and the vanished pass could not confirm death → no-op.
+  assertEquals(res.reason, "engine-unreachable");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "escalated");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+// --- Hardening the cross-check itself (#736 review) ---------------------------------------------
+// Two defects of the SAME class the first cut still carried — "never orphan a row we could not
+// positively confirm is gone" — plus the lock-hold the cross-check introduced:
+//   1. a MATCHING search item whose `state` was missing or outside the engine's lifecycle enum read as
+//      `false` ("gone"), so a malformed/partial engine answer folded live work;
+//   2. the probe (network I/O) was awaited INSIDE `src.tx(...)`, so a slow or unreachable engine held
+//      the SQLite write transaction open for the probe's full timeout PER CANDIDATE ROW — stalling
+//      every other writer, including boot — and an injected probe that threw aborted the whole pass.
+
+/** Answer `/v2/process-instances/search` with exactly `items` (200), so a probe's classification can
+ *  be read off one wire shape varying only in the item's `state`. */
+function searchItemsFetch(items: { processInstanceKey?: string | number; state?: string }[]): typeof fetch {
+  return (async () => new Response(JSON.stringify({ items }), { status: 200 })) as unknown as typeof fetch;
+}
+
+/** Probe the key `11644` against a stubbed engine search answer of `items`. */
+function probeAgainst(items: { processInstanceKey?: string | number; state?: string }[]): Promise<boolean | null> {
+  const probe = makeEngineActiveProbe({ restAddress: "http://engine.local/v2" }, { fetchImpl: searchItemsFetch(items) });
+  return probe("11644");
+}
+
+test("#736: the probe answers `true` for ACTIVE and `false` ONLY for a known-terminal engine state", async () => {
+  assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "ACTIVE" }]), true);
+  // The wire may carry the key as a JSON number and the state in any casing.
+  assertEquals(await probeAgainst([{ processInstanceKey: 11644, state: "active" }]), true);
+  for (const state of ["COMPLETED", "TERMINATED", "CANCELED", "FAILED"]) {
+    assertEquals(await probeAgainst([{ processInstanceKey: "11644", state }]), false, `${state} is a positive "gone"`);
+  }
+  // Absent from the read model is STILL a positive "gone": the engine answered and does not know it.
+  assertEquals(await probeAgainst([]), false);
+});
+
+test("RED→GREEN #736: a MISSING or UNRECOGNIZED engine state is UNKNOWN truth (`null` → spare), never `false`", async () => {
+  // RED (pre-fix): the probe answered `String(match.state ?? "").toUpperCase() === "ACTIVE"`, so a
+  // partial item (no `state`) or a state outside the enum this app knows read as "gone" and folded the
+  // row — contradicting the probe's own contract that malformed engine truth degrades to `null`.
+  assertEquals(await probeAgainst([{ processInstanceKey: "11644" }]), null, "a missing state is not a confirmed death");
+  assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "" }]), null);
+  assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "SUSPENDED" }]), null);
+  // An item for a DIFFERENT key is no match for this one, so that stays "absent" (gone), not unknown.
+  assertEquals(await probeAgainst([{ processInstanceKey: "99999" }]), false);
+});
+
+test("RED→GREEN #736: a malformed search item (no `state`) spares the candidate end-to-end", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // Past grace, no projection row, and the engine ANSWERS — but its item carries no lifecycle state, so
+  // engine truth is unestablished and the row must survive (RED pre-fix: folded to `orphaned`).
+  seedFeatureRun(raw, "nanobpm/nano-workforce#731", "escalated", "11644");
+  const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, [{ processInstanceKey: "11644" }]);
+
+  const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
+
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#731'").get() as {
+    status: string;
+  };
+  assertEquals(row.status, "escalated");
+  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
+});
+
+test("RED→GREEN #736: a probe that THROWS is unknown truth — it spares the row instead of aborting the pass", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "o/r#boom", "escalated", "71506");
+  // `engineActive` is injectable: an implementation that rejects means truth could NOT be established
+  // (spare), and must not bubble out of the pass.
+  const engineActive = async (): Promise<boolean | null> => {
+    throw new Error("engine exploded");
+  };
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
+
+  assertEquals(res.reason, "no-op");
+  assertEquals(res.orphanedCount, 0);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#boom'").get() as { status: string };
+  assertEquals(row.status, "escalated");
+  // The pass COMPLETED and recorded its run — pre-fix the throw escaped the transaction and rejected the
+  // whole reconcile, so no `reconcile_runs` row was written at all.
+  const run = raw.prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id='van-1'").get() as
+    | { reason: string; orphaned_count: number }
+    | undefined;
+  assertEquals(run?.reason, "no-op");
+  assertEquals(run?.orphaned_count, 0);
+});
+
+test("RED→GREEN #736: the probe (network I/O) is never awaited INSIDE the write transaction", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  seedFeatureRun(raw, "o/r#tx", "escalated", "71506");
+
+  // Wrap the gateway so the probe can report how deep in `src.tx(...)` it was awaited. RED (pre-fix):
+  // depth 1 — every candidate row held the SQLite write transaction open across a network round trip
+  // (up to the probe's full timeout on a slow/unreachable engine), delaying every other writer and
+  // slowing/locking boot. GREEN: depth 0 — the probes finish first, and only the guarded UPDATE +
+  // provenance writes run in a short transaction.
+  let depth = 0;
+  const observed: number[] = [];
+  const inner = data.open();
+  const tracked = {
+    open: () => ({
+      query: (sql: string, params?: unknown[]) => inner.query(sql, params),
+      exec: (sql: string, params?: unknown[]) => inner.exec(sql, params),
+      schema: () => inner.schema(),
+      table: (name: string, pk?: string) => inner.table(name, pk),
+      tx: async <T>(fn: (t: DataSource) => Promise<T>): Promise<T> => {
+        depth += 1;
+        try {
+          return await inner.tx(fn);
+        } finally {
+          depth -= 1;
+        }
+      },
+    }),
+  } as unknown as DataLayer;
+
+  const engineActive = async (): Promise<boolean | null> => {
+    observed.push(depth);
+    return false;
+  };
+
+  const res = await reconcileVanishedInstances(tracked, { now: AT, runId: "van-tx", engineActive });
+
+  // Exactly one entry: the seeded row also mirrors into `delivery_units` (same `process_key`), and one
+  // instance is probed once (see the next test) — at depth 0, outside any open write transaction.
+  assertEquals(observed, [0], "the engine-truth probe must be awaited outside any open write transaction");
+  // Hoisting the probe out of the transaction must not weaken the pass: a confirmed-gone row still folds.
+  assertEquals(res.orphanedCount, 1);
+  const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#tx'").get() as { status: string };
+  assertEquals(row.status, ORPHANED_STATUS);
+});
+
+test("#736: one instance backing several tracked rows is probed ONCE (engine truth is per instance)", async () => {
+  const { data, raw } = freshData();
+  ensureInstanceState(raw);
+  // A `feature_runs` row is ALSO mirrored into the `delivery_units` aggregate by DB trigger
+  // (db/migrations/089), carrying the same `process_key` — so one vanished instance yields TWO
+  // candidates. Engine truth is per instance, so it must be asked once, not once per row (each probe
+  // can block for its full timeout on a slow engine).
+  seedFeatureRun(raw, "o/r#mirror", "escalated", "71506");
+  const probed: string[] = [];
+  const engineActive = async (key: string): Promise<boolean | null> => {
+    probed.push(key);
+    return false;
+  };
+
+  const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-mirror", engineActive });
+
+  assertEquals(probed, ["71506"], "one probe per distinct instance key");
+  // Folding the base row re-projects its mirror (the sync trigger clears `dispatch_status`), so the
+  // mirror is not folded a second time for the same instance — no duplicate provenance.
+  assertEquals(res.orphaned.map((o) => o.table), ["feature_runs"]);
+  assertEquals(res.orphanedCount, 1);
+  const mirror = raw.prepare("SELECT dispatch_status FROM delivery_units WHERE legacy_key='o/r#mirror'").get() as {
+    dispatch_status: string | null;
+  };
+  assertNotEquals(mirror.dispatch_status, "dispatched");
+});

package/app/reconcile.ts

@@ -38,8 +38,13 @@
 // `reconcileVanishedInstances` drives every such row — active, dispatched, its key absent from
 // `_urban_instance_state`, and PAST A GRACE WINDOW (so a still-starting run not yet projected is
 // spared) — to the same `orphaned` terminal, with a DISTINCT provenance reason so an operator can
-// tell a vanished-instance orphan apart from an epoch-regression one. `runEngineReconcile` runs BOTH
-// passes, so startup and the operator command converge both failure modes in one call.
+// tell a vanished-instance orphan apart from an epoch-regression one. Because `_urban_instance_state`
+// is an app-side projection that can lag / be pruned / be rebuilt while the instance is still ACTIVE
+// on the engine, "no projection row" is NOT proof the instance vanished — so before folding, the pass
+// CROSS-CHECKS ENGINE TRUTH (`/v2/process-instances/search`, issue #736): an instance the engine still
+// reports ACTIVE (or whose truth cannot be established) is SPARED, closing the false-orphan class on
+// deployments whose engine omits the incarnation epoch. `runEngineReconcile` runs BOTH passes, so
+// startup and the operator command converge both failure modes in one call.
 //
 // The provenance is app-owned (not urban's `_urban_write_provenance`, which is a domain-free
 // insert-join sidecar written only inside a job): reconcile runs at boot / over HTTP, outside any
@@ -152,6 +157,15 @@ export interface VanishedReconcileOptions extends ReconcileOptions {
    *  still-starting run (not yet projected into `_urban_instance_state`) is not orphaned prematurely.
    *  Defaults to {@link DEFAULT_VANISHED_GRACE_MS}. */
   graceMs?: number;
+  /** Cross-check against ENGINE TRUTH before orphaning a candidate row (issue #736). Given the row's
+   *  `keyField` (process instance key), it reports whether the engine still considers the instance
+   *  ACTIVE. An ACTIVE instance is NEVER orphaned — the app-side `_urban_instance_state` projection is
+   *  merely lagging — and an instance whose truth could not be established (`null`) is spared too (we
+   *  never orphan what we could not confirm dead). Only a row the engine positively confirms is gone
+   *  (`false`) is folded. When omitted the pass falls back to projection-only behaviour (no live
+   *  cross-check); production always wires one via {@link runEngineReconcile}. See
+   *  {@link makeEngineActiveProbe}. */
+  engineActive?: EngineActiveProbe;
 }
 
 /** Read the incarnation epoch out of a `/v2/topology` body — `nano.incarnation` (or its `epoch`
@@ -189,6 +203,80 @@ export async function probeEngineEpoch(
   }
 }
 
+/** A cross-check against ENGINE TRUTH for one process instance key, used to spare a live instance from
+ *  the vanished-instance pass. Returns:
+ *   • `true`  — the engine reports the instance ACTIVE. It is live; the app-side `_urban_instance_state`
+ *               projection is merely lagging/pruned/rebuilding, so the row MUST NOT be orphaned.
+ *   • `false` — the engine answered and the instance is NOT active: absent from the read model, or in a
+ *               known terminal state ({@link ENGINE_TERMINAL_STATES}). It is genuinely gone in engine
+ *               truth → orphan-eligible.
+ *   • `null`  — engine truth could NOT be established (unreachable, non-2xx, malformed — including an
+ *               item whose `state` is missing or outside {@link ENGINE_TERMINAL_STATES}). We never
+ *               orphan a row we could not confirm dead, so the caller spares it (a transient outage
+ *               must never fold live work). */
+export type EngineActiveProbe = (processKey: string) => Promise<boolean | null>;
+
+/** One item of a `/v2/process-instances/search` result, narrowed to what the engine-truth cross-check
+ *  reads: the instance key (to match the row we probed for) and its lifecycle `state`. Keys are
+ *  stringified defensively (the wire may send a JSON number or string); `state` is the engine's
+ *  lifecycle enum (`ACTIVE`/`COMPLETED`/`CANCELED`/…). */
+interface InstanceSearchStateItem {
+  processInstanceKey?: string | number;
+  state?: string;
+}
+
+/** The lifecycle states that POSITIVELY mean "this instance is no longer running" — urban's
+ *  `ProcessInstanceState` terminals (`COMPLETED`/`TERMINATED`) plus the Camunda-8-parity v2 REST
+ *  terminals (`CANCELED`/`FAILED`), since the probe reads that raw surface rather than the typed
+ *  client. ONLY these may answer `false` (orphan-eligible): a `state` that is missing, empty, or
+ *  outside this set is a partial/malformed read this app cannot interpret, so it degrades to `null`
+ *  ("unknown" → the caller spares the row). Classifying an unrecognized state as "gone" would fold
+ *  live work off a wire shape we misread — the exact failure mode the #736 cross-check exists to stop. */
+const ENGINE_TERMINAL_STATES: ReadonlySet<string> = new Set(["COMPLETED", "TERMINATED", "CANCELED", "FAILED"]);
+
+/** Build an {@link EngineActiveProbe} that queries the engine's own `/v2/process-instances/search` for
+ *  a single process instance key and reports whether the engine still considers it ACTIVE. This is the
+ *  authoritative engine-truth check the vanished-instance pass consults before orphaning: an ACTIVE
+ *  engine instance must NEVER be orphaned regardless of the app-side projection (issue #736), so a
+ *  merlin-style deployment whose `_urban_instance_state` lags no longer false-orphans live work.
+ *  Never throws — every transport/parse failure, and every `state` this app cannot interpret (missing,
+ *  or outside {@link ENGINE_TERMINAL_STATES}), degrades to `null` ("unknown"), which the caller treats
+ *  as "spare" (we never orphan what we could not confirm dead). */
+export function makeEngineActiveProbe(
+  engineRest: { restAddress: string; token?: string },
+  opts: { fetchImpl?: typeof fetch; timeoutMs?: number } = {},
+): EngineActiveProbe {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = engineRest.restAddress.replace(/\/+$/, "");
+  const headers: Record<string, string> = { accept: "application/json", "content-type": "application/json" };
+  if (engineRest.token) headers.authorization = `Bearer ${engineRest.token}`;
+  return async (processKey: string): Promise<boolean | null> => {
+    try {
+      const res = await fetchImpl(`${base}/process-instances/search`, {
+        method: "POST",
+        headers,
+        body: JSON.stringify({ filter: { processInstanceKey: processKey }, page: { from: 0, limit: 10 } }),
+        signal: AbortSignal.timeout(opts.timeoutMs ?? 3000),
+      });
+      if (!res.ok) return null;
+      // biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
+      const body = (await res.json()) as { items?: InstanceSearchStateItem[] };
+      const items = body.items ?? [];
+      const match = items.find((it) => it.processInstanceKey != null && String(it.processInstanceKey) === processKey);
+      // Engine answered but the instance is absent from the read model → genuinely gone (not active).
+      if (!match) return false;
+      const state = String(match.state ?? "").trim().toUpperCase();
+      if (state === "ACTIVE") return true;
+      // A KNOWN terminal state is a positive "gone". Anything else — a missing/empty `state`, or one
+      // outside the enum this app can interpret — is a partial or malformed answer, NOT a confirmed
+      // death, so it degrades to `null` and the caller spares the row.
+      return ENGINE_TERMINAL_STATES.has(state) ? false : null;
+    } catch {
+      return null;
+    }
+  };
+}
+
 /** Double-quote a SQL identifier (table/column) so a manifest-declared name is safe to interpolate. */
 function q(id: string): string {
   return `"${id.replace(/"/g, '""')}"`;
@@ -334,19 +422,24 @@ function withinGrace(updated: unknown, nowMs: number, graceMs: number): boolean
   return nowMs - t < graceMs;
 }
 
-/** Orphan every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
- *  `_urban_instance_state` row — the instance is absent/unknown in engine truth (vanished, issue
- *  #630) — and whose last transition is older than the grace window. Records one
- *  `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). Runs inside
- *  the caller's transaction. */
-async function orphanVanishedRows(
+/** A vanished-instance candidate: the row selected for possible orphaning plus the binding shape that
+ *  resolves its physical schema. Selected OUTSIDE any transaction, because the engine-truth
+ *  cross-check that narrows these is network I/O (see {@link confirmVanishedGone}). */
+interface VanishedCandidate {
+  shape: BindingShape;
+  row: OrphanCandidate;
+}
+
+/** SELECT every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
+ *  `_urban_instance_state` row — the instance is absent/unknown in the app-side read model (vanished,
+ *  issue #630) — and whose last transition is older than the grace window. READ-ONLY and network-free,
+ *  so it is safe to run outside the orphaning transaction. */
+async function selectVanishedCandidates(
   src: DataSource,
-  runId: string,
-  at: string,
   nowMs: number,
   graceMs: number,
-): Promise<OrphanedRow[]> {
-  const orphaned: OrphanedRow[] = [];
+): Promise<VanishedCandidate[]> {
+  const candidates: VanishedCandidate[] = [];
   for (const binding of engineBackedBindings()) {
     const shape = await resolveShape(src, binding);
     if (!shape) continue;
@@ -360,17 +453,79 @@ async function orphanVanishedRows(
         `AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s WHERE s.process_instance_key = b.${q(shape.keyField)})`,
       [...shape.active],
     );
-    // Re-assert "still no instance-state row" in the guarded UPDATE too, so an instance that reappears
-    // (the poller records it) between the SELECT above and the UPDATE wins the race.
-    const stillVanishedGuard =
-      ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
-      `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
     for (const row of rows) {
       if (shape.hasUpdatedAt && withinGrace(row.__updated, nowMs, graceMs)) continue;
-      const o = await orphanRow(src, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
-      if (o) orphaned.push(o);
+      candidates.push({ shape, row });
     }
   }
+  return candidates;
+}
+
+/** Narrow candidates to the ones ENGINE TRUTH positively confirms are gone (issue #736): the
+ *  `_urban_instance_state` projection is an app-side read model that can lag / be pruned / be rebuilt
+ *  while the instance is still ACTIVE on the engine, so "no projection row" is NOT "instance vanished".
+ *  An ACTIVE instance (`true`) is spared, and one whose truth could not be established (`null` — engine
+ *  unreachable, malformed answer, or a probe that THREW) is spared too: we never orphan a row we could
+ *  not positively confirm is gone, and an injected probe's failure must never abort the pass. Only a
+ *  `false` (engine confirms absent/terminated) survives. Runs OUTSIDE the DB transaction — this is
+ *  network I/O, and awaiting it under an open write transaction would hold the SQLite lock for up to
+ *  the probe's timeout PER ROW, stalling every other writer (including boot). Without a probe (omitted)
+ *  the pass falls back to projection-only behaviour, so every candidate survives. */
+async function confirmVanishedGone(
+  candidates: VanishedCandidate[],
+  engineActive?: EngineActiveProbe,
+): Promise<VanishedCandidate[]> {
+  if (!engineActive) return candidates;
+  // Engine truth is PER INSTANCE, not per row, and one instance can back several tracked rows: the
+  // `delivery_units` aggregate is a DB-trigger mirror of its legacy base row (db/migrations/089) and
+  // carries the same `process_key`, so both are candidates for one vanished instance. Probe each
+  // DISTINCT key once and apply that verdict to every row carrying it — the same answer, without
+  // doubling engine calls that can each block for the probe's full timeout.
+  const verdicts = new Map<string, boolean | null>();
+  const gone: VanishedCandidate[] = [];
+  for (const candidate of candidates) {
+    const key = candidate.row.__key == null ? null : String(candidate.row.__key);
+    // Defensive: the SELECT requires a populated key, so with no key there is nothing to cross-check
+    // and the projection-only verdict stands.
+    if (key == null) {
+      gone.push(candidate);
+      continue;
+    }
+    let verdict = verdicts.get(key);
+    if (verdict === undefined) {
+      try {
+        verdict = await engineActive(key);
+      } catch {
+        // A probe that throws established nothing — treat it exactly like an unreachable engine.
+        verdict = null;
+      }
+      verdicts.set(key, verdict);
+    }
+    if (verdict === false) gone.push(candidate);
+  }
+  return gone;
+}
+
+/** Fold each confirmed-gone candidate to the `orphaned` terminal, recording one
+ *  `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). WRITE-ONLY and
+ *  network-free, so the caller's transaction stays short. Runs inside the caller's transaction. */
+async function orphanVanishedCandidates(
+  t: DataSource,
+  candidates: VanishedCandidate[],
+  runId: string,
+  at: string,
+): Promise<OrphanedRow[]> {
+  const orphaned: OrphanedRow[] = [];
+  for (const { shape, row } of candidates) {
+    // Re-assert "still no instance-state row" in the guarded UPDATE (which also re-asserts the exact
+    // status read), so a row that went terminal — or an instance that reappeared, the poller recording
+    // it — between the out-of-transaction SELECT/probe and this UPDATE wins the race.
+    const stillVanishedGuard =
+      ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
+      `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
+    const o = await orphanRow(t, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
+    if (o) orphaned.push(o);
+  }
   return orphaned;
 }
 
@@ -475,9 +630,20 @@ export async function reconcileEngineBackedWork(
  *     it), every dispatched row would look vanished, so the pass is a hard no-op.
  *   • GUARDED — the same status-re-assert as the epoch pass, plus a still-vanished re-check, so a
  *     concurrent terminal write or a reappearing instance wins the race.
- *   • This pass reads the app's OWN last-known projection, not a live probe, so it acts correctly on a
- *     genuinely-vanished instance regardless of transient engine reachability (a live instance keeps
- *     its persisted ACTIVE row across a restart, so it is never mistaken for vanished).
+ *   • ENGINE-TRUTH CROSS-CHECK (issue #736) — the `_urban_instance_state` projection is an app-side
+ *     read model that can lag / be pruned / be rebuilt for an instance that is still ACTIVE on the
+ *     engine, so "no projection row" is NOT "instance vanished". Before folding, the pass consults
+ *     `opts.engineActive` (production wires one from the live engine via {@link makeEngineActiveProbe};
+ *     {@link runEngineReconcile}): an instance the engine reports ACTIVE — or whose truth could not be
+ *     established (engine unreachable, malformed answer, a probe that threw) — is SPARED. Only an
+ *     instance the engine positively confirms is gone is orphaned. Without a cross-check (omitted) the
+ *     pass falls back to projection-only.
+ *   • SHORT TRANSACTION — the cross-check is network I/O, so the candidate SELECT and every probe run
+ *     OUTSIDE `src.tx(...)`; the transaction covers only the guarded UPDATE + provenance writes. A
+ *     slow or unreachable engine therefore cannot hold the SQLite write lock open (for up to the
+ *     probe's timeout per candidate row), stalling other writers or boot. The guards make the split
+ *     safe: a row that went terminal, or an instance that reappeared in the projection, between the
+ *     out-of-transaction read and the in-transaction UPDATE wins the race and is not folded.
  */
 export async function reconcileVanishedInstances(
   data: DataLayer,
@@ -498,8 +664,13 @@ export async function reconcileVanishedInstances(
     return { runId, reason: "no-op", observedEpoch: null, recordedEpoch: null, orphanedCount: 0, orphaned: [] };
   }
 
+  // READ + PROBE first, transaction second: the engine-truth cross-check is network I/O and must never
+  // be awaited under an open write transaction (see SHORT TRANSACTION above).
+  const candidates = await selectVanishedCandidates(src, nowMs, graceMs);
+  const confirmedGone = await confirmVanishedGone(candidates, opts.engineActive);
+
   const orphaned = await src.tx(async (t) => {
-    const rows = await orphanVanishedRows(t, runId, at, nowMs, graceMs);
+    const rows = await orphanVanishedCandidates(t, confirmedGone, runId, at);
     const reason: ReconcileReason = rows.length > 0 ? "instance-vanished" : "no-op";
     await recordRun(t, { runId, at, observedEpoch: null, recordedEpoch: null, reason, orphanedCount: rows.length });
     return rows;
@@ -561,6 +732,12 @@ export async function runEngineReconcile(
   const vanished = await reconcileVanishedInstances(data, {
     ...opts,
     runId: `${epoch.runId}-vanished`,
+    // Cross-check ENGINE TRUTH before orphaning any vanished-instance candidate (issue #736): an
+    // instance the engine still reports ACTIVE is spared even when its `_urban_instance_state`
+    // projection is absent (the merlin false-orphan: the projection lags, the instance is live). A
+    // caller-supplied `engineActive` wins (tests inject a deterministic one); otherwise build one from
+    // the same engine address/token the epoch probe used.
+    engineActive: opts.engineActive ?? makeEngineActiveProbe(engineRest, { fetchImpl: opts.fetchImpl }),
   });
 
   const orphaned = [...epoch.orphaned, ...vanished.orphaned];