@nanobpm/nano-workforce 0.182.3 → 0.183.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -1,3 +1,15 @@
1
+ ## [0.183.1](https://github.com/nanobpm/nano-workforce/compare/v0.183.0...v0.183.1) (2026-09-07)
2
+
3
+ ### Bug Fixes
4
+
5
+ * **reconcile:** cross-check engine truth before orphaning a vanished instance ([#753](https://github.com/nanobpm/nano-workforce/issues/753)) ([a363fdb](https://github.com/nanobpm/nano-workforce/commit/a363fdbec1fd14641f2c4a2ef2718034a11a1e92)), closes [#630](https://github.com/nanobpm/nano-workforce/issues/630) [#736](https://github.com/nanobpm/nano-workforce/issues/736) [#736](https://github.com/nanobpm/nano-workforce/issues/736) [#736](https://github.com/nanobpm/nano-workforce/issues/736)
6
+
7
+ ## [0.183.0](https://github.com/nanobpm/nano-workforce/compare/v0.182.3...v0.183.0) (2026-09-07)
8
+
9
+ ### Features
10
+
11
+ * **agent:** mark senior:* tasks as engine-native external AgentTasks ([#748](https://github.com/nanobpm/nano-workforce/issues/748)) ([806eea5](https://github.com/nanobpm/nano-workforce/commit/806eea522490a999747ad4814fc92847dab3a6e4)), closes [#464](https://github.com/nanobpm/nano-workforce/issues/464) [#745](https://github.com/nanobpm/nano-workforce/issues/745) [#746](https://github.com/nanobpm/nano-workforce/issues/746) [#747](https://github.com/nanobpm/nano-workforce/issues/747) [Magikcraft/nano-bpm#1137](https://github.com/Magikcraft/nano-bpm/issues/1137)
12
+
1
13
  ## [0.182.3](https://github.com/nanobpm/nano-workforce/compare/v0.182.2...v0.182.3) (2026-09-06)
2
14
 
3
15
  ### Bug Fixes
@@ -0,0 +1,74 @@
1
+ // Tests for the engine-native AgentTask marker (issue #745, umbrella #746 — Camunda 8.10 parity),
2
+ // including the defect-class regression guard: every deployed prompt-bearing `senior:*` agent task
3
+ // must carry `<zeebe:agentDefinition agentType="external" />` alongside its `<zeebe:taskDefinition>`,
4
+ // so the worker harness mints an engine-native AgentInstance for it. A newly-added agent task that
5
+ // forgets the marker never persists durable AgentHistory — so it fails CI here instead of silently
6
+ // drifting.
7
+ import { readFileSync, readdirSync } from "node:fs";
8
+ import { test } from "node:test";
9
+ import { dirname, join, relative } from "node:path";
10
+ import { fileURLToPath } from "node:url";
11
+ import { assert, assertEquals } from "#test-assert";
12
+ import { agentTaskTypesMissingExternalMarker, promptBearingTaskTypes } from "./job-types.ts";
13
+
14
+ const PROCESSES_DIR = join(dirname(fileURLToPath(import.meta.url)), "../../../resources/processes");
15
+
16
+ // urban deploys `resources/` recursively (every file at any depth), so a process
17
+ // model added under a subdirectory would still deploy — walk recursively here too,
18
+ // or the guard would miss it and let an unmarked agent task slip through.
19
+ function bpmnFiles(): string[] {
20
+ const walk = (dir: string): string[] =>
21
+ readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
22
+ const full = join(dir, entry.name);
23
+ if (entry.isDirectory()) return walk(full);
24
+ return entry.name.endsWith(".bpmn") ? [relative(PROCESSES_DIR, full)] : [];
25
+ });
26
+ return walk(PROCESSES_DIR).sort();
27
+ }
28
+
29
+ test("agentTaskTypesMissingExternalMarker flags a prompt-bearing agent task with no external marker", () => {
30
+ const xml = `
31
+ <bpmn:serviceTask id="agent">
32
+ <bpmn:extensionElements>
33
+ <zeebe:taskDefinition type="senior:feature" />
34
+ <zeebe:linkedResources>
35
+ <zeebe:linkedResource resourceId="feature.md" resourceType="GenericScript" linkName="prompt" />
36
+ </zeebe:linkedResources>
37
+ </bpmn:extensionElements>
38
+ </bpmn:serviceTask>`;
39
+ assertEquals(agentTaskTypesMissingExternalMarker(xml), ["senior:feature"]);
40
+ });
41
+
42
+ test("agentTaskTypesMissingExternalMarker passes a marked agent task and ignores a plain host task", () => {
43
+ const xml = `
44
+ <bpmn:serviceTask id="host">
45
+ <bpmn:extensionElements>
46
+ <zeebe:taskDefinition type="pr.finalize" />
47
+ </bpmn:extensionElements>
48
+ </bpmn:serviceTask>
49
+ <bpmn:serviceTask id="agent">
50
+ <bpmn:extensionElements>
51
+ <zeebe:taskDefinition type="senior:feature" />
52
+ <zeebe:agentDefinition agentType="external" />
53
+ <zeebe:linkedResources>
54
+ <zeebe:linkedResource resourceId="feature.md" resourceType="GenericScript" linkName="prompt" />
55
+ </zeebe:linkedResources>
56
+ </bpmn:extensionElements>
57
+ </bpmn:serviceTask>`;
58
+ assertEquals(agentTaskTypesMissingExternalMarker(xml), []);
59
+ });
60
+
61
+ test("DEFECT-CLASS GUARD: every deployed prompt-bearing agent task carries the external AgentTask marker", () => {
62
+ let anyAgentTasks = false;
63
+ for (const file of bpmnFiles()) {
64
+ const xml = readFileSync(join(PROCESSES_DIR, file), "utf8");
65
+ if (promptBearingTaskTypes(xml).length > 0) anyAgentTasks = true;
66
+ const missing = agentTaskTypesMissingExternalMarker(xml);
67
+ assert(
68
+ missing.length === 0,
69
+ `${file}: agent task(s) missing <zeebe:agentDefinition agentType="external" /> — ${missing.join(", ")}`,
70
+ );
71
+ }
72
+ // Sanity: the models really do declare agent tasks (guard is not vacuously green).
73
+ assert(anyAgentTasks, "the deployed models declare prompt-bearing agent tasks");
74
+ });
@@ -49,6 +49,10 @@ export function jobTypeToRoutingToken(jobType: string): string | undefined {
49
49
  const SERVICE_TASK = /<(?:\w+:)?serviceTask\b[\s\S]*?<\/(?:\w+:)?serviceTask>/g;
50
50
  const TASK_DEFINITION_TYPE = /<(?:\w+:)?taskDefinition\b[^>]*\btype="([^"]*)"/;
51
51
  const PROMPT_LINK = /<(?:\w+:)?linkedResource\b[^>]*\blinkName="prompt"/;
52
+ // The engine-native AgentTask marker (issue #745): a `<zeebe:agentDefinition agentType="external" />`
53
+ // sibling of the `<zeebe:taskDefinition>` inside a `senior:*` agent task's extensionElements. It is
54
+ // what makes the element eligible for engine-native AgentInstance minting by the worker harness.
55
+ const EXTERNAL_AGENT_MARKER = /<(?:\w+:)?agentDefinition\b[^>]*\bagentType="external"/;
52
56
 
53
57
  /**
54
58
  * Scan one BPMN document for the job types of its PROMPT-BEARING service tasks — the deployed fleet
@@ -68,3 +72,25 @@ export function promptBearingTaskTypes(xml: string): string[] {
68
72
  }
69
73
  return types;
70
74
  }
75
+
76
+ /**
77
+ * Scan one BPMN document for the job types of PROMPT-BEARING agent service tasks that are MISSING the
78
+ * engine-native AgentTask marker `<zeebe:agentDefinition agentType="external" />` (issue #745). Every
79
+ * deployed `senior:*` agent task must carry the marker so the worker harness mints an AgentInstance
80
+ * for it; a newly-added agent task that forgets it is a silent drift surface (its run never persists
81
+ * durable AgentHistory), so the regression guard fails CI. Returns the offending task types in
82
+ * first-occurrence order (empty when every agent task is marked).
83
+ */
84
+ export function agentTaskTypesMissingExternalMarker(xml: string): string[] {
85
+ const seen = new Set<string>();
86
+ const missing: string[] = [];
87
+ for (const [block] of xml.matchAll(SERVICE_TASK)) {
88
+ if (!PROMPT_LINK.test(block)) continue;
89
+ if (EXTERNAL_AGENT_MARKER.test(block)) continue;
90
+ const type = block.match(TASK_DEFINITION_TYPE)?.[1];
91
+ if (type === undefined || type.length === 0 || seen.has(type)) continue;
92
+ seen.add(type);
93
+ missing.push(type);
94
+ }
95
+ return missing;
96
+ }
package/app/contracts.ts CHANGED
@@ -479,6 +479,14 @@ export const WIRE_CONTRACTS = {
479
479
  "The proxy-safe single-stream transcript READ URL (issue #744): `GET <base>/app/api/agentic/transcripts?stream=<id>&from=<n>` — the stream id rides a QUERY value, NEVER a path segment, because the Nano Console gateway proxy peels one percent-encoding layer before the app routes: an encoded slash (%2F) in a PATH segment arrives as a real / and splits a slash-bearing worker-instance id (`34:<instance>/<jobKey>`) into an extra segment, so the legacy `GET /app/api/agentic/transcripts/{stream}` route 404s behind the proxy (the cockpit past-session replay rendered empty). The path form stays served for back-compat and is proxy-safe ONLY for the slash-free `job:<jobKey>` ids it is seeded with (the worker-emitted `transcriptUrl` = `transcriptUrlBaseFor()` + `jobStream()`, a bare concatenation, must remain resolvable both directly and behind the proxy). ONE builder: `transcriptReadUrlFor()` in app/agentic/transcript-url.ts, served by ONE canonical read (`readSingleTranscript` in app/agentic/transcript-read.ts) shared by both routes; the browser adapter pages/cockpit/mount.js carries a hand-maintained twin (it cannot import server modules). Never put a stream id in a path segment again — do not re-declare a synonym scheme.",
480
480
  shape: "GET <transcriptsEndpoint>?stream=<percent-encoded stream id>[&from=<non-negative integer offset>] → AgenticTranscriptData | ErrorBody",
481
481
  },
482
+ "agentTask.agentDefinition": {
483
+ category: "wire",
484
+ name: "agentTask.agentDefinition",
485
+ owner: "resources/processes/*.bpmn",
486
+ semantics:
487
+ "The engine-native AgentTask marker (issue #745, umbrella #746 — Camunda 8.10 parity). Every `senior:*` agent service task carries `<zeebe:agentDefinition agentType=\"external\" />` INSIDE its `<bpmn:extensionElements>`, COEXISTING with the existing `<zeebe:taskDefinition type=\"senior:*\"/>` dispatch verb (the verb stays, per #464). The marker makes the element eligible for engine-native AgentInstance minting by the worker harness (jwulf/c8ctl-plugin-nano#194): the harness mints Create/Update/Complete AgentInstance/AgentHistory records against the pinned engine (`@nanobpm/engine-wasm` 0.8.6, broker REST, SDK) while the element still emits its NORMAL `senior:*` job. `agentType=\"external\"` means the agent runs OUTSIDE the engine (a remote fleet worker), not an engine-embedded model call. This is the PRODUCER half; the durable AgentInstance/AgentHistory it mints is read back by the Cockpit historical view via `searchAgentInstanceHistory` (the CONSUMER half, deferred until the broker read API is reachable from the app's EngineClient). It is authored in the hand-written BPMN semantic model, NOT the generated `<bpmndi:…>` DI, and survives `npm run layout` untouched. Add the marker to a NEW `senior:*` agent task — never a second/synonym marker element.",
488
+ shape: '<zeebe:agentDefinition agentType="external" /> (sibling of <zeebe:taskDefinition> in a senior:* service task\'s extensionElements)',
489
+ },
482
490
  } as const satisfies Record<string, WireContract>;
483
491
 
484
492
  export const TYPE_CONTRACTS = {
@@ -10,10 +10,12 @@
10
10
  // `makeGateway`), so the tables/columns/indexes reconcile reads and writes are the shipping schema.
11
11
  import { DatabaseSync } from "node:sqlite";
12
12
  import { test } from "node:test";
13
- import { assertEquals } from "#test-assert";
13
+ import type { DataLayer, GatewayDataSource as DataSource } from "@nanobpm/urban";
14
+ import { assertEquals, assertNotEquals } from "#test-assert";
14
15
  import { freshData } from "../test/reconcileDb.ts";
15
16
  import {
16
17
  DEFAULT_VANISHED_GRACE_MS,
18
+ makeEngineActiveProbe,
17
19
  ORPHANED_STATUS,
18
20
  parseEngineEpoch,
19
21
  RECONCILE_ORPHAN_REASON,
@@ -393,28 +395,115 @@ test("idempotent: a second vanished pass is a no-op (the orphaned row left activ
393
395
  assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 1);
394
396
  });
395
397
 
398
+ // --- Engine-truth cross-check before orphaning a "vanished" row (issue #736) -------------------
399
+ // The `_urban_instance_state` projection is an app-side read model that can lag / be pruned / be
400
+ // rebuilt while the instance is still ACTIVE on the engine. "No projection row" is therefore NOT
401
+ // "instance vanished": on merlin (whose engine exposes no incarnation epoch, so the robust epoch
402
+ // detector is disabled) this false-orphaned 3 concurrently-LIVE instances in one pass. The vanished
403
+ // pass now cross-checks ENGINE TRUTH via `engineActive` before folding — an ACTIVE instance is spared.
404
+
405
+ test("RED→GREEN #736: an engine-ACTIVE instance with no _urban_instance_state row (past grace) is NOT orphaned", async () => {
406
+ const { data, raw } = freshData();
407
+ ensureInstanceState(raw);
408
+ // The merlin repro: an inflight run past grace whose projection row is absent (lagging/pruned) but
409
+ // whose instance the engine still reports ACTIVE. RED (pre-fix): folded to `orphaned`. GREEN: spared.
410
+ seedFeatureRun(raw, "nanobpm/nano-workforce#336", "running", "11625");
411
+ // Engine truth says ACTIVE for this key.
412
+ const engineActive = async (key: string) => (key === "11625" ? true : false);
413
+
414
+ const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
415
+
416
+ assertEquals(res.reason, "no-op");
417
+ assertEquals(res.orphanedCount, 0);
418
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#336'").get() as {
419
+ status: string;
420
+ };
421
+ assertEquals(row.status, "running");
422
+ assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
423
+ });
424
+
425
+ test("#736: an engine-CONFIRMED-gone instance (engineActive=false) IS still orphaned", async () => {
426
+ const { data, raw } = freshData();
427
+ ensureInstanceState(raw);
428
+ seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
429
+ // Engine answered and the instance is absent/terminated — genuinely gone in engine truth.
430
+ const engineActive = async () => false;
431
+
432
+ const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
433
+
434
+ assertEquals(res.reason, "instance-vanished");
435
+ assertEquals(res.orphanedCount, 1);
436
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
437
+ status: string;
438
+ };
439
+ assertEquals(row.status, ORPHANED_STATUS);
440
+ });
441
+
442
+ test("#736: an instance whose engine truth is UNKNOWN (engineActive=null) is spared — never orphan unconfirmed", async () => {
443
+ const { data, raw } = freshData();
444
+ ensureInstanceState(raw);
445
+ seedFeatureRun(raw, "o/r#unknown", "escalated", "71506");
446
+ // The engine truth could not be established (unreachable / non-2xx / malformed) — we must NOT orphan.
447
+ const engineActive = async () => null;
448
+
449
+ const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
450
+
451
+ assertEquals(res.reason, "no-op");
452
+ assertEquals(res.orphanedCount, 0);
453
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#unknown'").get() as {
454
+ status: string;
455
+ };
456
+ assertEquals(row.status, "escalated");
457
+ });
458
+
396
459
  // --- Merged seam: runEngineReconcile (both passes, one result) --------------------------------
397
460
  // The operator/startup seam merges the epoch-regression and vanished-instance passes into ONE
398
461
  // result. This guards the merged behavior the two per-pass suites above don't reach: run-id
399
- // correlation (the vanished pass's provenance must be locatable from the returned `runId`) and
400
- // `reason` selection when the epoch pass is `engine-unreachable` yet vanished instances are orphaned.
462
+ // correlation (the vanished pass's provenance must be locatable from the returned `runId`), the
463
+ // engine-truth cross-check the seam wires from the live engine (#736), and `reason` selection.
464
+
465
+ /** A `/v2` fetch stub: `/topology` answers `topologyBody` (200), and `/process-instances/search`
466
+ * answers with `searchItems` (200) — the engine-truth cross-check the vanished pass runs (#736). */
467
+ function engineFetch(
468
+ topologyBody: unknown,
469
+ searchItems: { processInstanceKey?: string | number; state?: string }[],
470
+ ): typeof fetch {
471
+ return (async (url: string, init?: { method?: string }) => {
472
+ const u = String(url);
473
+ if (u.endsWith("/topology")) return new Response(JSON.stringify(topologyBody), { status: 200 });
474
+ if (u.endsWith("/process-instances/search") && init?.method === "POST") {
475
+ return new Response(JSON.stringify({ items: searchItems }), { status: 200 });
476
+ }
477
+ return new Response("not found", { status: 404 });
478
+ }) as unknown as typeof fetch;
479
+ }
480
+
481
+ test("runEngineReconcile #736: a reachable engine reporting the instance ACTIVE spares it (no false-orphan)", async () => {
482
+ const { data, raw } = freshData();
483
+ ensureInstanceState(raw);
484
+ // No projection row, past grace — but the engine (reachable, no epoch, like merlin) reports ACTIVE.
485
+ seedFeatureRun(raw, "nanobpm/nano-workforce#731", "escalated", "11644");
486
+ const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, [{ processInstanceKey: "11644", state: "ACTIVE" }]);
487
+
488
+ const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
489
+
490
+ assertEquals(res.orphanedCount, 0);
491
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#731'").get() as {
492
+ status: string;
493
+ };
494
+ assertEquals(row.status, "escalated");
495
+ assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
496
+ });
401
497
 
402
- test("runEngineReconcile: engine-unreachable epoch pass still folds vanished instances, with a correlatable run id", async () => {
498
+ test("runEngineReconcile #736: a reachable engine that no longer knows the instance folds it, with a correlatable run id", async () => {
403
499
  const { data, raw } = freshData();
404
500
  ensureInstanceState(raw);
405
- // A vanished orphan (escalated, past grace, instance absent from the read model).
501
+ // A genuinely-vanished orphan (escalated, past grace); the engine answers but the instance is absent.
406
502
  seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
503
+ const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, []);
407
504
 
408
- // The engine is unreachable the epoch probe fails, so the epoch pass reports `engine-unreachable`
409
- // and orphans nothing; the vanished pass must still act.
410
- const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
411
- const res = await runEngineReconcile(
412
- data,
413
- { restAddress: "http://engine.invalid" },
414
- { now: AT, fetchImpl },
415
- );
505
+ const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
416
506
 
417
- // The vanished pass acted even though the epoch pass could not reach the engine.
418
507
  assertEquals(res.reason, "instance-vanished");
419
508
  assertEquals(res.orphanedCount, 1);
420
509
  const orphan = raw
@@ -429,14 +518,189 @@ test("runEngineReconcile: engine-unreachable epoch pass still folds vanished ins
429
518
  .get() as { run_id: string };
430
519
  assertEquals(prov.run_id, `${res.runId}-vanished`);
431
520
 
432
- // Both passes recorded their own reconcile_runs row under correlatable ids.
433
- const epochRun = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id=?").get(res.runId) as
434
- | { reason: string }
435
- | undefined;
436
- assertEquals(epochRun?.reason, "engine-unreachable");
437
521
  const vanishedRun = raw
438
522
  .prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id=?")
439
523
  .get(`${res.runId}-vanished`) as { reason: string; orphaned_count: number } | undefined;
440
524
  assertEquals(vanishedRun?.reason, "instance-vanished");
441
525
  assertEquals(vanishedRun?.orphaned_count, 1);
442
526
  });
527
+
528
+ test("runEngineReconcile #736: an UNREACHABLE engine spares vanished candidates (truth unconfirmed → never orphan)", async () => {
529
+ const { data, raw } = freshData();
530
+ ensureInstanceState(raw);
531
+ // A candidate that looks vanished (escalated, past grace, no projection row) — but with the engine
532
+ // unreachable we cannot confirm it is gone, so it MUST be spared (issue #736): the old projection-only
533
+ // behavior would have orphaned it, potentially false-orphaning a live instance mid-outage.
534
+ seedFeatureRun(raw, "Magikcraft/nano-bpm#1051", "escalated", "71506");
535
+
536
+ const fetchImpl = (() => Promise.reject(new Error("ECONNREFUSED"))) as unknown as typeof fetch;
537
+ const res = await runEngineReconcile(data, { restAddress: "http://engine.invalid" }, { now: AT, fetchImpl });
538
+
539
+ // The epoch pass could not reach the engine, and the vanished pass could not confirm death → no-op.
540
+ assertEquals(res.reason, "engine-unreachable");
541
+ assertEquals(res.orphanedCount, 0);
542
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='Magikcraft/nano-bpm#1051'").get() as {
543
+ status: string;
544
+ };
545
+ assertEquals(row.status, "escalated");
546
+ assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
547
+ });
548
+
549
+ // --- Hardening the cross-check itself (#736 review) ---------------------------------------------
550
+ // Two defects of the SAME class the first cut still carried — "never orphan a row we could not
551
+ // positively confirm is gone" — plus the lock-hold the cross-check introduced:
552
+ // 1. a MATCHING search item whose `state` was missing or outside the engine's lifecycle enum read as
553
+ // `false` ("gone"), so a malformed/partial engine answer folded live work;
554
+ // 2. the probe (network I/O) was awaited INSIDE `src.tx(...)`, so a slow or unreachable engine held
555
+ // the SQLite write transaction open for the probe's full timeout PER CANDIDATE ROW — stalling
556
+ // every other writer, including boot — and an injected probe that threw aborted the whole pass.
557
+
558
+ /** Answer `/v2/process-instances/search` with exactly `items` (200), so a probe's classification can
559
+ * be read off one wire shape varying only in the item's `state`. */
560
+ function searchItemsFetch(items: { processInstanceKey?: string | number; state?: string }[]): typeof fetch {
561
+ return (async () => new Response(JSON.stringify({ items }), { status: 200 })) as unknown as typeof fetch;
562
+ }
563
+
564
+ /** Probe the key `11644` against a stubbed engine search answer of `items`. */
565
+ function probeAgainst(items: { processInstanceKey?: string | number; state?: string }[]): Promise<boolean | null> {
566
+ const probe = makeEngineActiveProbe({ restAddress: "http://engine.local/v2" }, { fetchImpl: searchItemsFetch(items) });
567
+ return probe("11644");
568
+ }
569
+
570
+ test("#736: the probe answers `true` for ACTIVE and `false` ONLY for a known-terminal engine state", async () => {
571
+ assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "ACTIVE" }]), true);
572
+ // The wire may carry the key as a JSON number and the state in any casing.
573
+ assertEquals(await probeAgainst([{ processInstanceKey: 11644, state: "active" }]), true);
574
+ for (const state of ["COMPLETED", "TERMINATED", "CANCELED", "FAILED"]) {
575
+ assertEquals(await probeAgainst([{ processInstanceKey: "11644", state }]), false, `${state} is a positive "gone"`);
576
+ }
577
+ // Absent from the read model is STILL a positive "gone": the engine answered and does not know it.
578
+ assertEquals(await probeAgainst([]), false);
579
+ });
580
+
581
+ test("RED→GREEN #736: a MISSING or UNRECOGNIZED engine state is UNKNOWN truth (`null` → spare), never `false`", async () => {
582
+ // RED (pre-fix): the probe answered `String(match.state ?? "").toUpperCase() === "ACTIVE"`, so a
583
+ // partial item (no `state`) or a state outside the enum this app knows read as "gone" and folded the
584
+ // row — contradicting the probe's own contract that malformed engine truth degrades to `null`.
585
+ assertEquals(await probeAgainst([{ processInstanceKey: "11644" }]), null, "a missing state is not a confirmed death");
586
+ assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "" }]), null);
587
+ assertEquals(await probeAgainst([{ processInstanceKey: "11644", state: "SUSPENDED" }]), null);
588
+ // An item for a DIFFERENT key is no match for this one, so that stays "absent" (gone), not unknown.
589
+ assertEquals(await probeAgainst([{ processInstanceKey: "99999" }]), false);
590
+ });
591
+
592
+ test("RED→GREEN #736: a malformed search item (no `state`) spares the candidate end-to-end", async () => {
593
+ const { data, raw } = freshData();
594
+ ensureInstanceState(raw);
595
+ // Past grace, no projection row, and the engine ANSWERS — but its item carries no lifecycle state, so
596
+ // engine truth is unestablished and the row must survive (RED pre-fix: folded to `orphaned`).
597
+ seedFeatureRun(raw, "nanobpm/nano-workforce#731", "escalated", "11644");
598
+ const fetchImpl = engineFetch({ nano: { engine: "merlin" } }, [{ processInstanceKey: "11644" }]);
599
+
600
+ const res = await runEngineReconcile(data, { restAddress: "http://engine.local/v2" }, { now: AT, fetchImpl });
601
+
602
+ assertEquals(res.orphanedCount, 0);
603
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='nanobpm/nano-workforce#731'").get() as {
604
+ status: string;
605
+ };
606
+ assertEquals(row.status, "escalated");
607
+ assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
608
+ });
609
+
610
+ test("RED→GREEN #736: a probe that THROWS is unknown truth — it spares the row instead of aborting the pass", async () => {
611
+ const { data, raw } = freshData();
612
+ ensureInstanceState(raw);
613
+ seedFeatureRun(raw, "o/r#boom", "escalated", "71506");
614
+ // `engineActive` is injectable: an implementation that rejects means truth could NOT be established
615
+ // (spare), and must not bubble out of the pass.
616
+ const engineActive = async (): Promise<boolean | null> => {
617
+ throw new Error("engine exploded");
618
+ };
619
+
620
+ const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-1", engineActive });
621
+
622
+ assertEquals(res.reason, "no-op");
623
+ assertEquals(res.orphanedCount, 0);
624
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#boom'").get() as { status: string };
625
+ assertEquals(row.status, "escalated");
626
+ // The pass COMPLETED and recorded its run — pre-fix the throw escaped the transaction and rejected the
627
+ // whole reconcile, so no `reconcile_runs` row was written at all.
628
+ const run = raw.prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id='van-1'").get() as
629
+ | { reason: string; orphaned_count: number }
630
+ | undefined;
631
+ assertEquals(run?.reason, "no-op");
632
+ assertEquals(run?.orphaned_count, 0);
633
+ });
634
+
635
+ test("RED→GREEN #736: the probe (network I/O) is never awaited INSIDE the write transaction", async () => {
636
+ const { data, raw } = freshData();
637
+ ensureInstanceState(raw);
638
+ seedFeatureRun(raw, "o/r#tx", "escalated", "71506");
639
+
640
+ // Wrap the gateway so the probe can report how deep in `src.tx(...)` it was awaited. RED (pre-fix):
641
+ // depth 1 — every candidate row held the SQLite write transaction open across a network round trip
642
+ // (up to the probe's full timeout on a slow/unreachable engine), delaying every other writer and
643
+ // slowing/locking boot. GREEN: depth 0 — the probes finish first, and only the guarded UPDATE +
644
+ // provenance writes run in a short transaction.
645
+ let depth = 0;
646
+ const observed: number[] = [];
647
+ const inner = data.open();
648
+ const tracked = {
649
+ open: () => ({
650
+ query: (sql: string, params?: unknown[]) => inner.query(sql, params),
651
+ exec: (sql: string, params?: unknown[]) => inner.exec(sql, params),
652
+ schema: () => inner.schema(),
653
+ table: (name: string, pk?: string) => inner.table(name, pk),
654
+ tx: async <T>(fn: (t: DataSource) => Promise<T>): Promise<T> => {
655
+ depth += 1;
656
+ try {
657
+ return await inner.tx(fn);
658
+ } finally {
659
+ depth -= 1;
660
+ }
661
+ },
662
+ }),
663
+ } as unknown as DataLayer;
664
+
665
+ const engineActive = async (): Promise<boolean | null> => {
666
+ observed.push(depth);
667
+ return false;
668
+ };
669
+
670
+ const res = await reconcileVanishedInstances(tracked, { now: AT, runId: "van-tx", engineActive });
671
+
672
+ // Exactly one entry: the seeded row also mirrors into `delivery_units` (same `process_key`), and one
673
+ // instance is probed once (see the next test) — at depth 0, outside any open write transaction.
674
+ assertEquals(observed, [0], "the engine-truth probe must be awaited outside any open write transaction");
675
+ // Hoisting the probe out of the transaction must not weaken the pass: a confirmed-gone row still folds.
676
+ assertEquals(res.orphanedCount, 1);
677
+ const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#tx'").get() as { status: string };
678
+ assertEquals(row.status, ORPHANED_STATUS);
679
+ });
680
+
681
+ test("#736: one instance backing several tracked rows is probed ONCE (engine truth is per instance)", async () => {
682
+ const { data, raw } = freshData();
683
+ ensureInstanceState(raw);
684
+ // A `feature_runs` row is ALSO mirrored into the `delivery_units` aggregate by DB trigger
685
+ // (db/migrations/089), carrying the same `process_key` — so one vanished instance yields TWO
686
+ // candidates. Engine truth is per instance, so it must be asked once, not once per row (each probe
687
+ // can block for its full timeout on a slow engine).
688
+ seedFeatureRun(raw, "o/r#mirror", "escalated", "71506");
689
+ const probed: string[] = [];
690
+ const engineActive = async (key: string): Promise<boolean | null> => {
691
+ probed.push(key);
692
+ return false;
693
+ };
694
+
695
+ const res = await reconcileVanishedInstances(data, { now: AT, runId: "van-mirror", engineActive });
696
+
697
+ assertEquals(probed, ["71506"], "one probe per distinct instance key");
698
+ // Folding the base row re-projects its mirror (the sync trigger clears `dispatch_status`), so the
699
+ // mirror is not folded a second time for the same instance — no duplicate provenance.
700
+ assertEquals(res.orphaned.map((o) => o.table), ["feature_runs"]);
701
+ assertEquals(res.orphanedCount, 1);
702
+ const mirror = raw.prepare("SELECT dispatch_status FROM delivery_units WHERE legacy_key='o/r#mirror'").get() as {
703
+ dispatch_status: string | null;
704
+ };
705
+ assertNotEquals(mirror.dispatch_status, "dispatched");
706
+ });
package/app/reconcile.ts CHANGED
@@ -38,8 +38,13 @@
38
38
  // `reconcileVanishedInstances` drives every such row — active, dispatched, its key absent from
39
39
  // `_urban_instance_state`, and PAST A GRACE WINDOW (so a still-starting run not yet projected is
40
40
  // spared) — to the same `orphaned` terminal, with a DISTINCT provenance reason so an operator can
41
- // tell a vanished-instance orphan apart from an epoch-regression one. `runEngineReconcile` runs BOTH
42
- // passes, so startup and the operator command converge both failure modes in one call.
41
+ // tell a vanished-instance orphan apart from an epoch-regression one. Because `_urban_instance_state`
42
+ // is an app-side projection that can lag / be pruned / be rebuilt while the instance is still ACTIVE
43
+ // on the engine, "no projection row" is NOT proof the instance vanished — so before folding, the pass
44
+ // CROSS-CHECKS ENGINE TRUTH (`/v2/process-instances/search`, issue #736): an instance the engine still
45
+ // reports ACTIVE (or whose truth cannot be established) is SPARED, closing the false-orphan class on
46
+ // deployments whose engine omits the incarnation epoch. `runEngineReconcile` runs BOTH passes, so
47
+ // startup and the operator command converge both failure modes in one call.
43
48
  //
44
49
  // The provenance is app-owned (not urban's `_urban_write_provenance`, which is a domain-free
45
50
  // insert-join sidecar written only inside a job): reconcile runs at boot / over HTTP, outside any
@@ -152,6 +157,15 @@ export interface VanishedReconcileOptions extends ReconcileOptions {
152
157
  * still-starting run (not yet projected into `_urban_instance_state`) is not orphaned prematurely.
153
158
  * Defaults to {@link DEFAULT_VANISHED_GRACE_MS}. */
154
159
  graceMs?: number;
160
+ /** Cross-check against ENGINE TRUTH before orphaning a candidate row (issue #736). Given the row's
161
+ * `keyField` (process instance key), it reports whether the engine still considers the instance
162
+ * ACTIVE. An ACTIVE instance is NEVER orphaned — the app-side `_urban_instance_state` projection is
163
+ * merely lagging — and an instance whose truth could not be established (`null`) is spared too (we
164
+ * never orphan what we could not confirm dead). Only a row the engine positively confirms is gone
165
+ * (`false`) is folded. When omitted the pass falls back to projection-only behaviour (no live
166
+ * cross-check); production always wires one via {@link runEngineReconcile}. See
167
+ * {@link makeEngineActiveProbe}. */
168
+ engineActive?: EngineActiveProbe;
155
169
  }
156
170
 
157
171
  /** Read the incarnation epoch out of a `/v2/topology` body — `nano.incarnation` (or its `epoch`
@@ -189,6 +203,80 @@ export async function probeEngineEpoch(
189
203
  }
190
204
  }
191
205
 
206
+ /** A cross-check against ENGINE TRUTH for one process instance key, used to spare a live instance from
207
+ * the vanished-instance pass. Returns:
208
+ * • `true` — the engine reports the instance ACTIVE. It is live; the app-side `_urban_instance_state`
209
+ * projection is merely lagging/pruned/rebuilding, so the row MUST NOT be orphaned.
210
+ * • `false` — the engine answered and the instance is NOT active: absent from the read model, or in a
211
+ * known terminal state ({@link ENGINE_TERMINAL_STATES}). It is genuinely gone in engine
212
+ * truth → orphan-eligible.
213
+ * • `null` — engine truth could NOT be established (unreachable, non-2xx, malformed — including an
214
+ * item whose `state` is missing or outside {@link ENGINE_TERMINAL_STATES}). We never
215
+ * orphan a row we could not confirm dead, so the caller spares it (a transient outage
216
+ * must never fold live work). */
217
+ export type EngineActiveProbe = (processKey: string) => Promise<boolean | null>;
218
+
219
+ /** One item of a `/v2/process-instances/search` result, narrowed to what the engine-truth cross-check
220
+ * reads: the instance key (to match the row we probed for) and its lifecycle `state`. Keys are
221
+ * stringified defensively (the wire may send a JSON number or string); `state` is the engine's
222
+ * lifecycle enum (`ACTIVE`/`COMPLETED`/`CANCELED`/…). */
223
+ interface InstanceSearchStateItem {
224
+ processInstanceKey?: string | number;
225
+ state?: string;
226
+ }
227
+
228
+ /** The lifecycle states that POSITIVELY mean "this instance is no longer running" — urban's
229
+ * `ProcessInstanceState` terminals (`COMPLETED`/`TERMINATED`) plus the Camunda-8-parity v2 REST
230
+ * terminals (`CANCELED`/`FAILED`), since the probe reads that raw surface rather than the typed
231
+ * client. ONLY these may answer `false` (orphan-eligible): a `state` that is missing, empty, or
232
+ * outside this set is a partial/malformed read this app cannot interpret, so it degrades to `null`
233
+ * ("unknown" → the caller spares the row). Classifying an unrecognized state as "gone" would fold
234
+ * live work off a wire shape we misread — the exact failure mode the #736 cross-check exists to stop. */
235
+ const ENGINE_TERMINAL_STATES: ReadonlySet<string> = new Set(["COMPLETED", "TERMINATED", "CANCELED", "FAILED"]);
236
+
237
+ /** Build an {@link EngineActiveProbe} that queries the engine's own `/v2/process-instances/search` for
238
+ * a single process instance key and reports whether the engine still considers it ACTIVE. This is the
239
+ * authoritative engine-truth check the vanished-instance pass consults before orphaning: an ACTIVE
240
+ * engine instance must NEVER be orphaned regardless of the app-side projection (issue #736), so a
241
+ * merlin-style deployment whose `_urban_instance_state` lags no longer false-orphans live work.
242
+ * Never throws — every transport/parse failure, and every `state` this app cannot interpret (missing,
243
+ * or outside {@link ENGINE_TERMINAL_STATES}), degrades to `null` ("unknown"), which the caller treats
244
+ * as "spare" (we never orphan what we could not confirm dead). */
245
+ export function makeEngineActiveProbe(
246
+ engineRest: { restAddress: string; token?: string },
247
+ opts: { fetchImpl?: typeof fetch; timeoutMs?: number } = {},
248
+ ): EngineActiveProbe {
249
+ const fetchImpl = opts.fetchImpl ?? fetch;
250
+ const base = engineRest.restAddress.replace(/\/+$/, "");
251
+ const headers: Record<string, string> = { accept: "application/json", "content-type": "application/json" };
252
+ if (engineRest.token) headers.authorization = `Bearer ${engineRest.token}`;
253
+ return async (processKey: string): Promise<boolean | null> => {
254
+ try {
255
+ const res = await fetchImpl(`${base}/process-instances/search`, {
256
+ method: "POST",
257
+ headers,
258
+ body: JSON.stringify({ filter: { processInstanceKey: processKey }, page: { from: 0, limit: 10 } }),
259
+ signal: AbortSignal.timeout(opts.timeoutMs ?? 3000),
260
+ });
261
+ if (!res.ok) return null;
262
+ // biome-ignore lint/plugin: runtime/framework contract boundary for external data shape
263
+ const body = (await res.json()) as { items?: InstanceSearchStateItem[] };
264
+ const items = body.items ?? [];
265
+ const match = items.find((it) => it.processInstanceKey != null && String(it.processInstanceKey) === processKey);
266
+ // Engine answered but the instance is absent from the read model → genuinely gone (not active).
267
+ if (!match) return false;
268
+ const state = String(match.state ?? "").trim().toUpperCase();
269
+ if (state === "ACTIVE") return true;
270
+ // A KNOWN terminal state is a positive "gone". Anything else — a missing/empty `state`, or one
271
+ // outside the enum this app can interpret — is a partial or malformed answer, NOT a confirmed
272
+ // death, so it degrades to `null` and the caller spares the row.
273
+ return ENGINE_TERMINAL_STATES.has(state) ? false : null;
274
+ } catch {
275
+ return null;
276
+ }
277
+ };
278
+ }
279
+
192
280
  /** Double-quote a SQL identifier (table/column) so a manifest-declared name is safe to interpolate. */
193
281
  function q(id: string): string {
194
282
  return `"${id.replace(/"/g, '""')}"`;
@@ -334,19 +422,24 @@ function withinGrace(updated: unknown, nowMs: number, graceMs: number): boolean
334
422
  return nowMs - t < graceMs;
335
423
  }
336
424
 
337
- /** Orphan every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
338
- * `_urban_instance_state` row the instance is absent/unknown in engine truth (vanished, issue
339
- * #630) and whose last transition is older than the grace window. Records one
340
- * `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). Runs inside
341
- * the caller's transaction. */
342
- async function orphanVanishedRows(
425
+ /** A vanished-instance candidate: the row selected for possible orphaning plus the binding shape that
426
+ * resolves its physical schema. Selected OUTSIDE any transaction, because the engine-truth
427
+ * cross-check that narrows these is network I/O (see {@link confirmVanishedGone}). */
428
+ interface VanishedCandidate {
429
+ shape: BindingShape;
430
+ row: OrphanCandidate;
431
+ }
432
+
433
+ /** SELECT every NON-terminal, engine-backed row whose `keyField` (process instance key) has NO
434
+ * `_urban_instance_state` row — the instance is absent/unknown in the app-side read model (vanished,
435
+ * issue #630) — and whose last transition is older than the grace window. READ-ONLY and network-free,
436
+ * so it is safe to run outside the orphaning transaction. */
437
+ async function selectVanishedCandidates(
343
438
  src: DataSource,
344
- runId: string,
345
- at: string,
346
439
  nowMs: number,
347
440
  graceMs: number,
348
- ): Promise<OrphanedRow[]> {
349
- const orphaned: OrphanedRow[] = [];
441
+ ): Promise<VanishedCandidate[]> {
442
+ const candidates: VanishedCandidate[] = [];
350
443
  for (const binding of engineBackedBindings()) {
351
444
  const shape = await resolveShape(src, binding);
352
445
  if (!shape) continue;
@@ -360,17 +453,79 @@ async function orphanVanishedRows(
360
453
  `AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s WHERE s.process_instance_key = b.${q(shape.keyField)})`,
361
454
  [...shape.active],
362
455
  );
363
- // Re-assert "still no instance-state row" in the guarded UPDATE too, so an instance that reappears
364
- // (the poller records it) between the SELECT above and the UPDATE wins the race.
365
- const stillVanishedGuard =
366
- ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
367
- `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
368
456
  for (const row of rows) {
369
457
  if (shape.hasUpdatedAt && withinGrace(row.__updated, nowMs, graceMs)) continue;
370
- const o = await orphanRow(src, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
371
- if (o) orphaned.push(o);
458
+ candidates.push({ shape, row });
372
459
  }
373
460
  }
461
+ return candidates;
462
+ }
463
+
464
+ /** Narrow candidates to the ones ENGINE TRUTH positively confirms are gone (issue #736): the
465
+ * `_urban_instance_state` projection is an app-side read model that can lag / be pruned / be rebuilt
466
+ * while the instance is still ACTIVE on the engine, so "no projection row" is NOT "instance vanished".
467
+ * An ACTIVE instance (`true`) is spared, and one whose truth could not be established (`null` — engine
468
+ * unreachable, malformed answer, or a probe that THREW) is spared too: we never orphan a row we could
469
+ * not positively confirm is gone, and an injected probe's failure must never abort the pass. Only a
470
+ * `false` (engine confirms absent/terminated) survives. Runs OUTSIDE the DB transaction — this is
471
+ * network I/O, and awaiting it under an open write transaction would hold the SQLite lock for up to
472
+ * the probe's timeout PER ROW, stalling every other writer (including boot). Without a probe (omitted)
473
+ * the pass falls back to projection-only behaviour, so every candidate survives. */
474
+ async function confirmVanishedGone(
475
+ candidates: VanishedCandidate[],
476
+ engineActive?: EngineActiveProbe,
477
+ ): Promise<VanishedCandidate[]> {
478
+ if (!engineActive) return candidates;
479
+ // Engine truth is PER INSTANCE, not per row, and one instance can back several tracked rows: the
480
+ // `delivery_units` aggregate is a DB-trigger mirror of its legacy base row (db/migrations/089) and
481
+ // carries the same `process_key`, so both are candidates for one vanished instance. Probe each
482
+ // DISTINCT key once and apply that verdict to every row carrying it — the same answer, without
483
+ // doubling engine calls that can each block for the probe's full timeout.
484
+ const verdicts = new Map<string, boolean | null>();
485
+ const gone: VanishedCandidate[] = [];
486
+ for (const candidate of candidates) {
487
+ const key = candidate.row.__key == null ? null : String(candidate.row.__key);
488
+ // Defensive: the SELECT requires a populated key, so with no key there is nothing to cross-check
489
+ // and the projection-only verdict stands.
490
+ if (key == null) {
491
+ gone.push(candidate);
492
+ continue;
493
+ }
494
+ let verdict = verdicts.get(key);
495
+ if (verdict === undefined) {
496
+ try {
497
+ verdict = await engineActive(key);
498
+ } catch {
499
+ // A probe that throws established nothing — treat it exactly like an unreachable engine.
500
+ verdict = null;
501
+ }
502
+ verdicts.set(key, verdict);
503
+ }
504
+ if (verdict === false) gone.push(candidate);
505
+ }
506
+ return gone;
507
+ }
508
+
509
+ /** Fold each confirmed-gone candidate to the `orphaned` terminal, recording one
510
+ * `reconcile_provenance` row per transition (reason {@link RECONCILE_VANISHED_REASON}). WRITE-ONLY and
511
+ * network-free, so the caller's transaction stays short. Runs inside the caller's transaction. */
512
+ async function orphanVanishedCandidates(
513
+ t: DataSource,
514
+ candidates: VanishedCandidate[],
515
+ runId: string,
516
+ at: string,
517
+ ): Promise<OrphanedRow[]> {
518
+ const orphaned: OrphanedRow[] = [];
519
+ for (const { shape, row } of candidates) {
520
+ // Re-assert "still no instance-state row" in the guarded UPDATE (which also re-asserts the exact
521
+ // status read), so a row that went terminal — or an instance that reappeared, the poller recording
522
+ // it — between the out-of-transaction SELECT/probe and this UPDATE wins the race.
523
+ const stillVanishedGuard =
524
+ ` AND NOT EXISTS (SELECT 1 FROM ${q(INSTANCE_STATE_TABLE)} s ` +
525
+ `WHERE s.process_instance_key = ${q(shape.table)}.${q(shape.keyField)})`;
526
+ const o = await orphanRow(t, shape, row, RECONCILE_VANISHED_REASON, null, runId, at, stillVanishedGuard);
527
+ if (o) orphaned.push(o);
528
+ }
374
529
  return orphaned;
375
530
  }
376
531
 
@@ -475,9 +630,20 @@ export async function reconcileEngineBackedWork(
475
630
  * it), every dispatched row would look vanished, so the pass is a hard no-op.
476
631
  * • GUARDED — the same status-re-assert as the epoch pass, plus a still-vanished re-check, so a
477
632
  * concurrent terminal write or a reappearing instance wins the race.
478
- * • This pass reads the app's OWN last-known projection, not a live probe, so it acts correctly on a
479
- * genuinely-vanished instance regardless of transient engine reachability (a live instance keeps
480
- * its persisted ACTIVE row across a restart, so it is never mistaken for vanished).
633
+ * • ENGINE-TRUTH CROSS-CHECK (issue #736) the `_urban_instance_state` projection is an app-side
634
+ * read model that can lag / be pruned / be rebuilt for an instance that is still ACTIVE on the
635
+ * engine, so "no projection row" is NOT "instance vanished". Before folding, the pass consults
636
+ * `opts.engineActive` (production wires one from the live engine via {@link makeEngineActiveProbe};
637
+ * {@link runEngineReconcile}): an instance the engine reports ACTIVE — or whose truth could not be
638
+ * established (engine unreachable, malformed answer, a probe that threw) — is SPARED. Only an
639
+ * instance the engine positively confirms is gone is orphaned. Without a cross-check (omitted) the
640
+ * pass falls back to projection-only.
641
+ * • SHORT TRANSACTION — the cross-check is network I/O, so the candidate SELECT and every probe run
642
+ * OUTSIDE `src.tx(...)`; the transaction covers only the guarded UPDATE + provenance writes. A
643
+ * slow or unreachable engine therefore cannot hold the SQLite write lock open (for up to the
644
+ * probe's timeout per candidate row), stalling other writers or boot. The guards make the split
645
+ * safe: a row that went terminal, or an instance that reappeared in the projection, between the
646
+ * out-of-transaction read and the in-transaction UPDATE wins the race and is not folded.
481
647
  */
482
648
  export async function reconcileVanishedInstances(
483
649
  data: DataLayer,
@@ -498,8 +664,13 @@ export async function reconcileVanishedInstances(
498
664
  return { runId, reason: "no-op", observedEpoch: null, recordedEpoch: null, orphanedCount: 0, orphaned: [] };
499
665
  }
500
666
 
667
+ // READ + PROBE first, transaction second: the engine-truth cross-check is network I/O and must never
668
+ // be awaited under an open write transaction (see SHORT TRANSACTION above).
669
+ const candidates = await selectVanishedCandidates(src, nowMs, graceMs);
670
+ const confirmedGone = await confirmVanishedGone(candidates, opts.engineActive);
671
+
501
672
  const orphaned = await src.tx(async (t) => {
502
- const rows = await orphanVanishedRows(t, runId, at, nowMs, graceMs);
673
+ const rows = await orphanVanishedCandidates(t, confirmedGone, runId, at);
503
674
  const reason: ReconcileReason = rows.length > 0 ? "instance-vanished" : "no-op";
504
675
  await recordRun(t, { runId, at, observedEpoch: null, recordedEpoch: null, reason, orphanedCount: rows.length });
505
676
  return rows;
@@ -561,6 +732,12 @@ export async function runEngineReconcile(
561
732
  const vanished = await reconcileVanishedInstances(data, {
562
733
  ...opts,
563
734
  runId: `${epoch.runId}-vanished`,
735
+ // Cross-check ENGINE TRUTH before orphaning any vanished-instance candidate (issue #736): an
736
+ // instance the engine still reports ACTIVE is spared even when its `_urban_instance_state`
737
+ // projection is absent (the merlin false-orphan: the projection lags, the instance is live). A
738
+ // caller-supplied `engineActive` wins (tests inject a deterministic one); otherwise build one from
739
+ // the same engine address/token the epoch probe used.
740
+ engineActive: opts.engineActive ?? makeEngineActiveProbe(engineRest, { fetchImpl: opts.fetchImpl }),
564
741
  });
565
742
 
566
743
  const orphaned = [...epoch.orphaned, ...vanished.orphaned];
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@nanobpm/nano-workforce",
3
- "version": "0.182.3",
3
+ "version": "0.183.1",
4
4
  "description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
5
5
  "type": "module",
6
6
  "main": "main.ts",
@@ -70,7 +70,7 @@
70
70
  },
71
71
  "devDependencies": {
72
72
  "@biomejs/biome": "^2.4.11",
73
- "@nanobpm/urban-testkit": "^1.2.0",
73
+ "@nanobpm/urban-testkit": "^1.3.0",
74
74
  "@nanobpm/workflow": "^0.14.0",
75
75
  "@semantic-release/changelog": "^7.0.0",
76
76
  "@semantic-release/git": "^11.0.0",
@@ -83,6 +83,7 @@
83
83
  "yaml": "^2.9.0"
84
84
  },
85
85
  "overrides": {
86
+ "@nanobpm/engine-wasm": "^0.9.0",
86
87
  "@semantic-release/npm": "^13.1.5"
87
88
  }
88
89
  }
@@ -100,6 +100,7 @@
100
100
  <bpmn:serviceTask id="review-round" name="Review round (agent)">
101
101
  <bpmn:extensionElements>
102
102
  <zeebe:taskDefinition type="senior:pr-review" />
103
+ <zeebe:agentDefinition agentType="external" />
103
104
  <zeebe:linkedResources>
104
105
  <zeebe:linkedResource resourceId="prompts/review-round.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
105
106
  </zeebe:linkedResources>
@@ -320,6 +321,7 @@
320
321
  <bpmn:serviceTask id="classify-scope" name="Scope classifier (agent)">
321
322
  <bpmn:extensionElements>
322
323
  <zeebe:taskDefinition type="senior:scope-classify" />
324
+ <zeebe:agentDefinition agentType="external" />
323
325
  <zeebe:linkedResources>
324
326
  <zeebe:linkedResource resourceId="prompts/scope-classify.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
325
327
  </zeebe:linkedResources>
@@ -16,6 +16,7 @@
16
16
  <bpmn:serviceTask id="implement-task" name="Implement (agent)">
17
17
  <bpmn:extensionElements>
18
18
  <zeebe:taskDefinition type="senior:feature" />
19
+ <zeebe:agentDefinition agentType="external" />
19
20
  <zeebe:linkedResources>
20
21
  <zeebe:linkedResource resourceId="prompts/feature.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
21
22
  </zeebe:linkedResources>
@@ -7,6 +7,7 @@
7
7
  <bpmn:serviceTask id="trial-merge" name="Trial merge (agent)">
8
8
  <bpmn:extensionElements>
9
9
  <zeebe:taskDefinition type="senior:trial-merge" />
10
+ <zeebe:agentDefinition agentType="external" />
10
11
  <zeebe:linkedResources>
11
12
  <zeebe:linkedResource resourceId="prompts/trial-merge.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
12
13
  </zeebe:linkedResources>
@@ -339,6 +339,7 @@
339
339
  <bpmn:serviceTask id="fix-ci" name="Fix CI (agent)">
340
340
  <bpmn:extensionElements>
341
341
  <zeebe:taskDefinition type="senior:fix-ci" />
342
+ <zeebe:agentDefinition agentType="external" />
342
343
  <zeebe:linkedResources>
343
344
  <zeebe:linkedResource resourceId="prompts/fix-ci.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
344
345
  </zeebe:linkedResources>
@@ -496,6 +497,7 @@
496
497
  <bpmn:serviceTask id="rebase" name="Rebase (agent)">
497
498
  <bpmn:extensionElements>
498
499
  <zeebe:taskDefinition type="senior:rebase" />
500
+ <zeebe:agentDefinition agentType="external" />
499
501
  <zeebe:linkedResources>
500
502
  <zeebe:linkedResource resourceId="prompts/rebase.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
501
503
  </zeebe:linkedResources>
@@ -260,6 +260,7 @@
260
260
  <bpmn:serviceTask id="plan" name="Plan (agent)">
261
261
  <bpmn:extensionElements>
262
262
  <zeebe:taskDefinition type="senior:plan" />
263
+ <zeebe:agentDefinition agentType="external" />
263
264
  <zeebe:linkedResources>
264
265
  <zeebe:linkedResource resourceId="prompts/plan.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
265
266
  </zeebe:linkedResources>
@@ -311,6 +312,7 @@
311
312
  <bpmn:serviceTask id="review-plan" name="Review plan (agent)">
312
313
  <bpmn:extensionElements>
313
314
  <zeebe:taskDefinition type="senior:plan-review" />
315
+ <zeebe:agentDefinition agentType="external" />
314
316
  <zeebe:linkedResources>
315
317
  <zeebe:linkedResource resourceId="prompts/plan-review.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
316
318
  </zeebe:linkedResources>
@@ -520,6 +522,7 @@
520
522
  <bpmn:serviceTask id="trial-merge" name="Trial merge (agent)">
521
523
  <bpmn:extensionElements>
522
524
  <zeebe:taskDefinition type="senior:trial-merge" />
525
+ <zeebe:agentDefinition agentType="external" />
523
526
  <zeebe:linkedResources>
524
527
  <zeebe:linkedResource resourceId="prompts/trial-merge.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
525
528
  </zeebe:linkedResources>
@@ -43,6 +43,7 @@
43
43
  <bpmn:serviceTask id="conformance" name="Verify implementation vs spec (agent)">
44
44
  <bpmn:extensionElements>
45
45
  <zeebe:taskDefinition type="senior:conformance" />
46
+ <zeebe:agentDefinition agentType="external" />
46
47
  <zeebe:linkedResources>
47
48
  <zeebe:linkedResource resourceId="prompts/conformance.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
48
49
  </zeebe:linkedResources>
@@ -91,6 +92,7 @@
91
92
  <bpmn:serviceTask id="synthesize" name="Synthesize &#38; promote (agent)">
92
93
  <bpmn:extensionElements>
93
94
  <zeebe:taskDefinition type="senior:retro" />
95
+ <zeebe:agentDefinition agentType="external" />
94
96
  <zeebe:linkedResources>
95
97
  <zeebe:linkedResource resourceId="prompts/retro.md" bindingType="latest" resourceType="GenericScript" linkName="prompt" />
96
98
  </zeebe:linkedResources>
@@ -9,18 +9,19 @@
9
9
  // and linked resources — with a legible red/green diff on mismatch.
10
10
  //
11
11
  // Ported models run a real `assertDerivationParity`; parked models (see
12
- // `./flows.ts`) are reported as skipped WITH their precise reason, in two
12
+ // `./flows.ts`) are reported as skipped WITH their precise reason, in three
13
13
  // blocker classes — class 1: multiple top-level start/end events; class 2:
14
- // arbitrary control-flow graph (`convergence-loop`). Companion diagnostics prove
15
- // each blocker is real against the goldens themselves. No golden is modified to
14
+ // arbitrary control-flow graph (`convergence-loop`); class 3: the engine-native
15
+ // agent-task marker (`retro`, issue #745). Companion diagnostics prove each
16
+ // blocker is real against the goldens themselves. No golden is modified to
16
17
  // force a match — the derivation must reproduce the checked-in file.
17
18
 
18
19
  import { test } from "node:test";
19
20
  import { readFileSync } from "node:fs";
20
21
  import { assert, assertEquals } from "#test-assert";
21
- import { assertDerivationParity, normalize } from "@nanobpm/workflow/test-support";
22
+ import { assertDerivationParity, diffModels, modelsEqual, normalize } from "@nanobpm/workflow/test-support";
22
23
  import { declarativeToBpmn, defineFlow } from "@nanobpm/workflow";
23
- import { PORTS } from "./flows.ts";
24
+ import { PORTS, retroFlow } from "./flows.ts";
24
25
 
25
26
  const ROOT = decodeURIComponent(new URL("../../", import.meta.url).pathname);
26
27
  const goldenPath = (model: string): string => `${ROOT}resources/processes/${model}.bpmn`;
@@ -98,8 +99,9 @@ test("class-1 blocked goldens genuinely have multiple top-level start/end events
98
99
  }
99
100
 
100
101
  // The two single-start/single-end goldens (retro, convergence-loop) clear
101
- // class 1; retro is a GREEN whole-model parity port (it runs through
102
- // `assertDerivationParity` above), convergence-loop is class-2 blocked (below).
102
+ // class 1; each is blocked on a LATER class instead retro on the class-3
103
+ // agent-task marker (proven below), convergence-loop on its class-2 arbitrary
104
+ // graph.
103
105
  for (const model of ["retro", "convergence-loop"]) {
104
106
  const xml = readFileSync(goldenPath(model), "utf8");
105
107
  assertEquals(countTag(xml, "startEvent"), 1, `${model} should have one start event`);
@@ -157,3 +159,38 @@ test("loop() inserts a gateway head, so back-edges cannot merge into a task", ()
157
159
  "the loop-body task cannot itself be the back-edge merge (it stays in<=1)",
158
160
  );
159
161
  });
162
+
163
+ // CLASS 3 — retro has a single top-level start/end (clears class 1) and a fully
164
+ // structured topology (clears class 2), but issue #745 added the engine-native
165
+ // AgentTask marker `<zeebe:agentDefinition agentType="external" />` to its two
166
+ // prompt-bearing `senior:*` tasks and the published compiler cannot emit it.
167
+ // Prove the blocker is real AND that it is the ONLY divergence, so the parked
168
+ // entry is a complete, verified port held ready — not an abandoned one.
169
+ test("retro's complete port differs from its golden by ONLY the agent-task marker", () => {
170
+ const golden = readFileSync(goldenPath("retro"), "utf8");
171
+ const derived = declarativeToBpmn(retroFlow);
172
+ const markers = (xml: string): number =>
173
+ (xml.match(/<(?:\w+:)?agentDefinition\b[^>]*\bagentType="external"/g) ?? []).length;
174
+
175
+ // (a) the golden really carries the marker, on BOTH of its agent tasks — and it
176
+ // cannot simply be dropped: app/agentic/vocab/agent-marker.test.ts is a
177
+ // defect-class guard requiring it on every deployed prompt-bearing agent task.
178
+ assertEquals(markers(golden), 2, "retro's golden should mark senior:conformance and senior:retro");
179
+
180
+ // (b) the published compiler emits none of it — the blocker is real, not a guess.
181
+ // When this starts failing, upstream task() grew marker support: un-park retro
182
+ // by threading `retroFlow` back into its PORTS entry in ./flows.ts.
183
+ assertEquals(markers(derived), 0, "@nanobpm/workflow can now emit <zeebe:agentDefinition/> — un-park retro");
184
+
185
+ // (c) strip exactly the marker lines from the golden and the port derives the
186
+ // WHOLE model green, through the shared harness's own normalize/equality —
187
+ // so nothing but the marker diverges.
188
+ const unmarked = golden.replace(/^[ \t]*<(?:\w+:)?agentDefinition\b[^>]*\/>[ \t]*\r?\n/gm, "");
189
+ assertEquals(markers(unmarked), 0, "the strip must remove every marker line");
190
+ const expected = normalize(unmarked);
191
+ const actual = normalize(derived);
192
+ assert(
193
+ modelsEqual(expected, actual),
194
+ `retro's port must derive its golden once the agent-task marker is stripped — residual drift:\n${diffModels(expected, actual)}`,
195
+ );
196
+ });
@@ -9,10 +9,9 @@
9
9
  // the structurally-derivable goldens at full whole-model parity, park the rest
10
10
  // pending an upstream construct, and do NOT relax to node-surface parity):
11
11
  //
12
- // • `retro` is a GREEN whole-model parity port (see below). The remaining seven
13
- // goldens are `blockedReason`-parked, in TWO distinct classes, each awaiting
14
- // an upstream `@nanobpm/workflow` (nano-ide) construct + re-release — never a
15
- // golden edit and never relaxed acceptance:
12
+ // • EVERY golden is currently `blockedReason`-parked, in THREE distinct
13
+ // classes, each awaiting an upstream `@nanobpm/workflow` (nano-ide) construct
14
+ // + re-release — never a golden edit and never relaxed acceptance:
16
15
  //
17
16
  // (1) MULTI top-level start/end (spine-demo, readiness-gate, feature,
18
17
  // merge-loop, plan-fanout, delivery-human). `@nanobpm/workflow` derives EXACTLY
@@ -35,6 +34,19 @@
35
34
  // arbitrary-graph / explicit-join (named-target) builder — a SUPERSET of
36
35
  // the class-(1) gap.
37
36
  //
37
+ // (3) ENGINE-NATIVE AGENT-TASK MARKER (retro). `retro` clears BOTH classes
38
+ // above and WAS a green whole-model parity port: its single start/end,
39
+ // linear pipeline, `deviations?` branch, userTask, data envelopes, prompt
40
+ // bindings and general `io` mappings all derive faithfully — the complete
41
+ // port is still authored below, and `derivation-parity.test.ts` proves it
42
+ // differs from its golden by NOTHING but the marker. Issue #745 then added
43
+ // `<zeebe:agentDefinition agentType="external" />` to every prompt-bearing
44
+ // `senior:*` task (a marker `app/agentic/vocab/agent-marker.test.ts`
45
+ // requires, so the golden cannot drop it), and the published compiler
46
+ // cannot emit it: `task()` accepts only `{ jobType, prompt, io }`, and no
47
+ // `agentDefinition`/`agentType` appears anywhere in
48
+ // `@nanobpm/workflow@0.14.0`. Needs an agent-marker option on `task()`.
49
+ //
38
50
  // A resumed run flips any parked model to a real `flow` once the corresponding
39
51
  // upstream construct lands and `@nanobpm/workflow` is bumped to carry it.
40
52
 
@@ -60,7 +72,7 @@ export type PortEntry =
60
72
  readonly blockedReason: string;
61
73
  };
62
74
 
63
- // ── retro (GREENwhole-model parity) ───────────────────────────────────────
75
+ // ── retro (PARKEDclass 3: agent-task marker; the port is otherwise COMPLETE)
64
76
  // retro is a single-start/single-end model: a linear gather → conformance →
65
77
  // record-conformance agent pipeline, a `deviations?` exclusive gateway guarding a
66
78
  // conformance-escalation subgraph (nano-workforce #355/#356), then a shared
@@ -72,6 +84,15 @@ export type PortEntry =
72
84
  // (nano-ide#405) — `w.task`+`io` for `record-conformance-ack`'s general
73
85
  // <zeebe:ioMapping> (inputs `=planKey`→planKey and
74
86
  // `=if (is defined(note)) then note else null`→note).
87
+ //
88
+ // The port below is therefore kept WHOLE and exported, not deleted: it is a
89
+ // faithful whole-model derivation whose ONLY divergence from the golden is the
90
+ // `<zeebe:agentDefinition agentType="external" />` marker issue #745 added to the
91
+ // two `senior:*` tasks (class 3 above). `derivation-parity.test.ts` asserts that
92
+ // divergence is exactly the marker, so the moment upstream `task()` grows an
93
+ // agent-marker option this diagnostic fires and the model is un-parked by
94
+ // threading `retroFlow` back into its PORTS entry — a one-line change, with the
95
+ // port already proven correct rather than re-authored from scratch.
75
96
 
76
97
  /** The typed data envelopes retro's non-agent service tasks lift into the model
77
98
  * (`nano:shape` + `io.nanobpm.dataEnvelope.in`), matching the golden's shapes. */
@@ -96,8 +117,10 @@ const ConformanceRecordIn = envelope("ConformanceRecordIn", {
96
117
  summary: { type: "string", optional: true },
97
118
  });
98
119
 
99
- /** The code-first port of `resources/processes/retro.bpmn`. */
100
- const retroFlow: DeclarativeFlow = defineFlow(
120
+ /** The code-first port of `resources/processes/retro.bpmn` — complete but for the
121
+ * class-3 agent-task marker, so it is exported for the parity diagnostic that
122
+ * pins the divergence to exactly that marker (and parked in PORTS below). */
123
+ export const retroFlow: DeclarativeFlow = defineFlow(
101
124
  "retro",
102
125
  {
103
126
  gather: { in: RetroGatherIn },
@@ -146,9 +169,26 @@ const MULTI_START_END_BLOCK =
146
169
  "cannot reproduce. Awaits an upstream terminal/explicit-end (+ multi-start) " +
147
170
  "construct in @nanobpm/workflow (nano-ide).";
148
171
 
172
+ /** The engine-native AgentTask marker limitation (issue #745), the `blockedReason`
173
+ * for a golden whose ONLY underivable feature is `<zeebe:agentDefinition
174
+ * agentType="external" />` on its agent service tasks. */
175
+ const AGENT_MARKER_BLOCK =
176
+ "blocked (agent-task marker): single top-level start/end and a fully " +
177
+ "structured topology, so this golden derives EXCEPT for the " +
178
+ "<zeebe:agentDefinition agentType=\"external\" /> marker issue #745 added to " +
179
+ "its prompt-bearing senior:* tasks — a marker the published compiler cannot " +
180
+ "emit (task() accepts only { jobType, prompt, io }, and no agentDefinition " +
181
+ "appears anywhere in @nanobpm/workflow@0.14.0). The golden cannot drop the " +
182
+ "marker: app/agentic/vocab/agent-marker.test.ts is a defect-class guard " +
183
+ "requiring it on every deployed prompt-bearing agent task. Proven in the test " +
184
+ "suite — the complete port (retroFlow, still authored here) differs from the " +
185
+ "golden by nothing else. Awaits an agent-marker option on task() upstream in " +
186
+ "@nanobpm/workflow (nano-ide); flip this entry back to `flow: retroFlow` when " +
187
+ "it lands.";
188
+
149
189
  /** All ports, keyed by model, in the epic's stated authoring order. */
150
190
  export const PORTS: readonly PortEntry[] = [
151
- { model: "retro", flow: retroFlow },
191
+ { model: "retro", blockedReason: AGENT_MARKER_BLOCK },
152
192
  {
153
193
  model: "spine-demo",
154
194
  blockedReason: `${MULTI_START_END_BLOCK} (spine-demo: 1 start, 2 ends)`,