@nanobpm/nano-workforce 0.162.2 → 0.163.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/app/enginePreflight.ts +15 -1
- package/app/instanceTracking.ts +17 -0
- package/app/reconcile.test.ts +212 -0
- package/app/reconcile.ts +326 -0
- package/db/migrations/092_engine_reconcile.sql +53 -0
- package/main.ts +25 -0
- package/openapi.yaml +92 -0
- package/operations/reconcileEngineState.test.ts +81 -0
- package/operations/reconcileEngineState.ts +44 -0
- package/package.json +1 -1
- package/test/reconcileDb.ts +61 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
## [0.163.0](https://github.com/nanobpm/nano-workforce/compare/v0.162.2...v0.163.0) (2026-08-30)
|
|
2
|
+
|
|
3
|
+
### Features
|
|
4
|
+
|
|
5
|
+
* **reconcile:** engine-reset reconciliation surface for inflight work ([#622](https://github.com/nanobpm/nano-workforce/issues/622)) ([#627](https://github.com/nanobpm/nano-workforce/issues/627)) ([a10ba65](https://github.com/nanobpm/nano-workforce/commit/a10ba650829b15228e35eb45f08ecef98eff69d8)), closes [Magikcraft/nano-bpm#1065](https://github.com/Magikcraft/nano-bpm/issues/1065)
|
|
6
|
+
|
|
1
7
|
## [0.162.2](https://github.com/nanobpm/nano-workforce/compare/v0.162.1...v0.162.2) (2026-08-30)
|
|
2
8
|
|
|
3
9
|
### Bug Fixes
|
package/app/enginePreflight.ts
CHANGED
|
@@ -56,7 +56,21 @@ export function resolveEngineAddress(
|
|
|
56
56
|
* stock Camunda 8 gateway, which returns the same shape without it.
|
|
57
57
|
*/
|
|
58
58
|
export interface TopologyProbe {
|
|
59
|
-
nano?:
|
|
59
|
+
nano?:
|
|
60
|
+
| {
|
|
61
|
+
engine?: string;
|
|
62
|
+
version?: string;
|
|
63
|
+
falconPath?: string;
|
|
64
|
+
/** Monotonic incarnation / epoch id the engine stamps at boot and re-mints on a
|
|
65
|
+
* reset/restore/rewind (companion to the versioned snapshot envelope,
|
|
66
|
+
* Magikcraft/nano-bpm#1068). The app persists the last-seen value; a REGRESSION is the
|
|
67
|
+
* robust "engine was reset → reconcile" signal (issue #622, app/reconcile.ts). */
|
|
68
|
+
incarnation?: number | string;
|
|
69
|
+
/** Alias for {@link incarnation} — accepted so the app tolerates either spelling the engine
|
|
70
|
+
* status endpoint settles on without a code change. */
|
|
71
|
+
epoch?: number | string;
|
|
72
|
+
}
|
|
73
|
+
| null;
|
|
60
74
|
gatewayVersion?: string;
|
|
61
75
|
}
|
|
62
76
|
|
package/app/instanceTracking.ts
CHANGED
|
@@ -63,6 +63,23 @@ export function activeStatusesFor(table: string): readonly string[] {
|
|
|
63
63
|
return binding.activeStatuses;
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
/** A tracked table's engine-instance key column (the `keyField` in nano.app.json — e.g.
|
|
67
|
+
* `process_key`), the single source of truth the app-side reconcile probes/orphans by so it can
|
|
68
|
+
* never drift from the reconciler's notion of "which column holds the engine instance key". */
|
|
69
|
+
export function keyFieldFor(table: string): string {
|
|
70
|
+
return trackingBindingFor(table).keyField;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Every `instanceTracking` binding — the full registry of ENGINE-BACKED base tables (each row is
|
|
74
|
+
* projected off a live engine process instance keyed by `keyField`). The app-side engine-reset
|
|
75
|
+
* reconcile (app/reconcile.ts) scans exactly this set: a row whose `statusField` is still in the
|
|
76
|
+
* binding's `activeStatuses` and whose `keyField` is populated is non-terminal engine-backed work,
|
|
77
|
+
* the only surface reconcile may drive to `orphaned`. Terminal rows and non-engine-backed surfaces
|
|
78
|
+
* (presence, append-only audit) are, by construction, not in this set and are never touched. */
|
|
79
|
+
export function engineBackedBindings(): readonly InstanceTracking[] {
|
|
80
|
+
return INSTANCE_TRACKING_BINDINGS;
|
|
81
|
+
}
|
|
82
|
+
|
|
66
83
|
/** The managed derived read-model VIEW name + effective-status column for a base table, resolved by
|
|
67
84
|
* urban's OWN target resolver so the app never drifts from the framework's `<table>__tracking` /
|
|
68
85
|
* `derived_status` naming (ADR 0065). */
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
// Red/green coverage for the app-side engine-reset reconciliation surface (issue #622).
|
|
2
|
+
//
|
|
3
|
+
// The core scenario the incident (Magikcraft/nano-bpm#1065) demanded a supported remedy for: the
|
|
4
|
+
// engine is reset and its incarnation epoch REGRESSES, while `app.db` still projects engine-backed
|
|
5
|
+
// inflight work (an active `feature_runs`/`delivery_graph_runs`/… row keyed on a now-dead
|
|
6
|
+
// `process_key`). Reconcile must drive exactly those rows to the defined `orphaned` terminal WITH
|
|
7
|
+
// PROVENANCE, leave terminal history + non-engine-backed rows untouched, and be idempotent.
|
|
8
|
+
//
|
|
9
|
+
// These run against the REAL migration set (092 applied to an in-memory SQLite via urban's own
|
|
10
|
+
// `makeGateway`), so the tables/columns/indexes reconcile reads and writes are the shipping schema.
|
|
11
|
+
import { DatabaseSync } from "node:sqlite";
|
|
12
|
+
import { test } from "node:test";
|
|
13
|
+
import { assertEquals } from "#test-assert";
|
|
14
|
+
import { freshData } from "../test/reconcileDb.ts";
|
|
15
|
+
import {
|
|
16
|
+
ORPHANED_STATUS,
|
|
17
|
+
parseEngineEpoch,
|
|
18
|
+
RECONCILE_ORPHAN_REASON,
|
|
19
|
+
reconcileEngineBackedWork,
|
|
20
|
+
} from "./reconcile.ts";
|
|
21
|
+
|
|
22
|
+
const AT = () => new Date("2026-02-02T00:00:00.000Z");
|
|
23
|
+
|
|
24
|
+
function seedFeatureRun(raw: DatabaseSync, key: string, status: string, processKey: string | null): void {
|
|
25
|
+
raw
|
|
26
|
+
.prepare(
|
|
27
|
+
`INSERT INTO feature_runs (feature_key, repo, issue_number, issue_url, base_branch, status, process_key, created_at, updated_at)
|
|
28
|
+
VALUES (?, 'o/r', 1, 'https://x', 'main', ?, ?, '2026-01-01', '2026-01-01')`,
|
|
29
|
+
)
|
|
30
|
+
.run(key, status, processKey);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function seedDeliveryGraphRun(raw: DatabaseSync, runKey: string, status: string, processKey: string | null): void {
|
|
34
|
+
raw
|
|
35
|
+
.prepare(
|
|
36
|
+
`INSERT INTO delivery_graph_runs (run_key, process_key, digest, status, created_at, updated_at)
|
|
37
|
+
VALUES (?, ?, 'deadbeef', ?, '2026-01-01', '2026-01-01')`,
|
|
38
|
+
)
|
|
39
|
+
.run(runKey, processKey, status);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
test("parseEngineEpoch reads nano.incarnation (or its epoch alias), else null", () => {
|
|
43
|
+
assertEquals(parseEngineEpoch({ nano: { incarnation: 7 } }), 7);
|
|
44
|
+
assertEquals(parseEngineEpoch({ nano: { epoch: "9" } }), 9);
|
|
45
|
+
assertEquals(parseEngineEpoch({ nano: { engine: "nano" } }), null);
|
|
46
|
+
assertEquals(parseEngineEpoch({ gatewayVersion: "8.6" }), null);
|
|
47
|
+
assertEquals(parseEngineEpoch(null), null);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("first observation SEEDS the epoch without orphaning anything", async () => {
|
|
51
|
+
const { data, raw } = freshData();
|
|
52
|
+
seedFeatureRun(raw, "o/r#1", "running", "pk-1");
|
|
53
|
+
|
|
54
|
+
const res = await reconcileEngineBackedWork(data, { reachable: true, epoch: 5 }, { now: AT, runId: "run-seed" });
|
|
55
|
+
|
|
56
|
+
assertEquals(res.reason, "seed-epoch");
|
|
57
|
+
assertEquals(res.orphanedCount, 0);
|
|
58
|
+
const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
|
|
59
|
+
assertEquals(row.status, "running");
|
|
60
|
+
const rec = raw.prepare("SELECT epoch FROM engine_incarnation WHERE id=1").get() as { epoch: number };
|
|
61
|
+
assertEquals(rec.epoch, 5);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("RED→GREEN: an epoch regression orphans dangling inflight rows with provenance", async () => {
|
|
65
|
+
const { data, raw } = freshData();
|
|
66
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
67
|
+
seedFeatureRun(raw, "o/r#1", "running", "41");
|
|
68
|
+
seedDeliveryGraphRun(raw, "graph-1", "running", "77");
|
|
69
|
+
|
|
70
|
+
// The engine was reset/rewound: its incarnation epoch regressed 10 → 2.
|
|
71
|
+
const res = await reconcileEngineBackedWork(data, { reachable: true, epoch: 2 }, { now: AT, runId: "run-1" });
|
|
72
|
+
|
|
73
|
+
assertEquals(res.reason, "epoch-regression");
|
|
74
|
+
assertEquals(res.orphanedCount, 2);
|
|
75
|
+
|
|
76
|
+
const fr = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
|
|
77
|
+
assertEquals(fr.status, ORPHANED_STATUS);
|
|
78
|
+
const dg = raw.prepare("SELECT status FROM delivery_graph_runs WHERE run_key='graph-1'").get() as { status: string };
|
|
79
|
+
assertEquals(dg.status, ORPHANED_STATUS);
|
|
80
|
+
|
|
81
|
+
const prov = raw
|
|
82
|
+
.prepare("SELECT * FROM reconcile_provenance WHERE source_table='feature_runs'")
|
|
83
|
+
.get() as Record<string, unknown>;
|
|
84
|
+
assertEquals(prov.to_status, ORPHANED_STATUS);
|
|
85
|
+
assertEquals(prov.from_status, "running");
|
|
86
|
+
assertEquals(prov.reason, RECONCILE_ORPHAN_REASON);
|
|
87
|
+
assertEquals(prov.observed_epoch, 2);
|
|
88
|
+
assertEquals(prov.run_id, "run-1");
|
|
89
|
+
assertEquals(prov.key_value, "41");
|
|
90
|
+
|
|
91
|
+
const run = raw.prepare("SELECT reason, orphaned_count FROM reconcile_runs WHERE run_id='run-1'").get() as {
|
|
92
|
+
reason: string;
|
|
93
|
+
orphaned_count: number;
|
|
94
|
+
};
|
|
95
|
+
assertEquals(run.reason, "epoch-regression");
|
|
96
|
+
assertEquals(run.orphaned_count, 2);
|
|
97
|
+
const rec = raw.prepare("SELECT epoch FROM engine_incarnation WHERE id=1").get() as { epoch: number };
|
|
98
|
+
assertEquals(rec.epoch, 2);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("terminal history and rows without a process_key are NEVER touched", async () => {
|
|
102
|
+
const { data, raw } = freshData();
|
|
103
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
104
|
+
seedFeatureRun(raw, "term#1", "merged", "88"); // terminal — not in activeStatuses
|
|
105
|
+
seedFeatureRun(raw, "await#1", "opened", "89"); // terminal-for-tracking
|
|
106
|
+
seedFeatureRun(raw, "nokeed#1", "running", null); // active but no engine key
|
|
107
|
+
|
|
108
|
+
const res = await reconcileEngineBackedWork(data, { reachable: true, epoch: 2 }, { now: AT, runId: "run-1" });
|
|
109
|
+
|
|
110
|
+
assertEquals(res.orphanedCount, 0);
|
|
111
|
+
const statuses = raw.prepare("SELECT feature_key, status FROM feature_runs ORDER BY feature_key").all() as {
|
|
112
|
+
feature_key: string;
|
|
113
|
+
status: string;
|
|
114
|
+
}[];
|
|
115
|
+
assertEquals(statuses.find((r) => r.feature_key === "term#1")?.status, "merged");
|
|
116
|
+
assertEquals(statuses.find((r) => r.feature_key === "await#1")?.status, "opened");
|
|
117
|
+
assertEquals(statuses.find((r) => r.feature_key === "nokeed#1")?.status, "running");
|
|
118
|
+
assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test("idempotent: a second pass with a matching epoch is a no-op", async () => {
|
|
122
|
+
const { data, raw } = freshData();
|
|
123
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
124
|
+
seedFeatureRun(raw, "o/r#1", "running", "41");
|
|
125
|
+
|
|
126
|
+
const first = await reconcileEngineBackedWork(data, { reachable: true, epoch: 2 }, { now: AT, runId: "run-1" });
|
|
127
|
+
assertEquals(first.orphanedCount, 1);
|
|
128
|
+
|
|
129
|
+
const second = await reconcileEngineBackedWork(data, { reachable: true, epoch: 2 }, { now: AT, runId: "run-2" });
|
|
130
|
+
assertEquals(second.reason, "no-op");
|
|
131
|
+
assertEquals(second.orphanedCount, 0);
|
|
132
|
+
|
|
133
|
+
const provCount = raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number };
|
|
134
|
+
assertEquals(provCount.c, 1);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test("an unreachable engine is a hard no-op — live work is never orphaned", async () => {
|
|
138
|
+
const { data, raw } = freshData();
|
|
139
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
140
|
+
seedFeatureRun(raw, "o/r#1", "running", "41");
|
|
141
|
+
|
|
142
|
+
const res = await reconcileEngineBackedWork(data, { reachable: false, epoch: null }, { now: AT, runId: "run-1" });
|
|
143
|
+
assertEquals(res.reason, "engine-unreachable");
|
|
144
|
+
assertEquals(res.orphanedCount, 0);
|
|
145
|
+
|
|
146
|
+
const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
|
|
147
|
+
assertEquals(row.status, "running");
|
|
148
|
+
const run = raw.prepare("SELECT reason FROM reconcile_runs WHERE run_id='run-1'").get() as { reason: string };
|
|
149
|
+
assertEquals(run.reason, "engine-unreachable");
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
test("RED→GREEN: a concurrent terminal transition wins — the guarded UPDATE never clobbers it", async () => {
|
|
153
|
+
const { data, raw } = freshData();
|
|
154
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
155
|
+
seedFeatureRun(raw, "o/r#1", "running", "41");
|
|
156
|
+
|
|
157
|
+
// Interpose a writer that flips the row to a newer terminal status AFTER reconcile has SELECTed it
|
|
158
|
+
// as "running" but BEFORE its UPDATE lands — the exact TOCTOU window. With a blind UPDATE-by-pk the
|
|
159
|
+
// reset would clobber `merged` back to `orphaned` (and write provenance); the guarded UPDATE (status
|
|
160
|
+
// re-asserted) sees `res.changed === 0` and leaves the terminal history untouched.
|
|
161
|
+
const gw = data.open();
|
|
162
|
+
let raced = false;
|
|
163
|
+
const wrapTx = (t: { query: (...a: unknown[]) => unknown; exec: (sql: string, params?: unknown[]) => unknown }) => ({
|
|
164
|
+
query: (...a: unknown[]) => t.query(...a),
|
|
165
|
+
exec: (sql: string, params?: unknown[]) => {
|
|
166
|
+
if (!raced && /^UPDATE/.test(sql.trim())) {
|
|
167
|
+
raced = true;
|
|
168
|
+
raw.prepare("UPDATE feature_runs SET status='merged' WHERE feature_key='o/r#1'").run();
|
|
169
|
+
}
|
|
170
|
+
return t.exec(sql, params);
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
const wrappedSrc = {
|
|
174
|
+
query: (...a: unknown[]) => (gw as { query: (...a: unknown[]) => unknown }).query(...a),
|
|
175
|
+
exec: (sql: string, params?: unknown[]) => (gw as { exec: (sql: string, params?: unknown[]) => unknown }).exec(sql, params),
|
|
176
|
+
tx: (fn: (t: unknown) => unknown) => (gw as { tx: (f: (t: unknown) => unknown) => unknown }).tx((t) => fn(wrapTx(t as never))),
|
|
177
|
+
};
|
|
178
|
+
const wrapped = { open: () => wrappedSrc } as unknown as DataLayer;
|
|
179
|
+
|
|
180
|
+
const res = await reconcileEngineBackedWork(wrapped, { reachable: true, epoch: 2 }, { now: AT, runId: "run-1" });
|
|
181
|
+
|
|
182
|
+
assertEquals(raced, true);
|
|
183
|
+
assertEquals(res.orphanedCount, 0);
|
|
184
|
+
const row = raw.prepare("SELECT status FROM feature_runs WHERE feature_key='o/r#1'").get() as { status: string };
|
|
185
|
+
assertEquals(row.status, "merged");
|
|
186
|
+
assertEquals((raw.prepare("SELECT COUNT(*) c FROM reconcile_provenance").get() as { c: number }).c, 0);
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
test("RED→GREEN: orphaning stamps updated_at so the transition timestamp isn't left stale", async () => {
|
|
190
|
+
const { data, raw } = freshData();
|
|
191
|
+
await reconcileEngineBackedWork(data, { reachable: true, epoch: 10 }, { now: AT, runId: "run-0" });
|
|
192
|
+
seedFeatureRun(raw, "o/r#1", "running", "41");
|
|
193
|
+
seedDeliveryGraphRun(raw, "graph-1", "running", "77");
|
|
194
|
+
|
|
195
|
+
// The seeded rows carry updated_at='2026-01-01'; AT() (the reconcile clock) is 2026-02-02. A blind
|
|
196
|
+
// `SET status='orphaned'` would leave updated_at at the stale seed value, misrepresenting when the
|
|
197
|
+
// row was orphaned to the UI/audits. The transition must refresh updated_at like every other one.
|
|
198
|
+
const res = await reconcileEngineBackedWork(data, { reachable: true, epoch: 2 }, { now: AT, runId: "run-1" });
|
|
199
|
+
|
|
200
|
+
assertEquals(res.orphanedCount, 2);
|
|
201
|
+
const at = AT().toISOString();
|
|
202
|
+
const fr = raw
|
|
203
|
+
.prepare("SELECT status, updated_at FROM feature_runs WHERE feature_key='o/r#1'")
|
|
204
|
+
.get() as { status: string; updated_at: string };
|
|
205
|
+
assertEquals(fr.status, ORPHANED_STATUS);
|
|
206
|
+
assertEquals(fr.updated_at, at);
|
|
207
|
+
const dg = raw
|
|
208
|
+
.prepare("SELECT status, updated_at FROM delivery_graph_runs WHERE run_key='graph-1'")
|
|
209
|
+
.get() as { status: string; updated_at: string };
|
|
210
|
+
assertEquals(dg.status, ORPHANED_STATUS);
|
|
211
|
+
assertEquals(dg.updated_at, at);
|
|
212
|
+
});
|
package/app/reconcile.ts
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// nano-workforce — the app-side engine-reset reconciliation surface (issue #622).
|
|
2
|
+
//
|
|
3
|
+
// When the Nano engine is reset, restored, or rolled to an incarnation whose key generator has
|
|
4
|
+
// rewound (Magikcraft/nano-bpm#1065), `app.db` keeps projecting ENGINE-BACKED inflight work that no
|
|
5
|
+
// longer exists on the engine: open user tasks pointing at dead instances, active runs keyed on a
|
|
6
|
+
// `process_key` the fresh engine has re-minted for something unrelated (the release-train run
|
|
7
|
+
// recorded `process_key=41`; the fresh engine re-minted 41 for an unrelated probe). Without a
|
|
8
|
+
// supported way to converge, the app silently trusts stale projections — orphaned human gates and
|
|
9
|
+
// key-collision identity confusion.
|
|
10
|
+
//
|
|
11
|
+
// `reconcile` is the first-class remedy. It runs ON STARTUP (main.ts) and ON DEMAND (the
|
|
12
|
+
// `reconcileEngineState` operator command), scoped NARROWLY to claimed inflight work:
|
|
13
|
+
//
|
|
14
|
+
// • Detection — engine INCARNATION EPOCH (preferred over fragile per-key 404 probing). The engine
|
|
15
|
+
// stamps a monotonic incarnation id at boot and exposes it on `/v2/topology`; the app persists
|
|
16
|
+
// the last-seen value (`engine_incarnation`). An epoch REGRESSION (observed < recorded) — the
|
|
17
|
+
// #1065 rewind signature — or its absence where one was recorded means "engine was reset/rewound
|
|
18
|
+
// → reconcile", ONE cheap check instead of N per-instance probes.
|
|
19
|
+
// • Convergence — for every NON-terminal, engine-backed app row (a nano.app.json instanceTracking
|
|
20
|
+
// binding whose `statusField` is still in its `activeStatuses` set and whose `keyField` is
|
|
21
|
+
// populated), drive the row to the defined `orphaned` terminal WITH PROVENANCE
|
|
22
|
+
// (`reconcile_provenance`: the reason, the observed engine epoch, and the reconcile run id) —
|
|
23
|
+
// instead of trusting a stale projection or silently dropping data.
|
|
24
|
+
// • Guardrails — TERMINAL rows (done/failed/merged/abandoned/…) and append-only / non-engine-backed
|
|
25
|
+
// surfaces (presence, audit, provenance) are NEVER mutated: reconcile only touches rows whose
|
|
26
|
+
// status is in a binding's `activeStatuses`. Every pass is recorded in `reconcile_runs`.
|
|
27
|
+
// • Idempotent — a second pass with a matching epoch is a no-op (nothing regressed, and every
|
|
28
|
+
// already-orphaned row has left its `activeStatuses`, so it is not re-scanned). An UNREACHABLE
|
|
29
|
+
// engine is a no-op too: reconcile NEVER orphans when it could not confirm a reset (a 401/5xx or
|
|
30
|
+
// a network error yields `reachable:false`, not a false "engine missing").
|
|
31
|
+
//
|
|
32
|
+
// The provenance is app-owned (not urban's `_urban_write_provenance`, which is a domain-free
|
|
33
|
+
// insert-join sidecar written only inside a job): reconcile runs at boot / over HTTP, outside any
|
|
34
|
+
// job, and needs to record the REASON + epoch + run id — which the app-owned `reconcile_provenance`
|
|
35
|
+
// table carries, and the existing `app.db` backup convention makes the whole mutation reversible.
|
|
36
|
+
|
|
37
|
+
import type { DataLayer, GatewayDataSource as DataSource } from "@nanobpm/urban";
|
|
38
|
+
import type { TopologyProbe } from "./enginePreflight.ts";
|
|
39
|
+
import { activeStatusesFor, baseStatusFieldFor, engineBackedBindings, keyFieldFor } from "./instanceTracking.ts";
|
|
40
|
+
|
|
41
|
+
/** The defined terminal state a reset-orphaned engine-backed row is driven to. Deliberately DISTINCT
|
|
42
|
+
* from a binding's natural terminal (`abandoned`/`failed`/…) so an operator can tell a row that was
|
|
43
|
+
* orphaned by an engine reset apart from one that drained normally. Not in any binding's
|
|
44
|
+
* `activeStatuses`, so an orphaned row is never re-scanned (idempotency) nor re-polled by the urban
|
|
45
|
+
* instance-tracking reconciler. */
|
|
46
|
+
export const ORPHANED_STATUS = "orphaned";
|
|
47
|
+
|
|
48
|
+
/** The provenance reason stamped on every orphaned transition: the engine was reset/rewound and the
|
|
49
|
+
* recorded incarnation epoch regressed (the #1065 signature). */
|
|
50
|
+
export const RECONCILE_ORPHAN_REASON = "engine-reset/epoch-regression";
|
|
51
|
+
|
|
52
|
+
/** The single-row epoch ledger + its append-only run/provenance sidecars (migration 092). */
|
|
53
|
+
const INCARNATION_TABLE = "engine_incarnation";
|
|
54
|
+
const RUNS_TABLE = "reconcile_runs";
|
|
55
|
+
const PROVENANCE_TABLE = "reconcile_provenance";
|
|
56
|
+
/** The conventional last-touched timestamp column stamped on every status transition; orphaning
|
|
57
|
+
* refreshes it too, but only on the tables that actually declare it (introspected per binding). */
|
|
58
|
+
const UPDATED_AT_COLUMN = "updated_at";
|
|
59
|
+
|
|
60
|
+
/** What a `/v2/topology` epoch probe observed. `reachable:false` means the engine could not be
|
|
61
|
+
* confirmed (network error, or a non-2xx like 401/5xx) — reconcile then does NOTHING, so a
|
|
62
|
+
* transient outage can never be mistaken for a reset and orphan live work. `reachable:true` with a
|
|
63
|
+
* null `epoch` means the engine answered but exposes no incarnation id (e.g. a stock Camunda 8
|
|
64
|
+
* gateway, or before Magikcraft/nano-bpm#1068 ships). That null is a no-op ONLY when no epoch was
|
|
65
|
+
* ever recorded; if a concrete epoch WAS recorded, a now-null observation reads as a regression
|
|
66
|
+
* ("the epoch disappeared" — the reset signature), so reconcile orphans inflight work. See the
|
|
67
|
+
* decision table on {@link reconcileEngineBackedWork}. */
|
|
68
|
+
export interface EngineEpochObservation {
|
|
69
|
+
reachable: boolean;
|
|
70
|
+
epoch: number | null;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Why a reconcile pass acted (or did not). */
|
|
74
|
+
export type ReconcileReason = "epoch-regression" | "seed-epoch" | "no-op" | "engine-unreachable";
|
|
75
|
+
|
|
76
|
+
/** One orphaned engine-backed row. */
|
|
77
|
+
export interface OrphanedRow {
|
|
78
|
+
table: string;
|
|
79
|
+
pk: string;
|
|
80
|
+
key: string | null;
|
|
81
|
+
fromStatus: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** The outcome of one reconcile pass — the same shape the run row records and the operator command
|
|
85
|
+
* returns. */
|
|
86
|
+
export interface ReconcileResult {
|
|
87
|
+
runId: string;
|
|
88
|
+
reason: ReconcileReason;
|
|
89
|
+
observedEpoch: number | null;
|
|
90
|
+
recordedEpoch: number | null;
|
|
91
|
+
orphanedCount: number;
|
|
92
|
+
orphaned: OrphanedRow[];
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
export interface ReconcileLog {
|
|
96
|
+
info(msg: string): void;
|
|
97
|
+
warn(msg: string): void;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export interface ReconcileOptions {
|
|
101
|
+
/** Injectable clock (defaults to `Date`), so tests are deterministic. */
|
|
102
|
+
now?: () => Date;
|
|
103
|
+
/** Injectable run id (defaults to a random UUID). */
|
|
104
|
+
runId?: string;
|
|
105
|
+
/** The data source name to reconcile (defaults to the DataLayer's default source). */
|
|
106
|
+
sourceName?: string;
|
|
107
|
+
log?: ReconcileLog;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/** Read the incarnation epoch out of a `/v2/topology` body — `nano.incarnation` (or its `epoch`
|
|
111
|
+
* alias), coerced from a number or a numeric string. Any other shape (absent, non-numeric) yields
|
|
112
|
+
* null: "the engine exposes no epoch". A null is a no-op ONLY when no epoch was previously recorded;
|
|
113
|
+
* when one WAS recorded, reconcile reads a now-null observation as a regression ("epoch disappeared"),
|
|
114
|
+
* not a no-op — see the decision table on {@link reconcileEngineBackedWork}. */
|
|
115
|
+
export function parseEngineEpoch(body: TopologyProbe | null | undefined): number | null {
|
|
116
|
+
const raw = body?.nano?.incarnation ?? body?.nano?.epoch;
|
|
117
|
+
if (raw == null) return null;
|
|
118
|
+
const n = typeof raw === "number" ? raw : Number(raw);
|
|
119
|
+
return Number.isFinite(n) ? n : null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Probe `/v2/topology` for the engine incarnation epoch. Never throws: a network error or a non-2xx
|
|
123
|
+
* yields `reachable:false` (reconcile then does nothing), so an outage can never orphan live work. */
|
|
124
|
+
export async function probeEngineEpoch(
|
|
125
|
+
restAddress: string,
|
|
126
|
+
opts: { token?: string; fetchImpl?: typeof fetch; timeoutMs?: number } = {},
|
|
127
|
+
): Promise<EngineEpochObservation> {
|
|
128
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
129
|
+
const url = `${restAddress.replace(/\/+$/, "")}/topology`;
|
|
130
|
+
const headers: Record<string, string> = { accept: "application/json" };
|
|
131
|
+
if (opts.token) headers.authorization = `Bearer ${opts.token}`;
|
|
132
|
+
try {
|
|
133
|
+
const res = await fetchImpl(url, {
|
|
134
|
+
headers,
|
|
135
|
+
signal: AbortSignal.timeout(opts.timeoutMs ?? 3000),
|
|
136
|
+
});
|
|
137
|
+
if (!res.ok) return { reachable: false, epoch: null };
|
|
138
|
+
const body: TopologyProbe = await res.json();
|
|
139
|
+
return { reachable: true, epoch: parseEngineEpoch(body) };
|
|
140
|
+
} catch {
|
|
141
|
+
return { reachable: false, epoch: null };
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/** Double-quote a SQL identifier (table/column) so a manifest-declared name is safe to interpolate. */
|
|
146
|
+
function q(id: string): string {
|
|
147
|
+
return `"${id.replace(/"/g, '""')}"`;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** The schema of `table` we need to orphan a row: its primary-key column (the first `pk`-flagged
|
|
151
|
+
* column from `PRAGMA table_info`, or `rowid` when the table declares none — so provenance always
|
|
152
|
+
* records a stable row identity) and whether it carries an `updated_at` column to stamp. */
|
|
153
|
+
async function tableShape(src: DataSource, table: string): Promise<{ pkCol: string; hasUpdatedAt: boolean }> {
|
|
154
|
+
const cols = await src.query<{ name: string; pk: number }>(`PRAGMA table_info(${q(table)})`);
|
|
155
|
+
const pk = cols.find((c) => Number(c.pk) > 0);
|
|
156
|
+
return { pkCol: pk?.name ?? "rowid", hasUpdatedAt: cols.some((c) => c.name === UPDATED_AT_COLUMN) };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** The recorded last-seen epoch, or null when none was ever recorded (no row, or a null epoch). */
|
|
160
|
+
async function readRecordedEpoch(src: DataSource): Promise<number | null> {
|
|
161
|
+
const rows = await src.query<{ epoch: number | null }>(
|
|
162
|
+
`SELECT epoch FROM ${INCARNATION_TABLE} WHERE id = 1`,
|
|
163
|
+
);
|
|
164
|
+
const epoch = rows.length ? rows[0].epoch : null;
|
|
165
|
+
return epoch == null ? null : Number(epoch);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** Persist (seed or advance) the last-seen epoch. Only ever called with a concrete number, so a
|
|
169
|
+
* recorded epoch always means "an epoch was actually observed". */
|
|
170
|
+
async function persistEpoch(src: DataSource, epoch: number, at: string): Promise<void> {
|
|
171
|
+
await src.exec(
|
|
172
|
+
`INSERT INTO ${INCARNATION_TABLE} (id, epoch, observed_at) VALUES (1, ?, ?) ` +
|
|
173
|
+
`ON CONFLICT(id) DO UPDATE SET epoch = excluded.epoch, observed_at = excluded.observed_at`,
|
|
174
|
+
[epoch, at],
|
|
175
|
+
);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/** Orphan every NON-terminal, engine-backed row across all instanceTracking bindings, recording one
|
|
179
|
+
* `reconcile_provenance` row per transition. Runs inside the caller's transaction. */
|
|
180
|
+
async function orphanEngineBackedRows(
|
|
181
|
+
src: DataSource,
|
|
182
|
+
runId: string,
|
|
183
|
+
observedEpoch: number | null,
|
|
184
|
+
at: string,
|
|
185
|
+
): Promise<OrphanedRow[]> {
|
|
186
|
+
const orphaned: OrphanedRow[] = [];
|
|
187
|
+
for (const binding of engineBackedBindings()) {
|
|
188
|
+
const table = binding.table;
|
|
189
|
+
// A binding with no active-status selector cannot classify "in-flight" — skip it rather than
|
|
190
|
+
// guess (activeStatusesFor would throw; we tolerate a selector-less binding).
|
|
191
|
+
if (!binding.activeStatuses?.length) continue;
|
|
192
|
+
const active = activeStatusesFor(table);
|
|
193
|
+
const statusField = baseStatusFieldFor(table);
|
|
194
|
+
const keyField = keyFieldFor(table);
|
|
195
|
+
const { pkCol, hasUpdatedAt } = await tableShape(src, table);
|
|
196
|
+
const placeholders = active.map(() => "?").join(", ");
|
|
197
|
+
const rows = await src.query<{ __pk: unknown; __key: unknown; __status: unknown }>(
|
|
198
|
+
`SELECT ${q(pkCol)} AS __pk, ${q(keyField)} AS __key, ${q(statusField)} AS __status ` +
|
|
199
|
+
`FROM ${q(table)} WHERE ${q(statusField)} IN (${placeholders}) AND ${q(keyField)} IS NOT NULL`,
|
|
200
|
+
[...active],
|
|
201
|
+
);
|
|
202
|
+
for (const row of rows) {
|
|
203
|
+
const pk = String(row.__pk);
|
|
204
|
+
const key = row.__key == null ? null : String(row.__key);
|
|
205
|
+
const fromStatus = String(row.__status);
|
|
206
|
+
// GUARDED update: re-assert the exact status we read AND a populated key, so a writer that
|
|
207
|
+
// flipped the row to a newer terminal status (or cleared its key) between the SELECT above and
|
|
208
|
+
// this UPDATE wins the race — we never clobber that terminal history back to `orphaned`. Only a
|
|
209
|
+
// row we actually transitioned (`res.changed > 0`) gets provenance and is counted. We also stamp
|
|
210
|
+
// `updated_at` (when the table has one) so the transition to `orphaned` refreshes the row's
|
|
211
|
+
// timestamp the same way every other status transition in the codebase does — leaving it stale
|
|
212
|
+
// would misrepresent the orphaning moment to the UI/audits.
|
|
213
|
+
const res = await src.exec(
|
|
214
|
+
`UPDATE ${q(table)} SET ${q(statusField)} = ?` +
|
|
215
|
+
(hasUpdatedAt ? `, ${q(UPDATED_AT_COLUMN)} = ?` : "") +
|
|
216
|
+
` WHERE ${q(pkCol)} = ? AND ${q(statusField)} = ? AND ${q(keyField)} IS NOT NULL`,
|
|
217
|
+
hasUpdatedAt
|
|
218
|
+
? [ORPHANED_STATUS, at, row.__pk, fromStatus]
|
|
219
|
+
: [ORPHANED_STATUS, row.__pk, fromStatus],
|
|
220
|
+
);
|
|
221
|
+
if (res.changed <= 0) continue;
|
|
222
|
+
await src.exec(
|
|
223
|
+
`INSERT INTO ${PROVENANCE_TABLE} ` +
|
|
224
|
+
`(run_id, source_table, pk_value, key_value, from_status, to_status, reason, observed_epoch, at) ` +
|
|
225
|
+
`VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
226
|
+
[runId, table, pk, key, fromStatus, ORPHANED_STATUS, RECONCILE_ORPHAN_REASON, observedEpoch, at],
|
|
227
|
+
);
|
|
228
|
+
orphaned.push({ table, pk, key, fromStatus });
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return orphaned;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Reconcile the app's engine-backed projections against one epoch observation. Pure of I/O beyond the
|
|
236
|
+
* data layer (the topology probe is {@link probeEngineEpoch}, injected as `observation`), so the
|
|
237
|
+
* red/green test drives it with a seeded row + a regressed epoch directly.
|
|
238
|
+
*
|
|
239
|
+
* Decision table (engine reachable):
|
|
240
|
+
* • recorded != null AND (observed == null OR observed < recorded) → REGRESSION: orphan inflight.
|
|
241
|
+
* • recorded == null AND observed != null → SEED: first epoch learned.
|
|
242
|
+
* • otherwise → NO-OP (incl. a matching epoch).
|
|
243
|
+
* The epoch is persisted whenever a concrete one was observed (seed, advance, or the fresh
|
|
244
|
+
* post-rewind incarnation), so the very next pass with that same epoch is a pure no-op.
|
|
245
|
+
*
|
|
246
|
+
* This surface is deliberately EPOCH-SCOPED: it only ever acts on the epoch signal. An engine that
|
|
247
|
+
* is reachable but exposes NO epoch and for which none was ever recorded (observed == null AND
|
|
248
|
+
* recorded == null) is, by contract, an intentional no-op — we have no reset signal to act on, and
|
|
249
|
+
* we never orphan live work speculatively. Converging engine-backed rows against an engine with no
|
|
250
|
+
* epoch support (e.g. via a per-instance existence probe) is out of contract for this surface.
|
|
251
|
+
*/
|
|
252
|
+
export async function reconcileEngineBackedWork(
|
|
253
|
+
data: DataLayer,
|
|
254
|
+
observation: EngineEpochObservation,
|
|
255
|
+
opts: ReconcileOptions = {},
|
|
256
|
+
): Promise<ReconcileResult> {
|
|
257
|
+
const src = data.open(opts.sourceName);
|
|
258
|
+
const at = (opts.now?.() ?? new Date()).toISOString();
|
|
259
|
+
const runId = opts.runId ?? crypto.randomUUID();
|
|
260
|
+
|
|
261
|
+
// An unreachable engine is a hard no-op: we could not confirm a reset, so we NEVER orphan.
|
|
262
|
+
if (!observation.reachable) {
|
|
263
|
+
const recorded = await readRecordedEpoch(src);
|
|
264
|
+
await recordRun(src, { runId, at, observedEpoch: null, recordedEpoch: recorded, reason: "engine-unreachable", orphanedCount: 0 });
|
|
265
|
+
opts.log?.warn("reconcile: engine unreachable — skipped (no rows orphaned; live work left intact).");
|
|
266
|
+
return { runId, reason: "engine-unreachable", observedEpoch: null, recordedEpoch: recorded, orphanedCount: 0, orphaned: [] };
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const observedEpoch = observation.epoch;
|
|
270
|
+
const recordedEpoch = await readRecordedEpoch(src);
|
|
271
|
+
const regression = recordedEpoch != null && (observedEpoch == null || observedEpoch < recordedEpoch);
|
|
272
|
+
|
|
273
|
+
const result = await src.tx(async (t) => {
|
|
274
|
+
let orphaned: OrphanedRow[] = [];
|
|
275
|
+
let reason: ReconcileReason;
|
|
276
|
+
if (regression) {
|
|
277
|
+
orphaned = await orphanEngineBackedRows(t, runId, observedEpoch, at);
|
|
278
|
+
reason = "epoch-regression";
|
|
279
|
+
} else if (recordedEpoch == null && observedEpoch != null) {
|
|
280
|
+
reason = "seed-epoch";
|
|
281
|
+
} else {
|
|
282
|
+
reason = "no-op";
|
|
283
|
+
}
|
|
284
|
+
if (observedEpoch != null) await persistEpoch(t, observedEpoch, at);
|
|
285
|
+
await recordRun(t, { runId, at, observedEpoch, recordedEpoch, reason, orphanedCount: orphaned.length });
|
|
286
|
+
return { reason, orphaned };
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
if (result.reason === "epoch-regression") {
|
|
290
|
+
opts.log?.warn(
|
|
291
|
+
`reconcile: engine epoch regressed ${recordedEpoch} → ${observedEpoch} (reset/rewind) — ` +
|
|
292
|
+
`orphaned ${result.orphaned.length} inflight row(s) [run ${runId}].`,
|
|
293
|
+
);
|
|
294
|
+
} else if (result.reason === "seed-epoch") {
|
|
295
|
+
opts.log?.info(`reconcile: recorded engine epoch ${observedEpoch} (first observation) [run ${runId}].`);
|
|
296
|
+
} else {
|
|
297
|
+
opts.log?.info(`reconcile: engine epoch ${observedEpoch ?? "n/a"} unchanged — no-op [run ${runId}].`);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return { runId, reason: result.reason, observedEpoch, recordedEpoch, orphanedCount: result.orphaned.length, orphaned: result.orphaned };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function recordRun(
|
|
304
|
+
src: DataSource,
|
|
305
|
+
run: { runId: string; at: string; observedEpoch: number | null; recordedEpoch: number | null; reason: ReconcileReason; orphanedCount: number },
|
|
306
|
+
): Promise<void> {
|
|
307
|
+
await src.exec(
|
|
308
|
+
`INSERT INTO ${RUNS_TABLE} (run_id, started_at, observed_epoch, recorded_epoch, reason, orphaned_count) ` +
|
|
309
|
+
`VALUES (?, ?, ?, ?, ?, ?)`,
|
|
310
|
+
[run.runId, run.at, run.observedEpoch, run.recordedEpoch, run.reason, run.orphanedCount],
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Probe the engine's incarnation epoch, then reconcile — the wiring both startup (main.ts) and the
|
|
315
|
+
* `reconcileEngineState` operator command share, so the two paths can never diverge. */
|
|
316
|
+
export async function runEngineReconcile(
|
|
317
|
+
data: DataLayer,
|
|
318
|
+
engineRest: { restAddress: string; token?: string },
|
|
319
|
+
opts: ReconcileOptions & { fetchImpl?: typeof fetch } = {},
|
|
320
|
+
): Promise<ReconcileResult> {
|
|
321
|
+
const observation = await probeEngineEpoch(engineRest.restAddress, {
|
|
322
|
+
token: engineRest.token,
|
|
323
|
+
fetchImpl: opts.fetchImpl,
|
|
324
|
+
});
|
|
325
|
+
return reconcileEngineBackedWork(data, observation, opts);
|
|
326
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
-- Engine-reset reconciliation surface (issue #622).
|
|
2
|
+
--
|
|
3
|
+
-- When the Nano engine is reset, restored, or rolled to an incarnation whose key generator has
|
|
4
|
+
-- rewound (Magikcraft/nano-bpm#1065), `app.db` keeps projecting engine-backed inflight work that no
|
|
5
|
+
-- longer exists on the engine. These three sidecars give the app a first-class, idempotent,
|
|
6
|
+
-- provenance-stamped `reconcile` surface (app/reconcile.ts) that runs on startup and on demand:
|
|
7
|
+
--
|
|
8
|
+
-- • engine_incarnation — the single-row last-seen engine incarnation/epoch id. The engine stamps
|
|
9
|
+
-- a monotonic incarnation id at boot (companion to the versioned snapshot envelope,
|
|
10
|
+
-- Magikcraft/nano-bpm#1068) and exposes it on `/v2/topology`. An epoch REGRESSION (the observed
|
|
11
|
+
-- epoch is lower than the recorded one) — or its absence where one was recorded — is the cheap,
|
|
12
|
+
-- robust "engine was reset/rewound → reconcile" signal, one check instead of N per-instance probes.
|
|
13
|
+
-- • reconcile_runs — one row per reconcile pass: the observed vs recorded epoch, the outcome
|
|
14
|
+
-- reason, and how many rows were orphaned. The append-only audit of every convergence.
|
|
15
|
+
-- • reconcile_provenance— one row per orphaned transition: which engine-backed app row
|
|
16
|
+
-- (source_table, pk_value, its engine instance key) moved from which status to `orphaned`, why
|
|
17
|
+
-- (engine-reset/epoch-regression), the observed engine epoch, and the owning reconcile run id.
|
|
18
|
+
-- This is the provenance that makes each mutation legible and reversible instead of a silent drop.
|
|
19
|
+
--
|
|
20
|
+
-- These are app-owned bookkeeping surfaces; reconcile itself never mutates append-only audit or
|
|
21
|
+
-- already-terminal history — only NON-terminal, engine-backed rows (nano.app.json instanceTracking
|
|
22
|
+
-- bindings) whose status is still in the binding's activeStatuses set.
|
|
23
|
+
|
|
24
|
+
CREATE TABLE IF NOT EXISTS engine_incarnation (
|
|
25
|
+
id INTEGER PRIMARY KEY CHECK (id = 1), -- single-row table: the app only tracks one engine
|
|
26
|
+
epoch INTEGER, -- last-seen engine incarnation/epoch id (NULL until the engine exposes one)
|
|
27
|
+
observed_at TEXT NOT NULL
|
|
28
|
+
);
|
|
29
|
+
|
|
30
|
+
CREATE TABLE IF NOT EXISTS reconcile_runs (
|
|
31
|
+
run_id TEXT PRIMARY KEY,
|
|
32
|
+
started_at TEXT NOT NULL,
|
|
33
|
+
observed_epoch INTEGER, -- the engine epoch observed on this run (NULL when the engine exposes none)
|
|
34
|
+
recorded_epoch INTEGER, -- the previously-recorded epoch (NULL on the first ever run)
|
|
35
|
+
reason TEXT NOT NULL, -- 'epoch-regression' | 'seed-epoch' | 'no-op' | 'engine-unreachable'
|
|
36
|
+
orphaned_count INTEGER NOT NULL DEFAULT 0
|
|
37
|
+
);
|
|
38
|
+
|
|
39
|
+
CREATE TABLE IF NOT EXISTS reconcile_provenance (
|
|
40
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
41
|
+
run_id TEXT NOT NULL,
|
|
42
|
+
source_table TEXT NOT NULL, -- the engine-backed base table the orphaned row lives in
|
|
43
|
+
pk_value TEXT NOT NULL, -- the row's primary-key value
|
|
44
|
+
key_value TEXT, -- the engine instance key (keyField, e.g. process_key) the row projected
|
|
45
|
+
from_status TEXT, -- the non-terminal status the row carried before reconcile
|
|
46
|
+
to_status TEXT NOT NULL, -- always the defined 'orphaned' terminal
|
|
47
|
+
reason TEXT NOT NULL, -- 'engine-reset/epoch-regression'
|
|
48
|
+
observed_epoch INTEGER, -- the engine epoch observed when the row was orphaned
|
|
49
|
+
at TEXT NOT NULL
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
CREATE INDEX IF NOT EXISTS ix_reconcile_provenance_run ON reconcile_provenance (run_id);
|
|
53
|
+
CREATE INDEX IF NOT EXISTS ix_reconcile_provenance_row ON reconcile_provenance (source_table, pk_value);
|
package/main.ts
CHANGED
|
@@ -22,6 +22,7 @@ import { createNanoSdkEngineClient, runFromEnv, selectHost } from "@nanobpm/urba
|
|
|
22
22
|
import { type AgenticChannelHandle, mountAgenticChannel } from "./app/agentic/channel.ts";
|
|
23
23
|
import { makeElementInstanceResolver } from "./app/agentic/element-instance.ts";
|
|
24
24
|
import { announceEngine, resolveEngineAddress } from "./app/enginePreflight.ts";
|
|
25
|
+
import { runEngineReconcile } from "./app/reconcile.ts";
|
|
25
26
|
import { MAX_ROUNDS, pollOnce } from "./app/service.ts";
|
|
26
27
|
import { envVar } from "./app/version.ts";
|
|
27
28
|
|
|
@@ -102,6 +103,30 @@ if (httpServer instanceof Server) {
|
|
|
102
103
|
app.log.warn("agentic channel not mounted: app.httpServer is not a node:http Server on this host");
|
|
103
104
|
}
|
|
104
105
|
|
|
106
|
+
// Engine-reset reconciliation (issue #622). On boot, compare the engine's incarnation epoch against
|
|
107
|
+
// the last-seen value; on a REGRESSION (the engine was reset/restored/rewound and re-minted its keys,
|
|
108
|
+
// Magikcraft/nano-bpm#1065) drive every dangling engine-backed inflight row to the defined `orphaned`
|
|
109
|
+
// terminal WITH PROVENANCE — BEFORE the pollers below start projecting off stale, dead instances.
|
|
110
|
+
// Guarded: an unreachable engine is a no-op (it never orphans live work), and any failure degrades to
|
|
111
|
+
// a warn so reconcile can never block boot.
|
|
112
|
+
if (app.data) {
|
|
113
|
+
try {
|
|
114
|
+
const reconciled = await runEngineReconcile(
|
|
115
|
+
app.data,
|
|
116
|
+
{ restAddress: engineAddress.restAddress, token: process.env.CAMUNDA_TOKEN },
|
|
117
|
+
{ log: { info: (m) => app.log.info(m), warn: (m) => app.log.warn(m) } },
|
|
118
|
+
);
|
|
119
|
+
if (reconciled.orphanedCount > 0) {
|
|
120
|
+
app.log.warn(
|
|
121
|
+
`startup reconcile: engine reset detected — orphaned ${reconciled.orphanedCount} engine-backed ` +
|
|
122
|
+
`inflight row(s) [run ${reconciled.runId}].`,
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
} catch (err) {
|
|
126
|
+
app.log.warn(`startup reconcile skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
105
130
|
// Review-ready poller. Self-scheduling (not setInterval) so a slow GitHub call can never
|
|
106
131
|
// overlap two passes (which could double-signal `readiness-ready`); the next pass is scheduled
|
|
107
132
|
// only after the previous one settles.
|
package/openapi.yaml
CHANGED
|
@@ -844,6 +844,64 @@ components:
|
|
|
844
844
|
type: string
|
|
845
845
|
uptimeSeconds:
|
|
846
846
|
type: integer
|
|
847
|
+
ReconcileReport:
|
|
848
|
+
type: object
|
|
849
|
+
description: The result of one engine-reset reconciliation pass (issue #622).
|
|
850
|
+
additionalProperties: false
|
|
851
|
+
required:
|
|
852
|
+
- runId
|
|
853
|
+
- reason
|
|
854
|
+
- observedEpoch
|
|
855
|
+
- recordedEpoch
|
|
856
|
+
- orphanedCount
|
|
857
|
+
- orphaned
|
|
858
|
+
properties:
|
|
859
|
+
runId:
|
|
860
|
+
type: string
|
|
861
|
+
description: The reconcile run id every orphaned transition's provenance is stamped with.
|
|
862
|
+
reason:
|
|
863
|
+
type: string
|
|
864
|
+
description: Why this pass acted (or did not).
|
|
865
|
+
enum:
|
|
866
|
+
- epoch-regression
|
|
867
|
+
- seed-epoch
|
|
868
|
+
- no-op
|
|
869
|
+
- engine-unreachable
|
|
870
|
+
observedEpoch:
|
|
871
|
+
type: integer
|
|
872
|
+
nullable: true
|
|
873
|
+
description: The engine incarnation epoch observed on this pass (null when the engine exposes
|
|
874
|
+
none, or was unreachable).
|
|
875
|
+
recordedEpoch:
|
|
876
|
+
type: integer
|
|
877
|
+
nullable: true
|
|
878
|
+
description: The previously-recorded epoch this pass compared against (null on the first run).
|
|
879
|
+
orphanedCount:
|
|
880
|
+
type: integer
|
|
881
|
+
description: How many engine-backed inflight rows were driven to `orphaned`.
|
|
882
|
+
orphaned:
|
|
883
|
+
type: array
|
|
884
|
+
description: The rows orphaned by this pass.
|
|
885
|
+
items:
|
|
886
|
+
type: object
|
|
887
|
+
additionalProperties: false
|
|
888
|
+
required:
|
|
889
|
+
- table
|
|
890
|
+
- pk
|
|
891
|
+
- key
|
|
892
|
+
- fromStatus
|
|
893
|
+
properties:
|
|
894
|
+
table:
|
|
895
|
+
type: string
|
|
896
|
+
pk:
|
|
897
|
+
type: string
|
|
898
|
+
key:
|
|
899
|
+
type: string
|
|
900
|
+
nullable: true
|
|
901
|
+
description: The engine instance key (e.g. process_key) the row projected.
|
|
902
|
+
fromStatus:
|
|
903
|
+
type: string
|
|
904
|
+
description: The non-terminal status the row carried before it was orphaned.
|
|
847
905
|
AgentInstructions:
|
|
848
906
|
type: object
|
|
849
907
|
description: The agent operator guide — how to drive (submit PRs/epics, answer escalations)
|
|
@@ -3189,6 +3247,40 @@ paths:
|
|
|
3189
3247
|
application/json:
|
|
3190
3248
|
schema:
|
|
3191
3249
|
$ref: "#/components/schemas/ErrorBody"
|
|
3250
|
+
/reconcile:
|
|
3251
|
+
post:
|
|
3252
|
+
operationId: reconcileEngineState
|
|
3253
|
+
summary: Reconcile engine-backed inflight projections after an engine reset/rewind (issue #622).
|
|
3254
|
+
description: >-
|
|
3255
|
+
The explicit operator command for the app-side reconciliation surface. Probes the engine's
|
|
3256
|
+
incarnation epoch (`/v2/topology`) and compares it to the last-seen value: on a REGRESSION (the
|
|
3257
|
+
#1065 reset/rewind signature) every NON-terminal engine-backed app row (an instanceTracking
|
|
3258
|
+
binding whose status is still active and whose engine key is populated) is driven to the
|
|
3259
|
+
defined `orphaned` terminal WITH PROVENANCE. Terminal history and non-engine-backed surfaces
|
|
3260
|
+
(presence, audit) are never touched. Idempotent — a second call with a matching epoch is a
|
|
3261
|
+
no-op — and safe: an unreachable engine orphans nothing. Runs automatically on startup too.
|
|
3262
|
+
security:
|
|
3263
|
+
- hookSecret: []
|
|
3264
|
+
- {}
|
|
3265
|
+
responses:
|
|
3266
|
+
"200":
|
|
3267
|
+
description: The reconcile pass result (what it observed and orphaned).
|
|
3268
|
+
content:
|
|
3269
|
+
application/json:
|
|
3270
|
+
schema:
|
|
3271
|
+
$ref: "#/components/schemas/ReconcileReport"
|
|
3272
|
+
"401":
|
|
3273
|
+
description: Missing/invalid shared secret (only when NANO_PR_WEBHOOK_SECRET is set).
|
|
3274
|
+
content:
|
|
3275
|
+
application/json:
|
|
3276
|
+
schema:
|
|
3277
|
+
$ref: "#/components/schemas/ErrorBody"
|
|
3278
|
+
"503":
|
|
3279
|
+
description: The app has no data source configured, so there is nothing to reconcile.
|
|
3280
|
+
content:
|
|
3281
|
+
application/json:
|
|
3282
|
+
schema:
|
|
3283
|
+
$ref: "#/components/schemas/ErrorBody"
|
|
3192
3284
|
/version:
|
|
3193
3285
|
get:
|
|
3194
3286
|
operationId: getVersion
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
// Delegate-level tests for POST /app/api/reconcile → operation `reconcileEngineState` (issue #622).
|
|
2
|
+
// Covers the shared-secret guard (401), the no-data-source guard (503), and a happy-path 200 that
|
|
3
|
+
// exercises the wiring to `runEngineReconcile` against the REAL shipping schema (the whole migration
|
|
4
|
+
// set on an in-memory SQLite) with the engine `/topology` probe stubbed — so the operator command's
|
|
5
|
+
// auth guard, status codes, and reconcile wiring are regression-covered by the Node test suite.
|
|
6
|
+
import { test } from "node:test";
|
|
7
|
+
import { assert, assertEquals } from "#test-assert";
|
|
8
|
+
import type { AppApi } from "@nanobpm/urban";
|
|
9
|
+
import { noopLog } from "../test/log.ts";
|
|
10
|
+
import { freshData } from "../test/reconcileDb.ts";
|
|
11
|
+
import handler from "./reconcileEngineState.ts";
|
|
12
|
+
|
|
13
|
+
function input(headers: Record<string, string> = {}) {
|
|
14
|
+
return {
|
|
15
|
+
req: {
|
|
16
|
+
method: "POST",
|
|
17
|
+
path: "/app/api/reconcile",
|
|
18
|
+
query: new URLSearchParams(),
|
|
19
|
+
headers: new Headers(headers),
|
|
20
|
+
text: async () => "",
|
|
21
|
+
} as any,
|
|
22
|
+
params: {},
|
|
23
|
+
query: {},
|
|
24
|
+
body: undefined,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Stub the global `/topology` probe so the delegate never touches the network; restore after. */
|
|
29
|
+
async function withEngineEpoch<T>(epoch: number | null, fn: () => Promise<T>): Promise<T> {
|
|
30
|
+
const prev = globalThis.fetch;
|
|
31
|
+
globalThis.fetch = (async () =>
|
|
32
|
+
({
|
|
33
|
+
ok: true,
|
|
34
|
+
json: async () => (epoch == null ? {} : { nano: { incarnation: epoch } }),
|
|
35
|
+
}) as unknown as Response) as typeof fetch;
|
|
36
|
+
try {
|
|
37
|
+
return await fn();
|
|
38
|
+
} finally {
|
|
39
|
+
globalThis.fetch = prev;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
test("returns 503 when no data source is configured", async () => {
|
|
44
|
+
const app = { log: noopLog() } as any as AppApi;
|
|
45
|
+
const res = (await handler(input(), app)) as any;
|
|
46
|
+
assertEquals(res.status, 503);
|
|
47
|
+
assert("error" in res.body);
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("first observation seeds the epoch and returns 200 with a reconcile result", async () => {
|
|
51
|
+
const { data } = freshData();
|
|
52
|
+
const app = { data, log: noopLog() } as any as AppApi;
|
|
53
|
+
const res = await withEngineEpoch(7, async () => (await handler(input(), app)) as any);
|
|
54
|
+
assertEquals(res.status, 200);
|
|
55
|
+
assertEquals(res.body.reason, "seed-epoch");
|
|
56
|
+
assertEquals(res.body.orphanedCount, 0);
|
|
57
|
+
assert(typeof res.body.runId === "string" && res.body.runId.length > 0);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("shared-secret guard rejects a missing/wrong secret when configured", async () => {
|
|
61
|
+
const prev = process.env["NANO_PR_WEBHOOK_SECRET"];
|
|
62
|
+
process.env["NANO_PR_WEBHOOK_SECRET"] = "s3cr3t";
|
|
63
|
+
try {
|
|
64
|
+
// SECRET is bound at import time, so import a cache-busted copy to observe the guard.
|
|
65
|
+
const mod = await import(`./reconcileEngineState.ts?guard=${Date.now()}`);
|
|
66
|
+
const guarded = mod.default as typeof handler;
|
|
67
|
+
const { data } = freshData();
|
|
68
|
+
const app = { data, log: noopLog() } as any as AppApi;
|
|
69
|
+
const bad = (await guarded(input(), app)) as any;
|
|
70
|
+
assertEquals(bad.status, 401);
|
|
71
|
+
const wrong = (await guarded(input({ "x-hook-secret": "nope" }), app)) as any;
|
|
72
|
+
assertEquals(wrong.status, 401);
|
|
73
|
+
const ok = await withEngineEpoch(7, async () =>
|
|
74
|
+
(await guarded(input({ "x-hook-secret": "s3cr3t" }), app)) as any,
|
|
75
|
+
);
|
|
76
|
+
assertEquals(ok.status, 200);
|
|
77
|
+
} finally {
|
|
78
|
+
if (prev === undefined) delete process.env["NANO_PR_WEBHOOK_SECRET"];
|
|
79
|
+
else process.env["NANO_PR_WEBHOOK_SECRET"] = prev;
|
|
80
|
+
}
|
|
81
|
+
});
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
// POST /app/api/reconcile → operationId `reconcileEngineState` (ADR 0058/0059, base /app/api).
|
|
2
|
+
//
|
|
3
|
+
// The explicit OPERATOR COMMAND for the app-side engine-reset reconciliation surface (issue #622) —
|
|
4
|
+
// the on-demand twin of the startup pass in main.ts, sharing the one `runEngineReconcile` seam so the
|
|
5
|
+
// two paths can never diverge. An operator (or a restore runbook) POSTs here after resetting /
|
|
6
|
+
// restoring / rewinding the engine to converge `app.db`: it probes the engine incarnation epoch and,
|
|
7
|
+
// on a regression, drives every dangling engine-backed inflight row to the defined `orphaned` terminal
|
|
8
|
+
// with provenance. Idempotent (a matching epoch is a no-op) and safe (an unreachable engine orphans
|
|
9
|
+
// nothing), so it is harmless to run at any time — a green "nothing to do" is the common case.
|
|
10
|
+
//
|
|
11
|
+
// The engine address is the canonical `resolveEngineAddress` (the same precedence the engine client
|
|
12
|
+
// and startup preflight use), so the operator command talks to exactly the engine the app runs
|
|
13
|
+
// against. The optional shared-secret guard stays HERE (the runtime does not enforce OpenAPI
|
|
14
|
+
// `security`): when NANO_PR_WEBHOOK_SECRET is set, callers must present it via the x-hook-secret header.
|
|
15
|
+
|
|
16
|
+
import { resolveEngineAddress } from "../app/enginePreflight.ts";
|
|
17
|
+
import { runEngineReconcile } from "../app/reconcile.ts";
|
|
18
|
+
import { envVar } from "../app/version.ts";
|
|
19
|
+
import { defineOperation } from "../nano-generated/operations.ts";
|
|
20
|
+
|
|
21
|
+
const SECRET = envVar("NANO_PR_WEBHOOK_SECRET") ?? "";
|
|
22
|
+
|
|
23
|
+
export default defineOperation("reconcileEngineState", async ({ req }, app) => {
|
|
24
|
+
if (SECRET && req.headers.get("x-hook-secret") !== SECRET) {
|
|
25
|
+
app.log.warn("reconcileEngineState rejected: missing/invalid shared secret");
|
|
26
|
+
return { status: 401, body: { error: "unauthorized" } };
|
|
27
|
+
}
|
|
28
|
+
if (!app.data) {
|
|
29
|
+
app.log.warn("reconcileEngineState: no data source configured — nothing to reconcile");
|
|
30
|
+
return { status: 503, body: { error: "no data source configured" } };
|
|
31
|
+
}
|
|
32
|
+
const engineAddress = resolveEngineAddress();
|
|
33
|
+
const result = await runEngineReconcile(
|
|
34
|
+
app.data,
|
|
35
|
+
{ restAddress: engineAddress.restAddress, token: envVar("CAMUNDA_TOKEN") ?? undefined },
|
|
36
|
+
{ log: { info: (m) => app.log.info(m), warn: (m) => app.log.warn(m) } },
|
|
37
|
+
);
|
|
38
|
+
app.log.info("reconcileEngineState complete", {
|
|
39
|
+
reason: result.reason,
|
|
40
|
+
orphanedCount: result.orphanedCount,
|
|
41
|
+
runId: result.runId,
|
|
42
|
+
});
|
|
43
|
+
return { status: 200, body: result };
|
|
44
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/nano-workforce",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.163.0",
|
|
4
4
|
"description": "Nano Workforce — an Agent Graph Orchestration application for Agentic SDLC: durable BPMN processes that coordinate a graph of AI agents across the software delivery lifecycle.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "main.ts",
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// A test-only DataLayer over a real in-memory `node:sqlite` db with the WHOLE migration set applied,
|
|
2
|
+
// for exercising the engine-reset reconciliation surface (`app/reconcile.ts`, issue #622) and its
|
|
3
|
+
// operator-command delegate (`operations/reconcileEngineState.ts`) against the REAL shipping schema —
|
|
4
|
+
// so the tables/columns/indexes reconcile reads and writes are exactly what deploys, not a fake.
|
|
5
|
+
//
|
|
6
|
+
// Canonical harness shared by `app/reconcile.test.ts` and `operations/reconcileEngineState.test.ts`
|
|
7
|
+
// (derivation over duplication: one in-memory DataLayer builder, not two divergent copies).
|
|
8
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
9
|
+
import { DatabaseSync } from "node:sqlite";
|
|
10
|
+
import { afterEach } from "node:test";
|
|
11
|
+
import { fileURLToPath } from "node:url";
|
|
12
|
+
import { type DataLayer, makeGateway, type SqliteDb } from "@nanobpm/urban";
|
|
13
|
+
import { applyMigrationSet } from "#test-migrations";
|
|
14
|
+
|
|
15
|
+
const MIGRATIONS_DIR = fileURLToPath(new URL("../db/migrations", import.meta.url));
|
|
16
|
+
|
|
17
|
+
// Every raw handle `freshData()` opens is tracked here and released after each test, so call sites
|
|
18
|
+
// (all of them) don't leak native SQLite handles across the run. Mirrors `test/worldDb.ts` and
|
|
19
|
+
// `test/blackboardDb.ts` (derivation over duplication: the same auto-close idiom, not a new one).
|
|
20
|
+
const openDbs = new Set<DatabaseSync>();
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
for (const raw of openDbs) {
|
|
23
|
+
if (openDbs.delete(raw)) raw.close();
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
/** Adapt a raw `node:sqlite` handle to urban's tiny `SqliteDb` seam so `makeGateway` yields the real
|
|
28
|
+
* record-oriented `DataSource` reconcile binds to (no fakes — the shipping gateway). */
|
|
29
|
+
export function sqliteDb(raw: DatabaseSync): SqliteDb {
|
|
30
|
+
return {
|
|
31
|
+
exec: (sql) => raw.exec(sql),
|
|
32
|
+
run: (sql, params = []) => {
|
|
33
|
+
const r = raw.prepare(sql).run(...(params as never[]));
|
|
34
|
+
return { changes: Number(r.changes), lastInsertRowid: r.lastInsertRowid };
|
|
35
|
+
},
|
|
36
|
+
all: <T = Record<string, unknown>>(sql: string, params: unknown[] = []) =>
|
|
37
|
+
raw.prepare(sql).all(...(params as never[])) as T[],
|
|
38
|
+
close: () => raw.close(),
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function readMigrationFiles(): { name: string; sql: string }[] {
|
|
43
|
+
return readdirSync(MIGRATIONS_DIR)
|
|
44
|
+
.filter((n) => n.endsWith(".sql"))
|
|
45
|
+
.map((name) => ({ name, sql: readFileSync(`${MIGRATIONS_DIR}/${name}`, "utf8") }));
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** A DataLayer over a fresh in-memory DB with the whole migration set applied. The raw handle is
|
|
49
|
+
* tracked and auto-closed after each test (see `openDbs`), and FK enforcement is enabled so the
|
|
50
|
+
* migrations are exercised under the real constraints they ship with. */
|
|
51
|
+
export function freshData(): { data: DataLayer; raw: DatabaseSync } {
|
|
52
|
+
const raw = new DatabaseSync(":memory:");
|
|
53
|
+
openDbs.add(raw);
|
|
54
|
+
// SQLite disables FK enforcement by default; enable it so migrations with foreign keys are
|
|
55
|
+
// exercised (and any FK violations surface) exactly as they would on the shipping schema.
|
|
56
|
+
raw.exec("PRAGMA foreign_keys = ON;");
|
|
57
|
+
applyMigrationSet(raw, readMigrationFiles());
|
|
58
|
+
const gw = makeGateway(sqliteDb(raw));
|
|
59
|
+
const data = { open: () => gw } as unknown as DataLayer;
|
|
60
|
+
return { data, raw };
|
|
61
|
+
}
|