@nanobpm/urban 0.65.0 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/runtime/core/manifest.d.ts +24 -1
- package/dist/runtime/core/modules/instance-tracking.d.ts +14 -0
- package/dist/runtime/core/modules/instance-tracking.js +108 -11
- package/dist/runtime/core/validate.js +15 -0
- package/package.json +1 -1
- package/src/runtime/core/manifest.ts +22 -1
- package/src/runtime/core/modules/instance-tracking.test.ts +192 -6
- package/src/runtime/core/modules/instance-tracking.ts +115 -11
- package/src/runtime/core/validate.test.ts +46 -0
- package/src/runtime/core/validate.ts +15 -0
|
@@ -2,7 +2,8 @@ import type { HostContext } from "./host.ts";
|
|
|
2
2
|
import type { AppManifest, InstanceTracking as SchemaInstanceTracking, Worker } from "@nanobpm/nano-app-schema";
|
|
3
3
|
export type { ActionDecl, AppManifest, AppUi, ChatSurface, Connection, Data, DataSource, DomainField, DomainType, LlmBinding, Models, PagesSurface, Runtime, Security, Surfaces, TaskInboxSurface, Trigger, TriggerAction, Worker, } from "@nanobpm/nano-app-schema";
|
|
4
4
|
/**
|
|
5
|
-
* `instanceTracking` binding, bridged to add the fail-open `terminalStatuses` selector
|
|
5
|
+
* `instanceTracking` binding, bridged to add the fail-open `terminalStatuses` selector and the
|
|
6
|
+
* `onWaitingHuman` wait-on-human edge (issue #355).
|
|
6
7
|
*
|
|
7
8
|
* `terminalStatuses` is landing in the canonical schema (`@nanobpm/nano-app-schema`,
|
|
8
9
|
* Magikcraft/nano-bpm#769): an exclusion list — the runtime polls every row whose
|
|
@@ -16,6 +17,28 @@ export type InstanceTracking = SchemaInstanceTracking & {
|
|
|
16
17
|
/** Values of `statusField` considered finished. When set, every row NOT in one of these
|
|
17
18
|
* is polled (fail-open). Requires `statusField`; mutually exclusive with `activeStatuses`. */
|
|
18
19
|
readonly terminalStatuses?: readonly string[];
|
|
20
|
+
/**
|
|
21
|
+
* The reconciliation applied to a row whose instance is **parked waiting on a human** — the
|
|
22
|
+
* wait-state twin of {@link SchemaInstanceTracking.onTerminated} (issue #355). An instance is
|
|
23
|
+
* waiting on a human *iff* it has an open user task, so the runtime derives this edge from
|
|
24
|
+
* engine truth (`openUserTasks`) rather than from a written column: on each poll, an active,
|
|
25
|
+
* non-terminated row whose instance has any open user task has this patch applied (typically
|
|
26
|
+
* `status = "awaiting_operator"`). Terminated wins over waiting-human (the {@link
|
|
27
|
+
* SchemaInstanceTracking.onTerminated} edge is applied first and excludes the key from this
|
|
28
|
+
* pass); a row with neither a terminated instance nor an open user task is left untouched, so
|
|
29
|
+
* the worker-owned transient status (running / converging / merging) survives.
|
|
30
|
+
*
|
|
31
|
+
* Optional and, like `terminalStatuses`, bridged locally until the canonical schema
|
|
32
|
+
* (`@nanobpm/nano-app-schema`) folds it in; drop this augmentation and re-export once the dep
|
|
33
|
+
* bump lands (No Drift Surfaces).
|
|
34
|
+
*/
|
|
35
|
+
readonly onWaitingHuman?: {
|
|
36
|
+
/** Column → literal value patch written to a row whose instance has an open user task
|
|
37
|
+
* (e.g. set the status to a "waiting on human" value). */
|
|
38
|
+
readonly set: {
|
|
39
|
+
readonly [k: string]: string | number | boolean | null;
|
|
40
|
+
};
|
|
41
|
+
};
|
|
19
42
|
};
|
|
20
43
|
/**
|
|
21
44
|
* A status selector (`activeStatuses`/`terminalStatuses`) counts as *configured* only when it is
|
|
@@ -3,6 +3,12 @@ import { type InstanceTracking } from "../manifest.ts";
|
|
|
3
3
|
import { type SchedulerDeps } from "./scheduler.ts";
|
|
4
4
|
/** Default poll interval when a binding does not set `pollMs`. */
|
|
5
5
|
export declare const DEFAULT_INSTANCE_TRACKING_POLL_MS = 15000;
|
|
6
|
+
/** Max in-flight `openUserTasks` probes per reconcile pass. The wait-on-human edge probes the
|
|
7
|
+
* engine once per active, non-terminated key; probing them sequentially makes a pass O(N) network
|
|
8
|
+
* round-trips and can make a single pass take longer than `pollMs` under a large active backlog.
|
|
9
|
+
* Probing in bounded-parallel batches makes wall-clock time scale with max latency rather than
|
|
10
|
+
* N×latency, while the cap keeps in-flight load on the engine bounded. */
|
|
11
|
+
export declare const WAITING_HUMAN_PROBE_CONCURRENCY = 8;
|
|
6
12
|
export interface InstanceTrackingHandle extends Mounted {
|
|
7
13
|
/** `table.keyField` labels of the armed bindings, for `inspect()`. */
|
|
8
14
|
readonly bindings: string[];
|
|
@@ -14,6 +20,14 @@ export interface InstanceTrackingHandle extends Mounted {
|
|
|
14
20
|
* once per TERMINATED key it discovers) and the cancel primitive (immediately when it terminates an
|
|
15
21
|
* instance) route through here, so the two can never drift on the patch or the log. */
|
|
16
22
|
export declare function reconcileTerminatedKey(api: Pick<AppApi, "data" | "log">, bindings: InstanceTracking[], processInstanceKey: string): Promise<number>;
|
|
23
|
+
/** Apply a binding's `onWaitingHuman.set` patch to the single row keyed by `processInstanceKey`
|
|
24
|
+
* — the wait-on-human twin of {@link reconcileTerminatedKey} (issue #355). The caller establishes
|
|
25
|
+
* the precondition (the instance is non-terminated and has an open user task); this is the single
|
|
26
|
+
* place that writes `onWaitingHuman.set` and logs the reconcile. Bindings without an
|
|
27
|
+
* `onWaitingHuman` edge are no-ops, so callers can fan this over every binding. It skips the write
|
|
28
|
+
* when the row already carries the patch (quiet-idempotent — a long-parked instance is re-polled
|
|
29
|
+
* every tick and must not re-log). Returns rows changed. */
|
|
30
|
+
export declare function reconcileWaitingHumanKey(api: Pick<AppApi, "data" | "log">, bindings: InstanceTracking[], processInstanceKey: string): Promise<number>;
|
|
17
31
|
/**
|
|
18
32
|
* Mount the `instanceTracking` reconciler: arm one poll loop per binding. Each loop selects
|
|
19
33
|
* the active rows, asks the engine which of their instances are terminated, and applies the
|
|
@@ -1,21 +1,45 @@
|
|
|
1
1
|
// instance-tracking — the process-instance lifecycle reconciler (manifest `instanceTracking`).
|
|
2
2
|
//
|
|
3
3
|
// The problem it closes: an Urban app derives a row's status from the workers that run
|
|
4
|
-
// along a process's *happy path*. When an instance instead reaches
|
|
5
|
-
// no
|
|
6
|
-
// runs, so the read-model row stays "active" forever and any UI derived from it lies.
|
|
4
|
+
// along a process's *happy path*. When an instance instead reaches an engine-truth state with
|
|
5
|
+
// no worker to record it, the read-model row lies. Two such edges are reconciled declaratively:
|
|
7
6
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
7
|
+
// - The **terminal** edge (`onTerminated`): each binding's poll asks the engine which of its
|
|
8
|
+
// active instances are `TERMINATED` and applies `onTerminated.set` (→ e.g. `abandoned`). A
|
|
9
|
+
// terminated instance runs no completion worker, so nothing else would ever close the row.
|
|
10
|
+
//
|
|
11
|
+
// - The **wait-on-human** edge (`onWaitingHuman`, issue #355): the symmetric primitive for
|
|
12
|
+
// "this instance is parked waiting on a human." That state is authoritatively knowable — an
|
|
13
|
+
// instance is waiting on a human *iff* it has an open user task (`openUserTasks` =
|
|
14
|
+
// `searchUserTasks({state:"CREATED"})`) — for every process, forever, with no per-element
|
|
15
|
+
// registry. On each poll, an active, non-terminated row whose instance has any open user task
|
|
16
|
+
// has `onWaitingHuman.set` applied (typically `status = "awaiting_operator"`). This retires the
|
|
17
|
+
// per-subject imperative pattern (workers writing the status on escalate/answer plus a bespoke
|
|
18
|
+
// self-heal poller): the parked status becomes a derived view over the one user-task
|
|
19
|
+
// projection, so there is no second surface to drift (No Drift Surfaces).
|
|
20
|
+
//
|
|
21
|
+
// Precedence per active row, all derived from engine truth (never a worker write):
|
|
22
|
+
// terminated -> onTerminated.set (e.g. abandoned) // terminal edge
|
|
23
|
+
// else has open user task -> onWaitingHuman.set (e.g. awaiting_operator) // wait-on-human edge
|
|
24
|
+
// else -> leave the worker-owned transient status (running / converging / …)
|
|
25
|
+
//
|
|
26
|
+
// It deliberately reconciles the terminal edge only on `TERMINATED` — the state that by
|
|
27
|
+
// definition runs no completion worker — and never on `COMPLETED`, whose terminal row-write the
|
|
28
|
+
// app's own finalize worker owns (reconciling it here would race that worker and could clobber a
|
|
29
|
+
// legitimately-completed row). The wait-on-human edge reads *open* user tasks via `openUserTasks`
|
|
30
|
+
// (never a bare `searchUserTasks`, which lags a completion and would latch `awaiting_operator`
|
|
31
|
+
// onto a just-answered task). The poll is driven through the shared `SchedulerDeps` seam so tests
|
|
32
|
+
// are deterministic.
|
|
15
33
|
import { isConfiguredStatusSelector } from "../manifest.js";
|
|
16
34
|
import { defaultScheduler, MAX_TIMER_DELAY_MS } from "./scheduler.js";
|
|
17
35
|
/** Default poll interval when a binding does not set `pollMs`. */
|
|
18
36
|
export const DEFAULT_INSTANCE_TRACKING_POLL_MS = 15_000;
|
|
37
|
+
/** Max in-flight `openUserTasks` probes per reconcile pass. The wait-on-human edge probes the
|
|
38
|
+
* engine once per active, non-terminated key; probing them sequentially makes a pass O(N) network
|
|
39
|
+
* round-trips and can make a single pass take longer than `pollMs` under a large active backlog.
|
|
40
|
+
* Probing in bounded-parallel batches makes wall-clock time scale with max latency rather than
|
|
41
|
+
* N×latency, while the cap keeps in-flight load on the engine bounded. */
|
|
42
|
+
export const WAITING_HUMAN_PROBE_CONCURRENCY = 8;
|
|
19
43
|
/** Apply a binding's `onTerminated.set` patch to the single row keyed by `processInstanceKey`.
|
|
20
44
|
* Only the binding whose table owns that key has a matching row, so callers can fan this over
|
|
21
45
|
* every binding and let the no-match ones be no-ops. Returns rows changed. The single place that
|
|
@@ -40,8 +64,55 @@ export async function reconcileTerminatedKey(api, bindings, processInstanceKey)
|
|
|
40
64
|
}
|
|
41
65
|
return reconciled;
|
|
42
66
|
}
|
|
67
|
+
/** Whether `row` already satisfies `patch` on every column it sets — used to make the
|
|
68
|
+
* wait-on-human writer quiet-idempotent. Unlike a terminated row (which flips to a terminal
|
|
69
|
+
* status and then drops out of the active set), a row parked at a user task stays active and is
|
|
70
|
+
* re-polled every tick, so re-writing/re-logging the same `awaiting_operator` patch on every poll
|
|
71
|
+
* would be noise; skip the write when it would be a no-op. */
|
|
72
|
+
function patchAlreadyApplied(row, set) {
|
|
73
|
+
return Object.entries(set).every(([col, value]) => row[col] === value);
|
|
74
|
+
}
|
|
75
|
+
/** Apply a binding's `onWaitingHuman.set` patch to the single row keyed by `processInstanceKey`
|
|
76
|
+
* — the wait-on-human twin of {@link reconcileTerminatedKey} (issue #355). The caller establishes
|
|
77
|
+
* the precondition (the instance is non-terminated and has an open user task); this is the single
|
|
78
|
+
* place that writes `onWaitingHuman.set` and logs the reconcile. Bindings without an
|
|
79
|
+
* `onWaitingHuman` edge are no-ops, so callers can fan this over every binding. It skips the write
|
|
80
|
+
* when the row already carries the patch (quiet-idempotent — a long-parked instance is re-polled
|
|
81
|
+
* every tick and must not re-log). Returns rows changed. */
|
|
82
|
+
export async function reconcileWaitingHumanKey(api, bindings, processInstanceKey) {
|
|
83
|
+
const key = String(processInstanceKey);
|
|
84
|
+
let reconciled = 0;
|
|
85
|
+
for (const binding of bindings) {
|
|
86
|
+
const onWaitingHuman = binding.onWaitingHuman;
|
|
87
|
+
if (!onWaitingHuman)
|
|
88
|
+
continue;
|
|
89
|
+
const table = api.data.table(binding.table, binding.keyField);
|
|
90
|
+
// Quiet-idempotence must consider EVERY row sharing this key, not just one. `keyField` need not
|
|
91
|
+
// be a unique column (e.g. a read model keyed on a non-PK `process_key`), and `table.update`
|
|
92
|
+
// patches ALL matching rows — so a `get()` (LIMIT 1) that happens to hit one already-patched row
|
|
93
|
+
// would skip the write and leave the OTHER matching rows unreconciled. Skip only when every
|
|
94
|
+
// matching row already carries the patch; otherwise fall through and re-patch them all.
|
|
95
|
+
const current = await table.find({ [binding.keyField]: key });
|
|
96
|
+
if (current.length > 0 && current.every((row) => patchAlreadyApplied(row, onWaitingHuman.set))) {
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
const changed = await table.update(key, onWaitingHuman.set);
|
|
100
|
+
if (changed > 0) {
|
|
101
|
+
reconciled += changed;
|
|
102
|
+
api.log("info", "instanceTracking: reconciled instance awaiting operator", {
|
|
103
|
+
table: binding.table,
|
|
104
|
+
keyField: binding.keyField,
|
|
105
|
+
processInstanceKey: key,
|
|
106
|
+
rowsChanged: changed,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return reconciled;
|
|
111
|
+
}
|
|
43
112
|
/** Poll one binding's instances once. Selects the active rows, asks the engine which of their
|
|
44
|
-
* instances are TERMINATED, and applies `onTerminated.set` to each
|
|
113
|
+
* instances are TERMINATED, and applies `onTerminated.set` to each; then, for the binding's
|
|
114
|
+
* wait-on-human edge (`onWaitingHuman`, issue #355), applies `onWaitingHuman.set` to every
|
|
115
|
+
* non-terminated active row whose instance has an open user task. Returns how many keys were
|
|
45
116
|
* scanned and how many rows were reconciled this tick. */
|
|
46
117
|
async function reconcileOnce(api, binding) {
|
|
47
118
|
const table = api.data.table(binding.table, binding.keyField);
|
|
@@ -97,13 +168,39 @@ async function reconcileOnce(api, binding) {
|
|
|
97
168
|
// writes `onTerminated.set` and logs the reconcile — so the poll path and the cancel primitive
|
|
98
169
|
// can never drift on the patch or its logging (No Drift Surfaces).
|
|
99
170
|
let reconciled = 0;
|
|
171
|
+
const terminatedKeys = new Set();
|
|
100
172
|
for (const snap of snapshots) {
|
|
101
173
|
if (snap.state !== "TERMINATED")
|
|
102
174
|
continue; // defensive; we asked for TERMINATED only
|
|
103
175
|
if (!keyToRows.has(snap.processInstanceKey))
|
|
104
176
|
continue; // not one we were tracking
|
|
177
|
+
terminatedKeys.add(snap.processInstanceKey);
|
|
105
178
|
reconciled += await reconcileTerminatedKey(api, [binding], snap.processInstanceKey);
|
|
106
179
|
}
|
|
180
|
+
// Wait-on-human edge (issue #355), symmetric to onTerminated. Precedence: terminated wins, so a
|
|
181
|
+
// key reconciled as terminated above is excluded here. An instance is "waiting on a human" iff
|
|
182
|
+
// it has an open user task — the authoritative engine truth — so read `openUserTasks` (never a
|
|
183
|
+
// bare `searchUserTasks`, which lags a completion and would latch `awaiting_operator` onto a
|
|
184
|
+
// just-answered task; issue #355 "instance B"). When there is no open user task the row is left
|
|
185
|
+
// untouched, so a worker-owned transient status (running / converging / merging) survives and a
|
|
186
|
+
// stale `awaiting_operator` is not re-latched. Because the condition is derived every tick, a
|
|
187
|
+
// re-escalation after an answer re-flips to `awaiting_operator` on the next poll by construction
|
|
188
|
+
// (issue #355 "instance A"). Probe in bounded-parallel batches (`WAITING_HUMAN_PROBE_CONCURRENCY`)
|
|
189
|
+
// so a pass over a large active backlog scales with max latency rather than N×latency.
|
|
190
|
+
if (binding.onWaitingHuman) {
|
|
191
|
+
const humanKeys = keys.filter((key) => !terminatedKeys.has(key)); // terminated already won
|
|
192
|
+
for (let i = 0; i < humanKeys.length; i += WAITING_HUMAN_PROBE_CONCURRENCY) {
|
|
193
|
+
const batch = humanKeys.slice(i, i + WAITING_HUMAN_PROBE_CONCURRENCY);
|
|
194
|
+
const perKey = await Promise.all(batch.map(async (key) => {
|
|
195
|
+
const openTasks = await api.engine.openUserTasks({ processInstanceKey: key });
|
|
196
|
+
if (openTasks.length === 0)
|
|
197
|
+
return 0; // not parked on a human ⇒ leave the transient status
|
|
198
|
+
return reconcileWaitingHumanKey(api, [binding], key);
|
|
199
|
+
}));
|
|
200
|
+
for (const n of perKey)
|
|
201
|
+
reconciled += n;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
107
204
|
return { scanned: keys.length, reconciled };
|
|
108
205
|
}
|
|
109
206
|
/**
|
|
@@ -183,6 +183,21 @@ export function collectManifestIssues(m) {
|
|
|
183
183
|
message: "missing onTerminated.set patch (a non-empty column → value map)",
|
|
184
184
|
});
|
|
185
185
|
}
|
|
186
|
+
// `onWaitingHuman` (issue #355) is the optional wait-on-human edge — the twin of
|
|
187
|
+
// `onTerminated`. It is optional (a binding may reconcile only the terminal edge), but when
|
|
188
|
+
// present its `set` must be a non-empty column → value patch for the same reason
|
|
189
|
+
// `onTerminated.set` must be: a reconciler with nothing to write is inert and almost
|
|
190
|
+
// certainly a mistake.
|
|
191
|
+
if (b?.onWaitingHuman !== undefined) {
|
|
192
|
+
const onWaitingHuman = isRecord(b.onWaitingHuman) ? b.onWaitingHuman : undefined;
|
|
193
|
+
const waitingSet = isRecord(onWaitingHuman?.set) ? onWaitingHuman.set : undefined;
|
|
194
|
+
if (!waitingSet || Object.keys(waitingSet).length === 0) {
|
|
195
|
+
issues.push({
|
|
196
|
+
path: `instanceTracking[${i}].onWaitingHuman.set`,
|
|
197
|
+
message: "missing onWaitingHuman.set patch (a non-empty column → value map)",
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
}
|
|
186
201
|
// `activeStatuses` (fail-closed allow-list) and `terminalStatuses` (fail-open exclusion) are
|
|
187
202
|
// both consumed as arrays of status strings. A malformed value — a bare string, or an array
|
|
188
203
|
// holding a non-string/empty entry — would misbehave at runtime (`new Set("abandoned")`
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@nanobpm/urban",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.66.0",
|
|
4
4
|
"description": "Urban: build and run code-first apps on Nano — runtime, derivation toolkit, and CLI in one. Author durable processes with @nanobpm/workflow (defineFlow), a typed datasource, forms, triggers and surfaces from a nano.app.json manifest, on Node or Deno.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -39,7 +39,8 @@ export type {
|
|
|
39
39
|
} from "@nanobpm/nano-app-schema";
|
|
40
40
|
|
|
41
41
|
/**
|
|
42
|
-
* `instanceTracking` binding, bridged to add the fail-open `terminalStatuses` selector
|
|
42
|
+
* `instanceTracking` binding, bridged to add the fail-open `terminalStatuses` selector and the
|
|
43
|
+
* `onWaitingHuman` wait-on-human edge (issue #355).
|
|
43
44
|
*
|
|
44
45
|
* `terminalStatuses` is landing in the canonical schema (`@nanobpm/nano-app-schema`,
|
|
45
46
|
* Magikcraft/nano-bpm#769): an exclusion list — the runtime polls every row whose
|
|
@@ -53,6 +54,26 @@ export type InstanceTracking = SchemaInstanceTracking & {
|
|
|
53
54
|
/** Values of `statusField` considered finished. When set, every row NOT in one of these
|
|
54
55
|
* is polled (fail-open). Requires `statusField`; mutually exclusive with `activeStatuses`. */
|
|
55
56
|
readonly terminalStatuses?: readonly string[];
|
|
57
|
+
/**
|
|
58
|
+
* The reconciliation applied to a row whose instance is **parked waiting on a human** — the
|
|
59
|
+
* wait-state twin of {@link SchemaInstanceTracking.onTerminated} (issue #355). An instance is
|
|
60
|
+
* waiting on a human *iff* it has an open user task, so the runtime derives this edge from
|
|
61
|
+
* engine truth (`openUserTasks`) rather than from a written column: on each poll, an active,
|
|
62
|
+
* non-terminated row whose instance has any open user task has this patch applied (typically
|
|
63
|
+
* `status = "awaiting_operator"`). Terminated wins over waiting-human (the {@link
|
|
64
|
+
* SchemaInstanceTracking.onTerminated} edge is applied first and excludes the key from this
|
|
65
|
+
* pass); a row with neither a terminated instance nor an open user task is left untouched, so
|
|
66
|
+
* the worker-owned transient status (running / converging / merging) survives.
|
|
67
|
+
*
|
|
68
|
+
* Optional and, like `terminalStatuses`, bridged locally until the canonical schema
|
|
69
|
+
* (`@nanobpm/nano-app-schema`) folds it in; drop this augmentation and re-export once the dep
|
|
70
|
+
* bump lands (No Drift Surfaces).
|
|
71
|
+
*/
|
|
72
|
+
readonly onWaitingHuman?: {
|
|
73
|
+
/** Column → literal value patch written to a row whose instance has an open user task
|
|
74
|
+
* (e.g. set the status to a "waiting on human" value). */
|
|
75
|
+
readonly set: { readonly [k: string]: string | number | boolean | null };
|
|
76
|
+
};
|
|
56
77
|
};
|
|
57
78
|
|
|
58
79
|
/**
|
|
@@ -9,11 +9,12 @@ import { makeGateway } from "./gateway.ts";
|
|
|
9
9
|
import type { Table } from "./gateway.ts";
|
|
10
10
|
import { DataLayer, type ProvisionedSource } from "./datasource.ts";
|
|
11
11
|
import type { AppApi } from "../context.ts";
|
|
12
|
-
import type { EngineClient, ProcessInstanceSnapshot } from "../host.ts";
|
|
12
|
+
import type { EngineClient, ProcessInstanceSnapshot, UserTaskFilter, UserTaskSummary } from "../host.ts";
|
|
13
13
|
import type { InstanceTracking } from "../manifest.ts";
|
|
14
14
|
import {
|
|
15
15
|
DEFAULT_INSTANCE_TRACKING_POLL_MS,
|
|
16
16
|
mountInstanceTracking,
|
|
17
|
+
WAITING_HUMAN_PROBE_CONCURRENCY,
|
|
17
18
|
} from "./instance-tracking.ts";
|
|
18
19
|
import { MAX_TIMER_DELAY_MS, type SchedulerDeps } from "./scheduler.ts";
|
|
19
20
|
|
|
@@ -63,13 +64,21 @@ function fakeScheduler(): SchedulerDeps & { advance: (ms: number) => Promise<voi
|
|
|
63
64
|
};
|
|
64
65
|
}
|
|
65
66
|
|
|
66
|
-
/** An EngineClient
|
|
67
|
-
*
|
|
68
|
-
|
|
67
|
+
/** An EngineClient driven by a fixed map of processInstanceKey → state (for searchProcessInstances)
|
|
68
|
+
* and an optional map of processInstanceKey → number-of-open-user-tasks (for openUserTasks, the
|
|
69
|
+
* wait-on-human edge). Records the keys asked for on each call. `openUserTasks` models the engine
|
|
70
|
+
* truth the reconciler derives `awaiting_operator` from — only *open* (CREATED) tasks count, so a
|
|
71
|
+
* key absent from `openTasks` (or mapped to 0) reports no parked human. */
|
|
72
|
+
function fakeEngine(
|
|
73
|
+
states: Record<string, ProcessInstanceSnapshot["state"]>,
|
|
74
|
+
openTasks: Record<string, number> = {},
|
|
75
|
+
): {
|
|
69
76
|
engine: EngineClient;
|
|
70
77
|
queries: string[][];
|
|
78
|
+
userTaskQueries: (string | undefined)[];
|
|
71
79
|
} {
|
|
72
80
|
const queries: string[][] = [];
|
|
81
|
+
const userTaskQueries: (string | undefined)[] = [];
|
|
73
82
|
const notUsed = (m: string) => (): never => {
|
|
74
83
|
throw new Error(`fakeEngine.${m} is not exercised by this test`);
|
|
75
84
|
};
|
|
@@ -91,18 +100,25 @@ function fakeEngine(states: Record<string, ProcessInstanceSnapshot["state"]>): {
|
|
|
91
100
|
}
|
|
92
101
|
return out;
|
|
93
102
|
},
|
|
103
|
+
async openUserTasks(filter?: UserTaskFilter): Promise<UserTaskSummary[]> {
|
|
104
|
+
const key = filter?.processInstanceKey;
|
|
105
|
+
userTaskQueries.push(key);
|
|
106
|
+
const n = key ? (openTasks[key] ?? 0) : 0;
|
|
107
|
+
return Array.from({ length: n }, (_unused, i): UserTaskSummary => ({
|
|
108
|
+
userTaskKey: `${key}-t${i}`,
|
|
109
|
+
}));
|
|
110
|
+
},
|
|
94
111
|
deployResources: notUsed("deployResources"),
|
|
95
112
|
createInstance: notUsed("createInstance"),
|
|
96
113
|
cancelInstance: notUsed("cancelInstance"),
|
|
97
114
|
publishMessage: notUsed("publishMessage"),
|
|
98
115
|
searchUserTasks: notUsed("searchUserTasks"),
|
|
99
|
-
openUserTasks: notUsed("openUserTasks"),
|
|
100
116
|
getForm: notUsed("getForm"),
|
|
101
117
|
completeUserTask: notUsed("completeUserTask"),
|
|
102
118
|
registerWorker: notUsed("registerWorker"),
|
|
103
119
|
close: notUsed("close"),
|
|
104
120
|
};
|
|
105
|
-
return { engine, queries };
|
|
121
|
+
return { engine, queries, userTaskQueries };
|
|
106
122
|
}
|
|
107
123
|
|
|
108
124
|
interface Harness {
|
|
@@ -428,6 +444,176 @@ for (const bad of [0, -1000, Number.NaN]) {
|
|
|
428
444
|
}
|
|
429
445
|
|
|
430
446
|
|
|
447
|
+
// ── Wait-on-human edge (onWaitingHuman, issue #355) ────────────────────────────────────────────
|
|
448
|
+
|
|
449
|
+
// A binding that reconciles both engine-truth edges: terminal (onTerminated) and wait-on-human
|
|
450
|
+
// (onWaitingHuman). terminalStatuses (fail-open) keeps every non-abandoned row polled so a stale
|
|
451
|
+
// worker-written transient status (converging/merging/running) is still reconciled to awaiting.
|
|
452
|
+
const waitingBinding = (over: Partial<InstanceTracking> = {}): InstanceTracking => ({
|
|
453
|
+
table: "plans",
|
|
454
|
+
keyField: "process_key",
|
|
455
|
+
statusField: "status",
|
|
456
|
+
terminalStatuses: ["abandoned"],
|
|
457
|
+
onTerminated: { set: { status: "abandoned", note: null } },
|
|
458
|
+
onWaitingHuman: { set: { status: "awaiting_operator" } },
|
|
459
|
+
pollMs: 1000,
|
|
460
|
+
...over,
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
test("derives awaiting_operator for a row whose instance has an open user task (forward)", async () => {
|
|
464
|
+
// The row carries a STALE worker-written transient status; the instance is parked at a user task.
|
|
465
|
+
// One reconcile pass must read it back as awaiting_operator — derived, not written.
|
|
466
|
+
const { engine } = fakeEngine({ pi1: "ACTIVE" }, { pi1: 1 });
|
|
467
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
468
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "converging", note: null });
|
|
469
|
+
});
|
|
470
|
+
const sched = fakeScheduler();
|
|
471
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
472
|
+
await sched.advance(1000);
|
|
473
|
+
assert.equal((await h.table.get("pi1"))?.status, "awaiting_operator");
|
|
474
|
+
await handle.stop();
|
|
475
|
+
await h.close();
|
|
476
|
+
});
|
|
477
|
+
|
|
478
|
+
test("leaves the worker-owned transient status when the instance has NO open user task (backward)", async () => {
|
|
479
|
+
// No open user task and not terminated ⇒ the reconciler must not force awaiting_operator; the
|
|
480
|
+
// worker-owned transient status survives. This is the "must not remain awaiting_operator once the
|
|
481
|
+
// task completes" leg: a just-answered instance (task completed, worker wrote converging) is not
|
|
482
|
+
// re-latched to awaiting_operator.
|
|
483
|
+
const { engine } = fakeEngine({ pi1: "ACTIVE" }, { pi1: 0 });
|
|
484
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
485
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "converging", note: null });
|
|
486
|
+
});
|
|
487
|
+
const sched = fakeScheduler();
|
|
488
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
489
|
+
await sched.advance(1000);
|
|
490
|
+
assert.equal((await h.table.get("pi1"))?.status, "converging");
|
|
491
|
+
await handle.stop();
|
|
492
|
+
await h.close();
|
|
493
|
+
});
|
|
494
|
+
|
|
495
|
+
test("re-escalation after an answer re-flips to awaiting_operator on the next poll (instance A)", async () => {
|
|
496
|
+
// The re-escalation-after-answer defect (nano-workforce#318): the row sits at converging after an
|
|
497
|
+
// answer; a re-escalation opens a NEW user task. Because the status is DERIVED every tick, the
|
|
498
|
+
// next poll re-flips it to awaiting_operator by construction — no bespoke poller, no stuck state.
|
|
499
|
+
const openTasks: Record<string, number> = { pi1: 0 }; // answered: task completed, none open
|
|
500
|
+
const { engine } = fakeEngine({ pi1: "ACTIVE" }, openTasks);
|
|
501
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
502
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "converging", note: null });
|
|
503
|
+
});
|
|
504
|
+
const sched = fakeScheduler();
|
|
505
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
506
|
+
await sched.advance(1000);
|
|
507
|
+
assert.equal((await h.table.get("pi1"))?.status, "converging"); // still converging: no open task
|
|
508
|
+
openTasks.pi1 = 1; // re-escalation: a new user task opens
|
|
509
|
+
await sched.advance(1000);
|
|
510
|
+
assert.equal((await h.table.get("pi1"))?.status, "awaiting_operator"); // re-flipped
|
|
511
|
+
await handle.stop();
|
|
512
|
+
await h.close();
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
test("terminated wins over an open user task (precedence)", async () => {
|
|
516
|
+
// A terminated instance that also (transiently) reports an open user task must reconcile to the
|
|
517
|
+
// terminal patch, never awaiting_operator — onTerminated has precedence.
|
|
518
|
+
const { engine, userTaskQueries } = fakeEngine({ pi1: "TERMINATED" }, { pi1: 1 });
|
|
519
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
520
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "converging", note: "x" });
|
|
521
|
+
});
|
|
522
|
+
const sched = fakeScheduler();
|
|
523
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
524
|
+
await sched.advance(1000);
|
|
525
|
+
const row = await h.table.get("pi1");
|
|
526
|
+
assert.equal(row?.status, "abandoned");
|
|
527
|
+
assert.equal(row?.note, null);
|
|
528
|
+
assert.deepEqual(userTaskQueries, []); // terminated key was excluded before the openUserTasks probe
|
|
529
|
+
await handle.stop();
|
|
530
|
+
await h.close();
|
|
531
|
+
});
|
|
532
|
+
|
|
533
|
+
test("does not re-write or re-log a row already at awaiting_operator (quiet-idempotent)", async () => {
|
|
534
|
+
// A long-parked instance is re-polled every tick. The wait-on-human writer must be a no-op when
|
|
535
|
+
// the row already carries the patch, so it does not re-log on every poll.
|
|
536
|
+
const { engine } = fakeEngine({ pi1: "ACTIVE" }, { pi1: 1 });
|
|
537
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
538
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "awaiting_operator", note: null });
|
|
539
|
+
});
|
|
540
|
+
const sched = fakeScheduler();
|
|
541
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
542
|
+
await sched.advance(1000);
|
|
543
|
+
await sched.advance(1000);
|
|
544
|
+
assert.equal((await h.table.get("pi1"))?.status, "awaiting_operator");
|
|
545
|
+
const awaitingLogs = h.logs.filter((l) => l.msg.includes("awaiting operator"));
|
|
546
|
+
assert.equal(awaitingLogs.length, 0); // no-op patch never logged
|
|
547
|
+
await handle.stop();
|
|
548
|
+
await h.close();
|
|
549
|
+
});
|
|
550
|
+
|
|
551
|
+
test("a binding without onWaitingHuman never probes open user tasks (edge is opt-in)", async () => {
|
|
552
|
+
const { engine, userTaskQueries } = fakeEngine({ pi1: "ACTIVE" }, { pi1: 1 });
|
|
553
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
554
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "dispatched", note: null });
|
|
555
|
+
});
|
|
556
|
+
const sched = fakeScheduler();
|
|
557
|
+
const handle = mount(h, [planBinding()], sched); // planBinding has no onWaitingHuman
|
|
558
|
+
await sched.advance(1000);
|
|
559
|
+
assert.deepEqual(userTaskQueries, []); // never probed
|
|
560
|
+
assert.equal((await h.table.get("pi1"))?.status, "dispatched"); // untouched
|
|
561
|
+
await handle.stop();
|
|
562
|
+
await h.close();
|
|
563
|
+
});
|
|
564
|
+
|
|
565
|
+
test("reconciles EVERY row sharing a non-unique key, even when one is already patched (multi-row)", async () => {
|
|
566
|
+
// `keyField` (process_key) is NOT unique here, and table.update patches ALL matching rows. The
|
|
567
|
+
// quiet-idempotence check must consider every matching row, not a single get() (LIMIT 1): with two
|
|
568
|
+
// rows on the same process_key — one already awaiting_operator, one still converging — a LIMIT-1
|
|
569
|
+
// probe that hits the patched row would skip the write and strand the other. Both must converge.
|
|
570
|
+
const { engine } = fakeEngine({ pi1: "ACTIVE" }, { pi1: 1 });
|
|
571
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
572
|
+
await t.insert({ plan_key: "p1", process_key: "pi1", status: "awaiting_operator", note: null });
|
|
573
|
+
await t.insert({ plan_key: "p2", process_key: "pi1", status: "converging", note: null });
|
|
574
|
+
});
|
|
575
|
+
const sched = fakeScheduler();
|
|
576
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
577
|
+
await sched.advance(1000);
|
|
578
|
+
const rows = await h.table.find({ process_key: "pi1" });
|
|
579
|
+
assert.deepEqual(
|
|
580
|
+
rows.map((r) => r.status).sort(),
|
|
581
|
+
["awaiting_operator", "awaiting_operator"],
|
|
582
|
+
);
|
|
583
|
+
await handle.stop();
|
|
584
|
+
await h.close();
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
test("reconciles every parked key across multiple probe batches (bounded-parallel probing)", async () => {
|
|
588
|
+
// More active keys than WAITING_HUMAN_PROBE_CONCURRENCY forces multiple probe batches. Every key
|
|
589
|
+
// with an open user task must still be probed and reconciled — the batching must not drop keys.
|
|
590
|
+
const total = WAITING_HUMAN_PROBE_CONCURRENCY * 2 + 3; // spans three batches
|
|
591
|
+
const states: Record<string, ProcessInstanceSnapshot["state"]> = {};
|
|
592
|
+
const openTasks: Record<string, number> = {};
|
|
593
|
+
for (let i = 0; i < total; i++) {
|
|
594
|
+
states[`pi${i}`] = "ACTIVE";
|
|
595
|
+
openTasks[`pi${i}`] = i % 2; // half parked on a human, half not
|
|
596
|
+
}
|
|
597
|
+
const { engine, userTaskQueries } = fakeEngine(states, openTasks);
|
|
598
|
+
const h = await withHarness(engine, PLANS_DDL, async (t) => {
|
|
599
|
+
for (let i = 0; i < total; i++) {
|
|
600
|
+
await t.insert({ plan_key: `p${i}`, process_key: `pi${i}`, status: "converging", note: null });
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
const sched = fakeScheduler();
|
|
604
|
+
const handle = mount(h, [waitingBinding()], sched);
|
|
605
|
+
await sched.advance(1000);
|
|
606
|
+
// Every active key was probed exactly once.
|
|
607
|
+
assert.equal(userTaskQueries.length, total);
|
|
608
|
+
assert.deepEqual([...userTaskQueries].sort(), Object.keys(states).sort());
|
|
609
|
+
for (let i = 0; i < total; i++) {
|
|
610
|
+
const expected = i % 2 === 1 ? "awaiting_operator" : "converging";
|
|
611
|
+
assert.equal((await h.table.get(`pi${i}`))?.status, expected, `pi${i}`);
|
|
612
|
+
}
|
|
613
|
+
await handle.stop();
|
|
614
|
+
await h.close();
|
|
615
|
+
});
|
|
616
|
+
|
|
431
617
|
function mount(h: Harness, bindings: InstanceTracking[], sched: SchedulerDeps) {
|
|
432
618
|
h.api.manifest.instanceTracking = bindings;
|
|
433
619
|
return mountInstanceTracking(
|
|
@@ -1,17 +1,35 @@
|
|
|
1
1
|
// instance-tracking — the process-instance lifecycle reconciler (manifest `instanceTracking`).
|
|
2
2
|
//
|
|
3
3
|
// The problem it closes: an Urban app derives a row's status from the workers that run
|
|
4
|
-
// along a process's *happy path*. When an instance instead reaches
|
|
5
|
-
// no
|
|
6
|
-
// runs, so the read-model row stays "active" forever and any UI derived from it lies.
|
|
4
|
+
// along a process's *happy path*. When an instance instead reaches an engine-truth state with
|
|
5
|
+
// no worker to record it, the read-model row lies. Two such edges are reconciled declaratively:
|
|
7
6
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
//
|
|
14
|
-
//
|
|
7
|
+
// - The **terminal** edge (`onTerminated`): each binding's poll asks the engine which of its
|
|
8
|
+
// active instances are `TERMINATED` and applies `onTerminated.set` (→ e.g. `abandoned`). A
|
|
9
|
+
// terminated instance runs no completion worker, so nothing else would ever close the row.
|
|
10
|
+
//
|
|
11
|
+
// - The **wait-on-human** edge (`onWaitingHuman`, issue #355): the symmetric primitive for
|
|
12
|
+
// "this instance is parked waiting on a human." That state is authoritatively knowable — an
|
|
13
|
+
// instance is waiting on a human *iff* it has an open user task (`openUserTasks` =
|
|
14
|
+
// `searchUserTasks({state:"CREATED"})`) — for every process, forever, with no per-element
|
|
15
|
+
// registry. On each poll, an active, non-terminated row whose instance has any open user task
|
|
16
|
+
// has `onWaitingHuman.set` applied (typically `status = "awaiting_operator"`). This retires the
|
|
17
|
+
// per-subject imperative pattern (workers writing the status on escalate/answer plus a bespoke
|
|
18
|
+
// self-heal poller): the parked status becomes a derived view over the one user-task
|
|
19
|
+
// projection, so there is no second surface to drift (No Drift Surfaces).
|
|
20
|
+
//
|
|
21
|
+
// Precedence per active row, all derived from engine truth (never a worker write):
|
|
22
|
+
// terminated -> onTerminated.set (e.g. abandoned) // terminal edge
|
|
23
|
+
// else has open user task -> onWaitingHuman.set (e.g. awaiting_operator) // wait-on-human edge
|
|
24
|
+
// else -> leave the worker-owned transient status (running / converging / …)
|
|
25
|
+
//
|
|
26
|
+
// It deliberately reconciles the terminal edge only on `TERMINATED` — the state that by
|
|
27
|
+
// definition runs no completion worker — and never on `COMPLETED`, whose terminal row-write the
|
|
28
|
+
// app's own finalize worker owns (reconciling it here would race that worker and could clobber a
|
|
29
|
+
// legitimately-completed row). The wait-on-human edge reads *open* user tasks via `openUserTasks`
|
|
30
|
+
// (never a bare `searchUserTasks`, which lags a completion and would latch `awaiting_operator`
|
|
31
|
+
// onto a just-answered task). The poll is driven through the shared `SchedulerDeps` seam so tests
|
|
32
|
+
// are deterministic.
|
|
15
33
|
|
|
16
34
|
import type { AppApi, Mounted, RuntimeContext } from "../context.ts";
|
|
17
35
|
import { type InstanceTracking, isConfiguredStatusSelector } from "../manifest.ts";
|
|
@@ -20,6 +38,13 @@ import { defaultScheduler, MAX_TIMER_DELAY_MS, type SchedulerDeps } from "./sche
|
|
|
20
38
|
/** Default poll interval when a binding does not set `pollMs`. */
|
|
21
39
|
export const DEFAULT_INSTANCE_TRACKING_POLL_MS = 15_000;
|
|
22
40
|
|
|
41
|
+
/** Max in-flight `openUserTasks` probes per reconcile pass. The wait-on-human edge probes the
|
|
42
|
+
* engine once per active, non-terminated key; probing them sequentially makes a pass O(N) network
|
|
43
|
+
* round-trips and can make a single pass take longer than `pollMs` under a large active backlog.
|
|
44
|
+
* Probing in bounded-parallel batches makes wall-clock time scale with max latency rather than
|
|
45
|
+
* N×latency, while the cap keeps in-flight load on the engine bounded. */
|
|
46
|
+
export const WAITING_HUMAN_PROBE_CONCURRENCY = 8;
|
|
47
|
+
|
|
23
48
|
export interface InstanceTrackingHandle extends Mounted {
|
|
24
49
|
/** `table.keyField` labels of the armed bindings, for `inspect()`. */
|
|
25
50
|
readonly bindings: string[];
|
|
@@ -56,8 +81,60 @@ export async function reconcileTerminatedKey(
|
|
|
56
81
|
return reconciled;
|
|
57
82
|
}
|
|
58
83
|
|
|
84
|
+
/** Whether `row` already satisfies `patch` on every column it sets — used to make the
|
|
85
|
+
* wait-on-human writer quiet-idempotent. Unlike a terminated row (which flips to a terminal
|
|
86
|
+
* status and then drops out of the active set), a row parked at a user task stays active and is
|
|
87
|
+
* re-polled every tick, so re-writing/re-logging the same `awaiting_operator` patch on every poll
|
|
88
|
+
* would be noise; skip the write when it would be a no-op. */
|
|
89
|
+
function patchAlreadyApplied(row: Row, set: Record<string, string | number | boolean | null>): boolean {
|
|
90
|
+
return Object.entries(set).every(([col, value]) => row[col] === value);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Apply a binding's `onWaitingHuman.set` patch to the single row keyed by `processInstanceKey`
|
|
94
|
+
* — the wait-on-human twin of {@link reconcileTerminatedKey} (issue #355). The caller establishes
|
|
95
|
+
* the precondition (the instance is non-terminated and has an open user task); this is the single
|
|
96
|
+
* place that writes `onWaitingHuman.set` and logs the reconcile. Bindings without an
|
|
97
|
+
* `onWaitingHuman` edge are no-ops, so callers can fan this over every binding. It skips the write
|
|
98
|
+
* when the row already carries the patch (quiet-idempotent — a long-parked instance is re-polled
|
|
99
|
+
* every tick and must not re-log). Returns rows changed. */
|
|
100
|
+
export async function reconcileWaitingHumanKey(
|
|
101
|
+
api: Pick<AppApi, "data" | "log">,
|
|
102
|
+
bindings: InstanceTracking[],
|
|
103
|
+
processInstanceKey: string,
|
|
104
|
+
): Promise<number> {
|
|
105
|
+
const key = String(processInstanceKey);
|
|
106
|
+
let reconciled = 0;
|
|
107
|
+
for (const binding of bindings) {
|
|
108
|
+
const onWaitingHuman = binding.onWaitingHuman;
|
|
109
|
+
if (!onWaitingHuman) continue;
|
|
110
|
+
const table = api.data.table<Row>(binding.table, binding.keyField);
|
|
111
|
+
// Quiet-idempotence must consider EVERY row sharing this key, not just one. `keyField` need not
|
|
112
|
+
// be a unique column (e.g. a read model keyed on a non-PK `process_key`), and `table.update`
|
|
113
|
+
// patches ALL matching rows — so a `get()` (LIMIT 1) that happens to hit one already-patched row
|
|
114
|
+
// would skip the write and leave the OTHER matching rows unreconciled. Skip only when every
|
|
115
|
+
// matching row already carries the patch; otherwise fall through and re-patch them all.
|
|
116
|
+
const current = await table.find({ [binding.keyField]: key });
|
|
117
|
+
if (current.length > 0 && current.every((row) => patchAlreadyApplied(row, onWaitingHuman.set))) {
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
const changed = await table.update(key, onWaitingHuman.set);
|
|
121
|
+
if (changed > 0) {
|
|
122
|
+
reconciled += changed;
|
|
123
|
+
api.log("info", "instanceTracking: reconciled instance awaiting operator", {
|
|
124
|
+
table: binding.table,
|
|
125
|
+
keyField: binding.keyField,
|
|
126
|
+
processInstanceKey: key,
|
|
127
|
+
rowsChanged: changed,
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return reconciled;
|
|
132
|
+
}
|
|
133
|
+
|
|
59
134
|
/** Poll one binding's instances once. Selects the active rows, asks the engine which of their
|
|
60
|
-
* instances are TERMINATED, and applies `onTerminated.set` to each
|
|
135
|
+
* instances are TERMINATED, and applies `onTerminated.set` to each; then, for the binding's
|
|
136
|
+
* wait-on-human edge (`onWaitingHuman`, issue #355), applies `onWaitingHuman.set` to every
|
|
137
|
+
* non-terminated active row whose instance has an open user task. Returns how many keys were
|
|
61
138
|
* scanned and how many rows were reconciled this tick. */
|
|
62
139
|
async function reconcileOnce(
|
|
63
140
|
api: AppApi,
|
|
@@ -116,11 +193,38 @@ async function reconcileOnce(
|
|
|
116
193
|
// writes `onTerminated.set` and logs the reconcile — so the poll path and the cancel primitive
|
|
117
194
|
// can never drift on the patch or its logging (No Drift Surfaces).
|
|
118
195
|
let reconciled = 0;
|
|
196
|
+
const terminatedKeys = new Set<string>();
|
|
119
197
|
for (const snap of snapshots) {
|
|
120
198
|
if (snap.state !== "TERMINATED") continue; // defensive; we asked for TERMINATED only
|
|
121
199
|
if (!keyToRows.has(snap.processInstanceKey)) continue; // not one we were tracking
|
|
200
|
+
terminatedKeys.add(snap.processInstanceKey);
|
|
122
201
|
reconciled += await reconcileTerminatedKey(api, [binding], snap.processInstanceKey);
|
|
123
202
|
}
|
|
203
|
+
|
|
204
|
+
// Wait-on-human edge (issue #355), symmetric to onTerminated. Precedence: terminated wins, so a
|
|
205
|
+
// key reconciled as terminated above is excluded here. An instance is "waiting on a human" iff
|
|
206
|
+
// it has an open user task — the authoritative engine truth — so read `openUserTasks` (never a
|
|
207
|
+
// bare `searchUserTasks`, which lags a completion and would latch `awaiting_operator` onto a
|
|
208
|
+
// just-answered task; issue #355 "instance B"). When there is no open user task the row is left
|
|
209
|
+
// untouched, so a worker-owned transient status (running / converging / merging) survives and a
|
|
210
|
+
// stale `awaiting_operator` is not re-latched. Because the condition is derived every tick, a
|
|
211
|
+
// re-escalation after an answer re-flips to `awaiting_operator` on the next poll by construction
|
|
212
|
+
// (issue #355 "instance A"). Probe in bounded-parallel batches (`WAITING_HUMAN_PROBE_CONCURRENCY`)
|
|
213
|
+
// so a pass over a large active backlog scales with max latency rather than N×latency.
|
|
214
|
+
if (binding.onWaitingHuman) {
|
|
215
|
+
const humanKeys = keys.filter((key) => !terminatedKeys.has(key)); // terminated already won
|
|
216
|
+
for (let i = 0; i < humanKeys.length; i += WAITING_HUMAN_PROBE_CONCURRENCY) {
|
|
217
|
+
const batch = humanKeys.slice(i, i + WAITING_HUMAN_PROBE_CONCURRENCY);
|
|
218
|
+
const perKey = await Promise.all(
|
|
219
|
+
batch.map(async (key): Promise<number> => {
|
|
220
|
+
const openTasks = await api.engine.openUserTasks({ processInstanceKey: key });
|
|
221
|
+
if (openTasks.length === 0) return 0; // not parked on a human ⇒ leave the transient status
|
|
222
|
+
return reconcileWaitingHumanKey(api, [binding], key);
|
|
223
|
+
}),
|
|
224
|
+
);
|
|
225
|
+
for (const n of perKey) reconciled += n;
|
|
226
|
+
}
|
|
227
|
+
}
|
|
124
228
|
return { scanned: keys.length, reconciled };
|
|
125
229
|
}
|
|
126
230
|
|
|
@@ -166,6 +166,52 @@ test("an instanceTracking binding missing table/keyField/onTerminated.set is fla
|
|
|
166
166
|
assert.ok(issues.some((i) => i.path === "instanceTracking[0].onTerminated.set"));
|
|
167
167
|
});
|
|
168
168
|
|
|
169
|
+
test("an instanceTracking binding with a well-formed onWaitingHuman.set has no issues", () => {
|
|
170
|
+
const issues = collectManifestIssues({
|
|
171
|
+
...valid,
|
|
172
|
+
instanceTracking: [
|
|
173
|
+
{
|
|
174
|
+
table: "plans",
|
|
175
|
+
keyField: "process_key",
|
|
176
|
+
statusField: "status",
|
|
177
|
+
terminalStatuses: ["abandoned"],
|
|
178
|
+
onTerminated: { set: { status: "abandoned" } },
|
|
179
|
+
onWaitingHuman: { set: { status: "awaiting_operator" } },
|
|
180
|
+
},
|
|
181
|
+
],
|
|
182
|
+
});
|
|
183
|
+
assert.deepEqual(issues, []);
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test("an instanceTracking onWaitingHuman with an empty set patch is flagged", () => {
|
|
187
|
+
const issues = collectManifestIssues({
|
|
188
|
+
...valid,
|
|
189
|
+
instanceTracking: [
|
|
190
|
+
{
|
|
191
|
+
table: "plans",
|
|
192
|
+
keyField: "process_key",
|
|
193
|
+
onTerminated: { set: { status: "abandoned" } },
|
|
194
|
+
onWaitingHuman: { set: {} },
|
|
195
|
+
},
|
|
196
|
+
],
|
|
197
|
+
});
|
|
198
|
+
assert.ok(issues.some((i) => i.path === "instanceTracking[0].onWaitingHuman.set"));
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
test("an instanceTracking binding omitting onWaitingHuman is valid (the edge is opt-in)", () => {
|
|
202
|
+
const issues = collectManifestIssues({
|
|
203
|
+
...valid,
|
|
204
|
+
instanceTracking: [
|
|
205
|
+
{
|
|
206
|
+
table: "plans",
|
|
207
|
+
keyField: "process_key",
|
|
208
|
+
onTerminated: { set: { status: "abandoned" } },
|
|
209
|
+
},
|
|
210
|
+
],
|
|
211
|
+
});
|
|
212
|
+
assert.deepEqual(issues, []);
|
|
213
|
+
});
|
|
214
|
+
|
|
169
215
|
test("instanceTracking activeStatuses without statusField is flagged", () => {
|
|
170
216
|
const issues = collectManifestIssues({
|
|
171
217
|
...valid,
|
|
@@ -201,6 +201,21 @@ export function collectManifestIssues(m: unknown): ValidationIssue[] {
|
|
|
201
201
|
message: "missing onTerminated.set patch (a non-empty column → value map)",
|
|
202
202
|
});
|
|
203
203
|
}
|
|
204
|
+
// `onWaitingHuman` (issue #355) is the optional wait-on-human edge — the twin of
|
|
205
|
+
// `onTerminated`. It is optional (a binding may reconcile only the terminal edge), but when
|
|
206
|
+
// present its `set` must be a non-empty column → value patch for the same reason
|
|
207
|
+
// `onTerminated.set` must be: a reconciler with nothing to write is inert and almost
|
|
208
|
+
// certainly a mistake.
|
|
209
|
+
if (b?.onWaitingHuman !== undefined) {
|
|
210
|
+
const onWaitingHuman = isRecord(b.onWaitingHuman) ? b.onWaitingHuman : undefined;
|
|
211
|
+
const waitingSet = isRecord(onWaitingHuman?.set) ? onWaitingHuman.set : undefined;
|
|
212
|
+
if (!waitingSet || Object.keys(waitingSet).length === 0) {
|
|
213
|
+
issues.push({
|
|
214
|
+
path: `instanceTracking[${i}].onWaitingHuman.set`,
|
|
215
|
+
message: "missing onWaitingHuman.set patch (a non-empty column → value map)",
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
}
|
|
204
219
|
// `activeStatuses` (fail-closed allow-list) and `terminalStatuses` (fail-open exclusion) are
|
|
205
220
|
// both consumed as arrays of status strings. A malformed value — a bare string, or an array
|
|
206
221
|
// holding a non-string/empty entry — would misbehave at runtime (`new Set("abandoned")`
|