@inflexa-ai/harness 0.44.0 → 0.45.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/conversation-agent.d.ts +10 -0
- package/dist/agents/conversation-agent.js +19 -1
- package/dist/agents/sandbox/shared.d.ts +23 -2
- package/dist/agents/sandbox/shared.js +51 -18
- package/dist/agents/sandbox/translational-safety-agent.js +26 -0
- package/dist/agents/sandbox/types.d.ts +24 -1
- package/dist/agents/sandbox-catalog.d.ts +14 -0
- package/dist/agents/sandbox-catalog.js +19 -0
- package/dist/app/approve-plan.d.ts +51 -0
- package/dist/app/approve-plan.js +84 -0
- package/dist/app/chat-turn.js +6 -3
- package/dist/app/data-profile-view.d.ts +1 -1
- package/dist/app/run-activity.js +4 -6
- package/dist/config/environment-stores.d.ts +4 -2
- package/dist/contracts/chat-events.d.ts +14 -2
- package/dist/contracts/chat-events.js +1 -1
- package/dist/contracts/chat-frame.d.ts +7 -0
- package/dist/contracts/chat-frame.js +24 -6
- package/dist/contracts/chat-parts.d.ts +73 -2
- package/dist/contracts/chat-parts.js +3 -1
- package/dist/contracts/index.d.ts +4 -2
- package/dist/contracts/index.js +1 -0
- package/dist/contracts/part-registry.js +3 -0
- package/dist/contracts/schemas/chat-events.d.ts +38 -0
- package/dist/contracts/schemas/chat-events.js +8 -0
- package/dist/contracts/schemas/chat-parts.d.ts +239 -15
- package/dist/contracts/schemas/chat-parts.js +80 -12
- package/dist/contracts/schemas/index.d.ts +2 -2
- package/dist/contracts/schemas/index.js +2 -2
- package/dist/contracts/subagent-activity.d.ts +12 -0
- package/dist/contracts/subagent-activity.js +76 -0
- package/dist/execution/artifact-metadata.js +1 -3
- package/dist/execution/dbos-workflow-purger.d.ts +2 -0
- package/dist/execution/dbos-workflow-purger.js +21 -14
- package/dist/execution/run-canceler.d.ts +33 -11
- package/dist/execution/run-canceler.js +130 -28
- package/dist/execution/run-end-record.d.ts +30 -0
- package/dist/execution/run-end-record.js +27 -0
- package/dist/execution/run-event-parts.d.ts +6 -5
- package/dist/execution/run-event-parts.js +6 -5
- package/dist/execution/run-event-stream.d.ts +18 -6
- package/dist/execution/run-event-stream.js +67 -17
- package/dist/execution/run-synthesis.d.ts +21 -0
- package/dist/execution/run-synthesis.js +77 -3
- package/dist/index.d.ts +28 -10
- package/dist/index.js +26 -2
- package/dist/lib/hooks.d.ts +1 -1
- package/dist/lib/metrics.d.ts +2 -2
- package/dist/loop/call-guard.d.ts +50 -0
- package/dist/loop/call-guard.js +90 -0
- package/dist/loop/compaction.d.ts +5 -0
- package/dist/loop/compaction.js +17 -0
- package/dist/loop/run-agent.d.ts +9 -0
- package/dist/loop/run-agent.js +16 -1
- package/dist/loop/run-to-terminal.js +4 -1
- package/dist/loop/types.d.ts +8 -0
- package/dist/memory/card-builders.d.ts +11 -0
- package/dist/memory/card-builders.js +35 -21
- package/dist/memory/conversation-display-recorder.d.ts +2 -2
- package/dist/memory/conversation-display-recorder.js +38 -4
- package/dist/memory/conversation-display-storage.d.ts +9 -2
- package/dist/memory/conversation-display-storage.js +18 -3
- package/dist/memory/thread-history.d.ts +42 -8
- package/dist/memory/thread-history.js +164 -45
- package/dist/memory/thread-store.d.ts +13 -0
- package/dist/memory/thread-store.js +15 -1
- package/dist/prompts/briefing.d.ts +57 -3
- package/dist/prompts/briefing.js +129 -2
- package/dist/prompts/conversation.d.ts +1 -1
- package/dist/prompts/conversation.js +3 -1
- package/dist/prompts/planner.js +13 -2
- package/dist/provenance/seam.d.ts +39 -20
- package/dist/provenance/seam.js +7 -5
- package/dist/providers/ai-sdk.js +93 -11
- package/dist/providers/types.d.ts +6 -0
- package/dist/reference-data/catalog.js +141 -2
- package/dist/report-model/authoring-grammar.d.ts +26 -26
- package/dist/report-model/pin-snapshot.d.ts +1 -1
- package/dist/report-model/pin-snapshot.js +2 -50
- package/dist/runtime/assemble.d.ts +8 -1
- package/dist/runtime/assemble.js +11 -0
- package/dist/runtime/boot.d.ts +3 -0
- package/dist/runtime/boot.js +10 -1
- package/dist/sandbox/image-packages.d.ts +26 -4
- package/dist/sandbox/image-packages.js +18 -4
- package/dist/sandbox/sandbox-step-translate.d.ts +23 -0
- package/dist/sandbox/sandbox-step-translate.js +24 -0
- package/dist/schemas/artifact-manifest.d.ts +1 -1
- package/dist/schemas/artifact-manifest.js +11 -9
- package/dist/schemas/data-profile-schemas.d.ts +12 -12
- package/dist/schemas/plan-schemas.d.ts +75 -21
- package/dist/schemas/plan-schemas.js +1 -1
- package/dist/schemas/run-synthesis.d.ts +2 -2
- package/dist/schemas/workflow-state.d.ts +113 -0
- package/dist/schemas/workflow-state.js +54 -0
- package/dist/state/analyses-metrics.d.ts +79 -7
- package/dist/state/analyses-metrics.js +141 -6
- package/dist/state/artifacts.d.ts +66 -3
- package/dist/state/artifacts.js +85 -7
- package/dist/state/data-profile.d.ts +4 -2
- package/dist/state/data-profile.js +7 -5
- package/dist/state/index.d.ts +9 -8
- package/dist/state/index.js +5 -5
- package/dist/state/migrations/20261012120000_index_the_runs_by_start.d.ts +8 -0
- package/dist/state/migrations/20261012120000_index_the_runs_by_start.js +12 -0
- package/dist/state/migrations/20261013120000_index_the_ended_runs_and_the_recent_threads.d.ts +8 -0
- package/dist/state/migrations/20261013120000_index_the_ended_runs_and_the_recent_threads.js +13 -0
- package/dist/state/migrations/20261013180000_narrow_the_active_run_index.d.ts +8 -0
- package/dist/state/migrations/20261013180000_narrow_the_active_run_index.js +13 -0
- package/dist/state/migrations/20261014120000_index_the_plans_by_parent_and_the_runs_by_plan.d.ts +8 -0
- package/dist/state/migrations/20261014120000_index_the_plans_by_parent_and_the_runs_by_plan.js +13 -0
- package/dist/state/migrations/index.js +8 -0
- package/dist/state/plans.d.ts +21 -0
- package/dist/state/plans.js +25 -0
- package/dist/state/runs.d.ts +79 -14
- package/dist/state/runs.js +150 -33
- package/dist/state/schema.d.ts +2 -0
- package/dist/state/step-executions.d.ts +9 -0
- package/dist/state/step-executions.js +22 -0
- package/dist/tasks/data-profile-activity.d.ts +7 -1
- package/dist/tasks/data-profile-activity.js +18 -9
- package/dist/tasks/data-profile.d.ts +15 -1
- package/dist/tasks/data-profile.js +16 -11
- package/dist/tools/ad-hoc-router.d.ts +2 -0
- package/dist/tools/ad-hoc-router.js +18 -2
- package/dist/tools/approval/gateway.js +28 -5
- package/dist/tools/capability-data.d.ts +36 -0
- package/dist/tools/capability-data.js +73 -0
- package/dist/tools/display/show-user.d.ts +1 -1
- package/dist/tools/execute-analysis.d.ts +80 -1
- package/dist/tools/execute-analysis.js +352 -110
- package/dist/tools/knowledge/check.d.ts +60 -0
- package/dist/tools/knowledge/check.js +92 -0
- package/dist/tools/knowledge/client.d.ts +382 -0
- package/dist/tools/knowledge/client.js +302 -0
- package/dist/tools/knowledge/environment.d.ts +78 -0
- package/dist/tools/knowledge/environment.js +105 -0
- package/dist/tools/knowledge/index.d.ts +49 -0
- package/dist/tools/knowledge/index.js +70 -0
- package/dist/tools/knowledge/local-slots.d.ts +62 -0
- package/dist/tools/knowledge/local-slots.js +177 -0
- package/dist/tools/knowledge/pin.d.ts +14 -0
- package/dist/tools/knowledge/pin.js +17 -0
- package/dist/tools/knowledge/recommend.d.ts +92 -0
- package/dist/tools/knowledge/recommend.js +98 -0
- package/dist/tools/knowledge/situation.d.ts +143 -0
- package/dist/tools/knowledge/situation.js +164 -0
- package/dist/tools/knowledge/skeleton.d.ts +69 -0
- package/dist/tools/knowledge/skeleton.js +202 -0
- package/dist/tools/knowledge/template.d.ts +137 -0
- package/dist/tools/knowledge/template.js +267 -0
- package/dist/tools/research/generate-plan.d.ts +23 -0
- package/dist/tools/research/generate-plan.js +203 -12
- package/dist/tools/research/guarded-address.d.ts +20 -0
- package/dist/tools/research/guarded-address.js +148 -0
- package/dist/tools/research/inspect-data-profile.d.ts +1 -1
- package/dist/tools/research/inspect-run.d.ts +1 -1
- package/dist/tools/research/inspect-run.js +7 -7
- package/dist/tools/research/searxng-engine.d.ts +40 -0
- package/dist/tools/research/searxng-engine.js +120 -0
- package/dist/tools/research/web-fetch.d.ts +124 -0
- package/dist/tools/research/web-fetch.js +627 -0
- package/dist/tools/research/web-search.d.ts +75 -0
- package/dist/tools/research/web-search.js +112 -0
- package/dist/tools/sandbox/list-available-packages.d.ts +7 -4
- package/dist/tools/sandbox/list-available-packages.js +12 -4
- package/dist/tools/sandbox/list-available-refs.d.ts +21 -0
- package/dist/tools/sandbox/list-available-refs.js +60 -3
- package/dist/tools/sandbox/report-blocker.js +5 -1
- package/dist/tools/terminal-text.d.ts +11 -0
- package/dist/tools/terminal-text.js +19 -0
- package/dist/tools/workspace/decision-record.d.ts +81 -0
- package/dist/tools/workspace/decision-record.js +117 -0
- package/dist/tools/workspace/edit-file.d.ts +3 -1
- package/dist/tools/workspace/edit-file.js +20 -2
- package/dist/tools/workspace/execute-command.d.ts +17 -0
- package/dist/tools/workspace/execute-command.js +28 -1
- package/dist/tools/workspace/index.d.ts +1 -0
- package/dist/tools/workspace/index.js +1 -0
- package/dist/tools/workspace/mutator.d.ts +1 -1
- package/dist/workflows/execute-analysis.d.ts +62 -21
- package/dist/workflows/execute-analysis.js +307 -79
- package/dist/workflows/plan/graph.d.ts +16 -0
- package/dist/workflows/plan/graph.js +36 -0
- package/dist/workflows/sandbox-step.d.ts +12 -0
- package/dist/workflows/sandbox-step.js +14 -12
- package/package.json +5 -1
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// The sub-agent activity of a chat turn, beside `applyChatFrame`. Browser-safe: each import from outside `contracts/` is a type import.
|
|
2
|
+
/** The most entries that one root tool call holds. Each later call of its sub-agents only adds to `dropped`. */
|
|
3
|
+
export const SUBAGENT_ACTIVITY_MAX_ENTRIES = 200;
|
|
4
|
+
// A call id is unique only inside the loop that made the call, thus the call path of that loop is part of the key.
|
|
5
|
+
// The path is the lineage of the sub-agent, the same read as `isRootFrame`. No control flow of the harness uses it.
|
|
6
|
+
function callKey(loopPath, callId) {
|
|
7
|
+
return JSON.stringify([loopPath, callId]);
|
|
8
|
+
}
|
|
9
|
+
export function createSubAgentActivityTracker() {
|
|
10
|
+
// A dropped call stays in the map, thus a deeper call below it still counts in `dropped`.
|
|
11
|
+
const calls = new Map();
|
|
12
|
+
const records = new Map();
|
|
13
|
+
const partOf = (rootId, record) => ({
|
|
14
|
+
type: "data-subagent-activity",
|
|
15
|
+
id: `subagent-activity-${rootId}`,
|
|
16
|
+
toolCallId: rootId,
|
|
17
|
+
entries: record.entries.map((entry) => ({ ...entry, callPath: [...entry.callPath] })),
|
|
18
|
+
dropped: record.dropped,
|
|
19
|
+
});
|
|
20
|
+
return {
|
|
21
|
+
track(frame, nowMs) {
|
|
22
|
+
if (frame.type !== "tool-started" && frame.type !== "tool-finished")
|
|
23
|
+
return null;
|
|
24
|
+
const { source } = frame;
|
|
25
|
+
if (frame.type === "tool-started") {
|
|
26
|
+
if (source.callPath.length <= 1) {
|
|
27
|
+
calls.set(callKey(source.callPath, frame.toolUseId), { rootId: frame.toolUseId, startedMs: nowMs });
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
if (source.invocationId === undefined)
|
|
31
|
+
return null;
|
|
32
|
+
const parent = calls.get(callKey(source.callPath.slice(0, -1), source.invocationId));
|
|
33
|
+
if (parent === undefined)
|
|
34
|
+
return null;
|
|
35
|
+
let record = records.get(parent.rootId);
|
|
36
|
+
if (record === undefined) {
|
|
37
|
+
record = { entries: [], dropped: 0 };
|
|
38
|
+
records.set(parent.rootId, record);
|
|
39
|
+
}
|
|
40
|
+
if (record.entries.length >= SUBAGENT_ACTIVITY_MAX_ENTRIES) {
|
|
41
|
+
record.dropped += 1;
|
|
42
|
+
calls.set(callKey(source.callPath, frame.toolUseId), { rootId: parent.rootId, startedMs: nowMs });
|
|
43
|
+
}
|
|
44
|
+
else {
|
|
45
|
+
record.entries.push({
|
|
46
|
+
agentId: source.agentId,
|
|
47
|
+
callPath: [...source.callPath],
|
|
48
|
+
tool: frame.name,
|
|
49
|
+
...(frame.detail === undefined ? {} : { label: frame.detail }),
|
|
50
|
+
startedAt: new Date(nowMs).toISOString(),
|
|
51
|
+
});
|
|
52
|
+
calls.set(callKey(source.callPath, frame.toolUseId), {
|
|
53
|
+
rootId: parent.rootId,
|
|
54
|
+
entryIndex: record.entries.length - 1,
|
|
55
|
+
startedMs: nowMs,
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
return partOf(parent.rootId, record);
|
|
59
|
+
}
|
|
60
|
+
if (source.callPath.length <= 1)
|
|
61
|
+
return null;
|
|
62
|
+
const call = calls.get(callKey(source.callPath, frame.toolUseId));
|
|
63
|
+
if (call?.entryIndex === undefined)
|
|
64
|
+
return null;
|
|
65
|
+
const record = records.get(call.rootId);
|
|
66
|
+
const entry = record?.entries[call.entryIndex];
|
|
67
|
+
if (record === undefined || entry === undefined)
|
|
68
|
+
return null;
|
|
69
|
+
entry.outcome = frame.outcome;
|
|
70
|
+
if (frame.detail !== undefined)
|
|
71
|
+
entry.label = frame.detail;
|
|
72
|
+
entry.endedAt = new Date(frame.durationMs === undefined ? nowMs : call.startedMs + frame.durationMs).toISOString();
|
|
73
|
+
return partOf(call.rootId, record);
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
@@ -78,9 +78,7 @@ function fileExtension(displayPath) {
|
|
|
78
78
|
* composed from facts already on disk so search still finds the file.
|
|
79
79
|
*/
|
|
80
80
|
function fallbackEntry(artifact, extra) {
|
|
81
|
-
|
|
82
|
-
// on a working-directory-relative display path.
|
|
83
|
-
const fileType = inferArtifactType(`/${artifact.displayPath}`);
|
|
81
|
+
const fileType = inferArtifactType(artifact.displayPath);
|
|
84
82
|
const format = fileExtension(artifact.displayPath);
|
|
85
83
|
const sizePart = artifact.sizeBytes != null ? `, ${artifact.sizeBytes} bytes` : "";
|
|
86
84
|
const description = `${artifact.displayPath} — ${fileType} file (${format}${sizePart}); automated description unavailable.`;
|
|
@@ -19,6 +19,7 @@
|
|
|
19
19
|
* and queue rows each hold an `ON DELETE CASCADE` foreign key to
|
|
20
20
|
* `workflow_status`, so they go with the status row.
|
|
21
21
|
*/
|
|
22
|
+
import { DBOSClient } from "@dbos-inc/dbos-sdk";
|
|
22
23
|
import type { Pool } from "pg";
|
|
23
24
|
import type { Logger } from "../lib/logger.js";
|
|
24
25
|
import type { WorkflowPurger } from "./workflow-purger.js";
|
|
@@ -27,4 +28,5 @@ export interface DbosWorkflowPurgerDeps {
|
|
|
27
28
|
readonly pool: Pool;
|
|
28
29
|
readonly logger?: Logger;
|
|
29
30
|
}
|
|
31
|
+
export declare function dbosClientFor(pool: Pool, logger: Logger): Promise<DBOSClient>;
|
|
30
32
|
export declare function createDbosWorkflowPurger({ pool, logger: injected }: DbosWorkflowPurgerDeps): WorkflowPurger;
|
|
@@ -64,23 +64,30 @@ function asEngineLogger(logger) {
|
|
|
64
64
|
error: (entry, metadata) => logger.error(render(entry), metadata?.stack === undefined ? undefined : { stack: metadata.stack }),
|
|
65
65
|
};
|
|
66
66
|
}
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
67
|
+
/**
|
|
68
|
+
* One client for each pool, shared by every purger and by the run canceler. A client registers
|
|
69
|
+
* `error` and `connect` handlers on the pool it is handed, so a client for each caller would
|
|
70
|
+
* accumulate handler pairs on a long-lived pool. Its `destroy()` ends that pool, so a client over
|
|
71
|
+
* an injected pool must never be destroyed either — which leaves a per-call client nothing to
|
|
72
|
+
* release. Creation is lazy so a host that never reaches the engine builds no client. The logger of
|
|
73
|
+
* the first caller for a pool becomes the engine logger of that pool.
|
|
74
|
+
*/
|
|
75
|
+
const clients = new WeakMap();
|
|
76
|
+
export function dbosClientFor(pool, logger) {
|
|
77
|
+
let client = clients.get(pool);
|
|
78
|
+
if (client === undefined) {
|
|
77
79
|
// `create` never initializes the system database, and initialization is the
|
|
78
80
|
// only path that reads the connection string once a pool is supplied — so the
|
|
79
81
|
// url is unreachable from this client rather than merely untaken, and there is
|
|
80
|
-
//
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
}
|
|
82
|
+
// no url to pass.
|
|
83
|
+
client = DBOSClient.create({ systemDatabaseUrl: "", systemDatabasePool: pool, logger: asEngineLogger(logger) });
|
|
84
|
+
clients.set(pool, client);
|
|
85
|
+
}
|
|
86
|
+
return client;
|
|
87
|
+
}
|
|
88
|
+
export function createDbosWorkflowPurger({ pool, logger: injected }) {
|
|
89
|
+
const logger = (injected ?? createNoopLogger()).named("workflow-purger");
|
|
90
|
+
const engine = () => dbosClientFor(pool, logger);
|
|
84
91
|
/**
|
|
85
92
|
* Whether the engine's schema exists at all — the only thing that separates a
|
|
86
93
|
* ledger the engine never created from one that is broken. It costs a round trip
|
|
@@ -1,18 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* RunCanceler — the encapsulated external-cancel path for an `executeAnalysis` run.
|
|
3
3
|
*
|
|
4
|
-
* A cancelled DBOS workflow can never
|
|
5
|
-
* terminal block (`collectAndComplete`)
|
|
6
|
-
* nothing converges: the run row stays `running` (blocking same-plan relaunch
|
|
7
|
-
* through `queryActiveRun`'s active-status dedup)
|
|
8
|
-
* sweep
|
|
9
|
-
*
|
|
10
|
-
* parent + children, then converge
|
|
4
|
+
* A cancelled DBOS workflow can never start another step, so the body's own
|
|
5
|
+
* terminal block (`collectAndComplete`) can be unreachable once the cancel lands
|
|
6
|
+
* and nothing converges: the run row stays `running` (blocking same-plan relaunch
|
|
7
|
+
* through `queryActiveRun`'s active-status dedup) and pending step rows never
|
|
8
|
+
* sweep. A step that started before the cancel still runs to its end, and its
|
|
9
|
+
* run status write is conditional on a `running` row. This module is the
|
|
10
|
+
* host-side counterpart: cancel the engine's parent + children, then converge
|
|
11
|
+
* the ledgers the workflow would have.
|
|
12
|
+
*
|
|
13
|
+
* The writer that wins the conditional terminal write of the run row owns the
|
|
14
|
+
* run end: the run emit, the charge close, and the mandate revoke. This module
|
|
15
|
+
* does those acts only when its write changed the row. A refused write means
|
|
16
|
+
* the body won the row and owns them; a failed write leaves the row `running`,
|
|
17
|
+
* so a second cancel converges the run. The step rows settle on every path.
|
|
18
|
+
*
|
|
19
|
+
* One exception: after a failed status write the body does the run end and
|
|
20
|
+
* records it as a workflow event (`run-end-record.ts`), while the row stays
|
|
21
|
+
* `running`. This module reads that record before its terminal write. A present
|
|
22
|
+
* record makes the terminal write carry the recorded status and error, and no
|
|
23
|
+
* act of the run end follows. A failed read writes nothing, so a second cancel
|
|
24
|
+
* converges the run.
|
|
11
25
|
*
|
|
12
26
|
* The engine capability stays internal — the production default is the
|
|
13
27
|
* purger's `DBOSClient`-backed child-cascading cancel, which needs no launched
|
|
14
28
|
* engine (see `dbos-workflow-purger.ts` for why the static `DBOS` facade is
|
|
15
|
-
* the wrong tool here). `cancelWorkflows` is an optional
|
|
29
|
+
* the wrong tool here). `cancelWorkflows` is an optional injected function over it.
|
|
16
30
|
*
|
|
17
31
|
* No stream writes: `DBOS.writeStream` writes only from inside the workflow
|
|
18
32
|
* that owns the stream, so the run-event reader (`run-event-stream.ts`)
|
|
@@ -22,6 +36,7 @@ import type { Pool } from "pg";
|
|
|
22
36
|
import type { AgentSession } from "../auth/types.js";
|
|
23
37
|
import type { RunCharge } from "../billing/run-charge.js";
|
|
24
38
|
import type { Logger } from "../lib/logger.js";
|
|
39
|
+
import type { ProvenanceSeam } from "../provenance/seam.js";
|
|
25
40
|
import type { RunStatus } from "../state/schema.js";
|
|
26
41
|
import type { RunAuthorizer } from "./run-authorizer.js";
|
|
27
42
|
export declare class UnknownRunError extends Error {
|
|
@@ -40,8 +55,10 @@ export interface CancelRunResult {
|
|
|
40
55
|
* (the conditional write refuses to clobber it).
|
|
41
56
|
*/
|
|
42
57
|
readonly finalStatus: RunStatus;
|
|
43
|
-
/** What actually converged. `mandate`
|
|
44
|
-
*
|
|
58
|
+
/** What actually converged. `charge` and `mandate` are vacuously true when
|
|
59
|
+
* there is nothing for this cancel to converge — a refused terminal write
|
|
60
|
+
* (the body owns them) and, for `mandate`, a row with no persisted jti — so
|
|
61
|
+
* hosts alert on any false flag. */
|
|
45
62
|
readonly converged: {
|
|
46
63
|
readonly steps: boolean;
|
|
47
64
|
readonly charge: boolean;
|
|
@@ -53,9 +70,14 @@ export interface RunCancelerDeps {
|
|
|
53
70
|
readonly runCharge: RunCharge;
|
|
54
71
|
readonly runAuthorizer: RunAuthorizer;
|
|
55
72
|
readonly logger?: Logger;
|
|
56
|
-
/**
|
|
73
|
+
/** Receives the end of a run that this canceler cancels. An absent binding, or a binding with no run emit, emits nothing. */
|
|
74
|
+
readonly provenance?: ProvenanceSeam;
|
|
75
|
+
/** The engine cancel. Production default: the workflow
|
|
57
76
|
* purger's `DBOSClient`-backed cancel, child-cascading and launch-free. */
|
|
58
77
|
readonly cancelWorkflows?: (workflowIds: readonly string[]) => Promise<void>;
|
|
78
|
+
/** The run end that the body recorded: `null` when it recorded none, a rejection when the read fails.
|
|
79
|
+
* The value is unchecked engine data. Production default: `DBOSClient.getEvent` with a zero timeout. */
|
|
80
|
+
readonly readRecordedRunEnd?: (runId: string) => Promise<unknown>;
|
|
59
81
|
}
|
|
60
82
|
export interface RunCanceler {
|
|
61
83
|
/**
|
|
@@ -1,18 +1,32 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* RunCanceler — the encapsulated external-cancel path for an `executeAnalysis` run.
|
|
3
3
|
*
|
|
4
|
-
* A cancelled DBOS workflow can never
|
|
5
|
-
* terminal block (`collectAndComplete`)
|
|
6
|
-
* nothing converges: the run row stays `running` (blocking same-plan relaunch
|
|
7
|
-
* through `queryActiveRun`'s active-status dedup)
|
|
8
|
-
* sweep
|
|
9
|
-
*
|
|
10
|
-
* parent + children, then converge
|
|
4
|
+
* A cancelled DBOS workflow can never start another step, so the body's own
|
|
5
|
+
* terminal block (`collectAndComplete`) can be unreachable once the cancel lands
|
|
6
|
+
* and nothing converges: the run row stays `running` (blocking same-plan relaunch
|
|
7
|
+
* through `queryActiveRun`'s active-status dedup) and pending step rows never
|
|
8
|
+
* sweep. A step that started before the cancel still runs to its end, and its
|
|
9
|
+
* run status write is conditional on a `running` row. This module is the
|
|
10
|
+
* host-side counterpart: cancel the engine's parent + children, then converge
|
|
11
|
+
* the ledgers the workflow would have.
|
|
12
|
+
*
|
|
13
|
+
* The writer that wins the conditional terminal write of the run row owns the
|
|
14
|
+
* run end: the run emit, the charge close, and the mandate revoke. This module
|
|
15
|
+
* does those acts only when its write changed the row. A refused write means
|
|
16
|
+
* the body won the row and owns them; a failed write leaves the row `running`,
|
|
17
|
+
* so a second cancel converges the run. The step rows settle on every path.
|
|
18
|
+
*
|
|
19
|
+
* One exception: after a failed status write the body does the run end and
|
|
20
|
+
* records it as a workflow event (`run-end-record.ts`), while the row stays
|
|
21
|
+
* `running`. This module reads that record before its terminal write. A present
|
|
22
|
+
* record makes the terminal write carry the recorded status and error, and no
|
|
23
|
+
* act of the run end follows. A failed read writes nothing, so a second cancel
|
|
24
|
+
* converges the run.
|
|
11
25
|
*
|
|
12
26
|
* The engine capability stays internal — the production default is the
|
|
13
27
|
* purger's `DBOSClient`-backed child-cascading cancel, which needs no launched
|
|
14
28
|
* engine (see `dbos-workflow-purger.ts` for why the static `DBOS` facade is
|
|
15
|
-
* the wrong tool here). `cancelWorkflows` is an optional
|
|
29
|
+
* the wrong tool here). `cancelWorkflows` is an optional injected function over it.
|
|
16
30
|
*
|
|
17
31
|
* No stream writes: `DBOS.writeStream` writes only from inside the workflow
|
|
18
32
|
* that owns the stream, so the run-event reader (`run-event-stream.ts`)
|
|
@@ -22,9 +36,10 @@ import { createNoopLogger } from "../lib/console-logger.js";
|
|
|
22
36
|
import { deliverNotice } from "../lib/hooks.js";
|
|
23
37
|
import { elapsedSinceIso, recordRunCompleted, recordStepCompleted } from "../lib/metrics.js";
|
|
24
38
|
import { unwrapOrThrow } from "../lib/result.js";
|
|
25
|
-
import { markRunCanceledIfActive, queryRun } from "../state/runs.js";
|
|
26
|
-
import { queryStepsByRun, sweepPendingStepExecutions } from "../state/step-executions.js";
|
|
27
|
-
import { createDbosWorkflowPurger } from "./dbos-workflow-purger.js";
|
|
39
|
+
import { markRunCanceledIfActive, queryRun, updateRunStatus } from "../state/runs.js";
|
|
40
|
+
import { cancelRunningStepExecutions, queryStepsByRun, sweepPendingStepExecutions } from "../state/step-executions.js";
|
|
41
|
+
import { createDbosWorkflowPurger, dbosClientFor } from "./dbos-workflow-purger.js";
|
|
42
|
+
import { parseRecordedRunEnd, RUN_END_EVENT_KEY } from "./run-end-record.js";
|
|
28
43
|
/** The reason stamped on the run row and the revoke — the same literal
|
|
29
44
|
* `collectAndComplete` records for a cancel it observes from inside. */
|
|
30
45
|
const EXTERNAL_CANCEL_REASON = "external_cancel";
|
|
@@ -36,7 +51,7 @@ export class UnknownRunError extends Error {
|
|
|
36
51
|
this.name = "UnknownRunError";
|
|
37
52
|
}
|
|
38
53
|
}
|
|
39
|
-
const TERMINAL = new Set(["completed", "partial", "failed", "canceled"]);
|
|
54
|
+
const TERMINAL = new Set(["completed", "partial", "failed", "canceled", "suspended_insufficient_funds"]);
|
|
40
55
|
export function createRunCanceler(deps) {
|
|
41
56
|
const { pool, runCharge, runAuthorizer } = deps;
|
|
42
57
|
const logger = (deps.logger ?? createNoopLogger()).named("run-canceler");
|
|
@@ -52,9 +67,22 @@ export function createRunCanceler(deps) {
|
|
|
52
67
|
})();
|
|
53
68
|
await engineCancel(workflowIds);
|
|
54
69
|
};
|
|
70
|
+
const readRecordedRunEnd = deps.readRecordedRunEnd ??
|
|
71
|
+
(async (runId) => (await dbosClientFor(pool, logger)).getEvent(runId, RUN_END_EVENT_KEY, 0));
|
|
55
72
|
/** Steps whose child started (a persisted child workflow id) and has not written its terminal row. */
|
|
56
73
|
const incompleteSteps = async (runId) => unwrapOrThrow(await queryStepsByRun(pool, runId)).filter((step) => step.childWorkflowId !== null && step.completedAt === null);
|
|
57
74
|
const childIds = (steps) => steps.flatMap((step) => (step.childWorkflowId !== null ? [step.childWorkflowId] : []));
|
|
75
|
+
const emitRunEvent = deps.provenance?.emitRunEvent;
|
|
76
|
+
const emitGuarded = (event, session) => {
|
|
77
|
+
if (emitRunEvent === undefined)
|
|
78
|
+
return;
|
|
79
|
+
try {
|
|
80
|
+
emitRunEvent(event, session);
|
|
81
|
+
}
|
|
82
|
+
catch (err) {
|
|
83
|
+
logger.error("the run emit of the provenance binding threw", { runId: event.runId, event: event.type, ...logger.errorFields(err) });
|
|
84
|
+
}
|
|
85
|
+
};
|
|
58
86
|
return {
|
|
59
87
|
async cancel(runId, session) {
|
|
60
88
|
const row = unwrapOrThrow(await queryRun(pool, runId));
|
|
@@ -89,38 +117,112 @@ export function createRunCanceler(deps) {
|
|
|
89
117
|
catch (err) {
|
|
90
118
|
logger.error("late-child cancel sweep failed", { runId, ...logger.errorFields(err) });
|
|
91
119
|
}
|
|
92
|
-
//
|
|
93
|
-
|
|
120
|
+
// The run end that the body recorded after a failed status write. A value
|
|
121
|
+
// that is not a recorded run end counts as a failed read.
|
|
122
|
+
let recorded = null;
|
|
123
|
+
let readFailed = false;
|
|
94
124
|
try {
|
|
95
|
-
|
|
125
|
+
const value = await readRecordedRunEnd(runId);
|
|
126
|
+
if (value !== null && value !== undefined) {
|
|
127
|
+
recorded = parseRecordedRunEnd(value);
|
|
128
|
+
if (recorded === null) {
|
|
129
|
+
readFailed = true;
|
|
130
|
+
logger.error("the recorded run end has the wrong shape", { runId });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
96
133
|
}
|
|
97
134
|
catch (err) {
|
|
98
|
-
|
|
135
|
+
readFailed = true;
|
|
136
|
+
logger.error("the read of the recorded run end failed", { runId, ...logger.errorFields(err) });
|
|
137
|
+
}
|
|
138
|
+
// Convergence — each phase isolated so one failure skips nothing after it.
|
|
139
|
+
// The terminal write has three outcomes: changed (this cancel won the row),
|
|
140
|
+
// refused (the body won it), failed (the row is unchanged). A failed read of
|
|
141
|
+
// the recorded run end is a failed write that never ran.
|
|
142
|
+
let writeOutcome = "failed";
|
|
143
|
+
if (!readFailed) {
|
|
144
|
+
try {
|
|
145
|
+
const changed = recorded === null
|
|
146
|
+
? unwrapOrThrow(await markRunCanceledIfActive(pool, runId, EXTERNAL_CANCEL_REASON))
|
|
147
|
+
: unwrapOrThrow(await updateRunStatus(pool, runId, recorded.status, recorded.error));
|
|
148
|
+
writeOutcome = changed ? "changed" : "refused";
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
logger.error("the terminal write of the run failed", { runId, ...logger.errorFields(err) });
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
const transitioned = writeOutcome === "changed";
|
|
155
|
+
// A terminal step of the workflow that started before the cancel can still
|
|
156
|
+
// write, but its write is conditional on a `running` row too. The first
|
|
157
|
+
// write wins, and the run-end acts ride behind the write that won: a
|
|
158
|
+
// refused write means the run settled on its own and its own terminal step
|
|
159
|
+
// does them. A cut child never reached its terminal row, so its step
|
|
160
|
+
// records here too. A recorded run end means the body did the acts, so
|
|
161
|
+
// only the run metric rides behind the repair write.
|
|
162
|
+
if (transitioned && recorded !== null) {
|
|
163
|
+
recordRunCompleted({ workflow: "analysis", status: recorded.status, durationMs: elapsedSinceIso(row.startedAt) });
|
|
99
164
|
}
|
|
100
|
-
|
|
101
|
-
// conditional write is the one terminal write of an external cancel, and
|
|
102
|
-
// the outcome metrics ride behind it: a refused write means the run
|
|
103
|
-
// settled on its own and its own terminal step recorded them. A cut
|
|
104
|
-
// child never reached its terminal row, so its step records here too.
|
|
105
|
-
if (transitioned) {
|
|
165
|
+
if (transitioned && recorded === null) {
|
|
106
166
|
recordRunCompleted({ workflow: "analysis", status: "canceled", durationMs: elapsedSinceIso(row.startedAt) });
|
|
107
167
|
for (const step of cutSteps ?? [])
|
|
108
168
|
recordStepCompleted({ agentId: step.agentId, status: "canceled" });
|
|
169
|
+
// The end of the run rides the same condition. One wall-clock read stamps every event:
|
|
170
|
+
// the canceler is not a workflow, so no replay fires these emits again.
|
|
171
|
+
const atMs = Date.now();
|
|
172
|
+
for (const step of cutSteps ?? []) {
|
|
173
|
+
const durationMs = elapsedSinceIso(step.startedAt, atMs);
|
|
174
|
+
emitGuarded({
|
|
175
|
+
type: "step_completed",
|
|
176
|
+
analysisId: row.analysisId,
|
|
177
|
+
runId,
|
|
178
|
+
stepId: step.stepId,
|
|
179
|
+
status: "canceled",
|
|
180
|
+
...(durationMs !== undefined ? { durationMs } : {}),
|
|
181
|
+
atMs,
|
|
182
|
+
}, session);
|
|
183
|
+
}
|
|
184
|
+
emitGuarded({
|
|
185
|
+
type: "run_completed",
|
|
186
|
+
analysisId: row.analysisId,
|
|
187
|
+
runId,
|
|
188
|
+
status: "canceled",
|
|
189
|
+
atMs,
|
|
190
|
+
durationMs: Math.max(0, atMs - Date.parse(row.startedAt)),
|
|
191
|
+
}, session);
|
|
109
192
|
}
|
|
110
|
-
let
|
|
193
|
+
let pendingSwept = false;
|
|
111
194
|
try {
|
|
112
195
|
unwrapOrThrow(await sweepPendingStepExecutions(pool, runId));
|
|
113
|
-
|
|
196
|
+
pendingSwept = true;
|
|
114
197
|
}
|
|
115
198
|
catch (err) {
|
|
116
199
|
logger.error("pending-step sweep failed", { runId, ...logger.errorFields(err) });
|
|
117
200
|
}
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
(await
|
|
201
|
+
let runningSettled = false;
|
|
202
|
+
try {
|
|
203
|
+
unwrapOrThrow(await cancelRunningStepExecutions(pool, runId, EXTERNAL_CANCEL_REASON));
|
|
204
|
+
runningSettled = true;
|
|
205
|
+
}
|
|
206
|
+
catch (err) {
|
|
207
|
+
logger.error("running-step settle failed", { runId, ...logger.errorFields(err) });
|
|
208
|
+
}
|
|
209
|
+
const steps = pendingSwept && runningSettled;
|
|
210
|
+
// Only the writer that won the row closes the charge and revokes. A refused
|
|
211
|
+
// write left both to the body; a failed write left the row `running`, so a
|
|
212
|
+
// second cancel converges the run. The flags of a refused write are true
|
|
213
|
+
// because there is nothing for this cancel to converge. With a recorded run
|
|
214
|
+
// end the body did both acts, and the flags are its results.
|
|
215
|
+
let charge = recorded !== null ? recorded.closed : writeOutcome === "refused";
|
|
216
|
+
let mandate = recorded !== null ? recorded.revoked : writeOutcome === "refused" || (writeOutcome === "failed" && row.mandateJti === null);
|
|
217
|
+
if (transitioned && recorded === null) {
|
|
218
|
+
charge = await deliverNotice(logger.with({ runId }), "RunCharge.close", runCharge.close({ analysisId: row.analysisId, runId, outcome: { kind: "canceled" }, session }));
|
|
219
|
+
mandate =
|
|
220
|
+
row.mandateJti === null ||
|
|
221
|
+
(await deliverNotice(logger.with({ runId, jti: row.mandateJti }), "RunAuthorizer.revokeByJti", runAuthorizer.revokeByJti({ jti: row.mandateJti, auth: session.auth }, EXTERNAL_CANCEL_REASON)));
|
|
222
|
+
}
|
|
121
223
|
// No transition means the run reached a terminal state on its own (or the
|
|
122
224
|
// write failed) — report the row as it stands rather than claiming `canceled`.
|
|
123
|
-
let finalStatus = "canceled";
|
|
225
|
+
let finalStatus = recorded?.status ?? "canceled";
|
|
124
226
|
if (!transitioned) {
|
|
125
227
|
const reread = unwrapOrThrow(await queryRun(pool, runId));
|
|
126
228
|
finalStatus = reread?.status ?? "canceled";
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
/**
|
|
3
|
+
* The workflow event under which the body of `executeAnalysis` records its run
|
|
4
|
+
* end after a failed status write. The run canceler reads it with
|
|
5
|
+
* `DBOSClient.getEvent`.
|
|
6
|
+
*/
|
|
7
|
+
export declare const RUN_END_EVENT_KEY = "run-end";
|
|
8
|
+
/**
|
|
9
|
+
* The run end that the body did: the status and the error that its failed
|
|
10
|
+
* status write tried to write, and the results of the charge close and of the
|
|
11
|
+
* revoke.
|
|
12
|
+
*/
|
|
13
|
+
export declare const RecordedRunEndSchema: z.ZodObject<{
|
|
14
|
+
status: z.ZodEnum<{
|
|
15
|
+
failed: "failed";
|
|
16
|
+
completed: "completed";
|
|
17
|
+
partial: "partial";
|
|
18
|
+
canceled: "canceled";
|
|
19
|
+
suspended_insufficient_funds: "suspended_insufficient_funds";
|
|
20
|
+
}>;
|
|
21
|
+
error: z.ZodNullable<z.ZodString>;
|
|
22
|
+
closed: z.ZodBoolean;
|
|
23
|
+
revoked: z.ZodBoolean;
|
|
24
|
+
}, z.core.$strip>;
|
|
25
|
+
export type RecordedRunEnd = z.infer<typeof RecordedRunEndSchema>;
|
|
26
|
+
/**
|
|
27
|
+
* The engine hands back deserialized data, thus a value of an event is unknown
|
|
28
|
+
* until this check. A value of the wrong shape gives `null`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function parseRecordedRunEnd(value: unknown): RecordedRunEnd | null;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import { RunStatus } from "../state/schema.js";
|
|
3
|
+
/**
|
|
4
|
+
* The workflow event under which the body of `executeAnalysis` records its run
|
|
5
|
+
* end after a failed status write. The run canceler reads it with
|
|
6
|
+
* `DBOSClient.getEvent`.
|
|
7
|
+
*/
|
|
8
|
+
export const RUN_END_EVENT_KEY = "run-end";
|
|
9
|
+
/**
|
|
10
|
+
* The run end that the body did: the status and the error that its failed
|
|
11
|
+
* status write tried to write, and the results of the charge close and of the
|
|
12
|
+
* revoke.
|
|
13
|
+
*/
|
|
14
|
+
export const RecordedRunEndSchema = z.object({
|
|
15
|
+
status: RunStatus.exclude(["running"]),
|
|
16
|
+
error: z.string().nullable(),
|
|
17
|
+
closed: z.boolean(),
|
|
18
|
+
revoked: z.boolean(),
|
|
19
|
+
});
|
|
20
|
+
/**
|
|
21
|
+
* The engine hands back deserialized data, thus a value of an event is unknown
|
|
22
|
+
* until this check. A value of the wrong shape gives `null`.
|
|
23
|
+
*/
|
|
24
|
+
export function parseRecordedRunEnd(value) {
|
|
25
|
+
const parsed = RecordedRunEndSchema.safeParse(value);
|
|
26
|
+
return parsed.success ? parsed.data : null;
|
|
27
|
+
}
|
|
@@ -19,11 +19,12 @@ import type { ChatPart } from "../contracts/chat-parts.js";
|
|
|
19
19
|
* newer than the reader.
|
|
20
20
|
*
|
|
21
21
|
* The registry is keyed by the discriminant alone, so this validates `type` and
|
|
22
|
-
* trusts the producer for the payload.
|
|
23
|
-
*
|
|
24
|
-
* ends of this stream are typed against
|
|
25
|
-
* the discriminant is the boundary that can
|
|
26
|
-
* something else entirely) and
|
|
22
|
+
* trusts the producer for the payload. The part schemas in `contracts/schemas/`
|
|
23
|
+
* are for a consumer at the edge of the package, which parses what it receives.
|
|
24
|
+
* Both ends of this stream are typed against the same union inside this package,
|
|
25
|
+
* so the discriminant is the boundary that can drift (a producer that emits
|
|
26
|
+
* something else entirely), and a part that fails its schema still reaches the
|
|
27
|
+
* consumer for its own check.
|
|
27
28
|
*/
|
|
28
29
|
export declare function parseRunEventPart(value: unknown): ChatPart | null;
|
|
29
30
|
/**
|
|
@@ -20,11 +20,12 @@ import { PART_REGISTRY } from "../contracts/part-registry.js";
|
|
|
20
20
|
* newer than the reader.
|
|
21
21
|
*
|
|
22
22
|
* The registry is keyed by the discriminant alone, so this validates `type` and
|
|
23
|
-
* trusts the producer for the payload.
|
|
24
|
-
*
|
|
25
|
-
* ends of this stream are typed against
|
|
26
|
-
* the discriminant is the boundary that can
|
|
27
|
-
* something else entirely) and
|
|
23
|
+
* trusts the producer for the payload. The part schemas in `contracts/schemas/`
|
|
24
|
+
* are for a consumer at the edge of the package, which parses what it receives.
|
|
25
|
+
* Both ends of this stream are typed against the same union inside this package,
|
|
26
|
+
* so the discriminant is the boundary that can drift (a producer that emits
|
|
27
|
+
* something else entirely), and a part that fails its schema still reaches the
|
|
28
|
+
* consumer for its own check.
|
|
28
29
|
*/
|
|
29
30
|
export function parseRunEventPart(value) {
|
|
30
31
|
if (typeof value !== "object" || value === null)
|
|
@@ -36,6 +36,11 @@ import type { ChatPart } from "../contracts/chat-parts.js";
|
|
|
36
36
|
import { type DbError } from "../lib/db-result.js";
|
|
37
37
|
import type { Logger } from "../lib/logger.js";
|
|
38
38
|
import type { RunStatus } from "../state/schema.js";
|
|
39
|
+
/** The part of a run row that the terminal part is made from. */
|
|
40
|
+
export interface RunEventRunRead {
|
|
41
|
+
readonly status: RunStatus;
|
|
42
|
+
readonly error: string | null;
|
|
43
|
+
}
|
|
39
44
|
/**
|
|
40
45
|
* Construction-time dependencies. The pool is the application pool the run
|
|
41
46
|
* ledger lives in; the logger is where every contained failure is reported, and
|
|
@@ -45,8 +50,13 @@ import type { RunStatus } from "../state/schema.js";
|
|
|
45
50
|
export interface RunEventStreamDeps {
|
|
46
51
|
readonly pool: Pool;
|
|
47
52
|
readonly logger?: Logger;
|
|
48
|
-
/** Reads the status of the run row, `null` when no row exists. Defaults to `queryRun` on `pool`. */
|
|
49
|
-
readonly
|
|
53
|
+
/** Reads the status and the error of the run row, `null` when no row exists. Defaults to `queryRun` on `pool`. */
|
|
54
|
+
readonly readRun?: (runId: string) => ResultAsync<RunEventRunRead | null, DbError>;
|
|
55
|
+
/**
|
|
56
|
+
* Reads the status of the parent workflow in the durability engine, `null` when no workflow exists. Defaults to
|
|
57
|
+
* `DBOS.getWorkflowStatus`.
|
|
58
|
+
*/
|
|
59
|
+
readonly readWorkflowStatus?: (runId: string) => Promise<string | null>;
|
|
50
60
|
/** The wait between two reads of the run row. Defaults to a timer. */
|
|
51
61
|
readonly sleep?: (ms: number) => Promise<void>;
|
|
52
62
|
}
|
|
@@ -78,10 +88,12 @@ export interface RunEventStream {
|
|
|
78
88
|
* aborts.
|
|
79
89
|
*
|
|
80
90
|
* A canceled workflow runs no more steps, thus it never writes its own
|
|
81
|
-
* terminal part, and the canceler cannot write a stream. When
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
91
|
+
* terminal part, and the canceler cannot write a stream. When every stream
|
|
92
|
+
* ends with no `data-run-completed` or `data-run-failed` part, the last
|
|
93
|
+
* part delivered is a `data-run-failed` part that the run row gives, or the
|
|
94
|
+
* status of the parent workflow when the row still reads `running`. A row
|
|
95
|
+
* that does not exist, a `completed` or `partial` row, an active workflow,
|
|
96
|
+
* and no workflow give no part.
|
|
85
97
|
*
|
|
86
98
|
* Parts from one workflow arrive in the order that workflow wrote them. No
|
|
87
99
|
* order is promised ACROSS the parent and its children: those workflows
|