@zaalipro/dsh-workflows 0.1.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/NOTICE.md +224 -0
- package/README.i18n.yaml +4 -0
- package/README.md +109 -0
- package/README.zh.md +109 -0
- package/cordis.patch.yml +10 -0
- package/docs/architecture.i18n.yaml +4 -0
- package/docs/architecture.md +177 -0
- package/docs/architecture.zh.md +177 -0
- package/docs/testing.i18n.yaml +4 -0
- package/docs/testing.md +125 -0
- package/docs/testing.zh.md +125 -0
- package/docs/user-guide.i18n.yaml +4 -0
- package/docs/user-guide.md +351 -0
- package/docs/user-guide.zh.md +351 -0
- package/lib/client-types/WorkflowMemberInspector.d.ts +28 -0
- package/lib/client-types/WorkflowMemberInspector.d.ts.map +1 -0
- package/lib/client-types/WorkflowMemberInspector.js +181 -0
- package/lib/client-types/WorkflowMemberInspector.js.map +1 -0
- package/lib/client-types/WorkflowRunPanel.d.ts +43 -0
- package/lib/client-types/WorkflowRunPanel.d.ts.map +1 -0
- package/lib/client-types/WorkflowRunPanel.js +130 -0
- package/lib/client-types/WorkflowRunPanel.js.map +1 -0
- package/lib/client-types/WorkflowsDashboard.d.ts +49 -0
- package/lib/client-types/WorkflowsDashboard.d.ts.map +1 -0
- package/lib/client-types/WorkflowsDashboard.js +942 -0
- package/lib/client-types/WorkflowsDashboard.js.map +1 -0
- package/lib/client-types/adapter.d.ts +60 -0
- package/lib/client-types/adapter.d.ts.map +1 -0
- package/lib/client-types/adapter.js +117 -0
- package/lib/client-types/adapter.js.map +1 -0
- package/lib/client-types/chat-renderer.d.ts +10 -0
- package/lib/client-types/chat-renderer.d.ts.map +1 -0
- package/lib/client-types/chat-renderer.js +11 -0
- package/lib/client-types/chat-renderer.js.map +1 -0
- package/lib/client-types/contract.d.ts +210 -0
- package/lib/client-types/contract.d.ts.map +1 -0
- package/lib/client-types/contract.js +37 -0
- package/lib/client-types/contract.js.map +1 -0
- package/lib/client-types/controller.d.ts +83 -0
- package/lib/client-types/controller.d.ts.map +1 -0
- package/lib/client-types/controller.js +705 -0
- package/lib/client-types/controller.js.map +1 -0
- package/lib/client-types/index.d.ts +19 -0
- package/lib/client-types/index.d.ts.map +1 -0
- package/lib/client-types/index.js +730 -0
- package/lib/client-types/index.js.map +1 -0
- package/lib/client-types/locales.d.ts +257 -0
- package/lib/client-types/locales.d.ts.map +1 -0
- package/lib/client-types/locales.js +270 -0
- package/lib/client-types/locales.js.map +1 -0
- package/lib/client-types/slot-components.d.ts +38 -0
- package/lib/client-types/slot-components.d.ts.map +1 -0
- package/lib/client-types/slot-components.js +22 -0
- package/lib/client-types/slot-components.js.map +1 -0
- package/lib/client-types/store.d.ts +69 -0
- package/lib/client-types/store.d.ts.map +1 -0
- package/lib/client-types/store.js +135 -0
- package/lib/client-types/store.js.map +1 -0
- package/lib/client-types/workflow-definition.d.ts +52 -0
- package/lib/client-types/workflow-definition.d.ts.map +1 -0
- package/lib/client-types/workflow-definition.js +135 -0
- package/lib/client-types/workflow-definition.js.map +1 -0
- package/lib/client.js +11064 -0
- package/lib/client.js.map +1 -0
- package/lib/compat-engine/index.js +1926 -0
- package/lib/compat-engine/index.js.map +1 -0
- package/lib/compat-engine/worker.cjs +1500 -0
- package/lib/compat-engine/worker.cjs.map +1 -0
- package/lib/typert.host.d.ts +3 -0
- package/lib/typert.host.js +1709 -0
- package/lib/typert.remote-client.d.ts +57 -0
- package/lib/typert.remote-client.js +1662 -0
- package/lib/types/commands/aliases.d.ts +4 -0
- package/lib/types/commands/aliases.d.ts.map +1 -0
- package/lib/types/commands/aliases.js +25 -0
- package/lib/types/commands/aliases.js.map +1 -0
- package/lib/types/commands/index.d.ts +66 -0
- package/lib/types/commands/index.d.ts.map +1 -0
- package/lib/types/commands/index.js +550 -0
- package/lib/types/commands/index.js.map +1 -0
- package/lib/types/commands/parser.d.ts +17 -0
- package/lib/types/commands/parser.d.ts.map +1 -0
- package/lib/types/commands/parser.js +45 -0
- package/lib/types/commands/parser.js.map +1 -0
- package/lib/types/config.d.ts +71 -0
- package/lib/types/config.d.ts.map +1 -0
- package/lib/types/config.js +109 -0
- package/lib/types/config.js.map +1 -0
- package/lib/types/index.d.ts +41 -0
- package/lib/types/index.d.ts.map +1 -0
- package/lib/types/index.js +483 -0
- package/lib/types/index.js.map +1 -0
- package/lib/types/invariant.d.ts +21 -0
- package/lib/types/invariant.d.ts.map +1 -0
- package/lib/types/invariant.js +85 -0
- package/lib/types/invariant.js.map +1 -0
- package/lib/types/registry/definition.d.ts +24 -0
- package/lib/types/registry/definition.d.ts.map +1 -0
- package/lib/types/registry/definition.js +189 -0
- package/lib/types/registry/definition.js.map +1 -0
- package/lib/types/registry/index.d.ts +100 -0
- package/lib/types/registry/index.d.ts.map +1 -0
- package/lib/types/registry/index.js +899 -0
- package/lib/types/registry/index.js.map +1 -0
- package/lib/types/registry/names.d.ts +19 -0
- package/lib/types/registry/names.d.ts.map +1 -0
- package/lib/types/registry/names.js +69 -0
- package/lib/types/registry/names.js.map +1 -0
- package/lib/types/registry/remote.d.ts +15 -0
- package/lib/types/registry/remote.d.ts.map +1 -0
- package/lib/types/registry/remote.js +128 -0
- package/lib/types/registry/remote.js.map +1 -0
- package/lib/types/registry/roots.d.ts +44 -0
- package/lib/types/registry/roots.d.ts.map +1 -0
- package/lib/types/registry/roots.js +118 -0
- package/lib/types/registry/roots.js.map +1 -0
- package/lib/types/registry/types.d.ts +62 -0
- package/lib/types/registry/types.d.ts.map +1 -0
- package/lib/types/registry/types.js +2 -0
- package/lib/types/registry/types.js.map +1 -0
- package/lib/types/registry/watchers.d.ts +70 -0
- package/lib/types/registry/watchers.d.ts.map +1 -0
- package/lib/types/registry/watchers.js +152 -0
- package/lib/types/registry/watchers.js.map +1 -0
- package/lib/types/remote-events.d.ts +42 -0
- package/lib/types/remote-events.d.ts.map +1 -0
- package/lib/types/remote-events.js +21 -0
- package/lib/types/remote-events.js.map +1 -0
- package/lib/types/run-recorder.d.ts +44 -0
- package/lib/types/run-recorder.d.ts.map +1 -0
- package/lib/types/run-recorder.js +356 -0
- package/lib/types/run-recorder.js.map +1 -0
- package/lib/types/supervisor/canned-validate.d.ts +25 -0
- package/lib/types/supervisor/canned-validate.d.ts.map +1 -0
- package/lib/types/supervisor/canned-validate.js +122 -0
- package/lib/types/supervisor/canned-validate.js.map +1 -0
- package/lib/types/supervisor/completion-notice.d.ts +49 -0
- package/lib/types/supervisor/completion-notice.d.ts.map +1 -0
- package/lib/types/supervisor/completion-notice.js +281 -0
- package/lib/types/supervisor/completion-notice.js.map +1 -0
- package/lib/types/supervisor/cursors.d.ts +68 -0
- package/lib/types/supervisor/cursors.d.ts.map +1 -0
- package/lib/types/supervisor/cursors.js +158 -0
- package/lib/types/supervisor/cursors.js.map +1 -0
- package/lib/types/supervisor/engine-compat.d.ts +48 -0
- package/lib/types/supervisor/engine-compat.d.ts.map +1 -0
- package/lib/types/supervisor/engine-compat.js +116 -0
- package/lib/types/supervisor/engine-compat.js.map +1 -0
- package/lib/types/supervisor/index.d.ts +183 -0
- package/lib/types/supervisor/index.d.ts.map +1 -0
- package/lib/types/supervisor/index.js +2040 -0
- package/lib/types/supervisor/index.js.map +1 -0
- package/lib/types/supervisor/parallel-compat.d.ts +36 -0
- package/lib/types/supervisor/parallel-compat.d.ts.map +1 -0
- package/lib/types/supervisor/parallel-compat.js +180 -0
- package/lib/types/supervisor/parallel-compat.js.map +1 -0
- package/lib/types/supervisor/remote.d.ts +25 -0
- package/lib/types/supervisor/remote.d.ts.map +1 -0
- package/lib/types/supervisor/remote.js +478 -0
- package/lib/types/supervisor/remote.js.map +1 -0
- package/lib/types/supervisor/storage/bounded-file.d.ts +41 -0
- package/lib/types/supervisor/storage/bounded-file.d.ts.map +1 -0
- package/lib/types/supervisor/storage/bounded-file.js +186 -0
- package/lib/types/supervisor/storage/bounded-file.js.map +1 -0
- package/lib/types/supervisor/storage/details-codec.d.ts +14 -0
- package/lib/types/supervisor/storage/details-codec.d.ts.map +1 -0
- package/lib/types/supervisor/storage/details-codec.js +359 -0
- package/lib/types/supervisor/storage/details-codec.js.map +1 -0
- package/lib/types/supervisor/storage/index.d.ts +26 -0
- package/lib/types/supervisor/storage/index.d.ts.map +1 -0
- package/lib/types/supervisor/storage/index.js +100 -0
- package/lib/types/supervisor/storage/index.js.map +1 -0
- package/lib/types/supervisor/storage/lease.d.ts +34 -0
- package/lib/types/supervisor/storage/lease.d.ts.map +1 -0
- package/lib/types/supervisor/storage/lease.js +289 -0
- package/lib/types/supervisor/storage/lease.js.map +1 -0
- package/lib/types/supervisor/storage/manifest-codec.d.ts +10 -0
- package/lib/types/supervisor/storage/manifest-codec.d.ts.map +1 -0
- package/lib/types/supervisor/storage/manifest-codec.js +335 -0
- package/lib/types/supervisor/storage/manifest-codec.js.map +1 -0
- package/lib/types/supervisor/storage/manifest-store.d.ts +99 -0
- package/lib/types/supervisor/storage/manifest-store.d.ts.map +1 -0
- package/lib/types/supervisor/storage/manifest-store.js +1606 -0
- package/lib/types/supervisor/storage/manifest-store.js.map +1 -0
- package/lib/types/supervisor/storage/manifest-types.d.ts +269 -0
- package/lib/types/supervisor/storage/manifest-types.d.ts.map +1 -0
- package/lib/types/supervisor/storage/manifest-types.js +2 -0
- package/lib/types/supervisor/storage/manifest-types.js.map +1 -0
- package/lib/types/supervisor/storage/private-root.d.ts +143 -0
- package/lib/types/supervisor/storage/private-root.d.ts.map +1 -0
- package/lib/types/supervisor/storage/private-root.js +860 -0
- package/lib/types/supervisor/storage/private-root.js.map +1 -0
- package/lib/types/supervisor/storage/recovery.d.ts +14 -0
- package/lib/types/supervisor/storage/recovery.d.ts.map +1 -0
- package/lib/types/supervisor/storage/recovery.js +126 -0
- package/lib/types/supervisor/storage/recovery.js.map +1 -0
- package/lib/types/supervisor/storage/run-files.d.ts +67 -0
- package/lib/types/supervisor/storage/run-files.d.ts.map +1 -0
- package/lib/types/supervisor/storage/run-files.js +346 -0
- package/lib/types/supervisor/storage/run-files.js.map +1 -0
- package/lib/types/supervisor/types.d.ts +295 -0
- package/lib/types/supervisor/types.d.ts.map +1 -0
- package/lib/types/supervisor/types.js +3 -0
- package/lib/types/supervisor/types.js.map +1 -0
- package/lib/types/supervisor/value-view.d.ts +19 -0
- package/lib/types/supervisor/value-view.d.ts.map +1 -0
- package/lib/types/supervisor/value-view.js +183 -0
- package/lib/types/supervisor/value-view.js.map +1 -0
- package/lib/types/tool/index.d.ts +86 -0
- package/lib/types/tool/index.d.ts.map +1 -0
- package/lib/types/tool/index.js +810 -0
- package/lib/types/tool/index.js.map +1 -0
- package/lib/types/tool/schema.d.ts +158 -0
- package/lib/types/tool/schema.d.ts.map +1 -0
- package/lib/types/tool/schema.js +137 -0
- package/lib/types/tool/schema.js.map +1 -0
- package/lib/types/types.d.ts +9 -0
- package/lib/types/types.d.ts.map +1 -0
- package/lib/types/types.js +3 -0
- package/lib/types/types.js.map +1 -0
- package/lib/types/user-questions.d.ts +9 -0
- package/lib/types/user-questions.d.ts.map +1 -0
- package/lib/types/user-questions.js +91 -0
- package/lib/types/user-questions.js.map +1 -0
- package/package.json +273 -0
- package/skills/create-workflow/SKILL.md +166 -0
- package/vendor/workflow-engine/LICENSE +21 -0
- package/vendor/workflow-engine/README.md +1 -0
|
@@ -0,0 +1,2040 @@
|
|
|
1
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
2
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
3
|
+
import { validateDefinitionEnvelope } from '../registry/definition.js';
|
|
4
|
+
import { openRunScratch } from './storage/run-files.js';
|
|
5
|
+
import { WorkflowCompletionNotifier } from './completion-notice.js';
|
|
6
|
+
import { childTranscriptValue, memberOutcomeWithTranscript, snapshotWorkflowJsonValue, workflowRunValueView, } from './value-view.js';
|
|
7
|
+
import { VALIDATION_NOTE, } from './types.js';
|
|
8
|
+
import { WorkflowPackageError } from '../invariant.js';
|
|
9
|
+
import { cannedStockValidate, repairObjectLiteralSemicolons } from './canned-validate.js';
|
|
10
|
+
import { scriptWithJobMapParallel, unwrapStockCompatibilityResult } from './parallel-compat.js';
|
|
11
|
+
import { adaptEngineHandle, rejectPartialEngineHandle, supportsEngineReplay, } from './engine-compat.js';
|
|
12
|
+
export * from './types.js';
|
|
13
|
+
export * from './value-view.js';
|
|
14
|
+
export * from './completion-notice.js';
|
|
15
|
+
export * from './engine-compat.js';
|
|
16
|
+
const TERMINAL = new Set(['completed', 'failed', 'cancelled', 'interrupted']);
|
|
17
|
+
const ACTIVE = new Set(['running', 'pausing', 'stopping', 'needs-input', 'paused', 'budget-limited']);
|
|
18
|
+
const SCRATCH_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/u;
|
|
19
|
+
const encoder = new TextEncoder();
|
|
20
|
+
function opaqueId() { return randomBytes(16).toString('hex'); }
|
|
21
|
+
function sessionOf(agent) {
|
|
22
|
+
const value = agent?.session?.id ?? agent?.session?.header?.id ?? agent?.sessionId;
|
|
23
|
+
if (typeof value !== 'string' || value.length === 0)
|
|
24
|
+
throw new WorkflowPackageError('workflow start requires a calling Agent with a Session', 'WORKFLOW_INVALID_STATE');
|
|
25
|
+
return value;
|
|
26
|
+
}
|
|
27
|
+
function renderThrown(error) {
|
|
28
|
+
try {
|
|
29
|
+
return error instanceof Error ? error.message : String(error);
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return '[unrenderable thrown value]';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function utf8Prefix(text, maxBytes) {
|
|
36
|
+
let output = '';
|
|
37
|
+
let bytes = 0;
|
|
38
|
+
for (const point of text) {
|
|
39
|
+
const size = encoder.encode(point).byteLength;
|
|
40
|
+
if (bytes + size > maxBytes)
|
|
41
|
+
break;
|
|
42
|
+
output += point;
|
|
43
|
+
bytes += size;
|
|
44
|
+
}
|
|
45
|
+
return output;
|
|
46
|
+
}
|
|
47
|
+
/** Relabel engine `workflow:${meta.name}` frames with the package filename. */
|
|
48
|
+
function rewriteValidationDiagnostic(message, filename, metaName) {
|
|
49
|
+
let rewritten = message;
|
|
50
|
+
if (metaName.length > 0)
|
|
51
|
+
rewritten = rewritten.replaceAll(`workflow:${metaName}`, filename);
|
|
52
|
+
if (/Unexpected token ';'/u.test(rewritten)) {
|
|
53
|
+
rewritten += '\nHint: this is plain JavaScript — object and array literals use commas between fields, not semicolons.';
|
|
54
|
+
}
|
|
55
|
+
if (rewritten.includes(filename))
|
|
56
|
+
return rewritten;
|
|
57
|
+
return `${filename}: ${rewritten}`;
|
|
58
|
+
}
|
|
59
|
+
/** Collapse every gate smoke-stop into one product string. */
|
|
60
|
+
function productWouldPause(value, maxBytes) {
|
|
61
|
+
const raw = utf8Prefix(String(value ?? ''), maxBytes);
|
|
62
|
+
const message = raw.replace(/^would (?:pause|await_user)(?:\s*\([^)]*\))?:\s*/iu, '').trim();
|
|
63
|
+
return `would pause: ${message}`;
|
|
64
|
+
}
|
|
65
|
+
function numericCursor(cursor, total) {
|
|
66
|
+
if (cursor === undefined)
|
|
67
|
+
return 0;
|
|
68
|
+
if (!/^\d+$/u.test(String(cursor)))
|
|
69
|
+
throw new WorkflowPackageError('workflow page cursor is invalid', 'WORKFLOW_CURSOR_INVALID');
|
|
70
|
+
const value = Number(cursor);
|
|
71
|
+
if (!Number.isSafeInteger(value) || value < 0 || value > total)
|
|
72
|
+
throw new WorkflowPackageError('workflow page cursor is invalid', 'WORKFLOW_CURSOR_INVALID');
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
function pageLimit(value) {
|
|
76
|
+
const result = value ?? 50;
|
|
77
|
+
if (!Number.isSafeInteger(result) || result < 1 || result > 200)
|
|
78
|
+
throw new WorkflowPackageError('workflow page limit must be a safe integer from 1 through 200', 'WORKFLOW_LIMIT');
|
|
79
|
+
return result;
|
|
80
|
+
}
|
|
81
|
+
function nextCursor(offset, count, total) {
|
|
82
|
+
return offset + count < total ? String(offset + count) : undefined;
|
|
83
|
+
}
|
|
84
|
+
function memberCounts(detail) {
|
|
85
|
+
const members = detail.members ?? [];
|
|
86
|
+
return {
|
|
87
|
+
total: members.length,
|
|
88
|
+
running: members.filter(member => member.status === 'running').length,
|
|
89
|
+
completed: members.filter(member => member.status === 'completed').length,
|
|
90
|
+
failed: members.filter(member => member.status === 'failed').length,
|
|
91
|
+
cancelled: members.filter(member => member.status === 'cancelled').length,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
function headView(head) {
|
|
95
|
+
const terminal = TERMINAL.has(head.status) && head.stopReason !== undefined
|
|
96
|
+
? {
|
|
97
|
+
stopReason: head.stopReason === 'budget-limited' ? 'error' : head.stopReason,
|
|
98
|
+
resultState: head.status === 'completed' ? 'available' : 'not-produced',
|
|
99
|
+
...(head.terminalPreview === undefined ? {} : { preview: head.terminalPreview }),
|
|
100
|
+
...(head.error === undefined ? {} : { error: head.error }),
|
|
101
|
+
}
|
|
102
|
+
: undefined;
|
|
103
|
+
return {
|
|
104
|
+
runId: head.runId,
|
|
105
|
+
displayName: head.displayName,
|
|
106
|
+
name: head.name,
|
|
107
|
+
description: head.description ?? '',
|
|
108
|
+
status: head.status,
|
|
109
|
+
...(head.phase === undefined ? {} : { phase: head.phase }),
|
|
110
|
+
budget: { ...head.budget }, memberCounts: { ...head.memberCounts },
|
|
111
|
+
startedAt: head.startedAt,
|
|
112
|
+
...(head.settledAt === undefined ? {} : { settledAt: head.settledAt }),
|
|
113
|
+
...(terminal === undefined ? {} : { terminal }),
|
|
114
|
+
allowedActions: (head.allowedActions ?? []),
|
|
115
|
+
revision: head.revision, detailRevision: head.detailRevision,
|
|
116
|
+
membersRevision: head.membersRevision, logsRevision: head.logsRevision,
|
|
117
|
+
resultRevision: head.resultRevision, artifactsRevision: head.artifactsRevision,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
function isJsonRecord(value) {
|
|
121
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
122
|
+
}
|
|
123
|
+
function valueViewFromStored(value, outcome, maxBytes) {
|
|
124
|
+
if (outcome === 'pending')
|
|
125
|
+
return { state: 'pending' };
|
|
126
|
+
if (outcome === 'not-produced')
|
|
127
|
+
return { state: 'not-produced' };
|
|
128
|
+
if (outcome === 'evicted')
|
|
129
|
+
return { state: 'evicted' };
|
|
130
|
+
// `available` is meaningful even when the value is JSON null. A malformed
|
|
131
|
+
// old sidecar is treated as an evicted projection rather than allowing an
|
|
132
|
+
// untrusted undefined value to cross the browser boundary.
|
|
133
|
+
if (value === undefined)
|
|
134
|
+
return { state: 'evicted' };
|
|
135
|
+
try {
|
|
136
|
+
return workflowRunValueView(value, maxBytes);
|
|
137
|
+
}
|
|
138
|
+
catch {
|
|
139
|
+
return { state: 'evicted' };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
/** Durable logical-run supervisor backed only by the compatible official engine. */
|
|
143
|
+
export class WorkflowSupervisor {
|
|
144
|
+
ctx;
|
|
145
|
+
workflowEngine;
|
|
146
|
+
static inject = ['workflowEngine', 'workflows'];
|
|
147
|
+
config;
|
|
148
|
+
store;
|
|
149
|
+
registry;
|
|
150
|
+
runs = new Map();
|
|
151
|
+
byDisplay = new Map();
|
|
152
|
+
/** Persisted rows recovered after process death have inspection authority
|
|
153
|
+
* but deliberately no executable `InternalRun`/Agent authority. */
|
|
154
|
+
recoveredById = new Map();
|
|
155
|
+
recoveredByDisplay = new Map();
|
|
156
|
+
executions = new Map();
|
|
157
|
+
sessionRevisions = new Map();
|
|
158
|
+
activeSessions = new Map();
|
|
159
|
+
activeTotal = 0;
|
|
160
|
+
pendingStarts = new Set();
|
|
161
|
+
closedOwners = new WeakSet();
|
|
162
|
+
ownerDisposals = new Map();
|
|
163
|
+
listenerDisposers = [];
|
|
164
|
+
notifier;
|
|
165
|
+
ownsStore;
|
|
166
|
+
feedEpoch = opaqueId();
|
|
167
|
+
initializePromise;
|
|
168
|
+
admission = true;
|
|
169
|
+
disposed = false;
|
|
170
|
+
disposal;
|
|
171
|
+
constructor(ctx, config = {}, store, workflowEngine = undefined) {
|
|
172
|
+
this.ctx = ctx;
|
|
173
|
+
this.workflowEngine = workflowEngine;
|
|
174
|
+
this.config = {
|
|
175
|
+
defaultAgentBudget: config.defaultAgentBudget ?? 128,
|
|
176
|
+
maxAgentBudget: config.maxAgentBudget ?? 1_024,
|
|
177
|
+
maxConcurrentAgents: config.maxConcurrentAgents ?? 32,
|
|
178
|
+
maxActiveRunsPerSession: config.maxActiveRunsPerSession ?? 64,
|
|
179
|
+
maxActiveRunsGlobal: config.maxActiveRunsGlobal ?? config.maxActiveRuns ?? 1_024,
|
|
180
|
+
saveScope: config.saveScope ?? 'project',
|
|
181
|
+
completionNoticeMaxBytes: config.completionNoticeMaxBytes ?? 16_384,
|
|
182
|
+
completionCohortMaxItems: config.completionCohortMaxItems ?? 20,
|
|
183
|
+
completionCohortMaxBytes: config.completionCohortMaxBytes ?? 262_144,
|
|
184
|
+
memberOutcomeMaxBytes: config.memberOutcomeMaxBytes ?? 131_072,
|
|
185
|
+
maxRetainedRunsPerSession: config.maxRetainedRunsPerSession ?? 256,
|
|
186
|
+
maxWorkflowNamesPerSession: config.maxWorkflowNamesPerSession ?? 4_096,
|
|
187
|
+
maxRecoveryEntries: config.maxRecoveryEntries ?? 4_096,
|
|
188
|
+
maxMembersPerRun: config.maxMembersPerRun ?? 2_048,
|
|
189
|
+
maxManifestBytes: config.maxManifestBytes ?? 8_388_608,
|
|
190
|
+
maxRunDetailsBytes: config.maxRunDetailsBytes ?? 33_554_432,
|
|
191
|
+
maxRunStoreBytes: config.maxRunStoreBytes ?? 536_870_912,
|
|
192
|
+
maxLogLines: config.maxLogLines ?? 4_096,
|
|
193
|
+
maxLogLineBytes: config.maxLogLineBytes ?? 65_536,
|
|
194
|
+
maxLogTotalBytes: config.maxLogTotalBytes ?? 33_554_432,
|
|
195
|
+
maxEventTextBytes: config.maxEventTextBytes ?? 65_536,
|
|
196
|
+
remoteHeadTextMaxBytes: config.remoteHeadTextMaxBytes ?? 131_072,
|
|
197
|
+
maxScriptProjectionBytes: config.maxScriptProjectionBytes ?? 1_048_576,
|
|
198
|
+
maxRetainedArtifactsPerRun: config.maxRetainedArtifactsPerRun ?? 256,
|
|
199
|
+
artifactChunkDefaultBytes: config.artifactChunkDefaultBytes ?? 32_768,
|
|
200
|
+
artifactChunkMaxBytes: config.artifactChunkMaxBytes ?? 131_072,
|
|
201
|
+
scratchMaxOperations: config.scratchMaxOperations ?? 4_096,
|
|
202
|
+
scratchMaxPendingOperations: config.scratchMaxPendingOperations ?? 64,
|
|
203
|
+
scratchMaxFiles: config.scratchMaxFiles ?? 64,
|
|
204
|
+
scratchMaxFileBytes: config.scratchMaxFileBytes ?? 1_048_576,
|
|
205
|
+
scratchMaxTotalBytes: config.scratchMaxTotalBytes ?? 8_388_608,
|
|
206
|
+
maxActiveRuns: config.maxActiveRuns ?? 1_024,
|
|
207
|
+
};
|
|
208
|
+
if (this.config.defaultAgentBudget > this.config.maxAgentBudget || this.config.maxAgentBudget > 1_024)
|
|
209
|
+
throw new RangeError('workflow supervisor agent budget configuration is invalid');
|
|
210
|
+
if (this.config.maxActiveRunsPerSession > this.config.maxActiveRunsGlobal)
|
|
211
|
+
throw new RangeError('maxActiveRunsPerSession must not exceed maxActiveRunsGlobal');
|
|
212
|
+
if (this.config.maxMembersPerRun < this.config.maxAgentBudget)
|
|
213
|
+
throw new RangeError('maxMembersPerRun must be at least maxAgentBudget');
|
|
214
|
+
this.ownsStore = store === undefined && ctx?.workflowStore === undefined;
|
|
215
|
+
this.store = store ?? ctx?.workflowStore;
|
|
216
|
+
if (this.store === undefined)
|
|
217
|
+
throw new WorkflowPackageError('workflow storage is unavailable', 'WORKFLOW_STORAGE_UNSUPPORTED');
|
|
218
|
+
this.registry = ctx?.workflows;
|
|
219
|
+
this.notifier = new WorkflowCompletionNotifier(ctx, this.store, {
|
|
220
|
+
maxBytes: this.config.completionNoticeMaxBytes,
|
|
221
|
+
maxItems: this.config.completionCohortMaxItems,
|
|
222
|
+
maxCohortBytes: this.config.completionCohortMaxBytes,
|
|
223
|
+
});
|
|
224
|
+
this.attachEngineObservers();
|
|
225
|
+
}
|
|
226
|
+
listen(event, callback) {
|
|
227
|
+
const result = this.ctx?.on?.(event, callback);
|
|
228
|
+
if (typeof result === 'function')
|
|
229
|
+
this.listenerDisposers.push(result);
|
|
230
|
+
}
|
|
231
|
+
attachEngineObservers() {
|
|
232
|
+
this.listen('workflow/phase', (info, title) => {
|
|
233
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
234
|
+
if (!this.currentAttempt(run, attempt))
|
|
235
|
+
return;
|
|
236
|
+
const phase = utf8Prefix(String(title), this.config.maxEventTextBytes);
|
|
237
|
+
await this.commitActive(run, { phase }, 'detail');
|
|
238
|
+
}));
|
|
239
|
+
});
|
|
240
|
+
this.listen('workflow/log', (info, message) => {
|
|
241
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
242
|
+
if (!this.currentAttempt(run, attempt))
|
|
243
|
+
return;
|
|
244
|
+
const text = utf8Prefix(String(message), Math.min(this.config.maxLogLineBytes, this.config.maxEventTextBytes));
|
|
245
|
+
const logs = [...(run.detail.logs ?? []), { index: (run.detail.logs?.at(-1)?.index ?? -1) + 1, text }];
|
|
246
|
+
let bytes = logs.reduce((total, line) => total + encoder.encode(line.text).byteLength, 0);
|
|
247
|
+
while (logs.length > this.config.maxLogLines || bytes > this.config.maxLogTotalBytes) {
|
|
248
|
+
const removed = logs.shift();
|
|
249
|
+
if (removed !== undefined)
|
|
250
|
+
bytes -= encoder.encode(removed.text).byteLength;
|
|
251
|
+
}
|
|
252
|
+
run.detail = { ...run.detail, logs };
|
|
253
|
+
await this.commitActive(run, {}, 'logs');
|
|
254
|
+
}));
|
|
255
|
+
});
|
|
256
|
+
this.listen('workflow/agent-start', (info, member) => {
|
|
257
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
258
|
+
if (!this.currentAttempt(run, attempt))
|
|
259
|
+
return;
|
|
260
|
+
const sequence = Number(member?.seq);
|
|
261
|
+
if (!Number.isSafeInteger(sequence) || sequence < 1 || run.detail.members?.some(item => item.seq === sequence))
|
|
262
|
+
return;
|
|
263
|
+
if ((run.detail.members?.length ?? 0) >= this.config.maxMembersPerRun) {
|
|
264
|
+
attempt.intent = 'stop';
|
|
265
|
+
attempt.handle.cancel('workflow member retention limit exceeded');
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
const stored = {
|
|
269
|
+
memberId: opaqueId(), seq: sequence,
|
|
270
|
+
label: utf8Prefix(String(member?.label ?? ''), this.config.maxEventTextBytes) || `member-${sequence}`,
|
|
271
|
+
...(member?.phase === undefined ? {} : { phase: utf8Prefix(String(member.phase), this.config.maxEventTextBytes) }),
|
|
272
|
+
status: 'running', outcome: 'pending',
|
|
273
|
+
...(typeof member?.childId === 'string' && member.childId.length > 0 ? { childSessionId: member.childId } : {}),
|
|
274
|
+
startedAt: Date.now(),
|
|
275
|
+
};
|
|
276
|
+
run.detail = { ...run.detail, members: [...(run.detail.members ?? []), stored] };
|
|
277
|
+
const spent = run.head.budget.spent + 1;
|
|
278
|
+
run.head = {
|
|
279
|
+
...run.head,
|
|
280
|
+
budget: { ...run.head.budget, spent, remaining: Math.max(0, run.head.budget.total - spent) },
|
|
281
|
+
};
|
|
282
|
+
await this.commitActive(run, { budget: run.head.budget }, 'members');
|
|
283
|
+
await this.publishLifecycle(run, 'workflows/member-start', this.info(run), this.memberLifecycle(stored));
|
|
284
|
+
}));
|
|
285
|
+
});
|
|
286
|
+
this.listen('workflow/journal-commit', (info, entry) => {
|
|
287
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
288
|
+
if (!this.currentAttempt(run, attempt) || entry?.kind !== 'agent' || !Number.isSafeInteger(entry?.seq))
|
|
289
|
+
return;
|
|
290
|
+
const members = [...(run.detail.members ?? [])];
|
|
291
|
+
const index = members.findIndex(member => member.seq === entry.seq);
|
|
292
|
+
if (index < 0)
|
|
293
|
+
return;
|
|
294
|
+
let value;
|
|
295
|
+
try {
|
|
296
|
+
value = snapshotWorkflowJsonValue(entry.result);
|
|
297
|
+
}
|
|
298
|
+
catch {
|
|
299
|
+
return;
|
|
300
|
+
}
|
|
301
|
+
members[index] = { ...members[index], outcome: 'available', value };
|
|
302
|
+
run.detail = { ...run.detail, members };
|
|
303
|
+
await this.commitActive(run, {}, 'members');
|
|
304
|
+
}));
|
|
305
|
+
});
|
|
306
|
+
this.listen('workflow/agent-end', (info, member) => {
|
|
307
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
308
|
+
if (!this.currentAttempt(run, attempt) || !Number.isSafeInteger(member?.seq))
|
|
309
|
+
return;
|
|
310
|
+
const members = [...(run.detail.members ?? [])];
|
|
311
|
+
const index = members.findIndex(candidate => candidate.seq === member.seq);
|
|
312
|
+
if (index < 0 || members[index].status !== 'running')
|
|
313
|
+
return;
|
|
314
|
+
const status = member?.outcome === 'completed' || member?.outcome === 'failed' || member?.outcome === 'cancelled'
|
|
315
|
+
? member.outcome : 'failed';
|
|
316
|
+
let captured;
|
|
317
|
+
if (members[index].outcome === 'pending') {
|
|
318
|
+
const raw = member.result ?? member.value;
|
|
319
|
+
if (raw !== undefined) {
|
|
320
|
+
try {
|
|
321
|
+
captured = snapshotWorkflowJsonValue(raw);
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
captured = undefined;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
captured ??= childTranscriptValue(this.ctx, member?.childId ?? members[index].childSessionId);
|
|
328
|
+
}
|
|
329
|
+
const stored = {
|
|
330
|
+
...members[index], status, settledAt: Date.now(),
|
|
331
|
+
...(captured !== undefined
|
|
332
|
+
? { outcome: 'available', value: captured }
|
|
333
|
+
: members[index].outcome === 'pending' ? { outcome: 'not-produced' } : {}),
|
|
334
|
+
};
|
|
335
|
+
members[index] = stored;
|
|
336
|
+
run.detail = { ...run.detail, members };
|
|
337
|
+
await this.commitActive(run, {}, 'members');
|
|
338
|
+
await this.publishLifecycle(run, 'workflows/member-end', this.info(run), this.memberLifecycle(stored));
|
|
339
|
+
}));
|
|
340
|
+
});
|
|
341
|
+
this.listen('workflow/gate', (info, gate) => {
|
|
342
|
+
this.withAttempt(info?.id, (run, attempt) => this.enqueue(run, async () => {
|
|
343
|
+
if (!this.currentAttempt(run, attempt) || run.head.status === 'stopping' || run.head.status === 'pausing')
|
|
344
|
+
return;
|
|
345
|
+
this.clearGate(run);
|
|
346
|
+
const bounded = {
|
|
347
|
+
kind: ['user', 'back_off', 'no_progress', 'verification', 'infra'].includes(String(gate?.kind)) ? gate.kind : 'infra',
|
|
348
|
+
message: utf8Prefix(String(gate?.message ?? ''), this.config.maxEventTextBytes),
|
|
349
|
+
resumable: gate?.resumable === true,
|
|
350
|
+
};
|
|
351
|
+
const record = {
|
|
352
|
+
generation: attempt.generation, executionId: attempt.executionId,
|
|
353
|
+
gateId: opaqueId(), gate: bounded, abort: new AbortController(),
|
|
354
|
+
};
|
|
355
|
+
run.gate = record;
|
|
356
|
+
await this.commitActive(run, { status: 'needs-input' }, 'detail');
|
|
357
|
+
this.emit('workflows/gate-request', {
|
|
358
|
+
info: this.info(run), executionId: record.executionId, gateId: record.gateId,
|
|
359
|
+
gate: bounded, parent: run.parent, signal: record.abort.signal,
|
|
360
|
+
});
|
|
361
|
+
}));
|
|
362
|
+
});
|
|
363
|
+
this.listen('agent/disposed', ({ agent }) => { void this.disposeOwner(agent); });
|
|
364
|
+
}
|
|
365
|
+
async initialize(signal) {
|
|
366
|
+
if (this.initializePromise === undefined) {
|
|
367
|
+
this.initializePromise = (async () => {
|
|
368
|
+
signal?.throwIfAborted();
|
|
369
|
+
const recovered = await this.store.initialize(signal);
|
|
370
|
+
for (const recoveredHead of recovered) {
|
|
371
|
+
const sessionId = typeof recoveredHead.sessionId === 'string' ? String(recoveredHead.sessionId) : undefined;
|
|
372
|
+
if (sessionId !== undefined) {
|
|
373
|
+
this.sessionRevisions.set(sessionId, Math.max(this.sessionRevisions.get(sessionId) ?? 0, recoveredHead.revision));
|
|
374
|
+
this.recoveredById.set(recoveredHead.runId, recoveredHead);
|
|
375
|
+
this.recoveredByDisplay.set(this.displayKey(sessionId, recoveredHead.displayName), recoveredHead);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
})();
|
|
379
|
+
}
|
|
380
|
+
return this.initializePromise;
|
|
381
|
+
}
|
|
382
|
+
emit(name, ...args) {
|
|
383
|
+
try {
|
|
384
|
+
if (typeof this.ctx?.emit === 'function')
|
|
385
|
+
this.ctx.emit(name, ...args);
|
|
386
|
+
else if (this.ctx?.events?.dispatch !== undefined) {
|
|
387
|
+
for (const callback of this.ctx.events.dispatch('emit', [name, ...args])) {
|
|
388
|
+
try {
|
|
389
|
+
void Promise.resolve(callback(...args)).catch(error => this.ctx?.logger?.warn?.(`workflow supervisor ${name} listener rejected`, error));
|
|
390
|
+
}
|
|
391
|
+
catch (error) {
|
|
392
|
+
this.ctx?.logger?.warn?.(`workflow supervisor ${name} listener threw`, error);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
catch (error) {
|
|
398
|
+
this.ctx?.logger?.warn?.(`workflow supervisor ${name} observer failed`, error);
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
publishLifecycle(run, name, ...args) {
|
|
402
|
+
const operation = run.lifecycleTail.then(() => { this.emit(name, ...args); });
|
|
403
|
+
run.lifecycleTail = operation.catch(error => { this.ctx?.logger?.warn?.('workflow lifecycle publication failed', error); });
|
|
404
|
+
return operation;
|
|
405
|
+
}
|
|
406
|
+
publishChange(run) {
|
|
407
|
+
const revision = (this.sessionRevisions.get(run.sessionId) ?? 0) + 1;
|
|
408
|
+
this.sessionRevisions.set(run.sessionId, revision);
|
|
409
|
+
this.emit('workflows/run-change', { kind: 'invalidate', sessionId: run.sessionId, revision });
|
|
410
|
+
}
|
|
411
|
+
enqueue(run, operation) {
|
|
412
|
+
const result = run.tail.then(operation, operation);
|
|
413
|
+
run.tail = result.then(() => undefined, error => { this.ctx?.logger?.warn?.('workflow run mutation failed', error); });
|
|
414
|
+
return result;
|
|
415
|
+
}
|
|
416
|
+
withAttempt(executionId, callback) {
|
|
417
|
+
if (typeof executionId !== 'string')
|
|
418
|
+
return;
|
|
419
|
+
const lookup = this.executions.get(executionId);
|
|
420
|
+
if (lookup === undefined)
|
|
421
|
+
return;
|
|
422
|
+
const attempt = lookup.run.attempt;
|
|
423
|
+
if (attempt === undefined || attempt.generation !== lookup.generation || attempt.executionId !== executionId)
|
|
424
|
+
return;
|
|
425
|
+
callback(lookup.run, attempt);
|
|
426
|
+
}
|
|
427
|
+
currentAttempt(run, attempt) {
|
|
428
|
+
return run.attempt === attempt && run.generation === attempt.generation;
|
|
429
|
+
}
|
|
430
|
+
info(run) {
|
|
431
|
+
return { id: run.id, displayName: run.head.displayName, name: run.meta.name };
|
|
432
|
+
}
|
|
433
|
+
memberLifecycle(member) {
|
|
434
|
+
return {
|
|
435
|
+
memberId: member.memberId, seq: member.seq, label: member.label,
|
|
436
|
+
...(member.phase === undefined ? {} : { phase: member.phase }),
|
|
437
|
+
childSessionId: member.childSessionId ?? '', status: member.status,
|
|
438
|
+
};
|
|
439
|
+
}
|
|
440
|
+
resolveSource(spec) {
|
|
441
|
+
if (spec.definition !== undefined) {
|
|
442
|
+
if (spec.script !== undefined || spec.meta !== undefined)
|
|
443
|
+
throw new WorkflowPackageError('workflow launch must select either a definition or an inline script plus meta', 'WORKFLOW_DEFINITION_INVALID');
|
|
444
|
+
const meta = {
|
|
445
|
+
name: spec.definition.name, description: spec.definition.description,
|
|
446
|
+
...(spec.definition.whenToUse === undefined ? {} : { whenToUse: spec.definition.whenToUse }),
|
|
447
|
+
...(spec.definition.phases === undefined ? {} : { phases: [...spec.definition.phases] }),
|
|
448
|
+
};
|
|
449
|
+
const clean = validateDefinitionEnvelope({ meta, script: spec.definition.script });
|
|
450
|
+
return { script: clean.script, meta: clean.meta, builtIn: spec.definition.scope === 'bundled' };
|
|
451
|
+
}
|
|
452
|
+
if (spec.script === undefined || spec.meta === undefined)
|
|
453
|
+
throw new WorkflowPackageError('inline workflow launch requires both script and meta', 'WORKFLOW_DEFINITION_INVALID');
|
|
454
|
+
const clean = validateDefinitionEnvelope({ meta: spec.meta, script: spec.script });
|
|
455
|
+
return { script: clean.script, meta: clean.meta, builtIn: false };
|
|
456
|
+
}
|
|
457
|
+
resolveBudget(value) {
|
|
458
|
+
const budget = value ?? this.config.defaultAgentBudget;
|
|
459
|
+
if (!Number.isSafeInteger(budget) || budget < 1 || budget > this.config.maxAgentBudget) {
|
|
460
|
+
throw new WorkflowPackageError(`agent_budget must be a safe integer from 1 through ${this.config.maxAgentBudget}`, 'WORKFLOW_LIMIT');
|
|
461
|
+
}
|
|
462
|
+
return budget;
|
|
463
|
+
}
|
|
464
|
+
snapshotArgs(value) {
|
|
465
|
+
const snapshot = snapshotWorkflowJsonValue(value ?? {});
|
|
466
|
+
if (snapshot === null || Array.isArray(snapshot) || typeof snapshot !== 'object')
|
|
467
|
+
throw new WorkflowPackageError('workflow args must be a JSON object (wrap arrays/scalars in a field)', 'WORKFLOW_DEFINITION_INVALID');
|
|
468
|
+
return snapshot;
|
|
469
|
+
}
|
|
470
|
+
reserveStart(parent) {
|
|
471
|
+
if (!this.admission || this.disposed)
|
|
472
|
+
throw new WorkflowPackageError('workflow supervisor is shutting down', 'WORKFLOW_INVALID_STATE');
|
|
473
|
+
if (typeof parent === 'object' && parent !== null && this.closedOwners.has(parent)) {
|
|
474
|
+
throw new WorkflowPackageError('workflow owner is shutting down', 'WORKFLOW_INVALID_STATE');
|
|
475
|
+
}
|
|
476
|
+
const sessionId = sessionOf(parent);
|
|
477
|
+
const sessionCount = (this.activeSessions.get(sessionId) ?? 0) + [...this.pendingStarts].filter(item => item.sessionId === sessionId).length;
|
|
478
|
+
if (sessionCount >= this.config.maxActiveRunsPerSession) {
|
|
479
|
+
throw new WorkflowPackageError(`workflow active-run limit reached for this Session (${this.config.maxActiveRunsPerSession})`, 'WORKFLOW_LIMIT');
|
|
480
|
+
}
|
|
481
|
+
if (this.activeTotal + this.pendingStarts.size >= this.config.maxActiveRunsGlobal) {
|
|
482
|
+
throw new WorkflowPackageError(`workflow global active-run limit reached (${this.config.maxActiveRunsGlobal})`, 'WORKFLOW_LIMIT');
|
|
483
|
+
}
|
|
484
|
+
let finish;
|
|
485
|
+
const done = new Promise(resolve => { finish = resolve; });
|
|
486
|
+
const record = { controller: new AbortController(), sessionId, parent, durable: false, done, finish };
|
|
487
|
+
this.pendingStarts.add(record);
|
|
488
|
+
return record;
|
|
489
|
+
}
|
|
490
|
+
releaseStart(record) {
|
|
491
|
+
if (!this.pendingStarts.delete(record))
|
|
492
|
+
return;
|
|
493
|
+
record.finish();
|
|
494
|
+
}
|
|
495
|
+
scratchLimits() {
|
|
496
|
+
return {
|
|
497
|
+
maxOperations: this.config.scratchMaxOperations,
|
|
498
|
+
maxPendingOperations: this.config.scratchMaxPendingOperations,
|
|
499
|
+
maxFiles: this.config.scratchMaxFiles,
|
|
500
|
+
maxFileBytes: this.config.scratchMaxFileBytes,
|
|
501
|
+
maxTotalBytes: this.config.scratchMaxTotalBytes,
|
|
502
|
+
};
|
|
503
|
+
}
|
|
504
|
+
async createScratch(head) {
|
|
505
|
+
const layout = this.ctx?.workflowStorage?.layout;
|
|
506
|
+
if (layout !== undefined)
|
|
507
|
+
return openRunScratch(layout, head.runDirectory, this.scratchLimits());
|
|
508
|
+
throw new WorkflowPackageError('descriptor-rooted workflow scratch access is unavailable', 'WORKFLOW_STORAGE_UNSUPPORTED');
|
|
509
|
+
}
|
|
510
|
+
/** Admit one run durably, attach a deferred attempt, then return before settlement. */
|
|
511
|
+
async start(spec) {
|
|
512
|
+
if (!spec.parent)
|
|
513
|
+
throw new WorkflowPackageError('workflow start requires a calling agent', 'WORKFLOW_INVALID_STATE');
|
|
514
|
+
const source = this.resolveSource(spec);
|
|
515
|
+
const args = this.snapshotArgs(spec.args);
|
|
516
|
+
const budget = this.resolveBudget(spec.agentBudget);
|
|
517
|
+
const pending = this.reserveStart(spec.parent);
|
|
518
|
+
const admissionSignal = spec.signal === undefined
|
|
519
|
+
? pending.controller.signal
|
|
520
|
+
: AbortSignal.any([spec.signal, pending.controller.signal]);
|
|
521
|
+
// This variable is switched by the store's durable callback, not by the
|
|
522
|
+
// insertion promise's eventual return. That distinction closes the tiny
|
|
523
|
+
// but important window where a caller abort can otherwise cancel setup
|
|
524
|
+
// after the manifest has become durable.
|
|
525
|
+
let setupSignal = admissionSignal;
|
|
526
|
+
let committedHead;
|
|
527
|
+
let runId;
|
|
528
|
+
let startedAt = 0;
|
|
529
|
+
let run;
|
|
530
|
+
try {
|
|
531
|
+
await this.initialize(admissionSignal);
|
|
532
|
+
admissionSignal.throwIfAborted();
|
|
533
|
+
runId = opaqueId();
|
|
534
|
+
startedAt = Date.now();
|
|
535
|
+
const inserted = await this.store.insertWithNextDisplayName({
|
|
536
|
+
sessionId: pending.sessionId, name: source.meta.name, runId,
|
|
537
|
+
script: source.script, description: source.meta.description,
|
|
538
|
+
args: args, budgetTotal: budget,
|
|
539
|
+
onDurable: head => {
|
|
540
|
+
pending.durable = true;
|
|
541
|
+
committedHead = head;
|
|
542
|
+
setupSignal = pending.controller.signal;
|
|
543
|
+
},
|
|
544
|
+
}, identity => {
|
|
545
|
+
const saveAvailable = !source.builtIn && !identity.numberedHandle;
|
|
546
|
+
return {
|
|
547
|
+
head: {
|
|
548
|
+
description: source.meta.description, status: 'running',
|
|
549
|
+
budget: { total: budget, spent: 0, remaining: budget },
|
|
550
|
+
memberCounts: { total: 0, running: 0, completed: 0, failed: 0, cancelled: 0 },
|
|
551
|
+
startedAt, detailRevision: 1, membersRevision: 1, logsRevision: 1,
|
|
552
|
+
resultRevision: 1, artifactsRevision: 1, executionAvailable: true,
|
|
553
|
+
saveAvailable, allowedActions: saveAvailable ? ['pause', 'stop', 'save'] : ['pause', 'stop'],
|
|
554
|
+
},
|
|
555
|
+
detail: {
|
|
556
|
+
members: [], logs: [], result: { state: 'pending' },
|
|
557
|
+
...(source.meta.phases === undefined ? {} : { phases: source.meta.phases.map(phase => ({ title: phase.title })) }),
|
|
558
|
+
artifacts: [],
|
|
559
|
+
},
|
|
560
|
+
};
|
|
561
|
+
}, admissionSignal);
|
|
562
|
+
// Stores predating the callback seam still reach this fallback; the
|
|
563
|
+
// built-in file store invokes it at the manifest commit point above.
|
|
564
|
+
pending.durable = true;
|
|
565
|
+
setupSignal = pending.controller.signal;
|
|
566
|
+
run = {
|
|
567
|
+
id: runId, sessionId: pending.sessionId, meta: source.meta, script: source.script,
|
|
568
|
+
args, parent: spec.parent, runDirectory: inserted.runDirectory,
|
|
569
|
+
ownerController: pending.controller,
|
|
570
|
+
scriptPath: inserted.scriptPath ?? '', builtIn: source.builtIn, startedAt,
|
|
571
|
+
head: inserted,
|
|
572
|
+
detail: {
|
|
573
|
+
members: [], logs: [], result: { state: 'pending' },
|
|
574
|
+
...(source.meta.phases === undefined ? {} : { phases: source.meta.phases.map(phase => ({ title: phase.title })) }),
|
|
575
|
+
artifacts: [],
|
|
576
|
+
},
|
|
577
|
+
generation: 0, tail: Promise.resolve(), lifecycleTail: Promise.resolve(), active: true,
|
|
578
|
+
};
|
|
579
|
+
setupSignal.throwIfAborted();
|
|
580
|
+
run.scratch = await this.createScratch(inserted);
|
|
581
|
+
const attempt = this.createAttempt(run);
|
|
582
|
+
run.attempt = attempt;
|
|
583
|
+
run.generation = attempt.generation;
|
|
584
|
+
this.executions.set(attempt.executionId, { run, generation: attempt.generation });
|
|
585
|
+
this.runs.set(run.id, run);
|
|
586
|
+
this.byDisplay.set(this.displayKey(run.sessionId, run.head.displayName), run);
|
|
587
|
+
if (!this.admission && run.closing === undefined)
|
|
588
|
+
run.closing = 'supervisor';
|
|
589
|
+
this.activeTotal += 1;
|
|
590
|
+
this.activeSessions.set(run.sessionId, (this.activeSessions.get(run.sessionId) ?? 0) + 1);
|
|
591
|
+
this.startObservation(run, attempt);
|
|
592
|
+
await this.publishLifecycle(run, 'workflows/run-start', this.info(run));
|
|
593
|
+
this.publishChange(run);
|
|
594
|
+
try {
|
|
595
|
+
this.releaseDeferredAttempt(run, attempt);
|
|
596
|
+
}
|
|
597
|
+
catch (error) {
|
|
598
|
+
attempt.intent = 'running';
|
|
599
|
+
attempt.handle.cancel('workflow deferred release failed');
|
|
600
|
+
await attempt.quiescent.catch(() => undefined);
|
|
601
|
+
await this.enqueue(run, () => this.failAfterAdmission(run, `workflow launch failed: ${renderThrown(error)}`));
|
|
602
|
+
throw error;
|
|
603
|
+
}
|
|
604
|
+
return { status: 'started', displayName: inserted.displayName, runId, scriptPath: inserted.scriptPath };
|
|
605
|
+
}
|
|
606
|
+
catch (error) {
|
|
607
|
+
// A store may fail during post-publication accounting. The callback
|
|
608
|
+
// still gave us the committed head, so retain/terminalize that history
|
|
609
|
+
// rather than pretending the insertion was rolled back.
|
|
610
|
+
if (pending.durable && run === undefined && committedHead !== undefined) {
|
|
611
|
+
run = {
|
|
612
|
+
id: runId,
|
|
613
|
+
sessionId: pending.sessionId, meta: source.meta, script: source.script,
|
|
614
|
+
args, parent: spec.parent, ownerController: pending.controller,
|
|
615
|
+
runDirectory: committedHead.runDirectory, scriptPath: committedHead.scriptPath ?? '',
|
|
616
|
+
builtIn: source.builtIn, startedAt,
|
|
617
|
+
head: committedHead,
|
|
618
|
+
detail: {
|
|
619
|
+
members: [], logs: [], result: { state: 'pending' },
|
|
620
|
+
...(source.meta.phases === undefined ? {} : { phases: source.meta.phases.map(phase => ({ title: phase.title })) }),
|
|
621
|
+
artifacts: [],
|
|
622
|
+
},
|
|
623
|
+
generation: 0, tail: Promise.resolve(), lifecycleTail: Promise.resolve(), active: true,
|
|
624
|
+
};
|
|
625
|
+
}
|
|
626
|
+
if (pending.durable && run !== undefined && !this.runs.has(run.id)) {
|
|
627
|
+
this.runs.set(run.id, run);
|
|
628
|
+
this.byDisplay.set(this.displayKey(run.sessionId, run.head.displayName), run);
|
|
629
|
+
if (!this.admission && run.closing === undefined)
|
|
630
|
+
run.closing = 'supervisor';
|
|
631
|
+
this.activeTotal += 1;
|
|
632
|
+
this.activeSessions.set(run.sessionId, (this.activeSessions.get(run.sessionId) ?? 0) + 1);
|
|
633
|
+
await this.failAfterAdmission(run, `workflow launch failed: ${renderThrown(error)}`).catch(failure => this.ctx?.logger?.warn?.('workflow admission terminalization failed', failure));
|
|
634
|
+
}
|
|
635
|
+
throw error;
|
|
636
|
+
}
|
|
637
|
+
finally {
|
|
638
|
+
this.releaseStart(pending);
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
createAttempt(run) {
|
|
642
|
+
const engine = this.workflowEngine ?? this.ctx?.workflowEngine;
|
|
643
|
+
if (engine === undefined || typeof engine.start !== 'function')
|
|
644
|
+
throw new WorkflowPackageError('workflow engine is unavailable', 'WORKFLOW_INVALID_STATE');
|
|
645
|
+
const generation = run.generation + 1;
|
|
646
|
+
const checkpoint = run.checkpoint;
|
|
647
|
+
const retainedSeq = (run.detail.members ?? []).reduce((maximum, member) => Math.max(maximum, member.seq), 0);
|
|
648
|
+
const replayCheckpoint = checkpoint === undefined ? undefined : {
|
|
649
|
+
journal: checkpoint.journal,
|
|
650
|
+
agentSpend: Math.max(checkpoint.agentSpend, run.head.budget.spent),
|
|
651
|
+
agentSeq: Math.max(checkpoint.agentSeq, checkpoint.agentSpend, run.head.budget.spent, retainedSeq),
|
|
652
|
+
};
|
|
653
|
+
const scratch = run.scratch;
|
|
654
|
+
const native = engine?.dshWorkflowsNative === true;
|
|
655
|
+
const compatibleScript = native ? run.script : scriptWithJobMapParallel(run.script, {
|
|
656
|
+
agentBudget: run.head.budget.total,
|
|
657
|
+
scratchMaxOperations: this.config.scratchMaxOperations,
|
|
658
|
+
scratchMaxFiles: this.config.scratchMaxFiles,
|
|
659
|
+
scratchMaxFileBytes: this.config.scratchMaxFileBytes,
|
|
660
|
+
scratchMaxTotalBytes: this.config.scratchMaxTotalBytes,
|
|
661
|
+
});
|
|
662
|
+
const request = {
|
|
663
|
+
script: compatibleScript, meta: run.meta, args: run.args,
|
|
664
|
+
maxTotalAgents: run.head.budget.total,
|
|
665
|
+
parent: run.parent,
|
|
666
|
+
signal: run.ownerController.signal,
|
|
667
|
+
deferStart: true,
|
|
668
|
+
...(replayCheckpoint === undefined ? { replay: {} } : { replay: { checkpoint: replayCheckpoint } }),
|
|
669
|
+
...(scratch === undefined ? {} : {
|
|
670
|
+
scratch: {
|
|
671
|
+
read: (name, signal) => scratch.read(name, signal),
|
|
672
|
+
write: (name, content, signal) => scratch.write(name, content, signal),
|
|
673
|
+
},
|
|
674
|
+
}),
|
|
675
|
+
};
|
|
676
|
+
let raw;
|
|
677
|
+
try {
|
|
678
|
+
raw = engine.start(request);
|
|
679
|
+
}
|
|
680
|
+
catch (error) {
|
|
681
|
+
throw new WorkflowPackageError(`workflow engine could not start: ${renderThrown(error)}`, 'WORKFLOW_INVALID_STATE', { cause: error });
|
|
682
|
+
}
|
|
683
|
+
const handle = adaptEngineHandle(raw);
|
|
684
|
+
if (handle === undefined) {
|
|
685
|
+
rejectPartialEngineHandle(raw);
|
|
686
|
+
throw new WorkflowPackageError('workflow engine returned an invalid run handle', 'WORKFLOW_INVALID_STATE');
|
|
687
|
+
}
|
|
688
|
+
const executionId = typeof handle.id === 'string' && handle.id.length > 0 ? handle.id : opaqueId();
|
|
689
|
+
const quiescent = (async () => {
|
|
690
|
+
let result;
|
|
691
|
+
let resultFailure;
|
|
692
|
+
try {
|
|
693
|
+
result = await handle.result;
|
|
694
|
+
if (result.stopReason === 'completed') {
|
|
695
|
+
const stock = unwrapStockCompatibilityResult(result.value);
|
|
696
|
+
if (stock !== undefined) {
|
|
697
|
+
if (scratch === undefined)
|
|
698
|
+
throw new WorkflowPackageError('workflow scratch access is unavailable', 'WORKFLOW_STORAGE_UNSUPPORTED');
|
|
699
|
+
for (const [name, content] of Object.entries(stock.scratch))
|
|
700
|
+
await scratch.write(name, content, run.ownerController.signal);
|
|
701
|
+
result = { ...result, value: stock.value };
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
catch (error) {
|
|
706
|
+
resultFailure = error;
|
|
707
|
+
}
|
|
708
|
+
let cleanupError;
|
|
709
|
+
try {
|
|
710
|
+
await handle.dispose();
|
|
711
|
+
}
|
|
712
|
+
catch (error) {
|
|
713
|
+
cleanupError = renderThrown(error);
|
|
714
|
+
}
|
|
715
|
+
let captured;
|
|
716
|
+
try {
|
|
717
|
+
captured = handle.checkpoint();
|
|
718
|
+
}
|
|
719
|
+
catch (error) {
|
|
720
|
+
// Every supervisor attempt opts into replay, so unavailable checkpoint
|
|
721
|
+
// authority after quiescence is actionable even for the first attempt.
|
|
722
|
+
if (resultFailure === undefined)
|
|
723
|
+
resultFailure = error;
|
|
724
|
+
}
|
|
725
|
+
if (resultFailure !== undefined)
|
|
726
|
+
throw resultFailure;
|
|
727
|
+
if (result === undefined)
|
|
728
|
+
throw new WorkflowPackageError('workflow attempt settled without a result', 'WORKFLOW_INVALID_STATE');
|
|
729
|
+
return { result, checkpoint: captured, ...(cleanupError === undefined ? {} : { cleanupError }) };
|
|
730
|
+
})();
|
|
731
|
+
const attempt = {
|
|
732
|
+
generation, executionId, handle, intent: 'running', quiescent, observation: Promise.resolve(),
|
|
733
|
+
};
|
|
734
|
+
// Keep the executor's promise observed even when a caller abandons a
|
|
735
|
+
// control wait. The supervisor remains owner after durable admission.
|
|
736
|
+
void quiescent.catch(error => this.ctx?.logger?.warn?.('workflow attempt settlement failed', error));
|
|
737
|
+
return attempt;
|
|
738
|
+
}
|
|
739
|
+
/** One-shot release is skipped after owner/supervisor teardown or cancel so
|
|
740
|
+
* a racing start/resume cannot Go a cancelled inert attempt. */
|
|
741
|
+
shouldReleaseAttempt(run, attempt) {
|
|
742
|
+
return run.closing === undefined
|
|
743
|
+
&& !this.disposed
|
|
744
|
+
&& !run.ownerController.signal.aborted
|
|
745
|
+
&& attempt.intent === 'running';
|
|
746
|
+
}
|
|
747
|
+
releaseDeferredAttempt(run, attempt) {
|
|
748
|
+
if (!this.shouldReleaseAttempt(run, attempt))
|
|
749
|
+
return;
|
|
750
|
+
attempt.handle.release();
|
|
751
|
+
}
|
|
752
|
+
startObservation(run, attempt) {
|
|
753
|
+
attempt.observation = attempt.quiescent.then(outcome => this.enqueue(run, () => this.settleAttempt(run, attempt, outcome)), error => this.enqueue(run, () => this.settleAttempt(run, attempt, {
|
|
754
|
+
result: { value: null, stopReason: 'error', error: renderThrown(error), agentsStarted: run.head.budget.spent },
|
|
755
|
+
}))).then(() => undefined);
|
|
756
|
+
void attempt.observation.catch(error => {
|
|
757
|
+
run.settlementFailure ??= error;
|
|
758
|
+
this.ctx?.logger?.warn?.('workflow attempt observer failed', error);
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
async settleAttempt(run, attempt, outcome) {
|
|
762
|
+
if (!this.currentAttempt(run, attempt))
|
|
763
|
+
return;
|
|
764
|
+
this.executions.delete(attempt.executionId);
|
|
765
|
+
run.attempt = undefined;
|
|
766
|
+
this.clearGate(run);
|
|
767
|
+
const result = outcome.result;
|
|
768
|
+
if (outcome.checkpoint !== undefined) {
|
|
769
|
+
try {
|
|
770
|
+
this.acceptCheckpoint(run, outcome.checkpoint);
|
|
771
|
+
}
|
|
772
|
+
catch (error) {
|
|
773
|
+
if (attempt.intent === 'running' || attempt.intent === 'pause') {
|
|
774
|
+
await this.terminalize(run, 'failed', `workflow replay checkpoint diverged: ${renderThrown(error)}`, undefined);
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
const spent = Math.max(run.head.budget.spent, Number(result.agentsStarted) || 0, outcome.checkpoint?.agentSpend ?? 0);
|
|
780
|
+
if (outcome.cleanupError !== undefined && attempt.intent !== 'teardown') {
|
|
781
|
+
await this.terminalize(run, 'failed', `workflow attempt cleanup failed: ${outcome.cleanupError}`, undefined, spent);
|
|
782
|
+
return;
|
|
783
|
+
}
|
|
784
|
+
if (attempt.intent === 'pause') {
|
|
785
|
+
if (outcome.checkpoint === undefined) {
|
|
786
|
+
await this.terminalize(run, 'failed', 'workflow checkpoint is unavailable after pause', undefined, spent);
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
run.head = { ...run.head, budget: { ...run.head.budget, spent, remaining: Math.max(0, run.head.budget.total - spent) } };
|
|
790
|
+
await this.commitActive(run, { status: 'paused', budget: run.head.budget }, 'result');
|
|
791
|
+
return;
|
|
792
|
+
}
|
|
793
|
+
if (attempt.intent === 'stop') {
|
|
794
|
+
await this.terminalize(run, 'cancelled', result.error ?? 'stopped by user', undefined, spent);
|
|
795
|
+
return;
|
|
796
|
+
}
|
|
797
|
+
if (attempt.intent === 'teardown') {
|
|
798
|
+
await this.terminalize(run, 'interrupted', 'Process exited before workflow settlement.', undefined, spent);
|
|
799
|
+
return;
|
|
800
|
+
}
|
|
801
|
+
if (result.errorCode === 'AGENT_CAP') {
|
|
802
|
+
const ended = this.settleRunningMembers(run);
|
|
803
|
+
run.head = { ...run.head, budget: { ...run.head.budget, spent, remaining: Math.max(0, run.head.budget.total - spent) } };
|
|
804
|
+
await this.commitActive(run, { status: 'budget-limited', budget: run.head.budget }, 'members');
|
|
805
|
+
for (const member of ended)
|
|
806
|
+
await this.publishLifecycle(run, 'workflows/member-end', this.info(run), member);
|
|
807
|
+
return;
|
|
808
|
+
}
|
|
809
|
+
// A worker cancellation without a supervisor intent is an execution
|
|
810
|
+
// failure (for example worker death or an unowned abort), not a user Stop.
|
|
811
|
+
const status = result.stopReason === 'completed' ? 'completed' : 'failed';
|
|
812
|
+
await this.terminalize(run, status, result.error, status === 'completed' ? result.value : undefined, spent);
|
|
813
|
+
}
|
|
814
|
+
acceptCheckpoint(run, checkpoint) {
|
|
815
|
+
if (!Array.isArray(checkpoint.journal)
|
|
816
|
+
|| !Number.isSafeInteger(checkpoint.agentSpend) || checkpoint.agentSpend < 0
|
|
817
|
+
|| !Number.isSafeInteger(checkpoint.agentSeq) || checkpoint.agentSeq < 0) {
|
|
818
|
+
throw new WorkflowPackageError('workflow checkpoint is invalid', 'WORKFLOW_INVALID_STATE');
|
|
819
|
+
}
|
|
820
|
+
const callIds = new Set();
|
|
821
|
+
const agentSequences = new Set();
|
|
822
|
+
let agentEntries = 0;
|
|
823
|
+
let maximumSequence = 0;
|
|
824
|
+
const hasOrdinals = checkpoint.journal.some(entry => entry?.ordinal !== undefined);
|
|
825
|
+
if (hasOrdinals && checkpoint.journal.some(entry => entry?.ordinal === undefined)) {
|
|
826
|
+
throw new WorkflowPackageError('workflow checkpoint mixes ordinal and legacy entries', 'WORKFLOW_INVALID_STATE');
|
|
827
|
+
}
|
|
828
|
+
const orderedJournal = hasOrdinals
|
|
829
|
+
? [...checkpoint.journal].sort((left, right) => Number(left.ordinal) - Number(right.ordinal))
|
|
830
|
+
: [...checkpoint.journal];
|
|
831
|
+
for (let index = 0; index < orderedJournal.length; index += 1) {
|
|
832
|
+
const entry = orderedJournal[index];
|
|
833
|
+
if (entry === null || typeof entry !== 'object'
|
|
834
|
+
|| !Array.isArray(entry.callId) || entry.callId.length === 0
|
|
835
|
+
|| entry.callId.some((part) => !Number.isSafeInteger(part) || part <= 0)
|
|
836
|
+
|| !/^[a-f0-9]{64}$/u.test(entry.fingerprint)
|
|
837
|
+
|| !['agent', 'phase', 'log', 'scratch-read', 'scratch-write', 'await-user'].includes(entry.kind)) {
|
|
838
|
+
throw new WorkflowPackageError('workflow checkpoint is invalid', 'WORKFLOW_INVALID_STATE');
|
|
839
|
+
}
|
|
840
|
+
if (hasOrdinals && (!Number.isSafeInteger(entry.ordinal) || entry.ordinal !== index + 1)) {
|
|
841
|
+
throw new WorkflowPackageError('workflow checkpoint ordinals are not contiguous commit order', 'WORKFLOW_INVALID_STATE');
|
|
842
|
+
}
|
|
843
|
+
const address = entry.callId.join('.');
|
|
844
|
+
if (callIds.has(address))
|
|
845
|
+
throw new WorkflowPackageError('workflow checkpoint contains a repeated call id', 'WORKFLOW_INVALID_STATE');
|
|
846
|
+
callIds.add(address);
|
|
847
|
+
if (entry.kind === 'agent') {
|
|
848
|
+
const seq = entry.seq;
|
|
849
|
+
if (!Number.isSafeInteger(seq) || Number(seq) <= 0 || agentSequences.has(Number(seq))) {
|
|
850
|
+
throw new WorkflowPackageError('workflow checkpoint contains an invalid agent sequence', 'WORKFLOW_INVALID_STATE');
|
|
851
|
+
}
|
|
852
|
+
agentSequences.add(Number(seq));
|
|
853
|
+
maximumSequence = Math.max(maximumSequence, Number(seq));
|
|
854
|
+
agentEntries += 1;
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
if (checkpoint.agentSpend < agentEntries || checkpoint.agentSeq < maximumSequence) {
|
|
858
|
+
throw new WorkflowPackageError('workflow checkpoint counters are inconsistent', 'WORKFLOW_INVALID_STATE');
|
|
859
|
+
}
|
|
860
|
+
// Array order is commit order. Concurrent calls can settle opposite their
|
|
861
|
+
// call-path order, so sorting by callId would destroy replay authority.
|
|
862
|
+
const incoming = orderedJournal.map(entry => ({
|
|
863
|
+
...entry,
|
|
864
|
+
callId: [...entry.callId],
|
|
865
|
+
}));
|
|
866
|
+
const retainedMaximum = (run.detail.members ?? []).reduce((maximum, member) => Math.max(maximum, member.seq), 0);
|
|
867
|
+
const normalizedSpend = Math.max(checkpoint.agentSpend, run.head.budget.spent);
|
|
868
|
+
const normalizedSequence = Math.max(checkpoint.agentSeq, normalizedSpend, maximumSequence, retainedMaximum);
|
|
869
|
+
if (run.checkpoint !== undefined) {
|
|
870
|
+
const previous = run.checkpoint.journal;
|
|
871
|
+
if (incoming.length < previous.length
|
|
872
|
+
|| normalizedSpend < run.checkpoint.agentSpend
|
|
873
|
+
|| normalizedSequence < run.checkpoint.agentSeq) {
|
|
874
|
+
throw new WorkflowPackageError('workflow replay journal diverged', 'WORKFLOW_INVALID_STATE');
|
|
875
|
+
}
|
|
876
|
+
for (let index = 0; index < previous.length; index += 1) {
|
|
877
|
+
if (!isDeepStrictEqual(previous[index], incoming[index]))
|
|
878
|
+
throw new WorkflowPackageError('workflow replay journal diverged', 'WORKFLOW_INVALID_STATE');
|
|
879
|
+
}
|
|
880
|
+
}
|
|
881
|
+
run.checkpoint = { journal: incoming, agentSpend: normalizedSpend, agentSeq: normalizedSequence };
|
|
882
|
+
}
|
|
883
|
+
clearGate(run) {
|
|
884
|
+
const gate = run.gate;
|
|
885
|
+
if (gate === undefined)
|
|
886
|
+
return;
|
|
887
|
+
run.gate = undefined;
|
|
888
|
+
gate.abort.abort('workflow gate is no longer current');
|
|
889
|
+
}
|
|
890
|
+
allowedActions(run, status) {
|
|
891
|
+
const save = !run.builtIn && !run.head.numberedHandle && status !== 'interrupted';
|
|
892
|
+
const pause = run.attempt === undefined || supportsEngineReplay(run.attempt.handle);
|
|
893
|
+
switch (status) {
|
|
894
|
+
case 'running': return pause
|
|
895
|
+
? save ? ['pause', 'stop', 'save'] : ['pause', 'stop']
|
|
896
|
+
: save ? ['stop', 'save'] : ['stop'];
|
|
897
|
+
case 'needs-input':
|
|
898
|
+
case 'paused': return save ? ['resume', 'stop', 'save'] : ['resume', 'stop'];
|
|
899
|
+
case 'budget-limited': return save ? ['stop', 'save'] : ['stop'];
|
|
900
|
+
case 'completed':
|
|
901
|
+
case 'failed':
|
|
902
|
+
case 'cancelled': return save ? ['save'] : [];
|
|
903
|
+
default: return [];
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
async commitActive(run, patch, aspect) {
|
|
907
|
+
if (TERMINAL.has(run.head.status))
|
|
908
|
+
return run.head;
|
|
909
|
+
const nextDetail = run.detail;
|
|
910
|
+
const counts = memberCounts(nextDetail);
|
|
911
|
+
const next = {
|
|
912
|
+
...run.head, ...patch, memberCounts: counts,
|
|
913
|
+
budget: patch.budget ?? run.head.budget,
|
|
914
|
+
allowedActions: this.allowedActions(run, patch.status ?? run.head.status),
|
|
915
|
+
saveAvailable: this.allowedActions(run, patch.status ?? run.head.status).includes('save'),
|
|
916
|
+
revision: run.head.revision + 1,
|
|
917
|
+
detailRevision: aspect === 'detail' ? run.head.detailRevision + 1 : run.head.detailRevision,
|
|
918
|
+
membersRevision: aspect === 'members' ? run.head.membersRevision + 1 : run.head.membersRevision,
|
|
919
|
+
logsRevision: aspect === 'logs' ? run.head.logsRevision + 1 : run.head.logsRevision,
|
|
920
|
+
resultRevision: aspect === 'result' ? run.head.resultRevision + 1 : run.head.resultRevision,
|
|
921
|
+
artifactsRevision: aspect === 'artifacts' ? run.head.artifactsRevision + 1 : run.head.artifactsRevision,
|
|
922
|
+
};
|
|
923
|
+
const { detail: _detail, completionNotice: _notice, scriptPath: _path, ...durable } = next;
|
|
924
|
+
const saved = await this.store.commitRun({ sessionId: run.sessionId, runId: run.id, expectedRevision: run.head.revision, head: durable, detail: nextDetail });
|
|
925
|
+
run.head = saved;
|
|
926
|
+
this.publishChange(run);
|
|
927
|
+
return saved;
|
|
928
|
+
}
|
|
929
|
+
async failAfterAdmission(run, reason) {
|
|
930
|
+
if (TERMINAL.has(run.head.status))
|
|
931
|
+
return;
|
|
932
|
+
await this.terminalize(run, 'failed', reason, undefined);
|
|
933
|
+
}
|
|
934
|
+
async terminalize(run, status, error, value, spent = run.head.budget.spent) {
|
|
935
|
+
if (TERMINAL.has(run.head.status))
|
|
936
|
+
return;
|
|
937
|
+
const runningBefore = [...(run.detail.members ?? [])].filter(member => member.status === 'running');
|
|
938
|
+
const members = [...(run.detail.members ?? [])].map(member => member.status === 'running'
|
|
939
|
+
? { ...member, status: 'cancelled', outcome: member.outcome === 'available' ? member.outcome : 'not-produced', settledAt: Date.now() }
|
|
940
|
+
: member);
|
|
941
|
+
let artifacts = run.detail.artifacts ?? [];
|
|
942
|
+
let report;
|
|
943
|
+
try {
|
|
944
|
+
await run.scratch?.list();
|
|
945
|
+
}
|
|
946
|
+
catch (drainError) {
|
|
947
|
+
this.ctx?.logger?.warn?.('workflow scratch drain failed during settlement', drainError);
|
|
948
|
+
}
|
|
949
|
+
try {
|
|
950
|
+
const snapshot = await this.scanArtifacts(run, run.head);
|
|
951
|
+
artifacts = snapshot.items.map(({ name, bytes }) => ({ name, bytes }));
|
|
952
|
+
}
|
|
953
|
+
catch (artifactError) {
|
|
954
|
+
this.ctx?.logger?.warn?.('workflow artifact snapshot failed during settlement', artifactError);
|
|
955
|
+
}
|
|
956
|
+
try {
|
|
957
|
+
if (typeof this.store.readRunReport === 'function')
|
|
958
|
+
report = await this.store.readRunReport(run.runDirectory, this.config.completionNoticeMaxBytes);
|
|
959
|
+
else {
|
|
960
|
+
const raw = await run.scratch?.read('report.md');
|
|
961
|
+
if (raw !== undefined)
|
|
962
|
+
report = utf8Prefix(raw, this.config.completionNoticeMaxBytes);
|
|
963
|
+
}
|
|
964
|
+
}
|
|
965
|
+
catch (reportError) {
|
|
966
|
+
this.ctx?.logger?.warn?.('workflow completion report read failed during settlement', reportError);
|
|
967
|
+
}
|
|
968
|
+
// The scan/report operations above are capability-rooted and bounded. A
|
|
969
|
+
// final scratch drain/disposal now closes the authority before the
|
|
970
|
+
// terminal manifest commit, so no admitted write can follow the roster.
|
|
971
|
+
await run.scratch?.dispose().catch(error2 => this.ctx?.logger?.warn?.('workflow scratch disposal failed', error2));
|
|
972
|
+
const artifactsChanged = !isDeepStrictEqual(artifacts, run.detail.artifacts ?? []);
|
|
973
|
+
run.detail = { ...run.detail, members,
|
|
974
|
+
result: status === 'completed'
|
|
975
|
+
? this.resultPayload(value)
|
|
976
|
+
: { state: 'not-produced' },
|
|
977
|
+
artifacts,
|
|
978
|
+
};
|
|
979
|
+
let terminalPreview;
|
|
980
|
+
if (status === 'completed') {
|
|
981
|
+
try {
|
|
982
|
+
const projection = workflowRunValueView(value, this.config.remoteHeadTextMaxBytes);
|
|
983
|
+
terminalPreview = projection.state === 'available'
|
|
984
|
+
? projection.content.kind === 'preview'
|
|
985
|
+
? projection.content.text
|
|
986
|
+
: utf8Prefix(JSON.stringify(projection.content.value), this.config.remoteHeadTextMaxBytes)
|
|
987
|
+
: undefined;
|
|
988
|
+
}
|
|
989
|
+
catch {
|
|
990
|
+
terminalPreview = undefined;
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
const terminalHead = {
|
|
994
|
+
...run.head,
|
|
995
|
+
status,
|
|
996
|
+
stopReason: status === 'completed' ? 'completed' : status === 'cancelled' ? 'cancelled' : status === 'interrupted' ? 'interrupted' : 'error',
|
|
997
|
+
...(error === undefined ? {} : { error: utf8Prefix(error, this.config.maxEventTextBytes) }),
|
|
998
|
+
...(terminalPreview === undefined ? {} : { terminalPreview }),
|
|
999
|
+
settledAt: Date.now(),
|
|
1000
|
+
budget: { ...run.head.budget, spent: Math.min(run.head.budget.total, spent), remaining: Math.max(0, run.head.budget.total - spent) },
|
|
1001
|
+
memberCounts: memberCounts(run.detail),
|
|
1002
|
+
allowedActions: this.allowedActions(run, status), executionAvailable: false,
|
|
1003
|
+
saveAvailable: this.allowedActions(run, status).includes('save'),
|
|
1004
|
+
revision: run.head.revision + 1,
|
|
1005
|
+
membersRevision: run.head.membersRevision + (runningBefore.length > 0 ? 1 : 0),
|
|
1006
|
+
resultRevision: run.head.resultRevision + 1,
|
|
1007
|
+
artifactsRevision: run.head.artifactsRevision + (artifactsChanged ? 1 : 0),
|
|
1008
|
+
};
|
|
1009
|
+
const { detail: _detail, completionNotice: _notice, scriptPath: _scriptPath, ...durable } = terminalHead;
|
|
1010
|
+
let saved;
|
|
1011
|
+
try {
|
|
1012
|
+
saved = await this.store.commitTerminalAndClaimNotice({ sessionId: run.sessionId, runId: run.id, expectedRevision: run.head.revision, head: durable, detail: run.detail });
|
|
1013
|
+
}
|
|
1014
|
+
catch (failure) {
|
|
1015
|
+
run.settlementFailure ??= failure;
|
|
1016
|
+
this.ctx?.logger?.warn?.('workflow terminal persistence failed', failure);
|
|
1017
|
+
// Do not expose a false terminal row as durable; retain the in-memory
|
|
1018
|
+
// state for teardown/retry and surface the storage failure to callers.
|
|
1019
|
+
throw failure;
|
|
1020
|
+
}
|
|
1021
|
+
run.settlementFailure = undefined;
|
|
1022
|
+
run.head = saved;
|
|
1023
|
+
run.active = false;
|
|
1024
|
+
this.activeTotal = Math.max(0, this.activeTotal - 1);
|
|
1025
|
+
this.activeSessions.set(run.sessionId, Math.max(0, (this.activeSessions.get(run.sessionId) ?? 1) - 1));
|
|
1026
|
+
this.clearGate(run);
|
|
1027
|
+
this.publishChange(run);
|
|
1028
|
+
const endedIds = new Set(runningBefore.map(member => member.memberId));
|
|
1029
|
+
const lifecycleMembers = members.filter(member => endedIds.has(member.memberId)).map(member => this.memberLifecycle(member));
|
|
1030
|
+
for (const member of lifecycleMembers)
|
|
1031
|
+
await this.publishLifecycle(run, 'workflows/member-end', this.info(run), member);
|
|
1032
|
+
const stopReason = status === 'failed' ? 'error' : status;
|
|
1033
|
+
await this.publishLifecycle(run, 'workflows/run-end', this.info(run), { stopReason, ...(error === undefined ? {} : { error }), agentsStarted: spent });
|
|
1034
|
+
this.notifier.reserve(run.id, run.parent);
|
|
1035
|
+
const delivery = (async () => {
|
|
1036
|
+
let noticeResult;
|
|
1037
|
+
if (status === 'completed') {
|
|
1038
|
+
try {
|
|
1039
|
+
noticeResult = workflowRunValueView(value, this.config.memberOutcomeMaxBytes);
|
|
1040
|
+
}
|
|
1041
|
+
catch {
|
|
1042
|
+
noticeResult = { state: 'not-produced' };
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
await this.notifier.notify({
|
|
1046
|
+
runId: run.id, sessionId: run.sessionId, displayName: run.head.displayName, status,
|
|
1047
|
+
parent: run.parent, head: run.head, error, result: noticeResult,
|
|
1048
|
+
...(report === undefined ? {} : { report }),
|
|
1049
|
+
});
|
|
1050
|
+
// Notice finalization is a durable head revision. Refresh that exact row
|
|
1051
|
+
// so subsequent list/control baselines do not merge a stale live head.
|
|
1052
|
+
const rows = await this.store.readSession(run.sessionId);
|
|
1053
|
+
const finalized = rows.find(candidate => candidate.runId === String(run.id));
|
|
1054
|
+
if (finalized !== undefined && finalized.revision > run.head.revision) {
|
|
1055
|
+
run.head = finalized;
|
|
1056
|
+
this.publishChange(run);
|
|
1057
|
+
}
|
|
1058
|
+
})();
|
|
1059
|
+
run.delivery = delivery;
|
|
1060
|
+
void delivery.catch(error2 => this.ctx?.logger?.warn?.('workflow completion notice failed', error2)).finally(() => {
|
|
1061
|
+
if (run.delivery === delivery)
|
|
1062
|
+
run.delivery = undefined;
|
|
1063
|
+
});
|
|
1064
|
+
}
|
|
1065
|
+
resultPayload(value) {
|
|
1066
|
+
try {
|
|
1067
|
+
const view = workflowRunValueView(value, this.config.memberOutcomeMaxBytes);
|
|
1068
|
+
return view.state === 'available' && view.content.kind === 'value'
|
|
1069
|
+
? { state: 'available', value: view.content.value, totalBytes: view.totalBytes, truncated: false }
|
|
1070
|
+
: view.state === 'available' && view.content.kind === 'preview'
|
|
1071
|
+
? { state: 'available', preview: view.content.text, totalBytes: view.totalBytes, truncated: true }
|
|
1072
|
+
: { state: view.state };
|
|
1073
|
+
}
|
|
1074
|
+
catch {
|
|
1075
|
+
return { state: 'available', preview: '[result could not be serialized]', totalBytes: 31, truncated: true };
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
displayKey(sessionId, displayName) { return `${sessionId}\0${displayName}`; }
|
|
1079
|
+
requireOwned(displayName, agent, interruptedMessage) {
|
|
1080
|
+
const sessionId = sessionOf(agent);
|
|
1081
|
+
const run = this.byDisplay.get(this.displayKey(sessionId, displayName));
|
|
1082
|
+
if (run === undefined) {
|
|
1083
|
+
const recovered = this.recoveredByDisplay.get(this.displayKey(sessionId, displayName));
|
|
1084
|
+
if (recovered !== undefined) {
|
|
1085
|
+
throw new WorkflowPackageError(interruptedMessage ?? `workflow "${displayName}" was interrupted by process exit and cannot resume`, 'WORKFLOW_INVALID_STATE');
|
|
1086
|
+
}
|
|
1087
|
+
throw new WorkflowPackageError(`workflow "${displayName}" was not found`, 'WORKFLOW_RUN_NOT_FOUND');
|
|
1088
|
+
}
|
|
1089
|
+
if (run.parent !== agent)
|
|
1090
|
+
throw new WorkflowPackageError(`workflow "${displayName}" is not owned by this Agent`, 'WORKFLOW_RUN_NOT_OWNED');
|
|
1091
|
+
return run;
|
|
1092
|
+
}
|
|
1093
|
+
requireOwnedId(runId, agent) {
|
|
1094
|
+
const run = this.runs.get(runId);
|
|
1095
|
+
if (run === undefined) {
|
|
1096
|
+
const recovered = this.recoveredById.get(String(runId));
|
|
1097
|
+
if (recovered !== undefined && recovered.sessionId === sessionOf(agent)) {
|
|
1098
|
+
throw new WorkflowPackageError(`workflow "${recovered.displayName}" was interrupted by process exit and cannot resume`, 'WORKFLOW_INVALID_STATE');
|
|
1099
|
+
}
|
|
1100
|
+
throw new WorkflowPackageError('workflow run was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1101
|
+
}
|
|
1102
|
+
if (run.parent !== agent || run.sessionId !== sessionOf(agent))
|
|
1103
|
+
throw new WorkflowPackageError('workflow run is not owned by this Agent', 'WORKFLOW_RUN_NOT_OWNED');
|
|
1104
|
+
return run;
|
|
1105
|
+
}
|
|
1106
|
+
async awaitCaller(promise, signal) {
|
|
1107
|
+
if (signal === undefined)
|
|
1108
|
+
return promise;
|
|
1109
|
+
signal.throwIfAborted();
|
|
1110
|
+
return await new Promise((resolve, reject) => {
|
|
1111
|
+
let done = false;
|
|
1112
|
+
const cleanup = () => signal.removeEventListener('abort', abort);
|
|
1113
|
+
const abort = () => {
|
|
1114
|
+
if (done)
|
|
1115
|
+
return;
|
|
1116
|
+
done = true;
|
|
1117
|
+
cleanup();
|
|
1118
|
+
reject(signal.reason ?? new DOMException('This operation was aborted', 'AbortError'));
|
|
1119
|
+
};
|
|
1120
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
1121
|
+
void promise.then(value => { if (!done) {
|
|
1122
|
+
done = true;
|
|
1123
|
+
cleanup();
|
|
1124
|
+
resolve(value);
|
|
1125
|
+
} }, error => { if (!done) {
|
|
1126
|
+
done = true;
|
|
1127
|
+
cleanup();
|
|
1128
|
+
reject(error);
|
|
1129
|
+
} });
|
|
1130
|
+
});
|
|
1131
|
+
}
|
|
1132
|
+
/** Quiesce and durably pause one running attempt. */
|
|
1133
|
+
async pause(displayName, agent, signal) {
|
|
1134
|
+
const run = this.requireOwned(displayName, agent, `workflow "${displayName}" is not running (interrupted)`);
|
|
1135
|
+
const owned = this.enqueue(run, async () => {
|
|
1136
|
+
if (run.head.status !== 'running' || run.attempt === undefined)
|
|
1137
|
+
throw new WorkflowPackageError(`workflow "${displayName}" is not running (${run.head.status})`, 'WORKFLOW_INVALID_STATE');
|
|
1138
|
+
const attempt = run.attempt;
|
|
1139
|
+
if (!supportsEngineReplay(attempt.handle)) {
|
|
1140
|
+
throw new WorkflowPackageError(`workflow "${displayName}" cannot pause because replay checkpoints are unavailable`, 'WORKFLOW_INVALID_STATE');
|
|
1141
|
+
}
|
|
1142
|
+
attempt.intent = 'pause';
|
|
1143
|
+
this.clearGate(run);
|
|
1144
|
+
await this.commitActive(run, { status: 'pausing' }, 'detail');
|
|
1145
|
+
try {
|
|
1146
|
+
attempt.handle.cancel('paused by user');
|
|
1147
|
+
}
|
|
1148
|
+
catch (error) {
|
|
1149
|
+
await this.terminalize(run, 'failed', `workflow pause cancellation failed: ${renderThrown(error)}`);
|
|
1150
|
+
return this.publicHead(run);
|
|
1151
|
+
}
|
|
1152
|
+
let outcome;
|
|
1153
|
+
try {
|
|
1154
|
+
outcome = await attempt.quiescent;
|
|
1155
|
+
}
|
|
1156
|
+
catch (error) {
|
|
1157
|
+
if (this.currentAttempt(run, attempt)) {
|
|
1158
|
+
this.executions.delete(attempt.executionId);
|
|
1159
|
+
run.attempt = undefined;
|
|
1160
|
+
await this.terminalize(run, 'failed', `workflow attempt settlement failed: ${renderThrown(error)}`);
|
|
1161
|
+
}
|
|
1162
|
+
return this.publicHead(run);
|
|
1163
|
+
}
|
|
1164
|
+
if (!this.currentAttempt(run, attempt))
|
|
1165
|
+
return headView(run.head);
|
|
1166
|
+
this.executions.delete(attempt.executionId);
|
|
1167
|
+
run.attempt = undefined;
|
|
1168
|
+
if (outcome.cleanupError !== undefined) {
|
|
1169
|
+
await this.terminalize(run, 'failed', `workflow attempt cleanup failed: ${outcome.cleanupError}`);
|
|
1170
|
+
return this.publicHead(run);
|
|
1171
|
+
}
|
|
1172
|
+
if (outcome.checkpoint === undefined) {
|
|
1173
|
+
await this.terminalize(run, 'failed', 'workflow checkpoint is unavailable after pause');
|
|
1174
|
+
return this.publicHead(run);
|
|
1175
|
+
}
|
|
1176
|
+
try {
|
|
1177
|
+
this.acceptCheckpoint(run, outcome.checkpoint);
|
|
1178
|
+
}
|
|
1179
|
+
catch (error) {
|
|
1180
|
+
await this.terminalize(run, 'failed', `workflow replay checkpoint diverged: ${renderThrown(error)}`);
|
|
1181
|
+
return this.publicHead(run);
|
|
1182
|
+
}
|
|
1183
|
+
const ended = this.settleRunningMembers(run);
|
|
1184
|
+
const spent = Math.max(run.head.budget.spent, outcome.checkpoint.agentSpend, outcome.result.agentsStarted);
|
|
1185
|
+
run.head = { ...run.head, budget: { ...run.head.budget, spent, remaining: Math.max(0, run.head.budget.total - spent) } };
|
|
1186
|
+
const saved = await this.commitActive(run, { status: 'paused', budget: run.head.budget }, 'result');
|
|
1187
|
+
for (const member of ended)
|
|
1188
|
+
await this.publishLifecycle(run, 'workflows/member-end', this.info(run), member);
|
|
1189
|
+
return headView(saved);
|
|
1190
|
+
});
|
|
1191
|
+
return this.awaitCaller(owned, signal);
|
|
1192
|
+
}
|
|
1193
|
+
settleRunningMembers(run) {
|
|
1194
|
+
const ended = [];
|
|
1195
|
+
const members = [...(run.detail.members ?? [])].map(member => {
|
|
1196
|
+
if (member.status !== 'running')
|
|
1197
|
+
return member;
|
|
1198
|
+
const settled = {
|
|
1199
|
+
...member, status: 'cancelled', settledAt: Date.now(),
|
|
1200
|
+
outcome: member.outcome === 'available' ? 'available' : 'not-produced',
|
|
1201
|
+
};
|
|
1202
|
+
ended.push(this.memberLifecycle(settled));
|
|
1203
|
+
return settled;
|
|
1204
|
+
});
|
|
1205
|
+
run.detail = { ...run.detail, members };
|
|
1206
|
+
return ended;
|
|
1207
|
+
}
|
|
1208
|
+
/** Stop one nonterminal run only after attempt and scratch cleanup. */
|
|
1209
|
+
async stop(displayName, agent, signal) {
|
|
1210
|
+
const run = this.requireOwned(displayName, agent, `workflow "${displayName}" already settled (interrupted)`);
|
|
1211
|
+
const owned = this.enqueue(run, async () => {
|
|
1212
|
+
if (TERMINAL.has(run.head.status))
|
|
1213
|
+
throw new WorkflowPackageError(`workflow "${displayName}" already settled (${run.head.status})`, 'WORKFLOW_INVALID_STATE');
|
|
1214
|
+
this.clearGate(run);
|
|
1215
|
+
const attempt = run.attempt;
|
|
1216
|
+
if (attempt !== undefined) {
|
|
1217
|
+
attempt.intent = 'stop';
|
|
1218
|
+
await this.commitActive(run, { status: 'stopping' }, 'detail');
|
|
1219
|
+
try {
|
|
1220
|
+
attempt.handle.cancel('stopped by user');
|
|
1221
|
+
}
|
|
1222
|
+
catch (error) {
|
|
1223
|
+
await this.terminalize(run, 'failed', `workflow stop cancellation failed: ${renderThrown(error)}`);
|
|
1224
|
+
return this.publicHead(run);
|
|
1225
|
+
}
|
|
1226
|
+
let outcome;
|
|
1227
|
+
try {
|
|
1228
|
+
outcome = await attempt.quiescent;
|
|
1229
|
+
}
|
|
1230
|
+
catch (error) {
|
|
1231
|
+
if (this.currentAttempt(run, attempt)) {
|
|
1232
|
+
this.executions.delete(attempt.executionId);
|
|
1233
|
+
run.attempt = undefined;
|
|
1234
|
+
await this.terminalize(run, 'failed', `workflow attempt settlement failed: ${renderThrown(error)}`);
|
|
1235
|
+
}
|
|
1236
|
+
return this.publicHead(run);
|
|
1237
|
+
}
|
|
1238
|
+
if (this.currentAttempt(run, attempt)) {
|
|
1239
|
+
this.executions.delete(attempt.executionId);
|
|
1240
|
+
run.attempt = undefined;
|
|
1241
|
+
if (outcome.checkpoint !== undefined) {
|
|
1242
|
+
try {
|
|
1243
|
+
this.acceptCheckpoint(run, outcome.checkpoint);
|
|
1244
|
+
}
|
|
1245
|
+
catch (error) {
|
|
1246
|
+
await this.terminalize(run, 'failed', `workflow replay checkpoint diverged: ${renderThrown(error)}`);
|
|
1247
|
+
return this.publicHead(run);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
if (outcome.cleanupError !== undefined) {
|
|
1251
|
+
await this.terminalize(run, 'failed', `workflow attempt cleanup failed: ${outcome.cleanupError}`);
|
|
1252
|
+
return this.publicHead(run);
|
|
1253
|
+
}
|
|
1254
|
+
await this.terminalize(run, 'cancelled', outcome.result.error ?? 'stopped by user', undefined, Math.max(run.head.budget.spent, outcome.result.agentsStarted, outcome.checkpoint?.agentSpend ?? 0));
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
else
|
|
1258
|
+
await this.terminalize(run, 'cancelled', 'stopped by user', undefined);
|
|
1259
|
+
return headView(run.head);
|
|
1260
|
+
});
|
|
1261
|
+
return this.awaitCaller(owned, signal);
|
|
1262
|
+
}
|
|
1263
|
+
/** Resume a paused run or acknowledge its current live gate. */
|
|
1264
|
+
async resume(displayName, agent, signal) {
|
|
1265
|
+
const run = this.requireOwned(displayName, agent);
|
|
1266
|
+
const owned = this.enqueue(run, () => this.resumeRecord(run, undefined, signal));
|
|
1267
|
+
return this.awaitCaller(owned, signal);
|
|
1268
|
+
}
|
|
1269
|
+
async resumeById(runId, agent, higherBudget, signal) {
|
|
1270
|
+
const run = this.requireOwnedId(runId, agent);
|
|
1271
|
+
const owned = this.enqueue(run, async () => {
|
|
1272
|
+
if (run.head.status === 'budget-limited') {
|
|
1273
|
+
if (!Number.isSafeInteger(higherBudget) || higherBudget <= run.head.budget.total || higherBudget > this.config.maxAgentBudget) {
|
|
1274
|
+
throw new WorkflowPackageError(`workflow "${run.head.displayName}" requires a higher agent_budget to resume`, 'WORKFLOW_INVALID_STATE');
|
|
1275
|
+
}
|
|
1276
|
+
return this.resumeRecord(run, higherBudget, signal);
|
|
1277
|
+
}
|
|
1278
|
+
if (higherBudget !== undefined && higherBudget !== run.head.budget.total)
|
|
1279
|
+
throw new WorkflowPackageError('agent_budget may be raised only when resuming a budget-limited workflow', 'WORKFLOW_INVALID_STATE');
|
|
1280
|
+
return this.resumeRecord(run, undefined, signal);
|
|
1281
|
+
});
|
|
1282
|
+
return this.awaitCaller(owned, signal);
|
|
1283
|
+
}
|
|
1284
|
+
async resumeRecord(run, higherBudget, signal) {
|
|
1285
|
+
signal?.throwIfAborted();
|
|
1286
|
+
if (this.disposed || !this.admission || run.closing !== undefined) {
|
|
1287
|
+
throw new WorkflowPackageError('workflow supervisor is shutting down', 'WORKFLOW_INVALID_STATE');
|
|
1288
|
+
}
|
|
1289
|
+
if (run.head.status === 'budget-limited' && higherBudget === undefined)
|
|
1290
|
+
throw new WorkflowPackageError(`workflow "${run.head.displayName}" requires a higher agent_budget to resume`, 'WORKFLOW_INVALID_STATE');
|
|
1291
|
+
if (run.head.status === 'needs-input') {
|
|
1292
|
+
const gate = run.gate;
|
|
1293
|
+
const attempt = run.attempt;
|
|
1294
|
+
if (gate === undefined || attempt === undefined)
|
|
1295
|
+
throw new WorkflowPackageError(`workflow "${run.head.displayName}" has no live input gate`, 'WORKFLOW_INVALID_STATE');
|
|
1296
|
+
if (gate.gate.resumable) {
|
|
1297
|
+
this.clearGate(run);
|
|
1298
|
+
const saved = await this.commitActive(run, { status: 'running' }, 'detail');
|
|
1299
|
+
attempt.handle.resume();
|
|
1300
|
+
return headView(saved);
|
|
1301
|
+
}
|
|
1302
|
+
return this.replaceGateAttempt(run, attempt, higherBudget);
|
|
1303
|
+
}
|
|
1304
|
+
if (run.head.status !== 'paused' && run.head.status !== 'budget-limited')
|
|
1305
|
+
throw new WorkflowPackageError(`workflow "${run.head.displayName}" cannot resume from ${run.head.status}`, 'WORKFLOW_INVALID_STATE');
|
|
1306
|
+
if (run.checkpoint === undefined)
|
|
1307
|
+
throw new WorkflowPackageError(`workflow "${run.head.displayName}" has no same-process replay authority`, 'WORKFLOW_INVALID_STATE');
|
|
1308
|
+
const oldBudget = run.head.budget;
|
|
1309
|
+
const oldStatus = run.head.status;
|
|
1310
|
+
if (higherBudget !== undefined)
|
|
1311
|
+
run.head = { ...run.head, budget: { total: higherBudget, spent: oldBudget.spent, remaining: Math.max(0, higherBudget - oldBudget.spent) } };
|
|
1312
|
+
let attempt;
|
|
1313
|
+
let committed = false;
|
|
1314
|
+
try {
|
|
1315
|
+
attempt = this.createAttempt(run);
|
|
1316
|
+
run.attempt = attempt;
|
|
1317
|
+
run.generation = attempt.generation;
|
|
1318
|
+
this.executions.set(attempt.executionId, { run, generation: attempt.generation });
|
|
1319
|
+
if (!this.admission && run.closing === undefined)
|
|
1320
|
+
run.closing = 'supervisor';
|
|
1321
|
+
this.startObservation(run, attempt);
|
|
1322
|
+
const saved = await this.commitActive(run, { status: 'running', budget: run.head.budget }, 'result');
|
|
1323
|
+
committed = true;
|
|
1324
|
+
this.releaseDeferredAttempt(run, attempt);
|
|
1325
|
+
return headView(saved);
|
|
1326
|
+
}
|
|
1327
|
+
catch (error) {
|
|
1328
|
+
if (attempt !== undefined) {
|
|
1329
|
+
this.executions.delete(attempt.executionId);
|
|
1330
|
+
if (run.attempt === attempt)
|
|
1331
|
+
run.attempt = undefined;
|
|
1332
|
+
attempt.handle.cancel('workflow resume rolled back');
|
|
1333
|
+
await attempt.quiescent.catch(() => undefined);
|
|
1334
|
+
}
|
|
1335
|
+
run.head = { ...run.head, budget: oldBudget };
|
|
1336
|
+
if (committed && !TERMINAL.has(run.head.status)) {
|
|
1337
|
+
try {
|
|
1338
|
+
await this.commitActive(run, { status: oldStatus, budget: oldBudget }, 'result');
|
|
1339
|
+
}
|
|
1340
|
+
catch (rollbackError) {
|
|
1341
|
+
this.ctx?.logger?.warn?.('workflow resume rollback persistence failed', rollbackError);
|
|
1342
|
+
await this.terminalize(run, 'failed', `workflow resume failed: ${renderThrown(error)}`).catch(() => undefined);
|
|
1343
|
+
}
|
|
1344
|
+
}
|
|
1345
|
+
throw error;
|
|
1346
|
+
}
|
|
1347
|
+
}
|
|
1348
|
+
async replaceGateAttempt(run, attempt, higherBudget) {
|
|
1349
|
+
attempt.intent = 'pause';
|
|
1350
|
+
this.clearGate(run);
|
|
1351
|
+
await this.commitActive(run, { status: 'pausing' }, 'detail');
|
|
1352
|
+
attempt.handle.cancel('replaying workflow pause gate');
|
|
1353
|
+
const outcome = await attempt.quiescent;
|
|
1354
|
+
if (!this.currentAttempt(run, attempt))
|
|
1355
|
+
return headView(run.head);
|
|
1356
|
+
this.executions.delete(attempt.executionId);
|
|
1357
|
+
run.attempt = undefined;
|
|
1358
|
+
if (outcome.cleanupError !== undefined) {
|
|
1359
|
+
await this.terminalize(run, 'failed', `workflow attempt cleanup failed: ${outcome.cleanupError}`);
|
|
1360
|
+
return this.publicHead(run);
|
|
1361
|
+
}
|
|
1362
|
+
if (outcome.checkpoint === undefined) {
|
|
1363
|
+
await this.terminalize(run, 'failed', 'workflow checkpoint is unavailable after gate');
|
|
1364
|
+
return this.publicHead(run);
|
|
1365
|
+
}
|
|
1366
|
+
try {
|
|
1367
|
+
this.acceptCheckpoint(run, outcome.checkpoint);
|
|
1368
|
+
}
|
|
1369
|
+
catch (error) {
|
|
1370
|
+
await this.terminalize(run, 'failed', `workflow replay checkpoint diverged: ${renderThrown(error)}`);
|
|
1371
|
+
return this.publicHead(run);
|
|
1372
|
+
}
|
|
1373
|
+
const ended = this.settleRunningMembers(run);
|
|
1374
|
+
await this.commitActive(run, { status: 'paused' }, 'result');
|
|
1375
|
+
for (const member of ended)
|
|
1376
|
+
await this.publishLifecycle(run, 'workflows/member-end', this.info(run), member);
|
|
1377
|
+
return this.resumeRecord(run, higherBudget);
|
|
1378
|
+
}
|
|
1379
|
+
/** Acknowledge only the exact still-current gate fence. */
|
|
1380
|
+
async resumeGate(runId, executionId, gateId, agent, signal) {
|
|
1381
|
+
const run = this.runs.get(runId);
|
|
1382
|
+
if (run === undefined || run.parent !== agent || run.sessionId !== sessionOf(agent))
|
|
1383
|
+
return false;
|
|
1384
|
+
const operation = this.enqueue(run, async () => {
|
|
1385
|
+
signal?.throwIfAborted();
|
|
1386
|
+
const gate = run.gate;
|
|
1387
|
+
const attempt = run.attempt;
|
|
1388
|
+
if (run.head.status !== 'needs-input' || gate === undefined || attempt === undefined)
|
|
1389
|
+
return false;
|
|
1390
|
+
if (gate.executionId !== executionId || gate.gateId !== gateId || gate.generation !== attempt.generation || !this.currentAttempt(run, attempt))
|
|
1391
|
+
return false;
|
|
1392
|
+
if (gate.gate.resumable) {
|
|
1393
|
+
this.clearGate(run);
|
|
1394
|
+
await this.commitActive(run, { status: 'running' }, 'detail');
|
|
1395
|
+
attempt.handle.resume();
|
|
1396
|
+
}
|
|
1397
|
+
else
|
|
1398
|
+
await this.replaceGateAttempt(run, attempt);
|
|
1399
|
+
return true;
|
|
1400
|
+
});
|
|
1401
|
+
return this.awaitCaller(operation, signal);
|
|
1402
|
+
}
|
|
1403
|
+
/** Save a safe, current editable projection without changing live authority. */
|
|
1404
|
+
async save(displayName, agent, scope = this.config.saveScope, signal) {
|
|
1405
|
+
const run = this.requireOwned(displayName, agent, `workflow "${displayName}" cannot be saved after process interruption`);
|
|
1406
|
+
return this.awaitCaller(this.enqueue(run, async () => {
|
|
1407
|
+
signal?.throwIfAborted();
|
|
1408
|
+
if (run.builtIn)
|
|
1409
|
+
throw new WorkflowPackageError(`workflow "${displayName}" is a built-in: save an edited copy under a new meta.name`, 'WORKFLOW_INVALID_STATE');
|
|
1410
|
+
if (run.head.numberedHandle)
|
|
1411
|
+
throw new WorkflowPackageError(`workflow "${displayName}" is a numbered handle: save an edited copy under a new unique meta.name`, 'WORKFLOW_INVALID_STATE');
|
|
1412
|
+
if (run.head.status === 'interrupted')
|
|
1413
|
+
throw new WorkflowPackageError(`workflow "${displayName}" cannot be saved after process interruption`, 'WORKFLOW_INVALID_STATE');
|
|
1414
|
+
if (!this.registry)
|
|
1415
|
+
throw new WorkflowPackageError('workflow registry is unavailable', 'WORKFLOW_REGISTRY_DISABLED');
|
|
1416
|
+
const readProjection = this.store.readRunScript;
|
|
1417
|
+
// Small in-memory stores used by embedders/tests may not expose a file
|
|
1418
|
+
// capability; their immutable source is the only projection available.
|
|
1419
|
+
// A Host-backed FileWorkflowRunStore always takes the descriptor branch.
|
|
1420
|
+
const script = typeof readProjection === 'function'
|
|
1421
|
+
? await readProjection.call(this.store, run.runDirectory, this.config.maxScriptProjectionBytes, signal)
|
|
1422
|
+
: run.script;
|
|
1423
|
+
const saved = await this.registry.save({ meta: run.meta, script }, { scope, cwd: agent?.session?.header?.cwd, signal });
|
|
1424
|
+
return saved.path;
|
|
1425
|
+
}), signal);
|
|
1426
|
+
}
|
|
1427
|
+
publicHead(run) {
|
|
1428
|
+
const raw = headView(run.head);
|
|
1429
|
+
// Recovered rows deliberately have no live Agent or editable authority.
|
|
1430
|
+
// Never let persisted `allowedActions` turn into a capability after a
|
|
1431
|
+
// process restart.
|
|
1432
|
+
const actions = run.parent === undefined ? [] : this.allowedActions(run, raw.status);
|
|
1433
|
+
return {
|
|
1434
|
+
...raw,
|
|
1435
|
+
allowedActions: actions,
|
|
1436
|
+
};
|
|
1437
|
+
}
|
|
1438
|
+
publicHeadRecord(head) {
|
|
1439
|
+
const raw = headView(head);
|
|
1440
|
+
return {
|
|
1441
|
+
...raw,
|
|
1442
|
+
allowedActions: head.executionAvailable === false ? [] : raw.allowedActions,
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
async storedHeadFor(agent, runId, signal) {
|
|
1446
|
+
const sessionId = sessionOf(agent);
|
|
1447
|
+
signal?.throwIfAborted();
|
|
1448
|
+
const live = this.runs.get(String(runId));
|
|
1449
|
+
if (live !== undefined) {
|
|
1450
|
+
if (live.sessionId !== sessionId)
|
|
1451
|
+
throw new WorkflowPackageError('workflow run was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1452
|
+
return live.head;
|
|
1453
|
+
}
|
|
1454
|
+
const rows = await this.store.readSession(sessionId, signal);
|
|
1455
|
+
const head = rows.find(candidate => candidate.runId === String(runId));
|
|
1456
|
+
if (head === undefined)
|
|
1457
|
+
throw new WorkflowPackageError('workflow run was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1458
|
+
return head;
|
|
1459
|
+
}
|
|
1460
|
+
async readDetailValue(runId, kind, request = {}, signal) {
|
|
1461
|
+
const result = await this.store.readDetails(runId, { kind, ...request }, signal);
|
|
1462
|
+
return result;
|
|
1463
|
+
}
|
|
1464
|
+
async readAllMembers(runId, signal) {
|
|
1465
|
+
const members = [];
|
|
1466
|
+
let cursor;
|
|
1467
|
+
do {
|
|
1468
|
+
signal?.throwIfAborted();
|
|
1469
|
+
const page = await this.readDetailValue(runId, 'members', { cursor, limit: 200 }, signal);
|
|
1470
|
+
if (Array.isArray(page.value))
|
|
1471
|
+
members.push(...page.value);
|
|
1472
|
+
cursor = page.nextCursor;
|
|
1473
|
+
if (members.length > this.config.maxMembersPerRun)
|
|
1474
|
+
throw new WorkflowPackageError('workflow member retention limit exceeded', 'WORKFLOW_STORAGE_LIMIT');
|
|
1475
|
+
} while (cursor !== undefined);
|
|
1476
|
+
return members;
|
|
1477
|
+
}
|
|
1478
|
+
memberHead(member) {
|
|
1479
|
+
return {
|
|
1480
|
+
memberId: member.memberId,
|
|
1481
|
+
seq: member.seq,
|
|
1482
|
+
label: member.label,
|
|
1483
|
+
...(member.phase === undefined ? {} : { phase: member.phase }),
|
|
1484
|
+
status: member.status,
|
|
1485
|
+
...(member.startedAt === undefined ? {} : { startedAt: member.startedAt }),
|
|
1486
|
+
...(member.settledAt === undefined ? {} : { settledAt: member.settledAt }),
|
|
1487
|
+
outcome: member.outcome,
|
|
1488
|
+
...(member.childSessionId === undefined ? {} : { childSessionId: member.childSessionId }),
|
|
1489
|
+
};
|
|
1490
|
+
}
|
|
1491
|
+
memberHeadWithTranscript(member) {
|
|
1492
|
+
const head = this.memberHead(member);
|
|
1493
|
+
return { ...head, outcome: memberOutcomeWithTranscript(this.ctx, member) };
|
|
1494
|
+
}
|
|
1495
|
+
async authorizedHead(agent, runId, signal) {
|
|
1496
|
+
const live = this.runs.get(String(runId));
|
|
1497
|
+
if (live !== undefined) {
|
|
1498
|
+
if (live.sessionId !== sessionOf(agent))
|
|
1499
|
+
throw new WorkflowPackageError('workflow run was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1500
|
+
return { head: live.head, live };
|
|
1501
|
+
}
|
|
1502
|
+
return { head: await this.storedHeadFor(agent, runId, signal) };
|
|
1503
|
+
}
|
|
1504
|
+
/** List the authorized Session's retained logical runs. */
|
|
1505
|
+
async list(agent, request = {}, signal) {
|
|
1506
|
+
const sessionId = sessionOf(agent);
|
|
1507
|
+
const rows = await this.store.readSession(sessionId, signal);
|
|
1508
|
+
signal?.throwIfAborted();
|
|
1509
|
+
const liveById = new Map([...this.runs.values()].filter(run => run.sessionId === sessionId).map(run => [String(run.id), run]));
|
|
1510
|
+
const merged = rows.map(head => liveById.get(head.runId)?.head ?? head);
|
|
1511
|
+
// A store implementation may return a newly recovered row that was not
|
|
1512
|
+
// present in the in-memory map. Keep list reads authoritative without
|
|
1513
|
+
// manufacturing executable authority for it.
|
|
1514
|
+
const known = new Set(merged.map(head => head.runId));
|
|
1515
|
+
for (const run of liveById.values())
|
|
1516
|
+
if (!known.has(String(run.id)))
|
|
1517
|
+
merged.push(run.head);
|
|
1518
|
+
merged.sort((left, right) => {
|
|
1519
|
+
const leftActive = ACTIVE.has(left.status) ? 0 : 1;
|
|
1520
|
+
const rightActive = ACTIVE.has(right.status) ? 0 : 1;
|
|
1521
|
+
if (leftActive !== rightActive)
|
|
1522
|
+
return leftActive - rightActive;
|
|
1523
|
+
if (leftActive === 0)
|
|
1524
|
+
return left.startedAt - right.startedAt || left.displayName.localeCompare(right.displayName);
|
|
1525
|
+
return (right.settledAt ?? 0) - (left.settledAt ?? 0)
|
|
1526
|
+
|| right.startedAt - left.startedAt
|
|
1527
|
+
|| left.displayName.localeCompare(right.displayName);
|
|
1528
|
+
});
|
|
1529
|
+
const total = merged.length;
|
|
1530
|
+
const limit = pageLimit(request.limit);
|
|
1531
|
+
const offset = numericCursor(request.cursor, total);
|
|
1532
|
+
const items = merged.slice(offset, offset + limit).map(head => {
|
|
1533
|
+
const run = this.runs.get(head.runId);
|
|
1534
|
+
return run === undefined ? this.publicHeadRecord(head) : this.publicHead(run);
|
|
1535
|
+
});
|
|
1536
|
+
const currentRevision = this.sessionRevisions.get(sessionId)
|
|
1537
|
+
?? merged.reduce((max, head) => Math.max(max, head.revision), 0);
|
|
1538
|
+
if (!this.sessionRevisions.has(sessionId))
|
|
1539
|
+
this.sessionRevisions.set(sessionId, currentRevision);
|
|
1540
|
+
return {
|
|
1541
|
+
epoch: this.feedEpoch,
|
|
1542
|
+
sessionRevision: currentRevision,
|
|
1543
|
+
items,
|
|
1544
|
+
total,
|
|
1545
|
+
...(nextCursor(offset, items.length, total) === undefined ? {} : { nextCursor: nextCursor(offset, items.length, total) }),
|
|
1546
|
+
};
|
|
1547
|
+
}
|
|
1548
|
+
/** Return selected-run metadata after Session authorization. */
|
|
1549
|
+
async detail(agent, runId, signal) {
|
|
1550
|
+
const found = await this.authorizedHead(agent, runId, signal);
|
|
1551
|
+
signal?.throwIfAborted();
|
|
1552
|
+
const run = found.live;
|
|
1553
|
+
let phases = run?.meta.phases?.map(phase => ({
|
|
1554
|
+
title: utf8Prefix(phase.title, this.config.maxEventTextBytes),
|
|
1555
|
+
...(phase.detail === undefined ? {} : { detail: utf8Prefix(phase.detail, this.config.maxEventTextBytes) }),
|
|
1556
|
+
...(phase.provider === undefined ? {} : { provider: utf8Prefix(phase.provider, this.config.maxEventTextBytes) }),
|
|
1557
|
+
...(phase.model === undefined ? {} : { model: utf8Prefix(phase.model, this.config.maxEventTextBytes) }),
|
|
1558
|
+
}));
|
|
1559
|
+
if (phases === undefined || phases.length === 0) {
|
|
1560
|
+
try {
|
|
1561
|
+
const stored = await this.readDetailValue(String(found.head.runId), 'phases', { limit: 200 }, signal);
|
|
1562
|
+
const rows = Array.isArray(stored.value) ? stored.value : [];
|
|
1563
|
+
const recovered = rows
|
|
1564
|
+
.filter((phase) => typeof phase?.title === 'string' && phase.title.length > 0)
|
|
1565
|
+
.map(phase => ({
|
|
1566
|
+
title: utf8Prefix(phase.title, this.config.maxEventTextBytes),
|
|
1567
|
+
...(typeof phase.detail === 'string' ? { detail: utf8Prefix(phase.detail, this.config.maxEventTextBytes) } : {}),
|
|
1568
|
+
}));
|
|
1569
|
+
if (recovered.length > 0)
|
|
1570
|
+
phases = recovered;
|
|
1571
|
+
}
|
|
1572
|
+
catch { /* completed runs without a phases sidecar stay phase-less */ }
|
|
1573
|
+
}
|
|
1574
|
+
const gate = run?.gate?.gate;
|
|
1575
|
+
const error = found.head.error;
|
|
1576
|
+
return {
|
|
1577
|
+
run: run === undefined ? this.publicHeadRecord(found.head) : this.publicHead(run),
|
|
1578
|
+
...(phases === undefined ? {} : { phases }),
|
|
1579
|
+
...(gate === undefined ? {} : { gate }),
|
|
1580
|
+
...(error === undefined ? {} : { error }),
|
|
1581
|
+
...(found.head.scriptPath === undefined ? {} : { scriptPath: found.head.scriptPath }),
|
|
1582
|
+
};
|
|
1583
|
+
}
|
|
1584
|
+
/** Return a bounded retained member page. */
|
|
1585
|
+
async members(agent, request, signal) {
|
|
1586
|
+
const found = await this.authorizedHead(agent, request.runId, signal);
|
|
1587
|
+
const limit = pageLimit(request.limit);
|
|
1588
|
+
const offset = numericCursor(request.cursor, Number.MAX_SAFE_INTEGER);
|
|
1589
|
+
const value = await this.readDetailValue(String(request.runId), 'members', {
|
|
1590
|
+
cursor: String(offset), limit,
|
|
1591
|
+
}, signal);
|
|
1592
|
+
const rows = Array.isArray(value.value) ? value.value : [];
|
|
1593
|
+
const items = rows.map(member => this.memberHeadWithTranscript(member));
|
|
1594
|
+
const total = value.total;
|
|
1595
|
+
const cursor = nextCursor(offset, items.length, total);
|
|
1596
|
+
return {
|
|
1597
|
+
items, total, revision: found.head.membersRevision,
|
|
1598
|
+
...(cursor === undefined ? {} : { nextCursor: cursor }),
|
|
1599
|
+
};
|
|
1600
|
+
}
|
|
1601
|
+
/** Return one retained member outcome, including JSON null when present. */
|
|
1602
|
+
async memberDetail(agent, request, signal) {
|
|
1603
|
+
await this.authorizedHead(agent, request.runId, signal);
|
|
1604
|
+
const members = await this.readAllMembers(String(request.runId), signal);
|
|
1605
|
+
const member = members.find(candidate => candidate.memberId === String(request.memberId));
|
|
1606
|
+
if (member === undefined)
|
|
1607
|
+
throw new WorkflowPackageError('workflow member was not found in this run', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1608
|
+
let outcome = valueViewFromStored(member.value, member.outcome, this.config.memberOutcomeMaxBytes);
|
|
1609
|
+
if (outcome.state === 'not-produced' || (outcome.state === 'pending' && member.status !== 'running')) {
|
|
1610
|
+
const recovered = childTranscriptValue(this.ctx, member.childSessionId);
|
|
1611
|
+
if (recovered !== undefined) {
|
|
1612
|
+
outcome = valueViewFromStored(recovered, 'available', this.config.memberOutcomeMaxBytes);
|
|
1613
|
+
}
|
|
1614
|
+
}
|
|
1615
|
+
return {
|
|
1616
|
+
member: this.memberHeadWithTranscript(member),
|
|
1617
|
+
...(member.childSessionId === undefined ? {} : { childSessionId: member.childSessionId }),
|
|
1618
|
+
outcome,
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
/** Return retained log lines with deterministic eviction metadata. */
|
|
1622
|
+
async logs(agent, request, signal) {
|
|
1623
|
+
const found = await this.authorizedHead(agent, request.runId, signal);
|
|
1624
|
+
const limit = pageLimit(request.limit);
|
|
1625
|
+
const rawOffset = numericCursor(request.cursor, Number.MAX_SAFE_INTEGER);
|
|
1626
|
+
const value = await this.readDetailValue(String(request.runId), 'logs', { cursor: String(rawOffset), limit }, signal);
|
|
1627
|
+
const rows = Array.isArray(value.value) ? value.value : [];
|
|
1628
|
+
const items = rows.map(line => ({ index: line.index, text: utf8Prefix(line.text, this.config.maxLogLineBytes) }));
|
|
1629
|
+
const retainedTotal = value.total;
|
|
1630
|
+
const omitted = value.omitted ?? (items.length > 0 ? items[0].index : 0);
|
|
1631
|
+
if (rawOffset > retainedTotal) {
|
|
1632
|
+
throw new WorkflowPackageError('workflow page cursor is invalid', 'WORKFLOW_CURSOR_INVALID');
|
|
1633
|
+
}
|
|
1634
|
+
const cursor = nextCursor(rawOffset, items.length, retainedTotal);
|
|
1635
|
+
return {
|
|
1636
|
+
items, nextCursor: cursor, evicted: omitted, total: omitted + retainedTotal,
|
|
1637
|
+
revision: found.head.logsRevision,
|
|
1638
|
+
};
|
|
1639
|
+
}
|
|
1640
|
+
/** Return the retained terminal result projection. */
|
|
1641
|
+
async result(agent, runId, signal) {
|
|
1642
|
+
const found = await this.authorizedHead(agent, runId, signal);
|
|
1643
|
+
const value = await this.readDetailValue(String(runId), 'result', {}, signal);
|
|
1644
|
+
const payload = value.value;
|
|
1645
|
+
let view;
|
|
1646
|
+
if (payload?.state === 'available' && Object.prototype.hasOwnProperty.call(payload, 'value')) {
|
|
1647
|
+
view = valueViewFromStored(payload.value, 'available', this.config.memberOutcomeMaxBytes);
|
|
1648
|
+
}
|
|
1649
|
+
else if (payload?.state === 'available' && typeof payload.preview === 'string') {
|
|
1650
|
+
view = {
|
|
1651
|
+
state: 'available',
|
|
1652
|
+
content: { kind: 'preview', text: payload.preview },
|
|
1653
|
+
totalBytes: Number(payload.totalBytes) || encoder.encode(payload.preview).byteLength,
|
|
1654
|
+
truncated: payload.truncated !== false,
|
|
1655
|
+
};
|
|
1656
|
+
}
|
|
1657
|
+
else if (payload?.state === 'pending' || payload?.state === 'not-produced' || payload?.state === 'evicted') {
|
|
1658
|
+
view = { state: payload.state };
|
|
1659
|
+
}
|
|
1660
|
+
else
|
|
1661
|
+
view = { state: TERMINAL.has(found.head.status) ? 'not-produced' : 'pending' };
|
|
1662
|
+
return { value: view, ...(found.head.error === undefined ? {} : { error: found.head.error }), revision: found.head.resultRevision };
|
|
1663
|
+
}
|
|
1664
|
+
async scanArtifacts(run, head, signal) {
|
|
1665
|
+
const capability = this.store.listRunArtifacts;
|
|
1666
|
+
if (typeof capability === 'function') {
|
|
1667
|
+
const inventory = await capability.call(this.store, head.runDirectory, this.config.maxRetainedArtifactsPerRun, signal);
|
|
1668
|
+
return {
|
|
1669
|
+
items: inventory.items.map(item => ({
|
|
1670
|
+
name: item.name, bytes: item.bytes,
|
|
1671
|
+
identity: `${item.identity.dev}:${item.identity.ino}:${item.identity.mtimeMs}:${item.identity.size}`,
|
|
1672
|
+
capabilityIdentity: item.identity,
|
|
1673
|
+
})),
|
|
1674
|
+
total: inventory.total,
|
|
1675
|
+
};
|
|
1676
|
+
}
|
|
1677
|
+
if (run?.scratch !== undefined) {
|
|
1678
|
+
const names = (await run.scratch.list(signal)).filter(name => SCRATCH_NAME.test(name)).sort();
|
|
1679
|
+
const records = [];
|
|
1680
|
+
for (const name of names) {
|
|
1681
|
+
signal?.throwIfAborted();
|
|
1682
|
+
const content = await run.scratch.read(name, signal);
|
|
1683
|
+
if (content === undefined)
|
|
1684
|
+
continue;
|
|
1685
|
+
const bytes = encoder.encode(content);
|
|
1686
|
+
const digest = createHash('sha256').update(bytes).digest('hex');
|
|
1687
|
+
records.push({ name, bytes: bytes.byteLength, identity: `scratch:${digest}` });
|
|
1688
|
+
}
|
|
1689
|
+
return { items: records.slice(0, this.config.maxRetainedArtifactsPerRun), total: records.length };
|
|
1690
|
+
}
|
|
1691
|
+
throw new WorkflowPackageError('descriptor-rooted workflow artifact access is unavailable', 'WORKFLOW_STORAGE_UNSUPPORTED');
|
|
1692
|
+
}
|
|
1693
|
+
/** Refresh and page bounded scratch artifact metadata. */
|
|
1694
|
+
async artifacts(agent, request, signal) {
|
|
1695
|
+
const found = await this.authorizedHead(agent, request.runId, signal);
|
|
1696
|
+
const scanned = await this.scanArtifacts(found.live, found.head, signal);
|
|
1697
|
+
const prior = found.live?.detail.artifacts ?? [];
|
|
1698
|
+
const next = scanned.items.map(item => ({ name: item.name, bytes: item.bytes }));
|
|
1699
|
+
if (found.live !== undefined && !isDeepStrictEqual(prior, next) && !TERMINAL.has(found.live.head.status)) {
|
|
1700
|
+
found.live.detail = { ...found.live.detail, artifacts: next };
|
|
1701
|
+
await this.commitActive(found.live, {}, 'artifacts');
|
|
1702
|
+
}
|
|
1703
|
+
const limit = pageLimit(request.limit);
|
|
1704
|
+
const offset = numericCursor(request.cursor, scanned.items.length);
|
|
1705
|
+
const items = scanned.items.slice(offset, offset + limit).map(item => ({ name: item.name, bytes: item.bytes }));
|
|
1706
|
+
const cursor = nextCursor(offset, items.length, scanned.items.length);
|
|
1707
|
+
return { items, total: scanned.total, omitted: Math.max(0, scanned.total - scanned.items.length), revision: found.head.artifactsRevision + (isDeepStrictEqual(prior, next) ? 0 : 1), ...(cursor === undefined ? {} : { nextCursor: cursor }) };
|
|
1708
|
+
}
|
|
1709
|
+
/** Read one UTF-8-safe artifact chunk through an opened no-follow handle. */
|
|
1710
|
+
async artifact(agent, request, signal) {
|
|
1711
|
+
const found = await this.authorizedHead(agent, request.runId, signal);
|
|
1712
|
+
if (!SCRATCH_NAME.test(request.name))
|
|
1713
|
+
throw new WorkflowPackageError('workflow scratch artifact was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1714
|
+
const scanned = await this.scanArtifacts(found.live, found.head, signal);
|
|
1715
|
+
const artifact = scanned.items.find(item => item.name === request.name);
|
|
1716
|
+
if (artifact === undefined)
|
|
1717
|
+
throw new WorkflowPackageError('workflow scratch artifact was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1718
|
+
const revision = found.live?.head.artifactsRevision ?? found.head.artifactsRevision;
|
|
1719
|
+
if (request.expectedRevision !== undefined && request.expectedRevision !== revision)
|
|
1720
|
+
throw new WorkflowPackageError('workflow artifact collection changed; refresh it before reading', 'WORKFLOW_STALE_REVISION');
|
|
1721
|
+
const offset = numericCursor(request.cursor, artifact.bytes);
|
|
1722
|
+
const maxBytes = request.maxBytes ?? this.config.artifactChunkDefaultBytes;
|
|
1723
|
+
if (!Number.isSafeInteger(maxBytes) || maxBytes < 4 || maxBytes > this.config.artifactChunkMaxBytes)
|
|
1724
|
+
throw new WorkflowPackageError(`workflow artifact maxBytes must be a safe integer from 4 through ${this.config.artifactChunkMaxBytes}`, 'WORKFLOW_LIMIT');
|
|
1725
|
+
const capability = this.store.readRunArtifact;
|
|
1726
|
+
const capabilityIdentity = artifact.capabilityIdentity;
|
|
1727
|
+
if (typeof capability === 'function' && capabilityIdentity !== undefined) {
|
|
1728
|
+
const read = await capability.call(this.store, found.head.runDirectory, request.name, offset, maxBytes, capabilityIdentity, signal);
|
|
1729
|
+
const next = offset + read.returnedBytes < read.totalBytes ? String(offset + read.returnedBytes) : undefined;
|
|
1730
|
+
return {
|
|
1731
|
+
artifact: { name: request.name, bytes: read.totalBytes }, text: read.text,
|
|
1732
|
+
offsetBytes: offset, returnedBytes: read.returnedBytes, totalBytes: read.totalBytes,
|
|
1733
|
+
revision, ...(next === undefined ? {} : { nextCursor: next }),
|
|
1734
|
+
};
|
|
1735
|
+
}
|
|
1736
|
+
if (found.live?.scratch !== undefined) {
|
|
1737
|
+
const content = await found.live.scratch.read(request.name, signal);
|
|
1738
|
+
if (content === undefined)
|
|
1739
|
+
throw new WorkflowPackageError('workflow scratch artifact was not found', 'WORKFLOW_RUN_NOT_FOUND');
|
|
1740
|
+
const bytes = encoder.encode(content);
|
|
1741
|
+
const identity = `scratch:${createHash('sha256').update(bytes).digest('hex')}`;
|
|
1742
|
+
if (identity !== artifact.identity)
|
|
1743
|
+
throw new WorkflowPackageError('workflow artifact changed; refresh it before reading', 'WORKFLOW_STALE_REVISION');
|
|
1744
|
+
const available = bytes.subarray(offset, Math.min(bytes.byteLength, offset + maxBytes));
|
|
1745
|
+
const decoder = new TextDecoder('utf-8', { fatal: true });
|
|
1746
|
+
let returned = available.byteLength;
|
|
1747
|
+
let text = '';
|
|
1748
|
+
for (let candidate = returned; candidate >= Math.max(0, returned - 3); candidate -= 1) {
|
|
1749
|
+
try {
|
|
1750
|
+
text = decoder.decode(available.subarray(0, candidate));
|
|
1751
|
+
returned = candidate;
|
|
1752
|
+
break;
|
|
1753
|
+
}
|
|
1754
|
+
catch { /* trim an incomplete trailing code point */ }
|
|
1755
|
+
}
|
|
1756
|
+
const next = offset + returned < bytes.byteLength ? String(offset + returned) : undefined;
|
|
1757
|
+
return { artifact: { name: artifact.name, bytes: bytes.byteLength }, text, offsetBytes: offset, returnedBytes: returned, totalBytes: bytes.byteLength, revision, ...(next === undefined ? {} : { nextCursor: next }) };
|
|
1758
|
+
}
|
|
1759
|
+
throw new WorkflowPackageError('descriptor-rooted workflow artifact access is unavailable', 'WORKFLOW_STORAGE_UNSUPPORTED');
|
|
1760
|
+
}
|
|
1761
|
+
/** Return one atomic lifecycle projection for the Chat recorder. */
|
|
1762
|
+
async recordingSnapshot(agent, runId, signal) {
|
|
1763
|
+
const sessionId = sessionOf(agent);
|
|
1764
|
+
signal?.throwIfAborted();
|
|
1765
|
+
const live = this.runs.get(String(runId));
|
|
1766
|
+
if (live !== undefined && live.sessionId === sessionId) {
|
|
1767
|
+
const stopReason = live.head.stopReason;
|
|
1768
|
+
return {
|
|
1769
|
+
info: this.info(live), run: this.publicHead(live),
|
|
1770
|
+
members: [...(live.detail.members ?? [])].sort((a, b) => a.seq - b.seq).map(member => this.memberLifecycle(member)),
|
|
1771
|
+
...(stopReason === undefined ? {} : {
|
|
1772
|
+
result: {
|
|
1773
|
+
stopReason: stopReason === 'budget-limited' ? 'error' : stopReason,
|
|
1774
|
+
...(live.head.error === undefined ? {} : { error: live.head.error }),
|
|
1775
|
+
agentsStarted: live.head.budget.spent,
|
|
1776
|
+
},
|
|
1777
|
+
}),
|
|
1778
|
+
};
|
|
1779
|
+
}
|
|
1780
|
+
const head = await this.storedHeadFor(agent, runId, signal).catch(error => {
|
|
1781
|
+
if (error instanceof WorkflowPackageError && error.code === 'WORKFLOW_RUN_NOT_FOUND')
|
|
1782
|
+
return undefined;
|
|
1783
|
+
throw error;
|
|
1784
|
+
});
|
|
1785
|
+
if (head === undefined)
|
|
1786
|
+
return undefined;
|
|
1787
|
+
const members = (await this.readAllMembers(String(runId), signal)).sort((a, b) => a.seq - b.seq);
|
|
1788
|
+
const stopReason = head.stopReason;
|
|
1789
|
+
const info = { id: head.runId, displayName: head.displayName, name: head.name };
|
|
1790
|
+
return {
|
|
1791
|
+
info,
|
|
1792
|
+
run: this.publicHeadRecord(head),
|
|
1793
|
+
members: members.map(member => this.memberLifecycle(member)),
|
|
1794
|
+
...(stopReason === undefined ? {} : {
|
|
1795
|
+
result: {
|
|
1796
|
+
stopReason: stopReason === 'budget-limited' ? 'error' : stopReason,
|
|
1797
|
+
...(head.error === undefined ? {} : { error: head.error }),
|
|
1798
|
+
agentsStarted: head.budget.spent,
|
|
1799
|
+
},
|
|
1800
|
+
}),
|
|
1801
|
+
};
|
|
1802
|
+
}
|
|
1803
|
+
/** Wait until work owned by one exact Agent reaches a quiescent fixed point. */
|
|
1804
|
+
async whenOwnerQuiescent(agent, signal) {
|
|
1805
|
+
for (;;) {
|
|
1806
|
+
signal?.throwIfAborted();
|
|
1807
|
+
const owned = [...this.runs.values()].filter(run => run.parent === agent);
|
|
1808
|
+
const waits = [];
|
|
1809
|
+
waits.push(...[...this.pendingStarts].filter(item => item.parent === agent).map(item => item.done));
|
|
1810
|
+
for (const run of owned) {
|
|
1811
|
+
waits.push(run.tail, run.lifecycleTail);
|
|
1812
|
+
if (run.attempt !== undefined && (run.head.status === 'running' || run.head.status === 'pausing' || run.head.status === 'stopping'))
|
|
1813
|
+
waits.push(run.attempt.observation);
|
|
1814
|
+
if (run.delivery !== undefined)
|
|
1815
|
+
waits.push(run.delivery);
|
|
1816
|
+
}
|
|
1817
|
+
waits.push(this.notifier.whenOwnerQuiescent(agent, signal));
|
|
1818
|
+
if (waits.length > 0)
|
|
1819
|
+
await Promise.allSettled(waits);
|
|
1820
|
+
signal?.throwIfAborted();
|
|
1821
|
+
const still = [...this.pendingStarts].some(item => item.parent === agent)
|
|
1822
|
+
|| [...this.runs.values()].some(run => run.parent === agent
|
|
1823
|
+
&& (run.delivery !== undefined
|
|
1824
|
+
|| run.head.status === 'running'
|
|
1825
|
+
|| run.head.status === 'pausing'
|
|
1826
|
+
|| run.head.status === 'stopping'));
|
|
1827
|
+
if (!still) {
|
|
1828
|
+
await this.notifier.whenOwnerQuiescent(agent, signal);
|
|
1829
|
+
return;
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
/** Close new admission at a synchronous linearization point. */
|
|
1834
|
+
closeAdmissionSync() {
|
|
1835
|
+
if (!this.admission)
|
|
1836
|
+
return;
|
|
1837
|
+
this.admission = false;
|
|
1838
|
+
for (const pending of this.pendingStarts)
|
|
1839
|
+
pending.controller.abort(new Error('workflow supervisor admission closed'));
|
|
1840
|
+
for (const run of this.runs.values()) {
|
|
1841
|
+
if (run.closing === undefined)
|
|
1842
|
+
run.closing = 'supervisor';
|
|
1843
|
+
}
|
|
1844
|
+
}
|
|
1845
|
+
/** Compatibility spelling used by lifecycle effects; the close itself is synchronous. */
|
|
1846
|
+
closeAdmission() { this.closeAdmissionSync(); }
|
|
1847
|
+
async disposeOwner(agent) {
|
|
1848
|
+
if (typeof agent === 'object' && agent !== null)
|
|
1849
|
+
this.closedOwners.add(agent);
|
|
1850
|
+
for (const pending of [...this.pendingStarts]) {
|
|
1851
|
+
if (pending.parent === agent)
|
|
1852
|
+
pending.controller.abort(new Error('workflow owner disposed'));
|
|
1853
|
+
}
|
|
1854
|
+
for (const run of this.runs.values()) {
|
|
1855
|
+
if (run.parent === agent && run.closing === undefined)
|
|
1856
|
+
run.closing = 'owner';
|
|
1857
|
+
}
|
|
1858
|
+
await Promise.allSettled([...this.pendingStarts].filter(item => item.parent === agent).map(item => item.done));
|
|
1859
|
+
const runs = [...this.runs.values()].filter(run => run.parent === agent && !TERMINAL.has(run.head.status));
|
|
1860
|
+
for (const run of runs) {
|
|
1861
|
+
const attempt = run.attempt;
|
|
1862
|
+
this.clearGate(run);
|
|
1863
|
+
if (attempt !== undefined) {
|
|
1864
|
+
attempt.intent = 'teardown';
|
|
1865
|
+
try {
|
|
1866
|
+
attempt.handle.cancel('workflow owner disposed');
|
|
1867
|
+
}
|
|
1868
|
+
catch { /* contained */ }
|
|
1869
|
+
await attempt.observation.catch(error => this.ctx?.logger?.warn?.('workflow owner disposal observation failed', error));
|
|
1870
|
+
}
|
|
1871
|
+
else {
|
|
1872
|
+
await this.enqueue(run, () => this.terminalize(run, 'interrupted', 'Process exited before workflow settlement.')).catch(error => this.ctx?.logger?.warn?.('workflow owner terminalization failed', error));
|
|
1873
|
+
}
|
|
1874
|
+
await run.tail.catch(() => undefined);
|
|
1875
|
+
await run.delivery?.catch(() => undefined);
|
|
1876
|
+
}
|
|
1877
|
+
await this.whenOwnerQuiescent(agent).catch(() => undefined);
|
|
1878
|
+
}
|
|
1879
|
+
/** Idempotent global teardown with admission, attempts, publications, and notices drained. */
|
|
1880
|
+
async dispose() {
|
|
1881
|
+
if (this.disposal !== undefined)
|
|
1882
|
+
return this.disposal;
|
|
1883
|
+
this.closeAdmissionSync();
|
|
1884
|
+
this.disposed = true;
|
|
1885
|
+
this.disposal = (async () => {
|
|
1886
|
+
let first;
|
|
1887
|
+
const remember = (error) => { first ??= error; };
|
|
1888
|
+
const settle = async (promises, preserveFailures = false) => {
|
|
1889
|
+
const outcomes = await Promise.allSettled(promises);
|
|
1890
|
+
if (preserveFailures) {
|
|
1891
|
+
for (const outcome of outcomes)
|
|
1892
|
+
if (outcome.status === 'rejected')
|
|
1893
|
+
remember(outcome.reason);
|
|
1894
|
+
}
|
|
1895
|
+
};
|
|
1896
|
+
// First make every pre-admission operation observe cancellation and wait
|
|
1897
|
+
// for its transaction promise. No directory/ordinal can be published
|
|
1898
|
+
// after this point.
|
|
1899
|
+
await Promise.allSettled([...this.pendingStarts].map(item => item.done));
|
|
1900
|
+
const runs = [...this.runs.values()];
|
|
1901
|
+
// An observer may have failed before dispose was requested and already
|
|
1902
|
+
// cleared run.attempt. Preserve that original persistence error before
|
|
1903
|
+
// making a bounded terminalization retry below.
|
|
1904
|
+
for (const run of runs)
|
|
1905
|
+
if (run.settlementFailure !== undefined)
|
|
1906
|
+
remember(run.settlementFailure);
|
|
1907
|
+
const directTerminalizations = [];
|
|
1908
|
+
const directlyTerminalized = new Set();
|
|
1909
|
+
for (const run of runs) {
|
|
1910
|
+
if (TERMINAL.has(run.head.status))
|
|
1911
|
+
continue;
|
|
1912
|
+
const attempt = run.attempt;
|
|
1913
|
+
this.clearGate(run);
|
|
1914
|
+
if (attempt !== undefined) {
|
|
1915
|
+
attempt.intent = 'teardown';
|
|
1916
|
+
try {
|
|
1917
|
+
attempt.handle.cancel('workflow supervisor disposed');
|
|
1918
|
+
}
|
|
1919
|
+
catch { /* contained */ }
|
|
1920
|
+
}
|
|
1921
|
+
else {
|
|
1922
|
+
directlyTerminalized.add(run);
|
|
1923
|
+
directTerminalizations.push(this.enqueue(run, () => this.terminalize(run, 'interrupted', 'Process exited before workflow settlement.')));
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
await settle(directTerminalizations, true);
|
|
1927
|
+
// Cancellation precedes all draining. Attempt observers own handle
|
|
1928
|
+
// disposal/checkpointing and clear run.attempt before terminal commit.
|
|
1929
|
+
const firstDrain = [];
|
|
1930
|
+
for (const run of runs) {
|
|
1931
|
+
firstDrain.push(run.tail, run.lifecycleTail);
|
|
1932
|
+
if (run.attempt !== undefined) {
|
|
1933
|
+
const observation = run.attempt.observation;
|
|
1934
|
+
void observation.catch(remember);
|
|
1935
|
+
firstDrain.push(observation);
|
|
1936
|
+
}
|
|
1937
|
+
if (run.delivery !== undefined)
|
|
1938
|
+
firstDrain.push(run.delivery);
|
|
1939
|
+
}
|
|
1940
|
+
await settle(firstDrain);
|
|
1941
|
+
for (const run of runs)
|
|
1942
|
+
if (run.settlementFailure !== undefined)
|
|
1943
|
+
remember(run.settlementFailure);
|
|
1944
|
+
// A failed observer leaves a deliberately nonterminal durable head and
|
|
1945
|
+
// no Attempt. Retry that exact terminal commit once, then fail forward.
|
|
1946
|
+
// Never use a fixed-point loop here: a permanent store failure cannot
|
|
1947
|
+
// make progress and must not strand aggregate storage/lease teardown.
|
|
1948
|
+
const retries = runs
|
|
1949
|
+
.filter(run => !directlyTerminalized.has(run) && !TERMINAL.has(run.head.status) && run.attempt === undefined)
|
|
1950
|
+
.map(run => this.enqueue(run, () => this.terminalize(run, 'interrupted', 'Process exited before workflow settlement.')));
|
|
1951
|
+
await settle(retries, true);
|
|
1952
|
+
const finalDrain = [];
|
|
1953
|
+
for (const run of runs) {
|
|
1954
|
+
finalDrain.push(run.tail, run.lifecycleTail);
|
|
1955
|
+
if (run.delivery !== undefined)
|
|
1956
|
+
finalDrain.push(run.delivery);
|
|
1957
|
+
}
|
|
1958
|
+
await settle(finalDrain);
|
|
1959
|
+
try {
|
|
1960
|
+
await this.notifier.dispose();
|
|
1961
|
+
}
|
|
1962
|
+
catch (error) {
|
|
1963
|
+
remember(error);
|
|
1964
|
+
}
|
|
1965
|
+
for (const remove of this.listenerDisposers.splice(0)) {
|
|
1966
|
+
try {
|
|
1967
|
+
remove();
|
|
1968
|
+
}
|
|
1969
|
+
catch { /* contained */ }
|
|
1970
|
+
}
|
|
1971
|
+
if (this.ownsStore) {
|
|
1972
|
+
try {
|
|
1973
|
+
await this.store.dispose();
|
|
1974
|
+
}
|
|
1975
|
+
catch (error) {
|
|
1976
|
+
remember(error);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
if (first !== undefined)
|
|
1980
|
+
throw first;
|
|
1981
|
+
})();
|
|
1982
|
+
return this.disposal;
|
|
1983
|
+
}
|
|
1984
|
+
/** Side-effect-free one-path validation through the configured evaluator API. */
|
|
1985
|
+
async validate(spec) {
|
|
1986
|
+
if (!spec.parent)
|
|
1987
|
+
return { ok: false, status: 'error', error: 'validate_only requires a calling agent' };
|
|
1988
|
+
let metaName = typeof spec.meta?.name === 'string'
|
|
1989
|
+
? spec.meta.name
|
|
1990
|
+
: typeof spec.definition?.name === 'string' ? spec.definition.name : '';
|
|
1991
|
+
try {
|
|
1992
|
+
const source = this.resolveSource(spec);
|
|
1993
|
+
metaName = source.meta.name;
|
|
1994
|
+
const budget = this.resolveBudget(spec.agentBudget);
|
|
1995
|
+
const args = this.snapshotArgs(spec.args);
|
|
1996
|
+
const script = repairObjectLiteralSemicolons(source.script);
|
|
1997
|
+
const engine = this.workflowEngine ?? this.ctx?.workflowEngine;
|
|
1998
|
+
const native = engine?.dshWorkflowsNative === true;
|
|
1999
|
+
const result = typeof engine?.validate === 'function'
|
|
2000
|
+
? await engine.validate({
|
|
2001
|
+
script: native ? script : scriptWithJobMapParallel(script, {
|
|
2002
|
+
agentBudget: budget,
|
|
2003
|
+
scratchMaxOperations: this.config.scratchMaxOperations,
|
|
2004
|
+
scratchMaxFiles: this.config.scratchMaxFiles,
|
|
2005
|
+
scratchMaxFileBytes: this.config.scratchMaxFileBytes,
|
|
2006
|
+
scratchMaxTotalBytes: this.config.scratchMaxTotalBytes,
|
|
2007
|
+
}),
|
|
2008
|
+
meta: source.meta,
|
|
2009
|
+
args,
|
|
2010
|
+
...(native ? { parent: spec.parent } : {}),
|
|
2011
|
+
maxTotalAgents: budget,
|
|
2012
|
+
...(spec.signal === undefined ? {} : { signal: spec.signal }),
|
|
2013
|
+
})
|
|
2014
|
+
: await cannedStockValidate(script, args, {
|
|
2015
|
+
filename: spec.filename ?? `workflow:${metaName}`,
|
|
2016
|
+
});
|
|
2017
|
+
return this.presentValidation(result, spec.filename, metaName);
|
|
2018
|
+
}
|
|
2019
|
+
catch (error) {
|
|
2020
|
+
if (spec.signal?.aborted)
|
|
2021
|
+
throw error;
|
|
2022
|
+
return { ok: false, status: 'error', error: rewriteValidationDiagnostic(renderThrown(error), spec.filename, metaName) };
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
presentValidation(result, filename, metaName) {
|
|
2026
|
+
if (result?.ok === true && result.status === 'completed') {
|
|
2027
|
+
return { ok: true, status: 'completed', value: snapshotWorkflowJsonValue(result.value), note: VALIDATION_NOTE };
|
|
2028
|
+
}
|
|
2029
|
+
if (result?.ok === true && result.status === 'would-pause') {
|
|
2030
|
+
return { ok: true, status: 'would-pause', value: productWouldPause(result.value, this.config.maxEventTextBytes), note: VALIDATION_NOTE };
|
|
2031
|
+
}
|
|
2032
|
+
const diagnostic = rewriteValidationDiagnostic(String(result?.error ?? 'workflow smoke check failed'), filename, metaName);
|
|
2033
|
+
return {
|
|
2034
|
+
ok: false, status: 'error', error: diagnostic,
|
|
2035
|
+
...(result?.errorCode === undefined ? {} : { errorCode: String(result.errorCode) }),
|
|
2036
|
+
};
|
|
2037
|
+
}
|
|
2038
|
+
}
|
|
2039
|
+
export default WorkflowSupervisor;
|
|
2040
|
+
//# sourceMappingURL=index.js.map
|