@hunterzhu/pulse-runtime 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/context/builder.d.ts +68 -0
- package/dist/context/builder.js +127 -0
- package/dist/context/index.d.ts +2 -0
- package/dist/context/index.js +2 -0
- package/dist/context/merger.d.ts +25 -0
- package/dist/context/merger.js +125 -0
- package/dist/core/actions.d.ts +1 -0
- package/dist/core/actions.js +1 -0
- package/dist/core/errors.d.ts +8 -0
- package/dist/core/errors.js +36 -0
- package/dist/core/events.d.ts +10 -0
- package/dist/core/events.js +24 -0
- package/dist/core/factory.d.ts +35 -0
- package/dist/core/factory.js +27 -0
- package/dist/core/inbox.d.ts +119 -0
- package/dist/core/inbox.js +217 -0
- package/dist/core/mutations.d.ts +80 -0
- package/dist/core/mutations.js +127 -0
- package/dist/core/records.d.ts +1 -0
- package/dist/core/records.js +1 -0
- package/dist/core/types.d.ts +615 -0
- package/dist/core/types.js +109 -0
- package/dist/dependencies/graph.d.ts +25 -0
- package/dist/dependencies/graph.js +92 -0
- package/dist/dependencies/index.d.ts +1 -0
- package/dist/dependencies/index.js +1 -0
- package/dist/dsl/context-proxy.d.ts +20 -0
- package/dist/dsl/context-proxy.js +64 -0
- package/dist/dsl/index.d.ts +4 -0
- package/dist/dsl/index.js +4 -0
- package/dist/dsl/program.d.ts +314 -0
- package/dist/dsl/program.js +756 -0
- package/dist/dsl/session.d.ts +45 -0
- package/dist/dsl/session.js +93 -0
- package/dist/dsl/templates-index.d.ts +1 -0
- package/dist/dsl/templates-index.js +1 -0
- package/dist/dsl/templates.d.ts +85 -0
- package/dist/dsl/templates.js +110 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +15 -0
- package/dist/lifecycle/index.d.ts +2 -0
- package/dist/lifecycle/index.js +2 -0
- package/dist/lifecycle/scopes.d.ts +38 -0
- package/dist/lifecycle/scopes.js +50 -0
- package/dist/lifecycle/watchdog.d.ts +16 -0
- package/dist/lifecycle/watchdog.js +66 -0
- package/dist/models/actions.d.ts +10 -0
- package/dist/models/actions.js +68 -0
- package/dist/models/index.d.ts +2 -0
- package/dist/models/index.js +2 -0
- package/dist/models/router.d.ts +187 -0
- package/dist/models/router.js +353 -0
- package/dist/scheduler/clock.d.ts +45 -0
- package/dist/scheduler/clock.js +92 -0
- package/dist/scheduler/decision.d.ts +72 -0
- package/dist/scheduler/decision.js +63 -0
- package/dist/scheduler/index.d.ts +6 -0
- package/dist/scheduler/index.js +6 -0
- package/dist/scheduler/locks.d.ts +18 -0
- package/dist/scheduler/locks.js +106 -0
- package/dist/scheduler/ready-queue.d.ts +32 -0
- package/dist/scheduler/ready-queue.js +40 -0
- package/dist/scheduler/runtime.d.ts +486 -0
- package/dist/scheduler/runtime.js +3445 -0
- package/dist/scheduler/telemetry.d.ts +111 -0
- package/dist/scheduler/telemetry.js +177 -0
- package/dist/scheduler/worker.d.ts +158 -0
- package/dist/scheduler/worker.js +744 -0
- package/dist/storage/artifacts.d.ts +17 -0
- package/dist/storage/artifacts.js +90 -0
- package/dist/storage/findings.d.ts +12 -0
- package/dist/storage/findings.js +70 -0
- package/dist/storage/index.d.ts +8 -0
- package/dist/storage/index.js +8 -0
- package/dist/storage/memory.d.ts +11 -0
- package/dist/storage/memory.js +21 -0
- package/dist/storage/mutation-log.d.ts +41 -0
- package/dist/storage/mutation-log.js +140 -0
- package/dist/storage/outbox.d.ts +30 -0
- package/dist/storage/outbox.js +59 -0
- package/dist/storage/persistence.d.ts +183 -0
- package/dist/storage/persistence.js +999 -0
- package/dist/storage/policy.d.ts +80 -0
- package/dist/storage/policy.js +268 -0
- package/dist/storage/session.d.ts +140 -0
- package/dist/storage/session.js +447 -0
- package/dist/tools/registry.d.ts +125 -0
- package/dist/tools/registry.js +308 -0
- package/dist/transitions/index.d.ts +2 -0
- package/dist/transitions/index.js +1 -0
- package/dist/transitions/validate.d.ts +4 -0
- package/dist/transitions/validate.js +1118 -0
- package/package.json +21 -0
|
@@ -0,0 +1,3445 @@
|
|
|
1
|
+
import { commitMutationTransaction, MutationLog } from '../storage/mutation-log.js';
|
|
2
|
+
import { buildAgent } from '../core/factory.js';
|
|
3
|
+
import { validateStep } from '../transitions/validate.js';
|
|
4
|
+
import { PriorityInheritance, ReadyQueue, readyItemFromLane, VirtualClock } from './index.js';
|
|
5
|
+
import { createRuntimeState, effectivePrivacy, isSideEffectful, privacyMetadataForDerivedRef, privacyTaintsForDerivedRefs, provenanceRefId, provenanceRefKind, replaceResumeInput, strictestPrivacy, validatePrivacyTaints } from '../core/types.js';
|
|
6
|
+
import { QuarantineScope } from '../lifecycle/scopes.js';
|
|
7
|
+
import { PulseSession } from '../dsl/session.js';
|
|
8
|
+
import { assertProgramPure, withPureStepGuard } from '../dsl/program.js';
|
|
9
|
+
import { FactInbox, ObservationInbox } from '../core/inbox.js';
|
|
10
|
+
import { observeProgress } from '../lifecycle/watchdog.js';
|
|
11
|
+
import { EffectOutbox } from '../storage/outbox.js';
|
|
12
|
+
import { exportRuntimeCheckpoint, exportRuntimePersistence, externalizeRuntimeResultBodies, externalizeRuntimeSnapshotBodies, hydrateRuntimeResultBodies, hydrateRuntimeSnapshotBodies, importRuntimePersistence, validateRuntimePersistenceSnapshot, withRuntimePersistenceIntegrity } from '../storage/persistence.js';
|
|
13
|
+
import { exportRuntimeLog, exportRuntimeLogTo, exportWarmStartSession } from '../storage/session.js';
|
|
14
|
+
import { ResourceLockManager } from './locks.js';
|
|
15
|
+
import { appendRuntimeEvent, normalizeRuntimeEvent } from '../core/events.js';
|
|
16
|
+
import { apply, forkRuntimeStateForAdmission } from '../core/mutations.js';
|
|
17
|
+
import { ContextMerger } from '../context/merger.js';
|
|
18
|
+
import { appendHistory, contentHash, historyPressure, stableSerialize } from '../context/builder.js';
|
|
19
|
+
import { assignRuntimeToolCallIds, InMemoryModelRegistry, ModelRouter, validateAdapterResult, validateJsonSchema } from '../models/router.js';
|
|
20
|
+
import { SessionStoragePolicy } from '../storage/policy.js';
|
|
21
|
+
import { collectRuntimeTelemetry } from './telemetry.js';
|
|
22
|
+
import { markArtifactPersisted, pinArtifact, prepareArtifactPublication, readArtifact, unpinArtifact } from '../storage/artifacts.js';
|
|
23
|
+
import { prepareFindingPublication } from '../storage/findings.js';
|
|
24
|
+
import { runtimeErrorFromCause } from '../core/errors.js';
|
|
25
|
+
import { RuntimeToolRegistry } from '../tools/registry.js';
|
|
26
|
+
import { SchedulerDecisionCoordinator, schedulerDecisionCandidateFromLane } from './decision.js';
|
|
27
|
+
function encodeEffectExecution(execution) {
|
|
28
|
+
const encoded = structuredClone(execution);
|
|
29
|
+
const artifact = execution.artifact;
|
|
30
|
+
if (artifact !== undefined) {
|
|
31
|
+
encoded.artifact = {
|
|
32
|
+
...artifact,
|
|
33
|
+
content: typeof artifact.content === 'string' ? artifact.content : { encoding: 'base64', value: Buffer.from(artifact.content).toString('base64') }
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
return encoded;
|
|
37
|
+
}
|
|
38
|
+
function decodeEffectExecution(value) {
|
|
39
|
+
const execution = structuredClone(value);
|
|
40
|
+
const artifact = execution.artifact;
|
|
41
|
+
if (artifact && typeof artifact === 'object' && !Array.isArray(artifact)) {
|
|
42
|
+
const content = artifact.content;
|
|
43
|
+
if (content && typeof content === 'object' && !Array.isArray(content) && content.encoding === 'base64' && typeof content.value === 'string') {
|
|
44
|
+
const base64 = content.value;
|
|
45
|
+
if (typeof base64 === 'string') {
|
|
46
|
+
;
|
|
47
|
+
execution.artifact.content = Buffer.from(base64, 'base64');
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return execution;
|
|
52
|
+
}
|
|
53
|
+
function resultMetadata(value) { return { sizeBytes: Buffer.byteLength(stableSerialize(value), 'utf8'), contentHash: contentHash(value) }; }
|
|
54
|
+
function asJsonValue(value) {
|
|
55
|
+
const serialized = JSON.stringify(value);
|
|
56
|
+
if (serialized === undefined)
|
|
57
|
+
throw new Error('MODEL_OUTPUT_NOT_SERIALIZABLE');
|
|
58
|
+
try {
|
|
59
|
+
return JSON.parse(serialized);
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
throw new Error('MODEL_OUTPUT_NOT_SERIALIZABLE');
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
function strictJsonValue(value, seen = new Set()) {
|
|
66
|
+
if (value === null || typeof value === 'string' || typeof value === 'boolean')
|
|
67
|
+
return value;
|
|
68
|
+
if (typeof value === 'number') {
|
|
69
|
+
if (!Number.isFinite(value))
|
|
70
|
+
throw new Error('TOOL_OUTPUT_NOT_SERIALIZABLE');
|
|
71
|
+
return value;
|
|
72
|
+
}
|
|
73
|
+
if (Array.isArray(value)) {
|
|
74
|
+
if (seen.has(value))
|
|
75
|
+
throw new Error('TOOL_OUTPUT_NOT_SERIALIZABLE');
|
|
76
|
+
seen.add(value);
|
|
77
|
+
try {
|
|
78
|
+
return value.map((item) => strictJsonValue(item, seen));
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
seen.delete(value);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (typeof value === 'object') {
|
|
85
|
+
if (value instanceof Uint8Array || value instanceof ArrayBuffer || value instanceof Date || Object.getPrototypeOf(value) !== Object.prototype)
|
|
86
|
+
throw new Error('TOOL_OUTPUT_NOT_SERIALIZABLE');
|
|
87
|
+
if (seen.has(value))
|
|
88
|
+
throw new Error('TOOL_OUTPUT_NOT_SERIALIZABLE');
|
|
89
|
+
seen.add(value);
|
|
90
|
+
try {
|
|
91
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, strictJsonValue(item, seen)]));
|
|
92
|
+
}
|
|
93
|
+
finally {
|
|
94
|
+
seen.delete(value);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
throw new Error('TOOL_OUTPUT_NOT_SERIALIZABLE');
|
|
98
|
+
}
|
|
99
|
+
function validateHostCommand(command) {
|
|
100
|
+
if (!command || typeof command !== 'object' || Array.isArray(command))
|
|
101
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
102
|
+
const value = command;
|
|
103
|
+
if (value.type === 'reply') {
|
|
104
|
+
if (typeof value.agentId !== 'string' || value.agentId.length === 0 || typeof value.effectId !== 'string' || value.effectId.length === 0)
|
|
105
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
106
|
+
try {
|
|
107
|
+
strictJsonValue(value.value);
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
throw new Error('INVALID_HOST_COMMAND_VALUE');
|
|
111
|
+
}
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
if (value.type === 'cancel' || value.type === 'cancel_effect') {
|
|
115
|
+
if (typeof value.agentId !== 'string' || value.agentId.length === 0 || typeof value.reason !== 'string' || value.reason.length === 0)
|
|
116
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
117
|
+
if (value.type === 'cancel_effect' && (typeof value.effectId !== 'string' || value.effectId.length === 0))
|
|
118
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
if (value.type === 'set_lane_priority') {
|
|
122
|
+
if (typeof value.laneId !== 'string' || value.laneId.length === 0 || typeof value.priority !== 'number' || !Number.isFinite(value.priority))
|
|
123
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
throw new Error('INVALID_HOST_COMMAND');
|
|
127
|
+
}
|
|
128
|
+
function artifactOutput(value) {
|
|
129
|
+
if (value instanceof Uint8Array)
|
|
130
|
+
return { mediaType: 'application/octet-stream', content: new Uint8Array(value) };
|
|
131
|
+
if (value instanceof ArrayBuffer)
|
|
132
|
+
return { mediaType: 'application/octet-stream', content: new Uint8Array(value) };
|
|
133
|
+
try {
|
|
134
|
+
const serialized = JSON.stringify(value);
|
|
135
|
+
if (serialized !== undefined)
|
|
136
|
+
return { mediaType: 'application/json', content: serialized };
|
|
137
|
+
}
|
|
138
|
+
catch { /* fall through to a bounded textual representation */ }
|
|
139
|
+
return { mediaType: 'text/plain', content: String(value) };
|
|
140
|
+
}
|
|
141
|
+
function priorityScore(priority) {
|
|
142
|
+
if (priority === undefined)
|
|
143
|
+
return undefined;
|
|
144
|
+
if (typeof priority === 'number') {
|
|
145
|
+
if (!Number.isFinite(priority))
|
|
146
|
+
throw new Error('INVALID_AGENT_PRIORITY');
|
|
147
|
+
return priority;
|
|
148
|
+
}
|
|
149
|
+
const scores = { background: -1, normal: 0, high: 1, urgent: 2 };
|
|
150
|
+
if (!(priority in scores))
|
|
151
|
+
throw new Error('INVALID_AGENT_PRIORITY');
|
|
152
|
+
return scores[priority];
|
|
153
|
+
}
|
|
154
|
+
function invalidConfig(field) { throw new Error(`INVALID_RUNTIME_CONFIG:${field}`); }
|
|
155
|
+
function optionalNonNegativeInteger(value, field) { if (value !== undefined && (!Number.isInteger(value) || value < 0))
|
|
156
|
+
invalidConfig(field); }
|
|
157
|
+
function optionalNonNegativeNumber(value, field) { if (value !== undefined && (typeof value !== 'number' || !Number.isFinite(value) || value < 0))
|
|
158
|
+
invalidConfig(field); }
|
|
159
|
+
function validateProgramShape(value) {
|
|
160
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
161
|
+
throw new Error('INVALID_PROGRAM');
|
|
162
|
+
const program = value;
|
|
163
|
+
if (typeof program.id !== 'string' || program.id.length === 0 || typeof program.version !== 'string' || program.version.length === 0 || typeof program.step !== 'function')
|
|
164
|
+
throw new Error('INVALID_PROGRAM');
|
|
165
|
+
if (program.entry !== undefined && (typeof program.entry !== 'string' || program.entry.length === 0))
|
|
166
|
+
throw new Error('INVALID_PROGRAM');
|
|
167
|
+
if (program.errorBoundary !== undefined && typeof program.errorBoundary !== 'function')
|
|
168
|
+
throw new Error('INVALID_PROGRAM');
|
|
169
|
+
if (program.seriesMember !== undefined && (!program.seriesMember || typeof program.seriesMember !== 'object' || Array.isArray(program.seriesMember)))
|
|
170
|
+
throw new Error('INVALID_PROGRAM');
|
|
171
|
+
if (program.seriesMemberProgram !== undefined && (!program.seriesMemberProgram || typeof program.seriesMemberProgram !== 'object' || Array.isArray(program.seriesMemberProgram)))
|
|
172
|
+
throw new Error('INVALID_PROGRAM');
|
|
173
|
+
if (program.seriesKeys !== undefined && (!Array.isArray(program.seriesKeys) || program.seriesKeys.length === 0 || program.seriesKeys.some((key) => typeof key !== 'string' || key.length === 0) || new Set(program.seriesKeys).size !== program.seriesKeys.length))
|
|
174
|
+
throw new Error('INVALID_PROGRAM');
|
|
175
|
+
if (program.seriesOnMemberFailure !== undefined && !['continue', 'abort'].includes(String(program.seriesOnMemberFailure)))
|
|
176
|
+
throw new Error('INVALID_PROGRAM');
|
|
177
|
+
}
|
|
178
|
+
function validateProgramRefShape(value) {
|
|
179
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
180
|
+
throw new Error('INVALID_PROGRAM_REF');
|
|
181
|
+
const ref = value;
|
|
182
|
+
if (typeof ref.programId !== 'string' || ref.programId.length === 0 || typeof ref.programVersion !== 'string' || ref.programVersion.length === 0)
|
|
183
|
+
throw new Error('INVALID_PROGRAM_REF');
|
|
184
|
+
if (ref.step !== undefined && (typeof ref.step !== 'string' || ref.step.length === 0))
|
|
185
|
+
throw new Error('INVALID_PROGRAM_REF');
|
|
186
|
+
if (ref.locals !== undefined)
|
|
187
|
+
try {
|
|
188
|
+
strictJsonValue(ref.locals);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
throw new Error('INVALID_PROGRAM_REF');
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function validateWarmStartShape(value) {
|
|
195
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
196
|
+
throw new Error('INVALID_WARM_START');
|
|
197
|
+
const warmStart = value;
|
|
198
|
+
if (warmStart.sessionId !== undefined && (typeof warmStart.sessionId !== 'string' || warmStart.sessionId.length === 0))
|
|
199
|
+
throw new Error('INVALID_WARM_START');
|
|
200
|
+
if (warmStart.agentId !== undefined && (typeof warmStart.agentId !== 'string' || warmStart.agentId.length === 0))
|
|
201
|
+
throw new Error('INVALID_WARM_START');
|
|
202
|
+
if (warmStart.sessionId === undefined && warmStart.agentId === undefined)
|
|
203
|
+
throw new Error('WARM_START_SESSION_REQUIRED');
|
|
204
|
+
if (warmStart.globalVersion !== undefined && warmStart.globalVersion !== 'latest' && warmStart.globalVersion !== 'final' && (!Number.isInteger(warmStart.globalVersion) || warmStart.globalVersion < 0))
|
|
205
|
+
throw new Error('INVALID_WARM_START');
|
|
206
|
+
if (warmStart.include !== undefined && !['facts', 'facts_and_findings'].includes(String(warmStart.include)))
|
|
207
|
+
throw new Error('INVALID_WARM_START');
|
|
208
|
+
if (warmStart.relevanceRefs !== undefined && (!Array.isArray(warmStart.relevanceRefs) || warmStart.relevanceRefs.some((ref) => typeof ref !== 'string' || ref.length === 0) || new Set(warmStart.relevanceRefs).size !== warmStart.relevanceRefs.length))
|
|
209
|
+
throw new Error('INVALID_WARM_START');
|
|
210
|
+
}
|
|
211
|
+
function validateRuntimeConfig(config) {
|
|
212
|
+
optionalNonNegativeInteger(config.maxLaneStepsPerTick, 'maxLaneStepsPerTick');
|
|
213
|
+
optionalNonNegativeNumber(config.maxTickMs, 'maxTickMs');
|
|
214
|
+
if (config.agingIntervalMs !== undefined && (!Number.isFinite(config.agingIntervalMs) || config.agingIntervalMs <= 0))
|
|
215
|
+
invalidConfig('agingIntervalMs');
|
|
216
|
+
if (config.agingCap !== undefined && (config.agingCap !== Number.POSITIVE_INFINITY && (typeof config.agingCap !== 'number' || !Number.isFinite(config.agingCap) || config.agingCap < 0)))
|
|
217
|
+
invalidConfig('agingCap');
|
|
218
|
+
optionalNonNegativeInteger(config.maxTotalLanes, 'maxTotalLanes');
|
|
219
|
+
optionalNonNegativeInteger(config.maxQueuedEffects, 'maxQueuedEffects');
|
|
220
|
+
if (config.maxRunning !== undefined)
|
|
221
|
+
for (const [key, value] of Object.entries(config.maxRunning))
|
|
222
|
+
if (!['llm', 'tool', 'agent', 'none'].includes(key) || (key === 'none' ? value !== Number.POSITIVE_INFINITY && (!Number.isInteger(value) || value < 0) : (!Number.isInteger(value) || value < 0)))
|
|
223
|
+
invalidConfig(`maxRunning.${key}`);
|
|
224
|
+
if (config.forkAffinity !== undefined && !['off', 'advise', 'coalesce'].includes(config.forkAffinity))
|
|
225
|
+
invalidConfig('forkAffinity');
|
|
226
|
+
optionalNonNegativeInteger(config.historySoftTokens, 'historySoftTokens');
|
|
227
|
+
optionalNonNegativeInteger(config.historyHardTokens, 'historyHardTokens');
|
|
228
|
+
if (config.historySoftTokens !== undefined && config.historyHardTokens !== undefined && config.historyHardTokens < config.historySoftTokens)
|
|
229
|
+
invalidConfig('historyHardTokens');
|
|
230
|
+
optionalNonNegativeInteger(config.maxResultSummaryBytes, 'maxResultSummaryBytes');
|
|
231
|
+
optionalNonNegativeInteger(config.maxConsecutiveControlErrors, 'maxConsecutiveControlErrors');
|
|
232
|
+
optionalNonNegativeNumber(config.maxRuntimeMs, 'maxRuntimeMs');
|
|
233
|
+
optionalNonNegativeInteger(config.maxAgentDepth, 'maxAgentDepth');
|
|
234
|
+
optionalNonNegativeInteger(config.watchdogNoProgressThreshold, 'watchdogNoProgressThreshold');
|
|
235
|
+
optionalNonNegativeInteger(config.watchdogRepeatedActionThreshold, 'watchdogRepeatedActionThreshold');
|
|
236
|
+
optionalNonNegativeInteger(config.maxPreparingLLMs, 'maxPreparingLLMs');
|
|
237
|
+
optionalNonNegativeInteger(config.maxPreparedLLMs, 'maxPreparedLLMs');
|
|
238
|
+
optionalNonNegativeInteger(config.writerPreferenceBound, 'writerPreferenceBound');
|
|
239
|
+
optionalNonNegativeInteger(config.maxObservationEntries, 'maxObservationEntries');
|
|
240
|
+
optionalNonNegativeInteger(config.maxObservationBytes, 'maxObservationBytes');
|
|
241
|
+
if (config.trustedSanitizerIds !== undefined && (!Array.isArray(config.trustedSanitizerIds) || new Set(config.trustedSanitizerIds).size !== config.trustedSanitizerIds.length || config.trustedSanitizerIds.some((id) => typeof id !== 'string' || id.length === 0)))
|
|
242
|
+
invalidConfig('trustedSanitizerIds');
|
|
243
|
+
if (config.sessionId !== undefined && (typeof config.sessionId !== 'string' || config.sessionId.length === 0))
|
|
244
|
+
invalidConfig('sessionId');
|
|
245
|
+
if (config.toolVersions !== undefined && (typeof config.toolVersions !== 'object' || config.toolVersions === null || Array.isArray(config.toolVersions) || Object.entries(config.toolVersions).some(([name, version]) => !name || typeof version !== 'string' || version.length === 0)))
|
|
246
|
+
invalidConfig('toolVersions');
|
|
247
|
+
if (config.policyVersion !== undefined && (typeof config.policyVersion !== 'string' || config.policyVersion.length === 0))
|
|
248
|
+
invalidConfig('policyVersion');
|
|
249
|
+
if (config.routerVersion !== undefined && (typeof config.routerVersion !== 'string' || config.routerVersion.length === 0))
|
|
250
|
+
invalidConfig('routerVersion');
|
|
251
|
+
if (config.persistenceExpectedDigest !== undefined && (typeof config.persistenceExpectedDigest !== 'string' || !/^[a-f0-9]{64}$/.test(config.persistenceExpectedDigest)))
|
|
252
|
+
invalidConfig('persistenceExpectedDigest');
|
|
253
|
+
if (config.auditLogPrivacy !== undefined && !['public', 'cloud_allowed', 'local_only'].includes(config.auditLogPrivacy))
|
|
254
|
+
invalidConfig('auditLogPrivacy');
|
|
255
|
+
if (config.schedulerDecision !== undefined) {
|
|
256
|
+
const decision = config.schedulerDecision;
|
|
257
|
+
if (decision === null || typeof decision !== 'object' || Array.isArray(decision))
|
|
258
|
+
invalidConfig('schedulerDecision');
|
|
259
|
+
if (decision.model !== undefined && (!decision.model || typeof decision.model !== 'object' || typeof decision.model.id !== 'string' || decision.model.id.length === 0 || typeof decision.model.decide !== 'function'))
|
|
260
|
+
invalidConfig('schedulerDecision.model');
|
|
261
|
+
if (decision.minCandidates !== undefined && (!Number.isInteger(decision.minCandidates) || decision.minCandidates < 2))
|
|
262
|
+
invalidConfig('schedulerDecision.minCandidates');
|
|
263
|
+
if (decision.candidateLimit !== undefined && (!Number.isInteger(decision.candidateLimit) || decision.candidateLimit < 1))
|
|
264
|
+
invalidConfig('schedulerDecision.candidateLimit');
|
|
265
|
+
if ((decision.candidateLimit ?? 8) < (decision.minCandidates ?? 3))
|
|
266
|
+
invalidConfig('schedulerDecision.candidateLimit');
|
|
267
|
+
if (decision.decisionTimeoutMs !== undefined && (!Number.isFinite(decision.decisionTimeoutMs) || decision.decisionTimeoutMs <= 0))
|
|
268
|
+
invalidConfig('schedulerDecision.decisionTimeoutMs');
|
|
269
|
+
if (decision.maxOutstandingDecisions !== undefined && (!Number.isInteger(decision.maxOutstandingDecisions) || decision.maxOutstandingDecisions < 1))
|
|
270
|
+
invalidConfig('schedulerDecision.maxOutstandingDecisions');
|
|
271
|
+
if (decision.maxReorderDistance !== undefined && (!Number.isInteger(decision.maxReorderDistance) || decision.maxReorderDistance < 0))
|
|
272
|
+
invalidConfig('schedulerDecision.maxReorderDistance');
|
|
273
|
+
if (decision.deterministicReserveEvery !== undefined && (!Number.isInteger(decision.deterministicReserveEvery) || decision.deterministicReserveEvery < 1))
|
|
274
|
+
invalidConfig('schedulerDecision.deterministicReserveEvery');
|
|
275
|
+
if (decision.includeGoals !== undefined && typeof decision.includeGoals !== 'boolean')
|
|
276
|
+
invalidConfig('schedulerDecision.includeGoals');
|
|
277
|
+
}
|
|
278
|
+
if (config.hostPolicy !== undefined && (config.hostPolicy === null || typeof config.hostPolicy !== 'object' || Array.isArray(config.hostPolicy)))
|
|
279
|
+
invalidConfig('hostPolicy');
|
|
280
|
+
if (config.hostPolicy?.allowCloud !== undefined && typeof config.hostPolicy.allowCloud !== 'boolean')
|
|
281
|
+
invalidConfig('hostPolicy.allowCloud');
|
|
282
|
+
if (config.storagePolicy !== undefined)
|
|
283
|
+
try {
|
|
284
|
+
new SessionStoragePolicy(config.storagePolicy);
|
|
285
|
+
}
|
|
286
|
+
catch {
|
|
287
|
+
invalidConfig('storagePolicy');
|
|
288
|
+
}
|
|
289
|
+
if (config.budget !== undefined) {
|
|
290
|
+
optionalNonNegativeInteger(config.budget.maxTotalAttempts, 'budget.maxTotalAttempts');
|
|
291
|
+
optionalNonNegativeInteger(config.budget.maxLLMAttempts, 'budget.maxLLMAttempts');
|
|
292
|
+
optionalNonNegativeInteger(config.budget.maxToolAttempts, 'budget.maxToolAttempts');
|
|
293
|
+
if (config.budget.maxCostByCurrency !== undefined && (typeof config.budget.maxCostByCurrency !== 'object' || config.budget.maxCostByCurrency === null || Array.isArray(config.budget.maxCostByCurrency) || Object.entries(config.budget.maxCostByCurrency).some(([currency, limit]) => !currency || typeof limit !== 'number' || !Number.isFinite(limit) || limit < 0)))
|
|
294
|
+
invalidConfig('budget.maxCostByCurrency');
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
export class ProgramRegistry {
|
|
298
|
+
records = new Map();
|
|
299
|
+
register(program) {
|
|
300
|
+
const pending = new Map();
|
|
301
|
+
const visiting = new Set();
|
|
302
|
+
const visit = (candidate) => {
|
|
303
|
+
validateProgramShape(candidate);
|
|
304
|
+
const key = `${candidate.id}@${candidate.version}`;
|
|
305
|
+
if (visiting.has(key))
|
|
306
|
+
throw new Error(`PROGRAM_REGISTRATION_CYCLE:${key}`);
|
|
307
|
+
if (pending.has(key))
|
|
308
|
+
return;
|
|
309
|
+
visiting.add(key);
|
|
310
|
+
assertProgramPure(candidate);
|
|
311
|
+
pending.set(key, candidate);
|
|
312
|
+
if (candidate.seriesMemberProgram)
|
|
313
|
+
visit(candidate.seriesMemberProgram);
|
|
314
|
+
visiting.delete(key);
|
|
315
|
+
};
|
|
316
|
+
visit(program);
|
|
317
|
+
for (const [key, candidate] of pending)
|
|
318
|
+
this.records.set(key, candidate);
|
|
319
|
+
}
|
|
320
|
+
get(programId, programVersion) {
|
|
321
|
+
return this.records.get(programVersion === undefined ? programId : `${programId}@${programVersion}`);
|
|
322
|
+
}
|
|
323
|
+
resolve(ref) {
|
|
324
|
+
const program = this.get(ref.programId, ref.programVersion);
|
|
325
|
+
if (!program)
|
|
326
|
+
throw new Error(`PROGRAM_NOT_REGISTERED:${ref.programId}@${ref.programVersion}`);
|
|
327
|
+
return program;
|
|
328
|
+
}
|
|
329
|
+
has(programId, programVersion) {
|
|
330
|
+
return this.records.has(programVersion === undefined ? programId : `${programId}@${programVersion}`);
|
|
331
|
+
}
|
|
332
|
+
entries() { return this.records.entries(); }
|
|
333
|
+
}
|
|
334
|
+
function warmStartGlobal(value, include, relevanceRefs) {
|
|
335
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
336
|
+
return structuredClone(value);
|
|
337
|
+
const output = structuredClone(value);
|
|
338
|
+
if (include === 'facts') {
|
|
339
|
+
delete output.findings;
|
|
340
|
+
}
|
|
341
|
+
else if (relevanceRefs !== undefined && Array.isArray(output.findings)) {
|
|
342
|
+
const refs = new Set(relevanceRefs);
|
|
343
|
+
output.findings = output.findings.filter((finding) => {
|
|
344
|
+
if (!finding || typeof finding !== 'object' || Array.isArray(finding))
|
|
345
|
+
return false;
|
|
346
|
+
const record = finding;
|
|
347
|
+
if (typeof record.ref === 'string' || typeof record.id === 'string')
|
|
348
|
+
return refs.has((record.ref ?? record.id));
|
|
349
|
+
return Array.isArray(record.derivedFrom) && record.derivedFrom.some((ref) => typeof ref === 'string' && refs.has(ref));
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
return output;
|
|
353
|
+
}
|
|
354
|
+
function outcomeForLane(lane) {
|
|
355
|
+
const unresolvedEffectIds = lane.unresolvedEffectIds === undefined || lane.unresolvedEffectIds.length === 0 ? {} : { unresolvedEffectIds: [...lane.unresolvedEffectIds] };
|
|
356
|
+
if (lane.status === 'succeeded')
|
|
357
|
+
return { status: 'succeeded', ...(lane.resultRef === undefined ? {} : { resultRef: lane.resultRef }), ...unresolvedEffectIds };
|
|
358
|
+
if (lane.status === 'failed')
|
|
359
|
+
return { status: 'failed', ...(lane.failure === undefined ? {} : { error: lane.failure.error }), ...unresolvedEffectIds };
|
|
360
|
+
if (lane.status === 'cancelled')
|
|
361
|
+
return { status: 'cancelled', reason: lane.cancelReason ?? 'CANCELLED', ...unresolvedEffectIds };
|
|
362
|
+
return undefined;
|
|
363
|
+
}
|
|
364
|
+
function runOutcome(lane, unresolvedEffectIds) {
|
|
365
|
+
const outcome = lane === undefined ? undefined : outcomeForLane(lane);
|
|
366
|
+
const unresolved = [...new Set([...(outcome?.unresolvedEffectIds ?? []), ...unresolvedEffectIds])];
|
|
367
|
+
if (outcome)
|
|
368
|
+
return { ...outcome, unresolvedEffectIds: unresolved };
|
|
369
|
+
return { status: 'failed', error: { code: 'RUNTIME_IDLE_BLOCKED', message: 'Runtime stopped before the root Lane reached a terminal state.' }, unresolvedEffectIds: unresolved };
|
|
370
|
+
}
|
|
371
|
+
function outcomeForSeriesMember(state, lane, key) {
|
|
372
|
+
const aggregate = outcomeForLane(lane);
|
|
373
|
+
if (!aggregate || lane.series === undefined || lane.resultRef === undefined)
|
|
374
|
+
return aggregate;
|
|
375
|
+
const value = state.results.get(lane.resultRef)?.value;
|
|
376
|
+
if (!value || typeof value !== 'object' || Array.isArray(value))
|
|
377
|
+
return aggregate;
|
|
378
|
+
const results = value.results;
|
|
379
|
+
if (!results || typeof results !== 'object' || Array.isArray(results))
|
|
380
|
+
return aggregate;
|
|
381
|
+
const member = results[key];
|
|
382
|
+
if (!member || typeof member !== 'object' || Array.isArray(member))
|
|
383
|
+
return aggregate;
|
|
384
|
+
const record = member;
|
|
385
|
+
const status = record.status;
|
|
386
|
+
if (status !== 'succeeded' && status !== 'failed' && status !== 'cancelled')
|
|
387
|
+
return aggregate;
|
|
388
|
+
return {
|
|
389
|
+
status,
|
|
390
|
+
...(record.result === undefined ? {} : { result: record.result }),
|
|
391
|
+
...(record.error && typeof record.error === 'object' && !Array.isArray(record.error) ? { error: record.error } : {}),
|
|
392
|
+
...(typeof record.reason === 'string' ? { reason: record.reason } : {}),
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
export class PulseRuntime {
|
|
396
|
+
state;
|
|
397
|
+
/** Host-facing read-only effect inspection. Returned records are detached snapshots. */
|
|
398
|
+
effects = {
|
|
399
|
+
inspect: (effectId) => {
|
|
400
|
+
const effect = this.state.effects.get(effectId);
|
|
401
|
+
return effect === undefined ? undefined : structuredClone(effect);
|
|
402
|
+
},
|
|
403
|
+
};
|
|
404
|
+
/** Host-facing read-only result lookup. Returned records are detached snapshots. */
|
|
405
|
+
results = {
|
|
406
|
+
get: (resultRef) => {
|
|
407
|
+
const result = this.state.results.get(resultRef);
|
|
408
|
+
return result === undefined ? undefined : structuredClone(result);
|
|
409
|
+
},
|
|
410
|
+
};
|
|
411
|
+
shuttingDown = false;
|
|
412
|
+
telemetryExporter;
|
|
413
|
+
auditLogSink;
|
|
414
|
+
auditLogPrivacy;
|
|
415
|
+
persistenceBackend;
|
|
416
|
+
sessionStore;
|
|
417
|
+
enforcingRecoveryPrograms;
|
|
418
|
+
toolVersions;
|
|
419
|
+
recoveryCompatibility;
|
|
420
|
+
policyVersion;
|
|
421
|
+
routerVersion;
|
|
422
|
+
budget;
|
|
423
|
+
persistenceDigest;
|
|
424
|
+
budgetCost = new Map();
|
|
425
|
+
persistencePending = Promise.resolve();
|
|
426
|
+
persistenceScheduled = false;
|
|
427
|
+
persistenceDirty = false;
|
|
428
|
+
dispatchPersistencePending = false;
|
|
429
|
+
dispatchPersistenceReady = false;
|
|
430
|
+
executionYieldPending = new Set();
|
|
431
|
+
mutationLog;
|
|
432
|
+
outbox;
|
|
433
|
+
clock;
|
|
434
|
+
ready;
|
|
435
|
+
quarantine = new QuarantineScope();
|
|
436
|
+
priorityInheritance = new PriorityInheritance();
|
|
437
|
+
resourceLocks;
|
|
438
|
+
storagePolicy;
|
|
439
|
+
factInbox;
|
|
440
|
+
observationInbox;
|
|
441
|
+
programs = new ProgramRegistry();
|
|
442
|
+
models;
|
|
443
|
+
modelRouter;
|
|
444
|
+
tools;
|
|
445
|
+
executions = new Map();
|
|
446
|
+
lockReleases = new Map();
|
|
447
|
+
waitDeadlineTimers = new Map();
|
|
448
|
+
lockBlocked = new Set();
|
|
449
|
+
grantedLockReleases = new Map();
|
|
450
|
+
/** Every lock request id an Effect ever issued (across attempts), keyed by Effect id, so terminal cleanup never misses a queued request. */
|
|
451
|
+
lockRequests = new Map();
|
|
452
|
+
/** Consecutive background persistence failures; reset on the next successful write. */
|
|
453
|
+
persistenceFailures = 0;
|
|
454
|
+
persistenceBackoff = false;
|
|
455
|
+
executor;
|
|
456
|
+
factInboxDedupeArchive;
|
|
457
|
+
customExecutor;
|
|
458
|
+
builtinHumanEffects;
|
|
459
|
+
enqueueSeq = 1;
|
|
460
|
+
maxSteps;
|
|
461
|
+
maxTickMs;
|
|
462
|
+
maxConsecutiveControlErrors;
|
|
463
|
+
maxRuntimeAt;
|
|
464
|
+
watchdogNoProgressThreshold;
|
|
465
|
+
watchdogRepeatedActionThreshold;
|
|
466
|
+
maxAgentDepth;
|
|
467
|
+
maxPreparingLLMs;
|
|
468
|
+
maxPreparedLLMs;
|
|
469
|
+
effectSubmissionPreparer;
|
|
470
|
+
schedulerDecisionCoordinator;
|
|
471
|
+
schedulerDecisionConfig;
|
|
472
|
+
schedulerDecisionModelId;
|
|
473
|
+
schedulerDecisionMaxReorderDistance;
|
|
474
|
+
schedulerDecisionReserveEvery;
|
|
475
|
+
schedulerDecisionIncludeGoals;
|
|
476
|
+
schedulerDecisionEpoch = 0;
|
|
477
|
+
schedulerDecisionRequestSeq = 1;
|
|
478
|
+
schedulerDecisionDispatches = 0;
|
|
479
|
+
schedulerDecisionCache;
|
|
480
|
+
schedulerDecisionRequests = new Map();
|
|
481
|
+
preparingLLMs = new Set();
|
|
482
|
+
sessionId;
|
|
483
|
+
hostCommandSeq = 1;
|
|
484
|
+
factWaiters = [];
|
|
485
|
+
wakeScheduled = false;
|
|
486
|
+
inDrain = false;
|
|
487
|
+
wakeError;
|
|
488
|
+
tickBudget;
|
|
489
|
+
static async restore(backend, config = {}) {
|
|
490
|
+
const loaded = await backend.load();
|
|
491
|
+
if (loaded?.snapshotBodies === 'external' && backend.snapshotStore === undefined)
|
|
492
|
+
throw new Error('RUNTIME_SNAPSHOT_STORE_REQUIRED');
|
|
493
|
+
const withSnapshots = loaded === undefined || backend.snapshotStore === undefined ? loaded : await hydrateRuntimeSnapshotBodies(loaded, backend.snapshotStore);
|
|
494
|
+
const snapshot = withSnapshots === undefined || backend.resultStore === undefined ? withSnapshots : await hydrateRuntimeResultBodies(withSnapshots, backend.resultStore);
|
|
495
|
+
if (snapshot?.resultBodies === 'external' && backend.resultStore === undefined)
|
|
496
|
+
throw new Error('RUNTIME_RESULT_STORE_REQUIRED');
|
|
497
|
+
const restoredConfig = snapshot === undefined ? config : { ...config, persistence: snapshot, ...(config.persistenceBackend === undefined || loaded?.integrity?.digest === undefined ? {} : { persistenceExpectedDigest: loaded.integrity.digest }) };
|
|
498
|
+
return new PulseRuntime({
|
|
499
|
+
...restoredConfig,
|
|
500
|
+
...(restoredConfig.factInboxDedupeArchive === undefined && backend.factInboxDedupeArchive !== undefined ? { factInboxDedupeArchive: backend.factInboxDedupeArchive } : {}),
|
|
501
|
+
...(restoredConfig.sessionStore === undefined && backend.sessionStore !== undefined ? { sessionStore: backend.sessionStore } : {}),
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
constructor(config = {}) {
|
|
505
|
+
validateRuntimeConfig(config);
|
|
506
|
+
const restored = config.persistence === undefined ? undefined : importRuntimePersistence(config.persistence);
|
|
507
|
+
this.enforcingRecoveryPrograms = restored !== undefined;
|
|
508
|
+
this.observationInbox = new ObservationInbox(config.maxObservationEntries ?? 4096, config.maxObservationBytes ?? 1_000_000);
|
|
509
|
+
this.models = config.models ?? config.modelRouter?.registry ?? new InMemoryModelRegistry();
|
|
510
|
+
if (config.modelRouter && config.hostPolicy?.allowCloud === false && config.modelRouter.hostPolicy.allowCloud)
|
|
511
|
+
throw new Error('HOST_POLICY_ROUTER_MISMATCH');
|
|
512
|
+
this.modelRouter = config.modelRouter ?? new ModelRouter(this.models, config.hostPolicy);
|
|
513
|
+
this.tools = config.tools ?? new RuntimeToolRegistry();
|
|
514
|
+
this.toolVersions = { ...(config.toolVersions ?? {}) };
|
|
515
|
+
this.policyVersion = config.policyVersion;
|
|
516
|
+
this.routerVersion = config.routerVersion;
|
|
517
|
+
this.state = restored?.state ?? createRuntimeState(config.maxTotalLanes ?? 64, { ...(config.maxQueuedEffects === undefined ? {} : { maxQueuedEffects: config.maxQueuedEffects }), ...(config.maxRunning === undefined ? {} : { maxRunning: config.maxRunning }), ...(config.forkAffinity === undefined ? {} : { forkAffinity: config.forkAffinity }), ...(config.historySoftTokens === undefined ? {} : { historySoftTokens: config.historySoftTokens }), ...(config.historyHardTokens === undefined ? {} : { historyHardTokens: config.historyHardTokens }), ...(config.maxResultSummaryBytes === undefined ? {} : { maxResultSummaryBytes: config.maxResultSummaryBytes }), ...(config.trustedSanitizerIds === undefined ? {} : { trustedSanitizerIds: config.trustedSanitizerIds }) });
|
|
518
|
+
if (config.trustedSanitizerIds)
|
|
519
|
+
for (const sanitizerId of config.trustedSanitizerIds)
|
|
520
|
+
this.state.trustedSanitizerIds.add(sanitizerId);
|
|
521
|
+
this.sessionId = config.sessionId ?? 'session-local';
|
|
522
|
+
this.storagePolicy = restored?.storagePolicy ?? new SessionStoragePolicy(config.storagePolicy);
|
|
523
|
+
this.resourceLocks = new ResourceLockManager(config.writerPreferenceBound ?? 1);
|
|
524
|
+
this.persistenceDigest = config.persistenceExpectedDigest ?? config.persistence?.integrity?.digest;
|
|
525
|
+
this.mutationLog = restored?.mutationLog ?? new MutationLog();
|
|
526
|
+
this.outbox = restored?.outbox ?? new EffectOutbox();
|
|
527
|
+
this.factInboxDedupeArchive = config.factInboxDedupeArchive ?? config.persistenceBackend?.factInboxDedupeArchive;
|
|
528
|
+
this.factInbox = restored?.factInbox === undefined
|
|
529
|
+
? new FactInbox(this.factInboxDedupeArchive === undefined ? {} : { dedupeArchive: this.factInboxDedupeArchive })
|
|
530
|
+
: FactInbox.fromSnapshot(restored.factInbox, this.factInboxDedupeArchive === undefined ? {} : { dedupeArchive: this.factInboxDedupeArchive });
|
|
531
|
+
for (const program of config.programs ?? [])
|
|
532
|
+
this.register(program);
|
|
533
|
+
this.recoveryCompatibility = restored?.compatibility;
|
|
534
|
+
const restoredCommandIds = this.factInbox.snapshot().seen.map((eventId) => /^host-command-(\d+)$/.exec(eventId)?.[1]).filter((value) => value !== undefined).map(Number);
|
|
535
|
+
if (restoredCommandIds.length)
|
|
536
|
+
this.hostCommandSeq = Math.max(...restoredCommandIds) + 1;
|
|
537
|
+
if (restored?.quarantine)
|
|
538
|
+
this.quarantine.restore(restored.quarantine);
|
|
539
|
+
if (restored) {
|
|
540
|
+
const recovery = this.outbox.recover(this.state);
|
|
541
|
+
for (const id of recovery.requeued)
|
|
542
|
+
this.emit({ type: 'outbox.requeued', data: id });
|
|
543
|
+
for (const id of recovery.unknown)
|
|
544
|
+
this.emit({ type: 'outbox.discarded', data: id });
|
|
545
|
+
}
|
|
546
|
+
this.clock = config.clock ?? new VirtualClock();
|
|
547
|
+
if (!restored && config.clock)
|
|
548
|
+
this.state.now = this.clock.now();
|
|
549
|
+
this.ready = new ReadyQueue(config.agingIntervalMs ?? 1000, config.agingCap ?? Number.POSITIVE_INFINITY);
|
|
550
|
+
if (restored) {
|
|
551
|
+
this.clock.set(this.state.now);
|
|
552
|
+
for (const lane of this.state.lanes.values())
|
|
553
|
+
if (lane.status === 'ready')
|
|
554
|
+
this.enqueueReadyItem(readyItemFromLane(lane));
|
|
555
|
+
for (const effect of this.state.effects.values()) {
|
|
556
|
+
const outboxEntry = this.outbox.get(`${effect.id}:${effect.attemptId}`);
|
|
557
|
+
if (effect.state === 'running' && (outboxEntry === undefined || outboxEntry.state === 'pending')) {
|
|
558
|
+
const recovered = structuredClone(effect);
|
|
559
|
+
const recoveryReason = isSideEffectful(effect.sideEffectPolicy) ? 'recovery_in_doubt' : 'recovery_requeue';
|
|
560
|
+
if (isSideEffectful(effect.sideEffectPolicy)) {
|
|
561
|
+
recovered.state = 'reconcile_required';
|
|
562
|
+
recovered.executionState = 'remote_unknown';
|
|
563
|
+
recovered.sideEffectState = 'unknown';
|
|
564
|
+
}
|
|
565
|
+
else {
|
|
566
|
+
recovered.state = 'queued';
|
|
567
|
+
recovered.executionState = 'local';
|
|
568
|
+
}
|
|
569
|
+
commitMutationTransaction(this.state, this.mutationLog, `recovery:${effect.id}:${effect.attemptId}:${recoveryReason}`, [{ op: 'setEffect', effectId: effect.id, record: recovered }], this.state.now, this.sessionId);
|
|
570
|
+
if (isSideEffectful(effect.sideEffectPolicy))
|
|
571
|
+
this.quarantine.add(effect.id, this.state.now, 'recovery_in_doubt');
|
|
572
|
+
}
|
|
573
|
+
if (effect.state === 'retry_wait' && effect.retryAt !== undefined) {
|
|
574
|
+
const attemptId = effect.attemptId;
|
|
575
|
+
this.scheduleRuntimeTimer(effect.retryAt, () => this.readyRetryEffect(effect.id, attemptId));
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
for (const effect of [...this.state.effects.values()].sort((left, right) => left.id.localeCompare(right.id)))
|
|
579
|
+
if (effect.state === 'reconcile_required' && effect.sideEffectState === 'unknown') {
|
|
580
|
+
const releases = [...(effect.locks ?? [])].sort((left, right) => left.resource.localeCompare(right.resource) || left.mode.localeCompare(right.mode)).map((lock, index) => this.resourceLocks.restoreHeld(lock.resource, lock.mode, `${effect.id}:${effect.attemptId}:recovery:${index}`));
|
|
581
|
+
if (releases.length)
|
|
582
|
+
this.lockReleases.set(effect.id, releases);
|
|
583
|
+
if (!this.quarantine.has(effect.id))
|
|
584
|
+
this.quarantine.add(effect.id, this.state.now, 'recovery_in_doubt');
|
|
585
|
+
}
|
|
586
|
+
for (const wait of this.state.waits.values())
|
|
587
|
+
if (wait.state === 'pending')
|
|
588
|
+
this.scheduleWaitDeadline(wait);
|
|
589
|
+
}
|
|
590
|
+
this.maxSteps = config.maxLaneStepsPerTick ?? 32;
|
|
591
|
+
this.maxTickMs = config.maxTickMs ?? 50;
|
|
592
|
+
this.maxConsecutiveControlErrors = config.maxConsecutiveControlErrors ?? 2;
|
|
593
|
+
this.watchdogNoProgressThreshold = config.watchdogNoProgressThreshold ?? 3;
|
|
594
|
+
this.watchdogRepeatedActionThreshold = config.watchdogRepeatedActionThreshold ?? 3;
|
|
595
|
+
this.maxAgentDepth = config.maxAgentDepth ?? 1;
|
|
596
|
+
this.maxPreparingLLMs = config.maxPreparingLLMs ?? 2;
|
|
597
|
+
this.maxPreparedLLMs = config.maxPreparedLLMs ?? 8;
|
|
598
|
+
this.effectSubmissionPreparer = config.effectSubmissionPreparer ?? ((submission) => this.prepareRegisteredToolSubmission(submission));
|
|
599
|
+
const schedulerDecision = config.schedulerDecision ?? {};
|
|
600
|
+
this.schedulerDecisionConfig = {
|
|
601
|
+
model: schedulerDecision.model,
|
|
602
|
+
minCandidates: schedulerDecision.minCandidates ?? 3,
|
|
603
|
+
candidateLimit: schedulerDecision.candidateLimit ?? 8,
|
|
604
|
+
decisionTimeoutMs: schedulerDecision.decisionTimeoutMs ?? 25,
|
|
605
|
+
maxOutstandingDecisions: schedulerDecision.maxOutstandingDecisions ?? 1,
|
|
606
|
+
maxReorderDistance: schedulerDecision.maxReorderDistance ?? 1,
|
|
607
|
+
deterministicReserveEvery: schedulerDecision.deterministicReserveEvery ?? 4,
|
|
608
|
+
includeGoals: schedulerDecision.includeGoals ?? false,
|
|
609
|
+
};
|
|
610
|
+
this.schedulerDecisionModelId = schedulerDecision.model?.id;
|
|
611
|
+
this.schedulerDecisionMaxReorderDistance = this.schedulerDecisionConfig.maxReorderDistance;
|
|
612
|
+
this.schedulerDecisionReserveEvery = this.schedulerDecisionConfig.deterministicReserveEvery;
|
|
613
|
+
this.schedulerDecisionIncludeGoals = this.schedulerDecisionConfig.includeGoals;
|
|
614
|
+
this.schedulerDecisionCoordinator = schedulerDecision.model === undefined ? undefined : new SchedulerDecisionCoordinator({
|
|
615
|
+
model: schedulerDecision.model,
|
|
616
|
+
timeoutMs: this.schedulerDecisionConfig.decisionTimeoutMs,
|
|
617
|
+
maxOutstanding: this.schedulerDecisionConfig.maxOutstandingDecisions,
|
|
618
|
+
});
|
|
619
|
+
this.telemetryExporter = config.telemetryExporter;
|
|
620
|
+
this.auditLogSink = config.auditLogSink;
|
|
621
|
+
this.auditLogPrivacy = config.auditLogPrivacy;
|
|
622
|
+
this.persistenceBackend = config.persistenceBackend;
|
|
623
|
+
this.sessionStore = config.sessionStore ?? config.persistenceBackend?.sessionStore;
|
|
624
|
+
this.budget = config.budget ?? {};
|
|
625
|
+
if (restored)
|
|
626
|
+
for (const event of this.state.events)
|
|
627
|
+
if (event.type === 'effect.execution_metadata')
|
|
628
|
+
this.recordBudgetMetadata(event.data ?? event.payload);
|
|
629
|
+
this.customExecutor = config.effectExecutor !== undefined;
|
|
630
|
+
this.builtinHumanEffects = config.builtinHumanEffects === true;
|
|
631
|
+
this.executor = config.effectExecutor ?? this.executeRegisteredEffect.bind(this);
|
|
632
|
+
if (restored)
|
|
633
|
+
this.clock.set(this.state.now);
|
|
634
|
+
if (config.maxRuntimeMs !== undefined) {
|
|
635
|
+
if (!Number.isFinite(config.maxRuntimeMs) || config.maxRuntimeMs < 0)
|
|
636
|
+
throw new Error('INVALID_MAX_RUNTIME');
|
|
637
|
+
this.maxRuntimeAt = this.clock.now() + config.maxRuntimeMs;
|
|
638
|
+
}
|
|
639
|
+
this.syncStoragePolicy();
|
|
640
|
+
}
|
|
641
|
+
register(program) { this.programs.register(program); }
|
|
642
|
+
createAgent(goalOrRequest, program, agentId) {
|
|
643
|
+
if (this.shuttingDown)
|
|
644
|
+
throw new Error('RUNTIME_SHUTTING_DOWN');
|
|
645
|
+
const request = typeof goalOrRequest === 'string' ? { goal: goalOrRequest, program: program, ...(agentId === undefined ? {} : { agentId }) } : goalOrRequest;
|
|
646
|
+
if (!request || typeof request !== 'object' || typeof request.goal !== 'string' || request.goal.length === 0)
|
|
647
|
+
throw new Error('INVALID_AGENT_GOAL');
|
|
648
|
+
if (!request.program || typeof request.program !== 'object' || Array.isArray(request.program))
|
|
649
|
+
throw new Error('INVALID_AGENT_PROGRAM');
|
|
650
|
+
const programRef = 'programId' in request.program ? request.program : undefined;
|
|
651
|
+
if (programRef !== undefined)
|
|
652
|
+
validateProgramRefShape(programRef);
|
|
653
|
+
else
|
|
654
|
+
validateProgramShape(request.program);
|
|
655
|
+
const rootProgram = programRef === undefined ? request.program : this.programs.resolve(programRef);
|
|
656
|
+
const rootPriority = priorityScore(request.priority);
|
|
657
|
+
const policyId = request.policy?.id ?? request.policyId;
|
|
658
|
+
const limitsId = request.limits?.id ?? request.limitsId;
|
|
659
|
+
if (request.policy !== undefined && (!request.policy || typeof request.policy !== 'object' || Array.isArray(request.policy) || typeof request.policy.id !== 'string' || request.policy.id.length === 0))
|
|
660
|
+
throw new Error('INVALID_AGENT_POLICY');
|
|
661
|
+
if (request.policyId !== undefined && (typeof request.policyId !== 'string' || request.policyId.length === 0))
|
|
662
|
+
throw new Error('INVALID_AGENT_POLICY');
|
|
663
|
+
if (request.limits !== undefined && (!request.limits || typeof request.limits !== 'object' || Array.isArray(request.limits)))
|
|
664
|
+
throw new Error('INVALID_AGENT_LIMITS');
|
|
665
|
+
if (request.limitsId !== undefined && (typeof request.limitsId !== 'string' || request.limitsId.length === 0))
|
|
666
|
+
throw new Error('INVALID_AGENT_LIMITS');
|
|
667
|
+
const timeoutMs = request.limits?.timeoutMs;
|
|
668
|
+
if (timeoutMs !== undefined && (!Number.isFinite(timeoutMs) || timeoutMs < 0))
|
|
669
|
+
throw new Error('INVALID_AGENT_TIMEOUT');
|
|
670
|
+
const maxActiveLanes = request.limits?.maxActiveLanes ?? request.maxActiveLanes;
|
|
671
|
+
if (maxActiveLanes !== undefined && (!Number.isInteger(maxActiveLanes) || maxActiveLanes < 1))
|
|
672
|
+
throw new Error('INVALID_AGENT_LIMITS');
|
|
673
|
+
const warmStart = request.warmStart;
|
|
674
|
+
if (warmStart !== undefined)
|
|
675
|
+
validateWarmStartShape(warmStart);
|
|
676
|
+
let initialGlobal;
|
|
677
|
+
let initialGlobalPrivacy;
|
|
678
|
+
let warmStartResultRefs = [];
|
|
679
|
+
let warmStartResults = [];
|
|
680
|
+
if (warmStart) {
|
|
681
|
+
const sourceSessionId = warmStart.sessionId ?? warmStart.agentId;
|
|
682
|
+
if (!sourceSessionId)
|
|
683
|
+
throw new Error('WARM_START_SESSION_REQUIRED');
|
|
684
|
+
const source = this.state.agents.get(sourceSessionId);
|
|
685
|
+
const stored = source === undefined ? this.sessionStore?.get(sourceSessionId) : undefined;
|
|
686
|
+
if (!source && !stored)
|
|
687
|
+
throw new Error(`WARM_START_SOURCE_NOT_FOUND:${sourceSessionId}`);
|
|
688
|
+
const sourceLatestVersion = source?.latestGlobalVersion ?? stored.agent.latestGlobalVersion;
|
|
689
|
+
const version = warmStart.globalVersion === 'latest' || warmStart.globalVersion === 'final' || warmStart.globalVersion === undefined ? sourceLatestVersion : warmStart.globalVersion;
|
|
690
|
+
const value = source?.globalVersions.get(version) ?? stored.agent.globalVersions.find(([candidate]) => candidate === version)?.[1];
|
|
691
|
+
if (value === undefined)
|
|
692
|
+
throw new Error(`WARM_START_VERSION_NOT_FOUND:${version}`);
|
|
693
|
+
initialGlobal = warmStartGlobal(value, warmStart.include ?? 'facts', warmStart.relevanceRefs);
|
|
694
|
+
initialGlobalPrivacy = source?.globalPrivacy?.get(version) === undefined
|
|
695
|
+
? stored?.agent.globalPrivacy?.find(([candidate]) => candidate === version)?.[1]
|
|
696
|
+
: structuredClone(source.globalPrivacy.get(version));
|
|
697
|
+
warmStartResultRefs = [...new Set(warmStart.relevanceRefs ?? [])];
|
|
698
|
+
const visibleResultRefs = source?.rootLaneId === undefined ? new Set(stored.visibleResultRefs) : this.state.lanes.get(source.rootLaneId)?.visibleResultRefs ?? new Set();
|
|
699
|
+
const storedResults = new Map(stored?.results ?? []);
|
|
700
|
+
for (const ref of warmStartResultRefs) {
|
|
701
|
+
if (!visibleResultRefs.has(ref))
|
|
702
|
+
throw new Error(`WARM_START_RESULT_NOT_VISIBLE:${ref}`);
|
|
703
|
+
if (!this.state.results.has(ref)) {
|
|
704
|
+
const result = storedResults.get(ref);
|
|
705
|
+
if (!result)
|
|
706
|
+
throw new Error(`WARM_START_RESULT_NOT_FOUND:${ref}`);
|
|
707
|
+
warmStartResults.push(structuredClone(result));
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (programRef === undefined)
|
|
712
|
+
this.register(rootProgram);
|
|
713
|
+
if (this.state.lanes.size >= this.state.maxTotalLanes)
|
|
714
|
+
throw new Error('MAX_TOTAL_LANES');
|
|
715
|
+
if (request.agentId !== undefined && this.state.agents.has(request.agentId))
|
|
716
|
+
throw new Error(`AGENT_ID_EXISTS:${request.agentId}`);
|
|
717
|
+
const parent = request.parentAgentId === undefined ? undefined : this.state.agents.get(request.parentAgentId);
|
|
718
|
+
if (request.parentAgentId !== undefined && !parent)
|
|
719
|
+
throw new Error(`PARENT_AGENT_NOT_FOUND:${request.parentAgentId}`);
|
|
720
|
+
const rootResume = programRef === undefined
|
|
721
|
+
? { programId: rootProgram.id, programVersion: rootProgram.version, step: rootProgram.entry ?? 'start', locals: {} }
|
|
722
|
+
: { programId: rootProgram.id, programVersion: rootProgram.version, step: programRef.step ?? rootProgram.entry ?? 'start', locals: programRef.locals ?? {} };
|
|
723
|
+
const { agent, root, nextIds } = buildAgent(this.state, request.goal, rootResume, { ...(request.agentId === undefined ? {} : { agentId: request.agentId }), ...(maxActiveLanes === undefined ? {} : { maxActiveLanes }), ...(rootPriority === undefined ? {} : { priority: rootPriority }), ...(initialGlobal === undefined ? {} : { initialGlobal }), ...(initialGlobalPrivacy === undefined ? {} : { initialGlobalPrivacy }), ...(request.parentAgentId === undefined ? {} : { parentAgentId: request.parentAgentId, depth: (parent?.depth ?? 0) + 1 }), ...(request.inheritedFloor === undefined ? {} : { inheritedFloor: request.inheritedFloor }), ...(policyId === undefined ? {} : { policyId }), ...(limitsId === undefined ? {} : { limitsId }), ...(timeoutMs === undefined ? {} : { deadlineAt: this.state.now + timeoutMs }) });
|
|
724
|
+
if (warmStartResultRefs.length)
|
|
725
|
+
root.visibleResultRefs = new Set(warmStartResultRefs);
|
|
726
|
+
if (rootProgram.seriesKeys?.length && programRef === undefined)
|
|
727
|
+
root.resume.locals = { $sdk: { series: { keys: [...rootProgram.seriesKeys], index: 0 } } };
|
|
728
|
+
root.enqueueSeq = this.enqueueSeq++;
|
|
729
|
+
agent.state = 'running';
|
|
730
|
+
const importedResultIds = new Set(warmStartResults.map((result) => result.id));
|
|
731
|
+
const nextIdsWithWarmStart = { ...nextIds };
|
|
732
|
+
for (const ref of importedResultIds) {
|
|
733
|
+
const match = /^result-(\d+)$/.exec(ref);
|
|
734
|
+
if (match)
|
|
735
|
+
nextIdsWithWarmStart.result = Math.max(nextIdsWithWarmStart.result, Number(match[1]) + 1);
|
|
736
|
+
}
|
|
737
|
+
const mutations = [
|
|
738
|
+
{ op: 'setAgent', agentId: agent.id, record: agent },
|
|
739
|
+
{ op: 'setLane', laneId: root.id, record: root },
|
|
740
|
+
...warmStartResults.map((result) => ({ op: 'publishResult', record: result })),
|
|
741
|
+
{ op: 'setNextIds', nextIds: nextIdsWithWarmStart },
|
|
742
|
+
];
|
|
743
|
+
this.assertStorageAdmission(mutations);
|
|
744
|
+
commitMutationTransaction(this.state, this.mutationLog, `agent:${agent.id}:created`, mutations, this.state.now, this.sessionId);
|
|
745
|
+
const committedRoot = this.state.lanes.get(root.id);
|
|
746
|
+
this.enqueueReadyItem(readyItemFromLane(committedRoot));
|
|
747
|
+
this.syncStoragePolicy();
|
|
748
|
+
this.schedulePersistence();
|
|
749
|
+
return { id: agent.id, agentId: agent.id, laneId: root.id };
|
|
750
|
+
}
|
|
751
|
+
start(agentId) { if (!this.state.agents.has(agentId))
|
|
752
|
+
throw new Error(`UNKNOWN_AGENT:${agentId}`); return new PulseSession(this, agentId); }
|
|
753
|
+
requestCancel(agentId, reason = 'USER_REQUESTED') {
|
|
754
|
+
if (!agentId)
|
|
755
|
+
throw new Error('INVALID_AGENT_ID');
|
|
756
|
+
if (!reason)
|
|
757
|
+
throw new Error('INVALID_CANCEL_REASON');
|
|
758
|
+
this.enqueueHostCommand({ type: 'cancel', agentId, reason });
|
|
759
|
+
}
|
|
760
|
+
setLanePriority(laneId, priority) {
|
|
761
|
+
if (!laneId)
|
|
762
|
+
throw new Error('INVALID_LANE_ID');
|
|
763
|
+
if (!Number.isFinite(priority))
|
|
764
|
+
throw new Error('INVALID_LANE_PRIORITY');
|
|
765
|
+
this.enqueueHostCommand({ type: 'set_lane_priority', laneId, priority });
|
|
766
|
+
}
|
|
767
|
+
effectHandle(effectId) {
|
|
768
|
+
const effect = this.state.effects.get(effectId);
|
|
769
|
+
if (!effect)
|
|
770
|
+
throw new Error(`UNKNOWN_EFFECT:${effectId}`);
|
|
771
|
+
return {
|
|
772
|
+
id: effectId,
|
|
773
|
+
status: () => {
|
|
774
|
+
const current = this.state.effects.get(effectId);
|
|
775
|
+
if (!current)
|
|
776
|
+
throw new Error(`UNKNOWN_EFFECT:${effectId}`);
|
|
777
|
+
return current.state;
|
|
778
|
+
},
|
|
779
|
+
requestCancel: (reason) => {
|
|
780
|
+
if (!reason)
|
|
781
|
+
throw new Error('INVALID_CANCEL_REASON');
|
|
782
|
+
this.enqueueHostCommand({ type: 'cancel_effect', agentId: effect.agentId, effectId, reason });
|
|
783
|
+
},
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
inspectLane(laneId) {
|
|
787
|
+
if (!laneId)
|
|
788
|
+
throw new Error('INVALID_LANE_ID');
|
|
789
|
+
return this.explain(laneId);
|
|
790
|
+
}
|
|
791
|
+
detachAgent(agentId) {
|
|
792
|
+
const agent = this.state.agents.get(agentId);
|
|
793
|
+
if (!agent)
|
|
794
|
+
throw new Error(`UNKNOWN_AGENT:${agentId}`);
|
|
795
|
+
const event = { type: 'agent.detached', agentId, data: { agentId } };
|
|
796
|
+
const nextAgent = structuredClone(agent);
|
|
797
|
+
nextAgent.detached = true;
|
|
798
|
+
const mutations = [{ op: 'setAgent', agentId, record: nextAgent }, { op: 'appendEvent', event }];
|
|
799
|
+
this.assertStorageAdmission(mutations);
|
|
800
|
+
commitMutationTransaction(this.state, this.mutationLog, `agent:${agentId}:detached`, mutations, this.state.now, this.sessionId);
|
|
801
|
+
Object.assign(agent, nextAgent);
|
|
802
|
+
this.state.agents.set(agentId, agent);
|
|
803
|
+
this.schedulePersistence();
|
|
804
|
+
return { agentId, rootLaneId: agent.rootLaneId, state: agent.state ?? 'created', detached: true };
|
|
805
|
+
}
|
|
806
|
+
attachAgent(agentId) {
|
|
807
|
+
const agent = this.state.agents.get(agentId);
|
|
808
|
+
if (!agent)
|
|
809
|
+
throw new Error(`UNKNOWN_AGENT:${agentId}`);
|
|
810
|
+
if (!agent.detached)
|
|
811
|
+
return;
|
|
812
|
+
const event = { type: 'agent.attached', agentId, data: { agentId } };
|
|
813
|
+
const nextAgent = structuredClone(agent);
|
|
814
|
+
delete nextAgent.detached;
|
|
815
|
+
const mutations = [{ op: 'setAgent', agentId, record: nextAgent }, { op: 'appendEvent', event }];
|
|
816
|
+
this.assertStorageAdmission(mutations);
|
|
817
|
+
commitMutationTransaction(this.state, this.mutationLog, `agent:${agentId}:attached`, mutations, this.state.now, this.sessionId);
|
|
818
|
+
Object.assign(agent, nextAgent);
|
|
819
|
+
delete agent.detached;
|
|
820
|
+
this.state.agents.set(agentId, agent);
|
|
821
|
+
this.schedulePersistence();
|
|
822
|
+
}
|
|
823
|
+
backgroundAgents() {
|
|
824
|
+
return [...this.state.agents.values()].filter((agent) => agent.detached === true).map((agent) => ({ agentId: agent.id, rootLaneId: agent.rootLaneId, state: agent.state ?? 'created', detached: true }));
|
|
825
|
+
}
|
|
826
|
+
exportPersistence() { return exportRuntimePersistence(this.state, this.mutationLog, this.outbox, this.quarantine, this.storagePolicy, this.factInbox.snapshot(), this.persistenceCompatibility()); }
|
|
827
|
+
createFactInboxDedupeArchiveBatch(through) { return this.factInbox.createDedupeArchiveBatch(through); }
|
|
828
|
+
compactFactInboxDedupeThrough(batch) {
|
|
829
|
+
const compacted = this.factInbox.compactDedupeThrough(batch);
|
|
830
|
+
if (compacted > 0)
|
|
831
|
+
this.schedulePersistence();
|
|
832
|
+
return compacted;
|
|
833
|
+
}
|
|
834
|
+
async persist(backend) {
|
|
835
|
+
const persistedPolicy = this.storagePolicy.clone();
|
|
836
|
+
persistedPolicy.markPersisted();
|
|
837
|
+
const exported = exportRuntimePersistence(this.persistenceState(), this.mutationLog, this.outbox, this.quarantine, persistedPolicy, this.factInbox.snapshot(), this.persistenceCompatibility());
|
|
838
|
+
// Never write a snapshot that the constructor would refuse to load; failing here is recoverable, a poisoned store is not.
|
|
839
|
+
validateRuntimePersistenceSnapshot(exported);
|
|
840
|
+
const withResults = backend.resultStore === undefined ? exported : await externalizeRuntimeResultBodies(exported, backend.resultStore);
|
|
841
|
+
const snapshot = backend.snapshotStore === undefined ? withResults : await externalizeRuntimeSnapshotBodies(withResults, backend.snapshotStore);
|
|
842
|
+
await backend.save(snapshot, backend === this.persistenceBackend ? this.persistenceDigest : undefined);
|
|
843
|
+
if (backend === this.persistenceBackend)
|
|
844
|
+
this.persistenceDigest = snapshot.integrity?.digest;
|
|
845
|
+
this.storagePolicy.markPersisted();
|
|
846
|
+
this.markArtifactsPersisted();
|
|
847
|
+
this.syncStoragePolicy();
|
|
848
|
+
}
|
|
849
|
+
async flushPersistence() {
|
|
850
|
+
if (!this.persistenceBackend)
|
|
851
|
+
return;
|
|
852
|
+
// An explicit flush is a request to try now, even while background retries are backing off.
|
|
853
|
+
this.persistenceBackoff = false;
|
|
854
|
+
this.schedulePersistence();
|
|
855
|
+
while (true) {
|
|
856
|
+
await this.persistencePending;
|
|
857
|
+
if (!this.persistenceDirty)
|
|
858
|
+
return;
|
|
859
|
+
this.schedulePersistence();
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
async checkpoint(backend, options = {}) {
|
|
863
|
+
const persistedPolicy = this.storagePolicy.clone();
|
|
864
|
+
persistedPolicy.markPersisted();
|
|
865
|
+
const factArchive = backend.factInboxDedupeArchive;
|
|
866
|
+
if (factArchive !== undefined && this.factInboxDedupeArchive === undefined) {
|
|
867
|
+
this.factInbox.attachDedupeArchive(factArchive);
|
|
868
|
+
this.factInboxDedupeArchive = factArchive;
|
|
869
|
+
}
|
|
870
|
+
if (factArchive !== undefined && factArchive !== this.factInboxDedupeArchive)
|
|
871
|
+
throw new Error('FACT_INBOX_DEDUPE_ARCHIVE_MISMATCH');
|
|
872
|
+
if (factArchive !== undefined) {
|
|
873
|
+
const inboxSnapshot = this.factInbox.snapshot();
|
|
874
|
+
const through = inboxSnapshot.nextSeq - 1;
|
|
875
|
+
if (through > this.factInbox.dedupeWatermark) {
|
|
876
|
+
const batch = this.factInbox.createDedupeArchiveBatch(through);
|
|
877
|
+
await factArchive.append(batch);
|
|
878
|
+
if (this.factInbox.size === 0) {
|
|
879
|
+
// The enclosing checkpoint persists the compacted inbox atomically;
|
|
880
|
+
// avoid scheduling a second background save while that write is open.
|
|
881
|
+
this.factInbox.compactDedupeThrough(batch);
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
}
|
|
885
|
+
const eventWatermark = options.compactEventsThrough ?? this.state.events.at(-1)?.seq;
|
|
886
|
+
if (backend.eventArchive !== undefined && eventWatermark !== undefined) {
|
|
887
|
+
const fromSeq = (this.state.eventsCompactedThrough ?? 0) + 1;
|
|
888
|
+
const events = this.state.events.filter((event) => event.seq >= fromSeq && event.seq <= eventWatermark);
|
|
889
|
+
if (events.length)
|
|
890
|
+
await backend.eventArchive.append(events);
|
|
891
|
+
}
|
|
892
|
+
const exported = exportRuntimeCheckpoint(this.persistenceState(), this.mutationLog, this.outbox, this.quarantine, persistedPolicy, eventWatermark === undefined ? {} : { compactEventsThrough: eventWatermark }, this.factInbox.snapshot(), this.persistenceCompatibility());
|
|
893
|
+
const archived = backend.eventArchive === undefined || eventWatermark === undefined ? exported : withRuntimePersistenceIntegrity({ ...exported, eventArchive: { through: eventWatermark } });
|
|
894
|
+
validateRuntimePersistenceSnapshot(archived);
|
|
895
|
+
const withResults = backend.resultStore === undefined ? archived : await externalizeRuntimeResultBodies(archived, backend.resultStore);
|
|
896
|
+
const snapshot = backend.snapshotStore === undefined ? withResults : await externalizeRuntimeSnapshotBodies(withResults, backend.snapshotStore);
|
|
897
|
+
await backend.save(snapshot, backend === this.persistenceBackend ? this.persistenceDigest : undefined);
|
|
898
|
+
if (backend === this.persistenceBackend)
|
|
899
|
+
this.persistenceDigest = snapshot.integrity?.digest;
|
|
900
|
+
this.storagePolicy.markPersisted();
|
|
901
|
+
this.markArtifactsPersisted();
|
|
902
|
+
this.syncStoragePolicy();
|
|
903
|
+
const watermark = snapshot.checkpoint?.logWatermark ?? 0;
|
|
904
|
+
if (watermark > 0 && this.mutationLog.lastSequence >= watermark)
|
|
905
|
+
this.mutationLog.truncateThrough(watermark);
|
|
906
|
+
if (eventWatermark !== undefined) {
|
|
907
|
+
this.state.events = this.state.events.filter((event) => event.seq > eventWatermark);
|
|
908
|
+
this.state.eventsCompactedThrough = Math.max(this.state.eventsCompactedThrough ?? 0, eventWatermark);
|
|
909
|
+
}
|
|
910
|
+
this.schedulePersistence();
|
|
911
|
+
return snapshot;
|
|
912
|
+
}
|
|
913
|
+
mergeProposals(agentId, proposalIds) {
|
|
914
|
+
const plan = new ContextMerger(this.state).plan(agentId, proposalIds);
|
|
915
|
+
if (plan.conflicts.length || plan.mutations.length === 0)
|
|
916
|
+
return plan;
|
|
917
|
+
commitMutationTransaction(this.state, this.mutationLog, `context-merge:${agentId}:${plan.version ?? this.state.now}`, plan.mutations, this.state.now, this.sessionId);
|
|
918
|
+
this.schedulePersistence();
|
|
919
|
+
return plan;
|
|
920
|
+
}
|
|
921
|
+
emit(event) {
|
|
922
|
+
// Admission for a lone event only needs the event itself: preview its normalized
|
|
923
|
+
// form and check it against a policy copy, instead of deep-cloning the whole state.
|
|
924
|
+
const preview = normalizeRuntimeEvent(event, this.state.nextIds.event, { sessionId: this.sessionId, timestamp: this.state.now });
|
|
925
|
+
this.storagePolicy.clone().put('event', `event:${preview.id}`, preview);
|
|
926
|
+
const emitted = appendRuntimeEvent(this.state, event, { sessionId: this.sessionId, timestamp: this.state.now });
|
|
927
|
+
this.syncStoragePolicy();
|
|
928
|
+
return emitted;
|
|
929
|
+
}
|
|
930
|
+
assertRecoveryPrograms() {
|
|
931
|
+
if (!this.enforcingRecoveryPrograms)
|
|
932
|
+
return;
|
|
933
|
+
if (this.recoveryCompatibility !== undefined)
|
|
934
|
+
this.assertRecoveryCompatibility(this.recoveryCompatibility);
|
|
935
|
+
for (const lane of this.state.lanes.values()) {
|
|
936
|
+
if (['succeeded', 'failed', 'cancelled'].includes(lane.status))
|
|
937
|
+
continue;
|
|
938
|
+
const key = `${lane.resume.programId}@${lane.resume.programVersion}`;
|
|
939
|
+
if (!this.programs.has(key))
|
|
940
|
+
throw new Error(`PROGRAM_VERSION_UNAVAILABLE:${key}`);
|
|
941
|
+
}
|
|
942
|
+
for (const effect of this.state.effects.values()) {
|
|
943
|
+
if (effect.outcome || ['succeeded', 'failed', 'cancelled'].includes(effect.state))
|
|
944
|
+
continue;
|
|
945
|
+
if (effect.kind !== 'tool' || effect.toolVersion === undefined)
|
|
946
|
+
continue;
|
|
947
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
948
|
+
const name = input.name;
|
|
949
|
+
if (typeof name !== 'string' || this.currentToolVersions()[name] !== effect.toolVersion)
|
|
950
|
+
throw new Error(`TOOL_VERSION_UNAVAILABLE:${typeof name === 'string' ? `${name}@${effect.toolVersion}` : effect.toolVersion}`);
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
persistenceCompatibility() {
|
|
954
|
+
const toolVersions = this.currentToolVersions();
|
|
955
|
+
return {
|
|
956
|
+
schemaVersion: 1,
|
|
957
|
+
programVersions: Object.fromEntries([...this.programs.entries()].map(([key, program]) => [key, program.version])),
|
|
958
|
+
toolVersions,
|
|
959
|
+
...(this.policyVersion === undefined ? {} : { policyVersion: this.policyVersion }),
|
|
960
|
+
...(this.routerVersion === undefined ? {} : { routerVersion: this.routerVersion }),
|
|
961
|
+
};
|
|
962
|
+
}
|
|
963
|
+
assertRecoveryCompatibility(expected) {
|
|
964
|
+
for (const [key, version] of Object.entries(expected.programVersions)) {
|
|
965
|
+
const program = this.programs.get(key);
|
|
966
|
+
if (!program || program.version !== version)
|
|
967
|
+
throw new Error(`PROGRAM_VERSION_UNAVAILABLE:${key}`);
|
|
968
|
+
}
|
|
969
|
+
const toolVersions = this.currentToolVersions();
|
|
970
|
+
for (const [name, version] of Object.entries(expected.toolVersions))
|
|
971
|
+
if (toolVersions[name] !== version)
|
|
972
|
+
throw new Error(`TOOL_VERSION_UNAVAILABLE:${name}@${version}`);
|
|
973
|
+
if (expected.policyVersion !== undefined && this.policyVersion !== expected.policyVersion)
|
|
974
|
+
throw new Error(`POLICY_VERSION_UNAVAILABLE:${expected.policyVersion}`);
|
|
975
|
+
if (expected.routerVersion !== undefined && this.routerVersion !== expected.routerVersion)
|
|
976
|
+
throw new Error(`ROUTER_VERSION_UNAVAILABLE:${expected.routerVersion}`);
|
|
977
|
+
}
|
|
978
|
+
currentToolVersions() {
|
|
979
|
+
return { ...this.toolVersions, ...Object.fromEntries(this.tools.list().map((manifest) => [manifest.name, manifest.version])) };
|
|
980
|
+
}
|
|
981
|
+
tryEmit(event) {
|
|
982
|
+
try {
|
|
983
|
+
this.assertStorageAdmission([{ op: 'appendEvent', event }]);
|
|
984
|
+
return this.emit(event);
|
|
985
|
+
}
|
|
986
|
+
catch {
|
|
987
|
+
return undefined;
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
rejectHostCommand(eventId, code) {
|
|
991
|
+
const mutations = [
|
|
992
|
+
{ op: 'appendEvent', event: { type: 'command.rejected', data: { eventId, code } } },
|
|
993
|
+
{ op: 'appendEvent', event: { type: 'command.applied', data: { eventId } } },
|
|
994
|
+
];
|
|
995
|
+
this.assertStorageAdmission(mutations);
|
|
996
|
+
commitMutationTransaction(this.state, this.mutationLog, `host-command:${eventId}:rejected`, mutations, this.state.now, this.sessionId);
|
|
997
|
+
}
|
|
998
|
+
prepareStepOutput(output, lane) {
|
|
999
|
+
const reasoningFloor = lane?.progressWatchdog?.interventionLevel !== undefined && lane.progressWatchdog.interventionLevel >= 2 ? 'high' : undefined;
|
|
1000
|
+
return {
|
|
1001
|
+
...output,
|
|
1002
|
+
actions: output.actions.map((action) => action.type === 'submit_effects' ? {
|
|
1003
|
+
...action,
|
|
1004
|
+
effects: action.effects.map((effect) => {
|
|
1005
|
+
const prepared = this.effectSubmissionPreparer(effect);
|
|
1006
|
+
if (reasoningFloor === undefined || prepared.kind !== 'llm' || !prepared.input || typeof prepared.input !== 'object' || Array.isArray(prepared.input))
|
|
1007
|
+
return prepared;
|
|
1008
|
+
const input = prepared.input;
|
|
1009
|
+
const existing = input.requirements && typeof input.requirements === 'object' && !Array.isArray(input.requirements) ? input.requirements : {};
|
|
1010
|
+
const current = existing.reasoning;
|
|
1011
|
+
if (current === 'high')
|
|
1012
|
+
return prepared;
|
|
1013
|
+
return { ...prepared, input: { ...input, requirements: { ...existing, reasoning: reasoningFloor } } };
|
|
1014
|
+
}),
|
|
1015
|
+
} : action),
|
|
1016
|
+
};
|
|
1017
|
+
}
|
|
1018
|
+
prepareRegisteredToolSubmission(submission) {
|
|
1019
|
+
if (submission.kind === 'llm') {
|
|
1020
|
+
const input = submission.input && typeof submission.input === 'object' && !Array.isArray(submission.input) ? submission.input : {};
|
|
1021
|
+
const rawQuery = input.toolDiscovery;
|
|
1022
|
+
if (rawQuery && typeof rawQuery === 'object' && !Array.isArray(rawQuery) && this.tools.list().length > 0) {
|
|
1023
|
+
const requestedId = typeof input.toolSetId === 'string' ? input.toolSetId : 'dynamic';
|
|
1024
|
+
const toolSet = this.tools.compileToolSet(requestedId, rawQuery);
|
|
1025
|
+
const tools = toolSet.tools.map((manifest) => ({ name: manifest.name, description: manifest.description, inputSchema: manifest.inputSchema }));
|
|
1026
|
+
return { ...submission, input: { ...input, toolSetId: `${toolSet.id}@${toolSet.version}`, tools: { tools } } };
|
|
1027
|
+
}
|
|
1028
|
+
return submission;
|
|
1029
|
+
}
|
|
1030
|
+
if (submission.kind !== 'tool')
|
|
1031
|
+
return submission;
|
|
1032
|
+
const input = submission.input && typeof submission.input === 'object' && !Array.isArray(submission.input) ? submission.input : {};
|
|
1033
|
+
if (typeof input.name !== 'string')
|
|
1034
|
+
return submission;
|
|
1035
|
+
if (this.tools.get(input.name) === undefined)
|
|
1036
|
+
return submission;
|
|
1037
|
+
const admission = this.tools.admission(input.name, input.arguments ?? {});
|
|
1038
|
+
return { ...submission, ...(submission.locks === undefined ? { locks: admission.locks } : {}), ...(submission.sideEffectPolicy === undefined ? { sideEffectPolicy: admission.sideEffectPolicy } : {}), ...(submission.attemptTimeoutMs === undefined ? { attemptTimeoutMs: admission.defaultTimeoutMs } : {}), ...(submission.toolVersion === undefined ? { toolVersion: admission.version } : {}) };
|
|
1039
|
+
}
|
|
1040
|
+
async executeRegisteredEffect(effect, signal, emitObservation) {
|
|
1041
|
+
if (effect.kind === 'tool') {
|
|
1042
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
1043
|
+
const name = input.name;
|
|
1044
|
+
if (typeof name !== 'string' || this.tools.get(name) === undefined)
|
|
1045
|
+
return { value: null, status: 'failed', executionState: 'failed', sideEffectState: 'none', error: { code: 'TOOL_NOT_REGISTERED', message: typeof name === 'string' ? `Tool ${name} is not registered.` : 'Tool effect requires a registered tool name.' } };
|
|
1046
|
+
const observations = [];
|
|
1047
|
+
const emit = (event) => {
|
|
1048
|
+
if (signal.aborted)
|
|
1049
|
+
return;
|
|
1050
|
+
const observation = { type: event.type, data: event.data };
|
|
1051
|
+
if (emitObservation)
|
|
1052
|
+
emitObservation(observation);
|
|
1053
|
+
else
|
|
1054
|
+
observations.push(observation);
|
|
1055
|
+
};
|
|
1056
|
+
const context = { toolCallId: effect.toolCallId ?? '', effectId: effect.id, attemptId: effect.attemptId, ...(effect.idempotencyKey === undefined ? {} : { idempotencyKey: effect.idempotencyKey }), agentId: effect.agentId, laneId: effect.ownerLaneId, signal, emit };
|
|
1057
|
+
const definition = this.tools.get(name);
|
|
1058
|
+
const argumentsValue = input.arguments ?? {};
|
|
1059
|
+
let executionRef;
|
|
1060
|
+
try {
|
|
1061
|
+
executionRef = this.tools.executionRef(name, argumentsValue, context);
|
|
1062
|
+
const detailed = await this.tools.executeDetailed(name, argumentsValue, context);
|
|
1063
|
+
let value;
|
|
1064
|
+
let artifact;
|
|
1065
|
+
try {
|
|
1066
|
+
value = strictJsonValue(detailed.output);
|
|
1067
|
+
}
|
|
1068
|
+
catch {
|
|
1069
|
+
value = null;
|
|
1070
|
+
artifact = artifactOutput(detailed.output);
|
|
1071
|
+
}
|
|
1072
|
+
return { value, ...(artifact === undefined ? {} : { artifact }), ...(detailed.normalized === undefined ? {} : { normalized: strictJsonValue(detailed.normalized) }), ...(detailed.summary === undefined ? {} : { summary: strictJsonValue(detailed.summary) }), sideEffectState: isSideEffectful(definition.manifest.sideEffectPolicy) ? 'applied' : 'none', executionState: 'succeeded', status: 'succeeded', ...(executionRef === undefined ? {} : { executionRef }), metadata: { toolVersion: detailed.manifest.version, retrySafety: detailed.manifest.retrySafety, defaultTimeoutMs: detailed.manifest.defaultTimeoutMs, observationCount: observations.length, ...(artifact === undefined ? {} : { artifactMediaType: artifact.mediaType }) }, ...(observations.length ? { observations } : {}) };
|
|
1073
|
+
}
|
|
1074
|
+
catch (cause) {
|
|
1075
|
+
if (signal.aborted && isSideEffectful(definition.manifest.sideEffectPolicy))
|
|
1076
|
+
return { value: null, executionState: 'remote_unknown', sideEffectState: 'unknown', ...(executionRef === undefined ? {} : { executionRef }), metadata: { toolVersion: definition.manifest.version, reconcileRequired: true }, ...(cause instanceof Error ? { error: { code: 'TOOL_CANCELLED_UNKNOWN', message: cause.message } } : {}) };
|
|
1077
|
+
const error = runtimeErrorFromCause(cause, 'TOOL_EXECUTION_FAILED');
|
|
1078
|
+
return { value: null, status: signal.aborted ? 'cancelled' : 'failed', executionState: 'failed', sideEffectState: 'none', ...(executionRef === undefined ? {} : { executionRef }), ...(cause instanceof Error ? { error } : {}), ...(observations.length ? { observations } : {}) };
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
if (effect.kind !== 'llm')
|
|
1082
|
+
return { value: null };
|
|
1083
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
1084
|
+
const task = input.task;
|
|
1085
|
+
const request = input.request;
|
|
1086
|
+
if (typeof task !== 'string' || !request || typeof request !== 'object' || Array.isArray(request))
|
|
1087
|
+
return { value: null, status: 'failed', executionState: 'failed', error: { code: 'INVALID_LLM_EFFECT_INPUT', message: 'LLM effect requires task and request.' } };
|
|
1088
|
+
const projection = request;
|
|
1089
|
+
const dynamicRequirements = input.requirements && typeof input.requirements === 'object' && !Array.isArray(input.requirements) ? input.requirements : {};
|
|
1090
|
+
const structuredRequirement = dynamicRequirements.structuredOutput;
|
|
1091
|
+
const structuredSchema = structuredRequirement && typeof structuredRequirement === 'object' && !Array.isArray(structuredRequirement) ? structuredRequirement.schema : undefined;
|
|
1092
|
+
if (structuredSchema !== undefined && (input.outputSchema === undefined || stableSerialize(structuredSchema) !== stableSerialize(input.outputSchema)))
|
|
1093
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: 'STRUCTURED_OUTPUT_CONTRACT_MISMATCH', message: 'requirements.structuredOutput.schema must equal outputSchema.' } };
|
|
1094
|
+
const requirements = {
|
|
1095
|
+
...(typeof dynamicRequirements.toolCalling === 'boolean' ? { toolCalling: dynamicRequirements.toolCalling } : {}),
|
|
1096
|
+
...(typeof dynamicRequirements.structuredOutput === 'boolean' ? { structuredOutput: dynamicRequirements.structuredOutput } : structuredSchema === undefined ? {} : { structuredOutput: true }),
|
|
1097
|
+
...(dynamicRequirements.reasoning === 'low' || dynamicRequirements.reasoning === 'medium' || dynamicRequirements.reasoning === 'high' ? { reasoning: dynamicRequirements.reasoning } : {}),
|
|
1098
|
+
...(typeof dynamicRequirements.maxOutputTokens === 'number' ? { maxOutputTokens: dynamicRequirements.maxOutputTokens } : {}),
|
|
1099
|
+
...(typeof dynamicRequirements.contextSize === 'number' ? { contextSize: dynamicRequirements.contextSize } : {}),
|
|
1100
|
+
};
|
|
1101
|
+
const candidates = this.modelRouter.routeProjection(task, projection, requirements);
|
|
1102
|
+
const routes = this.modelRouter.diagnostics(task, projection.privacy, requirements);
|
|
1103
|
+
const maxAttempts = effect.retryPolicy?.maxAttempts ?? candidates.length;
|
|
1104
|
+
const attemptNo = effect.attemptNo;
|
|
1105
|
+
const candidate = candidates[attemptNo - 1];
|
|
1106
|
+
const attemptId = effect.attemptId;
|
|
1107
|
+
const metadata = (attempt) => ({ selected: { id: candidate?.id ?? null, providerId: candidate?.providerId ?? null }, routes: asJsonValue(routes), attempts: [attempt] });
|
|
1108
|
+
const canFallback = candidate !== undefined && attemptNo < Math.max(0, maxAttempts) && candidates[attemptNo] !== undefined;
|
|
1109
|
+
if (candidate === undefined)
|
|
1110
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: candidates.length === 0 ? 'NO_ELIGIBLE_MODEL' : 'MODEL_ATTEMPT_LIMIT_REACHED', message: candidates.length === 0 ? 'No model candidate satisfies the task, privacy, capability, and context requirements.' : 'No additional routed model candidate is available for this Effect.' }, metadata: { routes: asJsonValue(routes), attempts: [] } };
|
|
1111
|
+
const attemptBase = { attemptId, attemptNo, modelId: candidate.id, providerId: candidate.providerId };
|
|
1112
|
+
if (candidate.adapter === undefined)
|
|
1113
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: 'MODEL_ADAPTER_NOT_BOUND', message: `No adapter is bound to routed model ${candidate.id}.`, ...(canFallback ? { retryable: true } : {}) }, metadata: metadata(attemptBase) };
|
|
1114
|
+
const startedAt = Date.now();
|
|
1115
|
+
try {
|
|
1116
|
+
const result = assignRuntimeToolCallIds(validateAdapterResult(await candidate.adapter.executeAttempt({ request: projection, signal, model: candidate.id, ...(input.outputSchema === undefined ? {} : { outputSchema: input.outputSchema }), ...(typeof requirements.maxOutputTokens === 'number' ? { maxOutputTokens: requirements.maxOutputTokens } : {}), ...(emitObservation === undefined ? {} : { onObservation: (chunk) => emitObservation({ type: 'chunk', data: chunk }) }) })), effect.id);
|
|
1117
|
+
const usage = result.usage === undefined ? { latencyMs: Math.max(0, Date.now() - startedAt) } : { ...result.usage, latencyMs: result.usage.latencyMs ?? Math.max(0, Date.now() - startedAt) };
|
|
1118
|
+
const attempt = { ...attemptBase, usage };
|
|
1119
|
+
if (result.finishReason === 'refusal') {
|
|
1120
|
+
this.modelRouter.recordFeedback({ modelId: candidate.id, providerId: candidate.providerId, outcome: 'refused', ...(result.usage === undefined ? {} : { usage: result.usage }) });
|
|
1121
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: 'MODEL_REFUSAL', message: result.refusal ?? 'Model refused the request.', ...(canFallback ? { retryable: true } : {}) }, metadata: metadata(attempt) };
|
|
1122
|
+
}
|
|
1123
|
+
if (result.finishReason === 'error') {
|
|
1124
|
+
this.modelRouter.recordFeedback({ modelId: candidate.id, providerId: candidate.providerId, outcome: 'failed', ...(result.usage === undefined ? {} : { usage: result.usage }) });
|
|
1125
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: 'MODEL_ERROR', message: 'Model adapter returned an error result.', ...(canFallback ? { retryable: true } : {}) }, metadata: metadata(attempt) };
|
|
1126
|
+
}
|
|
1127
|
+
const output = input.outputSchema === undefined ? result : result.structured ?? result.text;
|
|
1128
|
+
if (input.outputSchema !== undefined && !validateJsonSchema(output, input.outputSchema)) {
|
|
1129
|
+
this.modelRouter.recordFeedback({ modelId: candidate.id, providerId: candidate.providerId, outcome: 'schema_rejected', ...(result.usage === undefined ? {} : { usage: result.usage }) });
|
|
1130
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { code: 'OUTPUT_SCHEMA_VIOLATION', message: 'Provider output did not match the declared schema.', ...(canFallback ? { retryable: true } : {}) }, metadata: metadata(attempt), rejectedOutput: { value: asJsonValue(output), privacy: projection.privacy, derivedFrom: [...(effect.derivedFrom ?? [])] } };
|
|
1131
|
+
}
|
|
1132
|
+
this.modelRouter.recordFeedback({ modelId: candidate.id, providerId: candidate.providerId, outcome: 'succeeded', ...(result.usage === undefined ? {} : { usage: result.usage }) });
|
|
1133
|
+
return { value: asJsonValue(output), privacy: projection.privacy, sideEffectState: 'none', executionState: 'succeeded', metadata: metadata({ ...attempt, usage }) };
|
|
1134
|
+
}
|
|
1135
|
+
catch (cause) {
|
|
1136
|
+
this.modelRouter.recordFeedback({ modelId: candidate.id, providerId: candidate.providerId, outcome: 'failed' });
|
|
1137
|
+
const error = runtimeErrorFromCause(cause, 'MODEL_EXECUTION_FAILED');
|
|
1138
|
+
return { value: null, status: 'failed', executionState: 'failed', privacy: projection.privacy, error: { ...error, ...(canFallback && error.retryable !== false ? { retryable: true } : {}) }, metadata: metadata({ ...attemptBase, error: error.message }) };
|
|
1139
|
+
}
|
|
1140
|
+
}
|
|
1141
|
+
journalEffect(effect, transactionId, result, events = [], lane, correlation, artifact) {
|
|
1142
|
+
const mutations = [{ op: 'setEffect', effectId: effect.id, record: structuredClone(effect) }];
|
|
1143
|
+
if (artifact)
|
|
1144
|
+
mutations.push({ op: 'publishArtifact', record: structuredClone(artifact) });
|
|
1145
|
+
if (result)
|
|
1146
|
+
mutations.push({ op: 'publishResult', record: structuredClone(result) });
|
|
1147
|
+
if (lane)
|
|
1148
|
+
mutations.push({ op: 'setLane', laneId: lane.id, record: structuredClone(lane) });
|
|
1149
|
+
if (correlation)
|
|
1150
|
+
mutations.push({ op: 'setToolCallCorrelation', record: structuredClone(correlation) });
|
|
1151
|
+
for (const event of events) {
|
|
1152
|
+
if (event.txId === undefined)
|
|
1153
|
+
event.txId = transactionId;
|
|
1154
|
+
const { seq: _seq, ...input } = event;
|
|
1155
|
+
mutations.push({ op: 'appendEvent', event: input });
|
|
1156
|
+
}
|
|
1157
|
+
this.mutationLog.append(transactionId, mutations, this.state.now);
|
|
1158
|
+
}
|
|
1159
|
+
enqueueFact(fact, eventId) {
|
|
1160
|
+
const candidateInbox = FactInbox.fromSnapshot(this.factInbox.snapshot());
|
|
1161
|
+
if (!candidateInbox.enqueue(fact, eventId))
|
|
1162
|
+
return false;
|
|
1163
|
+
const candidatePolicy = this.storagePolicy.clone();
|
|
1164
|
+
this.syncStoragePolicy(candidatePolicy, this.state, candidateInbox);
|
|
1165
|
+
const envelope = this.factInbox.enqueue(fact, eventId);
|
|
1166
|
+
if (!envelope)
|
|
1167
|
+
return false;
|
|
1168
|
+
this.syncStoragePolicy();
|
|
1169
|
+
this.schedulePersistence();
|
|
1170
|
+
this.scheduleWake();
|
|
1171
|
+
for (const resolve of this.factWaiters.splice(0))
|
|
1172
|
+
resolve();
|
|
1173
|
+
return true;
|
|
1174
|
+
}
|
|
1175
|
+
enqueueHostCommand(command) {
|
|
1176
|
+
validateHostCommand(command);
|
|
1177
|
+
const eventId = `host-command-${this.hostCommandSeq}`;
|
|
1178
|
+
if (!this.enqueueFact(command, eventId))
|
|
1179
|
+
return;
|
|
1180
|
+
this.hostCommandSeq++;
|
|
1181
|
+
}
|
|
1182
|
+
enqueueEffectCompletion(effectId, attemptId, execution, status = 'succeeded', error, dispatchError) {
|
|
1183
|
+
const fact = { type: 'effect_completion', effectId, attemptId, execution: encodeEffectExecution(execution), status, ...(error === undefined ? {} : { error: error }), ...(dispatchError === undefined ? {} : { dispatchError: dispatchError }) };
|
|
1184
|
+
const eventId = `effect-completion:${effectId}:${attemptId}`;
|
|
1185
|
+
this.enqueueFact(fact, eventId);
|
|
1186
|
+
}
|
|
1187
|
+
enqueueLLMPreparation(effectId, generation, status, projectionRef) {
|
|
1188
|
+
const fact = { type: 'llm_preparation', effectId, generation, status, ...(projectionRef === undefined ? {} : { projectionRef }) };
|
|
1189
|
+
this.enqueueFact(fact, `llm-preparation:${effectId}:${generation}:${status}`);
|
|
1190
|
+
}
|
|
1191
|
+
enqueueEffectReconcile(effect, value, status, error) {
|
|
1192
|
+
const fact = { type: 'effect_reconcile', effectId: effect.id, attemptId: effect.attemptId, value: structuredClone(value), status, ...(error === undefined ? {} : { error: error }) };
|
|
1193
|
+
this.enqueueFact(fact, `effect-reconcile:${effect.id}:${effect.attemptId}:${status}`);
|
|
1194
|
+
}
|
|
1195
|
+
scheduleWake(force = false) {
|
|
1196
|
+
if (this.wakeScheduled || this.inDrain)
|
|
1197
|
+
return;
|
|
1198
|
+
if (!force && this.factInbox.size === 0)
|
|
1199
|
+
return;
|
|
1200
|
+
this.wakeScheduled = true;
|
|
1201
|
+
setImmediate(() => {
|
|
1202
|
+
this.wakeScheduled = false;
|
|
1203
|
+
if (this.inDrain || (!force && this.factInbox.size === 0))
|
|
1204
|
+
return;
|
|
1205
|
+
this.inDrain = true;
|
|
1206
|
+
try {
|
|
1207
|
+
this.tick();
|
|
1208
|
+
}
|
|
1209
|
+
catch (cause) {
|
|
1210
|
+
this.wakeError = cause;
|
|
1211
|
+
}
|
|
1212
|
+
finally {
|
|
1213
|
+
this.inDrain = false;
|
|
1214
|
+
if (this.wakeError === undefined && (this.factInbox.size > 0 || this.hasPendingTickCleanup()))
|
|
1215
|
+
this.scheduleWake(true);
|
|
1216
|
+
}
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
enqueueReadyItem(item) { this.ready.enqueue(item); this.schedulerDecisionEpoch++; }
|
|
1220
|
+
enqueueLane(laneId) { const lane = this.state.lanes.get(laneId); if (lane && lane.status === 'ready') {
|
|
1221
|
+
lane.enqueueSeq = this.enqueueSeq++;
|
|
1222
|
+
lane.readySince = this.state.now;
|
|
1223
|
+
this.enqueueReadyItem(readyItemFromLane(lane));
|
|
1224
|
+
} }
|
|
1225
|
+
requestSchedulerDecision(now, availableSlots, snapshot) {
|
|
1226
|
+
const coordinator = this.schedulerDecisionCoordinator;
|
|
1227
|
+
if (!coordinator || snapshot.length < this.schedulerDecisionConfig.minCandidates)
|
|
1228
|
+
return;
|
|
1229
|
+
const candidates = snapshot.slice(0, this.schedulerDecisionConfig.candidateLimit).flatMap((item) => {
|
|
1230
|
+
const lane = this.state.lanes.get(item.laneId);
|
|
1231
|
+
return lane === undefined || lane.status !== 'ready' ? [] : [schedulerDecisionCandidateFromLane(lane, item.effectivePriority, now, this.schedulerDecisionIncludeGoals, item.inheritedFloor)];
|
|
1232
|
+
});
|
|
1233
|
+
if (candidates.length < this.schedulerDecisionConfig.minCandidates)
|
|
1234
|
+
return;
|
|
1235
|
+
const decisionId = `scheduler-decision-${this.sessionId}-${this.schedulerDecisionRequestSeq++}`;
|
|
1236
|
+
const request = {
|
|
1237
|
+
schemaVersion: 1,
|
|
1238
|
+
decisionId,
|
|
1239
|
+
candidateEpoch: this.schedulerDecisionEpoch,
|
|
1240
|
+
now,
|
|
1241
|
+
availableSlots,
|
|
1242
|
+
candidates,
|
|
1243
|
+
};
|
|
1244
|
+
this.schedulerDecisionRequests.set(decisionId, { epoch: request.candidateEpoch, candidateIds: new Set(candidates.map((candidate) => candidate.laneId)) });
|
|
1245
|
+
if (!coordinator.request(request, (decision) => {
|
|
1246
|
+
if (decision.decisionId !== request.decisionId || decision.candidateEpoch !== request.candidateEpoch || decision.modelId !== this.schedulerDecisionModelId) {
|
|
1247
|
+
this.schedulerDecisionRequests.delete(request.decisionId);
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
try {
|
|
1251
|
+
const accepted = this.enqueueFact({ type: 'scheduler_decision', decisionId: decision.decisionId, candidateEpoch: decision.candidateEpoch, orderedLaneIds: [...decision.orderedLaneIds], modelId: decision.modelId }, `scheduler-decision:${decision.decisionId}`);
|
|
1252
|
+
if (!accepted)
|
|
1253
|
+
this.schedulerDecisionRequests.delete(decision.decisionId);
|
|
1254
|
+
}
|
|
1255
|
+
catch {
|
|
1256
|
+
this.schedulerDecisionRequests.delete(decision.decisionId);
|
|
1257
|
+
}
|
|
1258
|
+
}, () => this.schedulerDecisionRequests.delete(decisionId)))
|
|
1259
|
+
this.schedulerDecisionRequests.delete(decisionId);
|
|
1260
|
+
}
|
|
1261
|
+
applySchedulerDecision(fact) {
|
|
1262
|
+
const request = this.schedulerDecisionRequests.get(fact.decisionId);
|
|
1263
|
+
this.schedulerDecisionRequests.delete(fact.decisionId);
|
|
1264
|
+
if (this.schedulerDecisionModelId === undefined || fact.modelId !== this.schedulerDecisionModelId || fact.candidateEpoch !== this.schedulerDecisionEpoch)
|
|
1265
|
+
return false;
|
|
1266
|
+
if (!Array.isArray(fact.orderedLaneIds) || fact.orderedLaneIds.length === 0 || fact.orderedLaneIds.some((laneId) => typeof laneId !== 'string') || new Set(fact.orderedLaneIds).size !== fact.orderedLaneIds.length)
|
|
1267
|
+
return false;
|
|
1268
|
+
const currentReady = new Set(this.ready.snapshot(this.state.now).map((item) => item.laneId));
|
|
1269
|
+
const known = request?.candidateIds ?? currentReady;
|
|
1270
|
+
if (fact.orderedLaneIds.some((laneId) => !known.has(laneId)))
|
|
1271
|
+
return false;
|
|
1272
|
+
const orderedLaneIds = fact.orderedLaneIds.filter((laneId) => currentReady.has(laneId));
|
|
1273
|
+
if (orderedLaneIds.length === 0)
|
|
1274
|
+
return false;
|
|
1275
|
+
this.schedulerDecisionCache = { epoch: fact.candidateEpoch, decision: { decisionId: fact.decisionId, candidateEpoch: fact.candidateEpoch, orderedLaneIds, modelId: fact.modelId } };
|
|
1276
|
+
this.tryEmit({ type: 'scheduler.decision.accepted', data: { decisionId: fact.decisionId, modelId: fact.modelId, candidateEpoch: fact.candidateEpoch, orderedLaneIds } });
|
|
1277
|
+
return true;
|
|
1278
|
+
}
|
|
1279
|
+
selectReadyLane(now, availableSlots) {
|
|
1280
|
+
const snapshot = this.ready.snapshot(now);
|
|
1281
|
+
if (snapshot.length === 0)
|
|
1282
|
+
return undefined;
|
|
1283
|
+
let selected;
|
|
1284
|
+
const cached = this.schedulerDecisionCache;
|
|
1285
|
+
const reserveDeterministic = (this.schedulerDecisionDispatches + 1) % this.schedulerDecisionReserveEvery === 0;
|
|
1286
|
+
if (cached !== undefined && cached.epoch === this.schedulerDecisionEpoch && !reserveDeterministic) {
|
|
1287
|
+
const current = new Set(snapshot.map((item) => item.laneId));
|
|
1288
|
+
const next = cached.decision.orderedLaneIds.filter((laneId) => current.has(laneId));
|
|
1289
|
+
const candidate = next[0];
|
|
1290
|
+
const deterministicIndex = candidate === undefined ? -1 : snapshot.findIndex((item) => item.laneId === candidate);
|
|
1291
|
+
if (candidate !== undefined && deterministicIndex >= 0 && deterministicIndex <= this.schedulerDecisionMaxReorderDistance) {
|
|
1292
|
+
selected = candidate;
|
|
1293
|
+
this.schedulerDecisionCache = next.length > 1 ? { ...cached, decision: { ...cached.decision, orderedLaneIds: next.slice(1) } } : undefined;
|
|
1294
|
+
}
|
|
1295
|
+
else if (candidate === undefined)
|
|
1296
|
+
this.schedulerDecisionCache = undefined;
|
|
1297
|
+
}
|
|
1298
|
+
if (selected === undefined) {
|
|
1299
|
+
if (cached !== undefined && cached.epoch !== this.schedulerDecisionEpoch)
|
|
1300
|
+
this.schedulerDecisionCache = undefined;
|
|
1301
|
+
if (this.schedulerDecisionCache === undefined)
|
|
1302
|
+
this.requestSchedulerDecision(now, availableSlots, snapshot);
|
|
1303
|
+
selected = this.ready.dequeue(now);
|
|
1304
|
+
}
|
|
1305
|
+
else {
|
|
1306
|
+
selected = this.ready.dequeueSpecific(selected);
|
|
1307
|
+
}
|
|
1308
|
+
if (selected !== undefined)
|
|
1309
|
+
this.schedulerDecisionDispatches++;
|
|
1310
|
+
return selected;
|
|
1311
|
+
}
|
|
1312
|
+
scheduleRuntimeTimer(at, callback) {
|
|
1313
|
+
return this.clock.timers.schedule(at, callback);
|
|
1314
|
+
}
|
|
1315
|
+
scheduleRuntimeDelay(delayMs, callback) {
|
|
1316
|
+
return this.scheduleRuntimeTimer(this.clock.now() + delayMs, callback);
|
|
1317
|
+
}
|
|
1318
|
+
seriesStep(program, context, series) {
|
|
1319
|
+
const locals = context.lane.resume.locals && typeof context.lane.resume.locals === 'object' && !Array.isArray(context.lane.resume.locals) ? context.lane.resume.locals : {};
|
|
1320
|
+
const sdkValue = locals.$sdk && typeof locals.$sdk === 'object' && !Array.isArray(locals.$sdk) ? locals.$sdk : {};
|
|
1321
|
+
const seriesValue = sdkValue.series && typeof sdkValue.series === 'object' && !Array.isArray(sdkValue.series) ? sdkValue.series : {};
|
|
1322
|
+
const keys = Array.isArray(seriesValue.keys) ? seriesValue.keys.filter((key) => typeof key === 'string') : (series?.keys ?? program.seriesKeys ?? ['member']);
|
|
1323
|
+
const index = typeof seriesValue.index === 'number' && Number.isInteger(seriesValue.index) && seriesValue.index >= 0 ? seriesValue.index : 0;
|
|
1324
|
+
const memberRef = series?.member ?? (program.seriesMember ? { programId: program.seriesMember.programId, programVersion: program.seriesMember.programVersion, step: program.seriesMember.step ?? 'start', locals: program.seriesMember.locals ?? {} } : undefined);
|
|
1325
|
+
const member = memberRef === undefined ? undefined : this.programs.get(`${memberRef.programId}@${memberRef.programVersion}`);
|
|
1326
|
+
if (!member || memberRef === undefined)
|
|
1327
|
+
return { actions: [{ type: 'fail', error: { code: 'PROGRAM_NOT_REGISTERED', message: memberRef ? `${memberRef.programId}@${memberRef.programVersion}` : 'series member' } }], next: { programId: program.id, programVersion: program.version, step: 'start', locals } };
|
|
1328
|
+
if (index >= keys.length)
|
|
1329
|
+
return { actions: [{ type: 'complete', result: sdkValue.seriesResults ?? { results: {} } }], next: { programId: program.id, programVersion: program.version, step: 'start', locals } };
|
|
1330
|
+
const memberLocals = sdkValue.memberLocals ?? memberRef.locals ?? {};
|
|
1331
|
+
const memberLane = structuredClone(context.lane);
|
|
1332
|
+
memberLane.resume = { programId: member.id, programVersion: member.version, step: typeof sdkValue.memberStep === 'string' ? sdkValue.memberStep : memberRef.step ?? member.entry ?? 'start', locals: structuredClone(memberLocals) };
|
|
1333
|
+
memberLane.goal = series?.goals?.[keys[index]] ?? `${context.lane.goal} [series:${keys[index]}]`;
|
|
1334
|
+
const seriesResults = sdkValue.seriesResults && typeof sdkValue.seriesResults === 'object' && !Array.isArray(sdkValue.seriesResults) ? sdkValue.seriesResults : {};
|
|
1335
|
+
const nextLocals = (nextSdk) => ({ ...locals, $sdk: nextSdk });
|
|
1336
|
+
const nextAfterMember = (nextIndex) => { const nextSdk = { ...sdkValue, series: { keys, index: nextIndex }, seriesResults }; delete nextSdk.memberStep; delete nextSdk.memberLocals; return nextIndex >= keys.length ? { actions: [{ type: 'complete', result: { results: seriesResults } }], next: { programId: program.id, programVersion: program.version, step: 'start', locals: nextLocals(nextSdk) } } : { actions: [], next: { programId: program.id, programVersion: program.version, step: 'start', locals: nextLocals(nextSdk) } }; };
|
|
1337
|
+
const memberDependencies = series?.members?.[keys[index]]?.dependsOn ?? [];
|
|
1338
|
+
const dependencyObservations = Object.fromEntries(memberDependencies.map((dependency) => {
|
|
1339
|
+
const record = seriesResults[dependency.key];
|
|
1340
|
+
const value = record && typeof record === 'object' && !Array.isArray(record) ? record : {};
|
|
1341
|
+
const status = value.status === 'succeeded' || value.status === 'failed' || value.status === 'cancelled' ? value.status : undefined;
|
|
1342
|
+
const outcome = status === undefined ? { status: 'failed', error: { code: 'SERIES_DEPENDENCY_MISSING', message: `Series dependency ${dependency.key} is not settled.` } } : { status, ...(value.result === undefined ? {} : { result: value.result }), ...(value.error && typeof value.error === 'object' && !Array.isArray(value.error) ? { error: value.error } : {}), ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) };
|
|
1343
|
+
return [dependency.key, { state: 'settled', target: { kind: 'lane', id: `series:${dependency.key}` }, outcome }];
|
|
1344
|
+
}));
|
|
1345
|
+
const blocked = memberDependencies.find((dependency) => dependency.condition === 'success' && dependencyObservations[dependency.key]?.outcome.status !== 'succeeded');
|
|
1346
|
+
if (blocked) {
|
|
1347
|
+
seriesResults[keys[index]] = { status: 'failed', error: { code: 'DEPENDENCY_FAILED', message: `Series dependency ${blocked.key} did not succeed` } };
|
|
1348
|
+
if ((series?.onMemberFailure ?? program.seriesOnMemberFailure ?? 'continue') === 'abort')
|
|
1349
|
+
return { actions: [{ type: 'fail', error: { code: 'DEPENDENCY_FAILED', message: `Series dependency ${blocked.key} did not succeed` } }], next: { programId: program.id, programVersion: program.version, step: 'start', locals: nextLocals({ ...sdkValue, series: { keys, index }, seriesResults }) } };
|
|
1350
|
+
return nextAfterMember(index + 1);
|
|
1351
|
+
}
|
|
1352
|
+
const output = member.step({ ...context, lane: memberLane, ...(memberDependencies.length && sdkValue.memberStep === undefined ? { resumeInput: { type: 'wait', resolution: { waitId: `series:${keys[index]}`, status: 'satisfied', dependencies: dependencyObservations } } } : {}) });
|
|
1353
|
+
const terminal = output.actions.find((action) => action.type === 'complete' || action.type === 'fail');
|
|
1354
|
+
if (terminal?.type === 'fail') {
|
|
1355
|
+
seriesResults[keys[index]] = { status: 'failed', error: terminal.error };
|
|
1356
|
+
if ((series?.onMemberFailure ?? program.seriesOnMemberFailure ?? 'continue') === 'abort')
|
|
1357
|
+
return output;
|
|
1358
|
+
}
|
|
1359
|
+
else if (terminal?.type === 'complete') {
|
|
1360
|
+
seriesResults[keys[index]] = { status: 'succeeded', result: terminal.result };
|
|
1361
|
+
}
|
|
1362
|
+
if (terminal) {
|
|
1363
|
+
const nextIndex = index + 1;
|
|
1364
|
+
return nextAfterMember(nextIndex);
|
|
1365
|
+
}
|
|
1366
|
+
const nextSdk = { ...sdkValue, series: { keys, index }, memberStep: output.next.step, memberLocals: output.next.locals };
|
|
1367
|
+
return { ...output, next: { programId: program.id, programVersion: program.version, step: 'start', locals: nextLocals(nextSdk) } };
|
|
1368
|
+
}
|
|
1369
|
+
tick() {
|
|
1370
|
+
this.wakeError = undefined;
|
|
1371
|
+
this.assertRecoveryPrograms();
|
|
1372
|
+
this.state.now = this.clock.now();
|
|
1373
|
+
const tickStartedAt = performance.now();
|
|
1374
|
+
let tickOperations = 0;
|
|
1375
|
+
const canStartTickOperation = () => tickOperations === 0 || performance.now() - tickStartedAt < this.maxTickMs;
|
|
1376
|
+
this.tickBudget = { canStart: canStartTickOperation, consume: () => { tickOperations++; } };
|
|
1377
|
+
if (this.hasPendingTickCleanup())
|
|
1378
|
+
this.refreshWaits();
|
|
1379
|
+
while (this.factInbox.size > 0 && canStartTickOperation()) {
|
|
1380
|
+
const before = this.factInbox.snapshot();
|
|
1381
|
+
const envelope = this.factInbox.drain(1)[0];
|
|
1382
|
+
if (!envelope)
|
|
1383
|
+
break;
|
|
1384
|
+
try {
|
|
1385
|
+
let commandApplied = false;
|
|
1386
|
+
if (envelope.fact.type !== 'llm_preparation' && envelope.fact.type !== 'scheduler_decision' && !this.state.events.some((event) => event.id === envelope.eventId && event.type === 'command.enqueued')) {
|
|
1387
|
+
const enqueuedEvent = { id: envelope.eventId, type: 'command.enqueued', data: envelope.fact };
|
|
1388
|
+
if (envelope.fact.type === 'effect_completion')
|
|
1389
|
+
this.tryEmit(enqueuedEvent);
|
|
1390
|
+
else
|
|
1391
|
+
this.emit(enqueuedEvent);
|
|
1392
|
+
}
|
|
1393
|
+
if (envelope.fact.type === 'llm_preparation') {
|
|
1394
|
+
this.preparingLLMs.delete(envelope.fact.effectId);
|
|
1395
|
+
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1396
|
+
if (effect && !effect.outcome && effect.state === 'queued' && effect.preparation?.generation === envelope.fact.generation && !effect.cancelRequested) {
|
|
1397
|
+
const prepared = structuredClone(effect);
|
|
1398
|
+
prepared.preparation = { state: envelope.fact.status, generation: envelope.fact.generation, ...(envelope.fact.projectionRef === undefined ? {} : { projectionRef: envelope.fact.projectionRef }) };
|
|
1399
|
+
const mutations = [{ op: 'setEffect', effectId: effect.id, record: prepared }];
|
|
1400
|
+
if (envelope.fact.status === 'prepared')
|
|
1401
|
+
mutations.push({ op: 'appendEvent', event: { type: 'llm.request_prepared', effectId: effect.id, data: { generation: envelope.fact.generation, projectionRef: envelope.fact.projectionRef ?? null } } });
|
|
1402
|
+
this.assertStorageAdmission(mutations);
|
|
1403
|
+
commitMutationTransaction(this.state, this.mutationLog, `llm-preparation:${effect.id}:${envelope.fact.generation}:${envelope.fact.status}`, mutations, this.state.now, this.sessionId);
|
|
1404
|
+
Object.assign(effect, prepared);
|
|
1405
|
+
this.state.effects.set(effect.id, effect);
|
|
1406
|
+
commandApplied = true;
|
|
1407
|
+
}
|
|
1408
|
+
}
|
|
1409
|
+
else if (envelope.fact.type === 'scheduler_decision') {
|
|
1410
|
+
commandApplied = this.applySchedulerDecision(envelope.fact);
|
|
1411
|
+
}
|
|
1412
|
+
else if (envelope.fact.type === 'effect_completion') {
|
|
1413
|
+
if (envelope.fact.dispatchError !== undefined)
|
|
1414
|
+
this.tryEmit({ type: 'effect.dispatch_failed', effectId: envelope.fact.effectId, data: envelope.fact.dispatchError });
|
|
1415
|
+
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1416
|
+
if (!effect || effect.attemptId !== envelope.fact.attemptId) {
|
|
1417
|
+
this.tryEmit({ type: 'attempt.late_emit', effectId: envelope.fact.effectId, attemptId: envelope.fact.attemptId, data: { kind: 'completion', status: effect?.outcome?.status ?? effect?.state ?? 'missing' } });
|
|
1418
|
+
this.executions.delete(envelope.fact.effectId);
|
|
1419
|
+
this.refreshWaits();
|
|
1420
|
+
}
|
|
1421
|
+
else {
|
|
1422
|
+
commandApplied = this.completeEffect(envelope.fact.effectId, decodeEffectExecution(envelope.fact.execution), envelope.fact.status, envelope.fact.error);
|
|
1423
|
+
this.executions.delete(envelope.fact.effectId);
|
|
1424
|
+
this.refreshWaits();
|
|
1425
|
+
}
|
|
1426
|
+
}
|
|
1427
|
+
else if (envelope.fact.type === 'effect_reconcile') {
|
|
1428
|
+
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1429
|
+
if (!effect || effect.attemptId !== envelope.fact.attemptId || effect.state !== 'reconcile_required') {
|
|
1430
|
+
this.tryEmit({ type: 'attempt.late_emit', effectId: envelope.fact.effectId, attemptId: envelope.fact.attemptId, data: { kind: 'reconcile', status: effect?.outcome?.status ?? effect?.state ?? 'missing' } });
|
|
1431
|
+
}
|
|
1432
|
+
else {
|
|
1433
|
+
this.reconcileEffect(envelope.fact.effectId, envelope.fact.value, envelope.fact.status, envelope.fact.error);
|
|
1434
|
+
commandApplied = true;
|
|
1435
|
+
}
|
|
1436
|
+
}
|
|
1437
|
+
else if (envelope.fact.type === 'reply') {
|
|
1438
|
+
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1439
|
+
if (effect?.agentId === envelope.fact.agentId && effect.kind === 'human' && !effect.outcome)
|
|
1440
|
+
commandApplied = this.completeEffect(envelope.fact.effectId, { value: envelope.fact.value }, 'succeeded', undefined, [{ op: 'appendEvent', event: { type: 'command.applied', data: { eventId: envelope.eventId } } }]);
|
|
1441
|
+
else {
|
|
1442
|
+
this.rejectHostCommand(envelope.eventId, effect?.agentId !== envelope.fact.agentId ? 'EFFECT_NOT_OWNED' : 'EFFECT_NOT_REPLYABLE');
|
|
1443
|
+
commandApplied = true;
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
else if (envelope.fact.type === 'cancel')
|
|
1447
|
+
commandApplied = this.cancelAgent(envelope.fact.agentId, envelope.fact.reason, [{ op: 'appendEvent', event: { type: 'command.applied', data: { eventId: envelope.eventId } } }], `host-command:${envelope.eventId}`);
|
|
1448
|
+
else if (envelope.fact.type === 'cancel_effect') {
|
|
1449
|
+
const effect = this.state.effects.get(envelope.fact.effectId);
|
|
1450
|
+
if (!effect || effect.agentId !== envelope.fact.agentId) {
|
|
1451
|
+
this.rejectHostCommand(envelope.eventId, 'EFFECT_NOT_OWNED');
|
|
1452
|
+
commandApplied = true;
|
|
1453
|
+
}
|
|
1454
|
+
else if (effect.outcome) {
|
|
1455
|
+
this.rejectHostCommand(envelope.eventId, 'EFFECT_ALREADY_SETTLED');
|
|
1456
|
+
commandApplied = true;
|
|
1457
|
+
}
|
|
1458
|
+
else
|
|
1459
|
+
commandApplied = this.cancelEffect(envelope.fact.effectId, 0, envelope.fact.reason, [{ op: 'appendEvent', event: { type: 'command.applied', data: { eventId: envelope.eventId } } }]);
|
|
1460
|
+
}
|
|
1461
|
+
else {
|
|
1462
|
+
const lane = this.state.lanes.get(envelope.fact.laneId);
|
|
1463
|
+
if (!lane) {
|
|
1464
|
+
this.rejectHostCommand(envelope.eventId, 'LANE_NOT_FOUND');
|
|
1465
|
+
commandApplied = true;
|
|
1466
|
+
}
|
|
1467
|
+
else if (['succeeded', 'failed', 'cancelled'].includes(lane.status)) {
|
|
1468
|
+
this.rejectHostCommand(envelope.eventId, 'LANE_TERMINAL');
|
|
1469
|
+
commandApplied = true;
|
|
1470
|
+
}
|
|
1471
|
+
else {
|
|
1472
|
+
const nextLane = structuredClone(lane);
|
|
1473
|
+
nextLane.priority = envelope.fact.priority;
|
|
1474
|
+
nextLane.version++;
|
|
1475
|
+
const event = { type: 'lane.priority_changed', laneId: lane.id, data: { previous: lane.priority, priority: nextLane.priority } };
|
|
1476
|
+
const appliedEvent = { type: 'command.applied', data: { eventId: envelope.eventId } };
|
|
1477
|
+
const mutations = [{ op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event }, { op: 'appendEvent', event: appliedEvent }];
|
|
1478
|
+
this.assertStorageAdmission(mutations);
|
|
1479
|
+
commitMutationTransaction(this.state, this.mutationLog, `host-command:${envelope.eventId}`, mutations, this.state.now, this.sessionId);
|
|
1480
|
+
Object.assign(lane, nextLane);
|
|
1481
|
+
this.state.lanes.set(lane.id, lane);
|
|
1482
|
+
if (lane.status === 'ready')
|
|
1483
|
+
this.enqueueReadyItem(readyItemFromLane(lane));
|
|
1484
|
+
commandApplied = true;
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
if (!commandApplied && envelope.fact.type !== 'effect_completion' && envelope.fact.type !== 'scheduler_decision')
|
|
1488
|
+
this.emit({ type: 'command.applied', data: { eventId: envelope.eventId } });
|
|
1489
|
+
}
|
|
1490
|
+
catch (cause) {
|
|
1491
|
+
this.factInbox.restore(before);
|
|
1492
|
+
throw cause;
|
|
1493
|
+
}
|
|
1494
|
+
tickOperations++;
|
|
1495
|
+
}
|
|
1496
|
+
if (this.maxRuntimeAt !== undefined)
|
|
1497
|
+
for (const agent of this.state.agents.values())
|
|
1498
|
+
if (agent.state === 'running' && this.state.now >= this.maxRuntimeAt)
|
|
1499
|
+
this.cancelAgent(agent.id, 'TIMEOUT');
|
|
1500
|
+
for (const agent of this.state.agents.values())
|
|
1501
|
+
if (agent.state === 'running' && agent.deadlineAt !== undefined && this.state.now >= agent.deadlineAt)
|
|
1502
|
+
this.cancelAgent(agent.id, 'TIMEOUT');
|
|
1503
|
+
while (canStartTickOperation()) {
|
|
1504
|
+
const timer = this.clock.timers.due(this.state.now, 1)[0];
|
|
1505
|
+
if (!timer)
|
|
1506
|
+
break;
|
|
1507
|
+
timer.callback();
|
|
1508
|
+
tickOperations++;
|
|
1509
|
+
}
|
|
1510
|
+
let progressed = 0;
|
|
1511
|
+
while (progressed < this.maxSteps && canStartTickOperation()) {
|
|
1512
|
+
const laneId = this.selectReadyLane(this.state.now, this.maxSteps - progressed);
|
|
1513
|
+
if (!laneId)
|
|
1514
|
+
break;
|
|
1515
|
+
const lane = this.state.lanes.get(laneId);
|
|
1516
|
+
if (!lane || lane.status !== 'ready')
|
|
1517
|
+
continue;
|
|
1518
|
+
tickOperations++;
|
|
1519
|
+
const currentPressure = historyPressure(lane.context.history, this.state.historySoftTokens, this.state.historyHardTokens);
|
|
1520
|
+
const stepLane = structuredClone(lane);
|
|
1521
|
+
if (currentPressure)
|
|
1522
|
+
stepLane.historyPressure = currentPressure;
|
|
1523
|
+
else
|
|
1524
|
+
delete stepLane.historyPressure;
|
|
1525
|
+
const program = this.programs.get(`${lane.resume.programId}@${lane.resume.programVersion}`);
|
|
1526
|
+
if (!program) {
|
|
1527
|
+
this.failLane(lane, { code: 'PROGRAM_NOT_REGISTERED', message: `${lane.resume.programId}@${lane.resume.programVersion}` });
|
|
1528
|
+
continue;
|
|
1529
|
+
}
|
|
1530
|
+
let output;
|
|
1531
|
+
const stepContext = { lane: stepLane, state: structuredClone(this.state), ...(lane.pendingResumeInput ? { resumeInput: structuredClone(lane.pendingResumeInput) } : {}), now: this.state.now, observe: (event) => { this.observationInbox.enqueue({ ...event, agentId: lane.agentId, laneId: lane.id, timestamp: this.state.now }); } };
|
|
1532
|
+
try {
|
|
1533
|
+
output = withPureStepGuard(() => lane.series || program.seriesMember ? this.seriesStep(program, stepContext, lane.series) : program.step(stepContext));
|
|
1534
|
+
}
|
|
1535
|
+
catch (cause) {
|
|
1536
|
+
const failure = runtimeErrorFromCause(cause, 'STEP_FAILED');
|
|
1537
|
+
if (!program.errorBoundary) {
|
|
1538
|
+
this.failLane(lane, failure);
|
|
1539
|
+
continue;
|
|
1540
|
+
}
|
|
1541
|
+
try {
|
|
1542
|
+
output = withPureStepGuard(() => program.errorBoundary(failure, stepContext));
|
|
1543
|
+
}
|
|
1544
|
+
catch (boundaryCause) {
|
|
1545
|
+
this.failLane(lane, { code: 'ERROR_BOUNDARY_FAILED', message: boundaryCause instanceof Error ? boundaryCause.message : String(boundaryCause) });
|
|
1546
|
+
continue;
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
if (output && typeof output === 'object' && typeof output.then === 'function') {
|
|
1550
|
+
this.failLane(lane, { code: 'ASYNC_STEP_FORBIDDEN', message: 'LaneProgram.step() must return synchronously; external work belongs in an Effect.' });
|
|
1551
|
+
continue;
|
|
1552
|
+
}
|
|
1553
|
+
let preparedOutput;
|
|
1554
|
+
try {
|
|
1555
|
+
preparedOutput = this.prepareStepOutput(output, lane);
|
|
1556
|
+
}
|
|
1557
|
+
catch (cause) {
|
|
1558
|
+
const message = cause instanceof Error ? cause.message : String(cause);
|
|
1559
|
+
const candidate = cause && typeof cause === 'object' ? cause : undefined;
|
|
1560
|
+
const code = typeof candidate?.code === 'string' ? candidate.code : message.split(':', 1)[0] || 'STEP_OUTPUT_PREPARATION_FAILED';
|
|
1561
|
+
const rejection = { code, message };
|
|
1562
|
+
const consecutive = (lane.consecutiveControlErrors ?? 0) + 1;
|
|
1563
|
+
const controlInput = { type: 'control_error', error: rejection, ...(lane.pendingResumeInput ? { original: lane.pendingResumeInput } : {}) };
|
|
1564
|
+
if (consecutive >= this.maxConsecutiveControlErrors)
|
|
1565
|
+
this.failLane(lane, { code: 'CONTROL_ERROR_LOOP', message: 'Lane exceeded the consecutive control error limit.', details: { lastError: rejection } });
|
|
1566
|
+
else
|
|
1567
|
+
this.commitLaneControlInput(lane, controlInput, { type: 'step.rejected', laneId: lane.id, data: rejection }, { consecutiveControlErrors: consecutive });
|
|
1568
|
+
progressed++;
|
|
1569
|
+
continue;
|
|
1570
|
+
}
|
|
1571
|
+
const result = validateStep(this.state, lane.id, preparedOutput);
|
|
1572
|
+
if ('rejection' in result) {
|
|
1573
|
+
const consecutive = (lane.consecutiveControlErrors ?? 0) + 1;
|
|
1574
|
+
const controlInput = { type: 'control_error', error: result.rejection, ...(lane.pendingResumeInput ? { original: lane.pendingResumeInput } : {}) };
|
|
1575
|
+
if (result.rejection.code === 'FORK_AFFINITY_COLLAPSIBLE') {
|
|
1576
|
+
this.commitLaneControlInput(lane, controlInput, { type: 'fork.affinity_advice', laneId: lane.id, data: result.rejection });
|
|
1577
|
+
}
|
|
1578
|
+
else {
|
|
1579
|
+
if (consecutive >= this.maxConsecutiveControlErrors)
|
|
1580
|
+
this.failLane(lane, { code: 'CONTROL_ERROR_LOOP', message: 'Lane exceeded the consecutive control error limit.', details: { lastError: result.rejection } });
|
|
1581
|
+
else
|
|
1582
|
+
this.commitLaneControlInput(lane, controlInput, { type: 'step.rejected', laneId: lane.id, data: result.rejection }, { consecutiveControlErrors: consecutive });
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
else {
|
|
1586
|
+
let watchdogObservation;
|
|
1587
|
+
if (preparedOutput.actions.some((action) => action.type === 'submit_effects')) {
|
|
1588
|
+
const observation = observeProgress(lane, preparedOutput, this.state, lane.progressWatchdog, { noProgressThreshold: this.watchdogNoProgressThreshold, repeatedActionThreshold: this.watchdogRepeatedActionThreshold, admission: true });
|
|
1589
|
+
if (observation.rejected) {
|
|
1590
|
+
if (observation.state.interventionLevel >= 3)
|
|
1591
|
+
this.failLane(lane, observation.rejected, { progressWatchdog: observation.state });
|
|
1592
|
+
else
|
|
1593
|
+
this.commitLaneControlInput(lane, { type: 'control_error', error: observation.rejected, ...(lane.pendingResumeInput ? { original: lane.pendingResumeInput } : {}) }, { type: 'progress.intervention_applied', laneId: lane.id, data: observation.rejected }, { progressWatchdog: observation.state });
|
|
1594
|
+
progressed++;
|
|
1595
|
+
continue;
|
|
1596
|
+
}
|
|
1597
|
+
watchdogObservation = observation;
|
|
1598
|
+
}
|
|
1599
|
+
const watchdog = watchdogObservation ?? observeProgress(lane, preparedOutput, this.state, lane.progressWatchdog, { noProgressThreshold: this.watchdogNoProgressThreshold, repeatedActionThreshold: this.watchdogRepeatedActionThreshold });
|
|
1600
|
+
const stepMutations = result.mutations.map((mutation) => {
|
|
1601
|
+
if (mutation.op !== 'setLane' || mutation.laneId !== lane.id)
|
|
1602
|
+
return mutation;
|
|
1603
|
+
const nextLane = structuredClone(mutation.record);
|
|
1604
|
+
delete nextLane.consecutiveControlErrors;
|
|
1605
|
+
replaceResumeInput(nextLane, undefined);
|
|
1606
|
+
nextLane.progressWatchdog = watchdog.state;
|
|
1607
|
+
return { ...mutation, record: nextLane };
|
|
1608
|
+
});
|
|
1609
|
+
if (!watchdog.progressed)
|
|
1610
|
+
stepMutations.push({ op: 'appendEvent', event: { type: watchdog.state.interventionLevel >= 3 ? 'progress.no_progress_detected' : 'progress.intervention_applied', laneId: lane.id, data: { noProgressCount: watchdog.state.noProgressCount, interventionLevel: watchdog.state.interventionLevel } } });
|
|
1611
|
+
try {
|
|
1612
|
+
this.assertStorageAdmission(stepMutations);
|
|
1613
|
+
}
|
|
1614
|
+
catch (cause) {
|
|
1615
|
+
const storageError = { code: 'SESSION_STORAGE_LIMIT_EXCEEDED', message: cause instanceof Error ? cause.message : String(cause) };
|
|
1616
|
+
this.failLane(lane, storageError);
|
|
1617
|
+
progressed++;
|
|
1618
|
+
continue;
|
|
1619
|
+
}
|
|
1620
|
+
commitMutationTransaction(this.state, this.mutationLog, `step:${lane.id}:${lane.version + 1}`, stepMutations, this.state.now, this.sessionId);
|
|
1621
|
+
for (const mutation of stepMutations)
|
|
1622
|
+
if (mutation.op === 'insertEffect')
|
|
1623
|
+
this.outbox.enqueue(mutation.record, this.state.now);
|
|
1624
|
+
for (const mutation of stepMutations)
|
|
1625
|
+
if (mutation.op === 'insertWait')
|
|
1626
|
+
this.scheduleWaitDeadline(mutation.record);
|
|
1627
|
+
const updated = this.state.lanes.get(lane.id);
|
|
1628
|
+
if (updated) {
|
|
1629
|
+
if (watchdog.state.interventionLevel >= 3 && !['succeeded', 'failed', 'cancelled'].includes(updated.status))
|
|
1630
|
+
this.failLane(updated, { code: 'NO_PROGRESS_DETECTED', message: 'Lane made no observable progress within the watchdog threshold.' });
|
|
1631
|
+
}
|
|
1632
|
+
if (updated?.status === 'ready')
|
|
1633
|
+
this.enqueueLane(updated.id);
|
|
1634
|
+
this.enqueueNewReadyLanes();
|
|
1635
|
+
this.refreshWaits();
|
|
1636
|
+
this.propagateCancelledLanes();
|
|
1637
|
+
this.syncStoragePolicy();
|
|
1638
|
+
}
|
|
1639
|
+
this.dispatchQueuedEffects();
|
|
1640
|
+
progressed++;
|
|
1641
|
+
}
|
|
1642
|
+
this.dispatchQueuedEffects();
|
|
1643
|
+
this.completeFinishedChildAgents();
|
|
1644
|
+
this.finalizeCancellations();
|
|
1645
|
+
this.syncStoragePolicy();
|
|
1646
|
+
this.schedulePersistence();
|
|
1647
|
+
const pendingTickCleanup = this.hasPendingTickCleanup();
|
|
1648
|
+
this.tickBudget = undefined;
|
|
1649
|
+
if (pendingTickCleanup)
|
|
1650
|
+
this.scheduleWake(true);
|
|
1651
|
+
return progressed;
|
|
1652
|
+
}
|
|
1653
|
+
async run(agentOrMaxTicks = 10_000, requestedMaxTicks = 10_000) {
|
|
1654
|
+
if (typeof agentOrMaxTicks === 'string')
|
|
1655
|
+
return this.runAgent(agentOrMaxTicks, requestedMaxTicks);
|
|
1656
|
+
for (let tick = 0; tick < agentOrMaxTicks; tick++) {
|
|
1657
|
+
const work = this.tick();
|
|
1658
|
+
await this.flushPersistence();
|
|
1659
|
+
if (this.executionYieldPending.size) {
|
|
1660
|
+
this.executionYieldPending.clear();
|
|
1661
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1662
|
+
}
|
|
1663
|
+
if (this.ready.size === 0 && this.executions.size === 0 && !this.hasQueuedEffects() && !this.hasPendingTickCleanup()) {
|
|
1664
|
+
if (this.preparingLLMs.size) {
|
|
1665
|
+
await Promise.resolve();
|
|
1666
|
+
continue;
|
|
1667
|
+
}
|
|
1668
|
+
if (this.factInbox.size > 0)
|
|
1669
|
+
continue;
|
|
1670
|
+
if (this.hasPendingHostInteraction()) {
|
|
1671
|
+
await this.waitForFact();
|
|
1672
|
+
continue;
|
|
1673
|
+
}
|
|
1674
|
+
const nextAt = this.clock.timers.nextAt();
|
|
1675
|
+
if (nextAt !== undefined) {
|
|
1676
|
+
if (nextAt > this.clock.now()) {
|
|
1677
|
+
if (this.clock.waitUntil)
|
|
1678
|
+
await this.clock.waitUntil(nextAt);
|
|
1679
|
+
else
|
|
1680
|
+
this.clock.set(nextAt);
|
|
1681
|
+
}
|
|
1682
|
+
;
|
|
1683
|
+
continue;
|
|
1684
|
+
}
|
|
1685
|
+
break;
|
|
1686
|
+
}
|
|
1687
|
+
if (work === 0 && this.executions.size) {
|
|
1688
|
+
const nextAt = this.clock.timers.nextAt();
|
|
1689
|
+
if (nextAt !== undefined && nextAt > this.clock.now()) {
|
|
1690
|
+
if (this.clock.waitUntil)
|
|
1691
|
+
await Promise.race([this.clock.waitUntil(nextAt), ...[...this.executions.values()].map((execution) => execution.promise)]);
|
|
1692
|
+
else
|
|
1693
|
+
this.clock.set(nextAt);
|
|
1694
|
+
continue;
|
|
1695
|
+
}
|
|
1696
|
+
await Promise.race([...this.executions.values()].map((execution) => execution.promise));
|
|
1697
|
+
}
|
|
1698
|
+
else if (work === 0 && this.factInbox.size === 0 && this.hasPendingHostInteraction() && !this.hasQueuedEffects())
|
|
1699
|
+
await this.waitForFact();
|
|
1700
|
+
else if (work === 0)
|
|
1701
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1702
|
+
}
|
|
1703
|
+
const root = [...this.state.lanes.values()].find((lane) => lane.ownerLaneId === undefined);
|
|
1704
|
+
const status = root?.status === 'succeeded' ? 'succeeded' : root?.status === 'cancelled' ? 'cancelled' : 'failed';
|
|
1705
|
+
if (root && !['succeeded', 'failed', 'cancelled'].includes(root.status))
|
|
1706
|
+
this.emit({ type: 'runtime.idle_blocked', laneId: root.id, data: { status: root.status } });
|
|
1707
|
+
const agent = root ? this.state.agents.get(root.agentId) : undefined;
|
|
1708
|
+
if (agent && ['succeeded', 'failed', 'cancelled'].includes(root?.status ?? 'failed'))
|
|
1709
|
+
this.commitAgentState(agent.id, status, `agent:${agent.id}:run-settled:${root?.version ?? this.state.now}`);
|
|
1710
|
+
await this.flushPersistence();
|
|
1711
|
+
return runOutcome(root, this.quarantine.unresolvedEffectIds);
|
|
1712
|
+
}
|
|
1713
|
+
async runAgent(agentId, maxTicks = 10_000) {
|
|
1714
|
+
const agent = this.state.agents.get(agentId);
|
|
1715
|
+
if (!agent)
|
|
1716
|
+
throw new Error(`UNKNOWN_AGENT:${agentId}`);
|
|
1717
|
+
for (let tick = 0; tick < maxTicks; tick++) {
|
|
1718
|
+
const work = this.tick();
|
|
1719
|
+
await this.flushPersistence();
|
|
1720
|
+
if (this.executionYieldPending.size) {
|
|
1721
|
+
this.executionYieldPending.clear();
|
|
1722
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1723
|
+
}
|
|
1724
|
+
const root = this.state.lanes.get(agent.rootLaneId);
|
|
1725
|
+
if (root && ['succeeded', 'failed', 'cancelled'].includes(root.status)) {
|
|
1726
|
+
const status = root.status === 'succeeded' ? 'succeeded' : root.status === 'cancelled' ? 'cancelled' : 'failed';
|
|
1727
|
+
this.commitAgentState(agent.id, status, `agent:${agent.id}:run-agent-settled:${root.version}`);
|
|
1728
|
+
const effectIds = new Set([...this.state.effects.values()].filter((effect) => effect.agentId === agentId).map((effect) => effect.id));
|
|
1729
|
+
await this.flushPersistence();
|
|
1730
|
+
return runOutcome(root, this.quarantine.unresolvedEffectIds.filter((effectId) => effectIds.has(effectId)));
|
|
1731
|
+
}
|
|
1732
|
+
if (this.ready.size === 0 && this.executions.size === 0 && !this.hasQueuedEffects() && !this.hasPendingTickCleanup()) {
|
|
1733
|
+
if (this.preparingLLMs.size) {
|
|
1734
|
+
await Promise.resolve();
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
if (this.factInbox.size > 0)
|
|
1738
|
+
continue;
|
|
1739
|
+
if (this.hasPendingHostInteraction(agentId)) {
|
|
1740
|
+
await this.waitForFact();
|
|
1741
|
+
continue;
|
|
1742
|
+
}
|
|
1743
|
+
const nextAt = this.clock.timers.nextAt();
|
|
1744
|
+
if (nextAt !== undefined) {
|
|
1745
|
+
if (nextAt > this.clock.now()) {
|
|
1746
|
+
if (this.clock.waitUntil)
|
|
1747
|
+
await this.clock.waitUntil(nextAt);
|
|
1748
|
+
else
|
|
1749
|
+
this.clock.set(nextAt);
|
|
1750
|
+
}
|
|
1751
|
+
;
|
|
1752
|
+
continue;
|
|
1753
|
+
}
|
|
1754
|
+
break;
|
|
1755
|
+
}
|
|
1756
|
+
if (work === 0 && this.executions.size) {
|
|
1757
|
+
const executions = [...this.executions.entries()].filter(([effectId]) => this.state.effects.get(effectId)?.agentId === agentId).map(([, execution]) => execution.promise);
|
|
1758
|
+
const nextAt = this.clock.timers.nextAt();
|
|
1759
|
+
if (nextAt !== undefined && nextAt > this.clock.now()) {
|
|
1760
|
+
if (this.clock.waitUntil)
|
|
1761
|
+
await Promise.race([this.clock.waitUntil(nextAt), ...executions]);
|
|
1762
|
+
else
|
|
1763
|
+
this.clock.set(nextAt);
|
|
1764
|
+
continue;
|
|
1765
|
+
}
|
|
1766
|
+
if (executions.length)
|
|
1767
|
+
await Promise.race(executions);
|
|
1768
|
+
else
|
|
1769
|
+
await Promise.resolve();
|
|
1770
|
+
}
|
|
1771
|
+
else if (work === 0 && this.factInbox.size === 0 && this.hasPendingHostInteraction(agentId) && !this.hasQueuedEffects())
|
|
1772
|
+
await this.waitForFact();
|
|
1773
|
+
else if (work === 0)
|
|
1774
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1775
|
+
}
|
|
1776
|
+
const root = this.state.lanes.get(agent.rootLaneId);
|
|
1777
|
+
if (root && !['succeeded', 'failed', 'cancelled'].includes(root.status))
|
|
1778
|
+
this.emit({ type: 'runtime.idle_blocked', laneId: root.id, data: { status: root.status } });
|
|
1779
|
+
const effectIds = new Set([...this.state.effects.values()].filter((effect) => effect.agentId === agentId).map((effect) => effect.id));
|
|
1780
|
+
return runOutcome(root, this.quarantine.unresolvedEffectIds.filter((effectId) => effectIds.has(effectId)));
|
|
1781
|
+
}
|
|
1782
|
+
async waitForIdle() { while (this.ready.size || this.executions.size || this.preparingLLMs.size || this.hasQueuedEffects() || this.factInbox.size || this.hasPendingTickCleanup() || this.hasDueTimer()) {
|
|
1783
|
+
this.tick();
|
|
1784
|
+
if (this.executions.size)
|
|
1785
|
+
await Promise.race([...this.executions.values()].map((execution) => execution.promise));
|
|
1786
|
+
else if (this.preparingLLMs.size)
|
|
1787
|
+
await Promise.resolve();
|
|
1788
|
+
else if (this.hasQueuedEffects() || this.factInbox.size || this.hasPendingTickCleanup() || this.hasDueTimer())
|
|
1789
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1790
|
+
} await this.flushPersistence(); }
|
|
1791
|
+
async shutdown(timeoutMs = 5_000) {
|
|
1792
|
+
this.shuttingDown = true;
|
|
1793
|
+
this.schedulerDecisionCoordinator?.cancel();
|
|
1794
|
+
this.schedulerDecisionRequests.clear();
|
|
1795
|
+
this.schedulerDecisionCache = undefined;
|
|
1796
|
+
for (const agent of this.state.agents.values())
|
|
1797
|
+
if (agent.state === 'running' || agent.state === 'cancelling')
|
|
1798
|
+
this.cancelAgent(agent.id, 'USER_REQUESTED');
|
|
1799
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
1800
|
+
while ((this.ready.size || this.executions.size || this.preparingLLMs.size || this.hasQueuedEffects() || this.factInbox.size || this.hasPendingTickCleanup() || this.hasDueTimer()) && Date.now() < deadline) {
|
|
1801
|
+
this.tick();
|
|
1802
|
+
if (this.executions.size)
|
|
1803
|
+
await Promise.race([...this.executions.values()].map((execution) => execution.promise).concat([new Promise((resolve) => setTimeout(resolve, Math.min(10, Math.max(0, deadline - Date.now()))))]));
|
|
1804
|
+
else if (this.factInbox.size)
|
|
1805
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1806
|
+
else if (this.preparingLLMs.size)
|
|
1807
|
+
await Promise.race([this.waitForFact(), new Promise((resolve) => setTimeout(resolve, Math.min(10, Math.max(0, deadline - Date.now()))))]);
|
|
1808
|
+
else if (this.hasQueuedEffects())
|
|
1809
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
1810
|
+
}
|
|
1811
|
+
await this.flushPersistence();
|
|
1812
|
+
const unresolved = [...this.state.effects.values()].filter((effect) => !effect.outcome && effect.state !== 'cancelled').map((effect) => effect.id);
|
|
1813
|
+
return { status: unresolved.length || this.executions.size ? 'timed_out' : 'stopped', unresolvedEffectIds: [...new Set([...unresolved, ...this.quarantine.unresolvedEffectIds])], quarantine: this.quarantine.unresolvedEffectIds };
|
|
1814
|
+
}
|
|
1815
|
+
inspect() { const explanation = this.explain(); return { ...explanation, quarantineEntries: this.quarantine.snapshot(), observationsPending: this.observationInbox.size }; }
|
|
1816
|
+
telemetry() { return collectRuntimeTelemetry(this.state); }
|
|
1817
|
+
async exportTelemetry(timestamp = Date.now()) {
|
|
1818
|
+
const snapshot = this.telemetry();
|
|
1819
|
+
if (this.telemetryExporter)
|
|
1820
|
+
await this.telemetryExporter.publish({ schemaVersion: 1, timestamp, snapshot });
|
|
1821
|
+
return snapshot;
|
|
1822
|
+
}
|
|
1823
|
+
async exportAuditLog(options = {}) {
|
|
1824
|
+
const maxPrivacy = options.maxPrivacy ?? this.auditLogPrivacy;
|
|
1825
|
+
const effective = maxPrivacy === undefined ? options : { ...options, maxPrivacy };
|
|
1826
|
+
return this.auditLogSink === undefined ? exportRuntimeLog(this.state, effective) : exportRuntimeLogTo(this.state, this.auditLogSink, effective);
|
|
1827
|
+
}
|
|
1828
|
+
assertStorageAdmission(mutations) {
|
|
1829
|
+
const candidate = forkRuntimeStateForAdmission(this.state, mutations);
|
|
1830
|
+
apply(candidate, mutations, { sessionId: this.sessionId, timestamp: candidate.now });
|
|
1831
|
+
const policy = this.storagePolicy.clone();
|
|
1832
|
+
this.syncStoragePolicy(policy, candidate);
|
|
1833
|
+
}
|
|
1834
|
+
schedulePersistence() {
|
|
1835
|
+
if (!this.persistenceBackend)
|
|
1836
|
+
return;
|
|
1837
|
+
this.persistenceDirty = true;
|
|
1838
|
+
if (this.persistenceScheduled || this.persistenceBackoff)
|
|
1839
|
+
return;
|
|
1840
|
+
this.persistenceScheduled = true;
|
|
1841
|
+
this.persistencePending = this.persistencePending.catch(() => undefined).then(async () => {
|
|
1842
|
+
while (this.persistenceDirty) {
|
|
1843
|
+
this.persistenceDirty = false;
|
|
1844
|
+
try {
|
|
1845
|
+
await this.persist(this.persistenceBackend);
|
|
1846
|
+
if (this.persistenceFailures > 0) {
|
|
1847
|
+
this.persistenceFailures = 0;
|
|
1848
|
+
this.tryEmit({ type: 'persistence.recovered' });
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
catch (cause) {
|
|
1852
|
+
// Surface the failure instead of dropping it: hosts observe `persistence.failed`,
|
|
1853
|
+
// awaiting `flushPersistence()` callers see the rejection, and the state stays
|
|
1854
|
+
// dirty so a backoff timer retries instead of hot-looping against a dead backend.
|
|
1855
|
+
this.persistenceFailures += 1;
|
|
1856
|
+
this.persistenceDirty = true;
|
|
1857
|
+
this.persistenceBackoff = true;
|
|
1858
|
+
this.tryEmit({ type: 'persistence.failed', data: { attempt: this.persistenceFailures, error: cause instanceof Error ? cause.message : String(cause) } });
|
|
1859
|
+
this.scheduleRuntimeDelay(Math.min(30_000, 250 * 2 ** Math.min(7, this.persistenceFailures - 1)), () => { this.persistenceBackoff = false; if (this.persistenceDirty)
|
|
1860
|
+
this.schedulePersistence(); });
|
|
1861
|
+
throw cause;
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
}).finally(() => {
|
|
1865
|
+
this.persistenceScheduled = false;
|
|
1866
|
+
if (this.persistenceDirty && !this.persistenceBackoff)
|
|
1867
|
+
this.schedulePersistence();
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
persistenceState() {
|
|
1871
|
+
const state = structuredClone(this.state);
|
|
1872
|
+
for (const artifact of state.artifacts.values())
|
|
1873
|
+
artifact.storageState = 'persisted';
|
|
1874
|
+
return state;
|
|
1875
|
+
}
|
|
1876
|
+
markArtifactsPersisted() {
|
|
1877
|
+
for (const artifact of this.state.artifacts.values())
|
|
1878
|
+
artifact.storageState = 'persisted';
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* Reconcile the storage policy with the (candidate) runtime state.
|
|
1882
|
+
*
|
|
1883
|
+
* Cost model: keys whose content is immutable once written (events, versioned
|
|
1884
|
+
* Lane/global snapshots, LLM request snapshots, resume inputs, fact envelopes,
|
|
1885
|
+
* published results) are only serialized and hashed on first sight; later
|
|
1886
|
+
* syncs skip them with a map lookup. Only genuinely mutable records (waits,
|
|
1887
|
+
* artifacts) are re-put every time. Pins are recomputed from scratch, which is
|
|
1888
|
+
* a set-diff over keys, not over bodies.
|
|
1889
|
+
*/
|
|
1890
|
+
syncStoragePolicy(policy = this.storagePolicy, state = this.state, factInbox = this.factInbox) {
|
|
1891
|
+
const transactional = policy === this.storagePolicy;
|
|
1892
|
+
const target = transactional ? policy.clone() : policy;
|
|
1893
|
+
const residency = new Map();
|
|
1894
|
+
const pinKeys = new Set();
|
|
1895
|
+
const activeGlobalVersions = new Set();
|
|
1896
|
+
for (const lane of state.lanes.values()) {
|
|
1897
|
+
const active = !['succeeded', 'failed', 'cancelled'].includes(lane.status);
|
|
1898
|
+
if (active)
|
|
1899
|
+
pinKeys.add(`snapshot:lane:${lane.id}:${lane.context.version}`);
|
|
1900
|
+
if (active)
|
|
1901
|
+
for (const ref of lane.visibleResultRefs ?? [])
|
|
1902
|
+
pinKeys.add(`result:${ref}`);
|
|
1903
|
+
if (active && lane.contextSnapshotVersion !== undefined)
|
|
1904
|
+
activeGlobalVersions.add(`${lane.agentId}:${lane.contextSnapshotVersion}`);
|
|
1905
|
+
for (const record of lane.context.history)
|
|
1906
|
+
for (const ref of record.resultRefs)
|
|
1907
|
+
pinKeys.add(`result:${ref}`);
|
|
1908
|
+
if (lane.activeWaitId)
|
|
1909
|
+
pinKeys.add(`snapshot:wait:${lane.activeWaitId}`);
|
|
1910
|
+
if (lane.pendingResumeInput)
|
|
1911
|
+
pinKeys.add(`snapshot:resume:${lane.id}:${lane.version}`);
|
|
1912
|
+
}
|
|
1913
|
+
for (const agent of state.agents.values())
|
|
1914
|
+
for (const [version] of agent.globalVersions)
|
|
1915
|
+
if (activeGlobalVersions.has(`${agent.id}:${version}`))
|
|
1916
|
+
pinKeys.add(`snapshot:global:${agent.id}:${version}`);
|
|
1917
|
+
for (const wait of state.waits.values())
|
|
1918
|
+
if (wait.state === 'pending')
|
|
1919
|
+
pinKeys.add(`snapshot:wait:${wait.id}`);
|
|
1920
|
+
for (const artifact of state.artifacts.values()) {
|
|
1921
|
+
if (artifact.pinCount > 0)
|
|
1922
|
+
pinKeys.add(`artifact:${artifact.ref}`);
|
|
1923
|
+
}
|
|
1924
|
+
for (const effect of state.effects.values()) {
|
|
1925
|
+
if (!effect.outcome && effect.kind === 'llm')
|
|
1926
|
+
pinKeys.add(`snapshot:request:${effect.id}:${effect.attemptId}`);
|
|
1927
|
+
if (!effect.outcome)
|
|
1928
|
+
for (const ref of effect.derivedFrom ?? [])
|
|
1929
|
+
pinKeys.add(`${provenanceRefKind(ref) === 'artifact' ? 'artifact' : 'result'}:${provenanceRefId(ref)}`);
|
|
1930
|
+
}
|
|
1931
|
+
const factQueue = factInbox.snapshot().queue;
|
|
1932
|
+
for (const envelope of factQueue)
|
|
1933
|
+
pinKeys.add(`snapshot:fact:${envelope.eventId}`);
|
|
1934
|
+
target.replacePinSource('runtime', pinKeys);
|
|
1935
|
+
const liveSnapshotKeys = new Set();
|
|
1936
|
+
const putOnce = (kind, key, value) => { if (kind === 'snapshot')
|
|
1937
|
+
liveSnapshotKeys.add(key); if (!target.has(key))
|
|
1938
|
+
target.put(kind, key, value()); };
|
|
1939
|
+
for (const lane of state.lanes.values())
|
|
1940
|
+
putOnce('snapshot', `snapshot:lane:${lane.id}:${lane.context.version}`, () => ({ laneId: lane.id, version: lane.context.version, context: lane.context, resume: lane.resume }));
|
|
1941
|
+
for (const agent of state.agents.values())
|
|
1942
|
+
for (const [version, value] of agent.globalVersions)
|
|
1943
|
+
putOnce('snapshot', `snapshot:global:${agent.id}:${version}`, () => ({ agentId: agent.id, version, value }));
|
|
1944
|
+
for (const wait of state.waits.values()) {
|
|
1945
|
+
const key = `snapshot:wait:${wait.id}`;
|
|
1946
|
+
liveSnapshotKeys.add(key);
|
|
1947
|
+
target.put('snapshot', key, wait);
|
|
1948
|
+
}
|
|
1949
|
+
for (const effect of state.effects.values())
|
|
1950
|
+
if (!effect.outcome && effect.kind === 'llm')
|
|
1951
|
+
putOnce('snapshot', `snapshot:request:${effect.id}:${effect.attemptId}`, () => ({ effectId: effect.id, attemptId: effect.attemptId, input: effect.input }));
|
|
1952
|
+
for (const result of state.results.values()) {
|
|
1953
|
+
const key = `result:${result.id}`;
|
|
1954
|
+
putOnce('result', key, () => {
|
|
1955
|
+
const policyValue = structuredClone(result);
|
|
1956
|
+
delete policyValue.storageState;
|
|
1957
|
+
delete policyValue.pinCount;
|
|
1958
|
+
return policyValue;
|
|
1959
|
+
});
|
|
1960
|
+
const stored = target.record(key);
|
|
1961
|
+
residency.set(result.id, { storageState: stored.storageState === 'memory' ? 'memory' : 'persisted', pinCount: stored.pinCount });
|
|
1962
|
+
}
|
|
1963
|
+
for (const artifact of state.artifacts.values())
|
|
1964
|
+
target.put('artifact', `artifact:${artifact.ref}`, artifact);
|
|
1965
|
+
for (const event of state.events)
|
|
1966
|
+
putOnce('event', `event:${event.id}`, () => event);
|
|
1967
|
+
for (const lane of state.lanes.values())
|
|
1968
|
+
if (lane.pendingResumeInput)
|
|
1969
|
+
putOnce('snapshot', `snapshot:resume:${lane.id}:${lane.version}`, () => lane.pendingResumeInput);
|
|
1970
|
+
for (const envelope of factQueue)
|
|
1971
|
+
putOnce('snapshot', `snapshot:fact:${envelope.eventId}`, () => envelope);
|
|
1972
|
+
// Snapshots that no live state refers to any more (superseded Lane versions, consumed
|
|
1973
|
+
// resume inputs, settled request/wait/fact snapshots) are dropped so the record table
|
|
1974
|
+
// does not grow without bound. `remove` refuses pinned records, so nothing referenced
|
|
1975
|
+
// by the pin set above can disappear.
|
|
1976
|
+
for (const key of [...target.keys()])
|
|
1977
|
+
if (key.startsWith('snapshot:') && !liveSnapshotKeys.has(key))
|
|
1978
|
+
target.remove(key);
|
|
1979
|
+
if (transactional)
|
|
1980
|
+
policy.adopt(target);
|
|
1981
|
+
if (transactional) {
|
|
1982
|
+
for (const [id, value] of residency) {
|
|
1983
|
+
const result = state.results.get(id);
|
|
1984
|
+
if (result)
|
|
1985
|
+
Object.assign(result, value);
|
|
1986
|
+
}
|
|
1987
|
+
if (this.sessionStore)
|
|
1988
|
+
for (const agent of state.agents.values())
|
|
1989
|
+
this.sessionStore.put(exportWarmStartSession(state, agent.id));
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
hasQueuedEffects() { return [...this.state.effects.values()].some((effect) => effect.state === 'queued' && !this.executions.has(effect.id)); }
|
|
1993
|
+
hasDueTimer() {
|
|
1994
|
+
const nextAt = this.clock.timers.nextAt();
|
|
1995
|
+
return nextAt !== undefined && nextAt <= this.clock.now();
|
|
1996
|
+
}
|
|
1997
|
+
hasPendingTickCleanup() {
|
|
1998
|
+
if ([...this.state.lanes.values()].some((lane) => lane.status === 'cancelling'))
|
|
1999
|
+
return true;
|
|
2000
|
+
for (const agent of this.state.agents.values()) {
|
|
2001
|
+
if (agent.state !== 'cancelling')
|
|
2002
|
+
continue;
|
|
2003
|
+
const lanes = [...this.state.lanes.values()].filter((lane) => lane.agentId === agent.id);
|
|
2004
|
+
if (lanes.some((lane) => !['succeeded', 'failed', 'cancelled'].includes(lane.status)))
|
|
2005
|
+
continue;
|
|
2006
|
+
const effects = [...this.state.effects.values()].filter((effect) => effect.agentId === agent.id);
|
|
2007
|
+
if (!effects.some((effect) => !effect.outcome && !['cancelled', 'reconcile_required', 'succeeded', 'failed'].includes(effect.state)))
|
|
2008
|
+
return true;
|
|
2009
|
+
}
|
|
2010
|
+
for (const wait of this.state.waits.values()) {
|
|
2011
|
+
if (wait.state !== 'pending')
|
|
2012
|
+
continue;
|
|
2013
|
+
for (const dependency of wait.spec.dependencies) {
|
|
2014
|
+
const target = dependency.target;
|
|
2015
|
+
if (target.kind === 'effect' && this.state.effects.get(target.id)?.outcome !== undefined)
|
|
2016
|
+
return true;
|
|
2017
|
+
if (target.kind === 'lane') {
|
|
2018
|
+
const lane = this.state.lanes.get(target.id);
|
|
2019
|
+
if (lane !== undefined && ['succeeded', 'failed', 'cancelled'].includes(lane.status))
|
|
2020
|
+
return true;
|
|
2021
|
+
}
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
return false;
|
|
2025
|
+
}
|
|
2026
|
+
hasPendingHostInteraction(agentId) { return [...this.state.effects.values()].some((effect) => effect.kind === 'human' && !effect.outcome && (agentId === undefined || effect.agentId === agentId)); }
|
|
2027
|
+
waitForFact() { return new Promise((resolve) => this.factWaiters.push(resolve)); }
|
|
2028
|
+
completeFinishedChildAgents() {
|
|
2029
|
+
for (const effect of this.state.effects.values()) {
|
|
2030
|
+
if (effect.kind !== 'agent' || !effect.childAgentId || effect.outcome)
|
|
2031
|
+
continue;
|
|
2032
|
+
const child = this.state.agents.get(effect.childAgentId);
|
|
2033
|
+
const root = child ? this.state.lanes.get(child.rootLaneId) : undefined;
|
|
2034
|
+
if (!child || !root || !['succeeded', 'failed', 'cancelled'].includes(root.status))
|
|
2035
|
+
continue;
|
|
2036
|
+
const status = root.status === 'succeeded' ? 'succeeded' : root.status === 'cancelled' ? 'cancelled' : 'failed';
|
|
2037
|
+
const nextChild = structuredClone(child);
|
|
2038
|
+
nextChild.state = status;
|
|
2039
|
+
this.completeEffect(effect.id, { value: { agentId: child.id, status } }, status, status === 'failed' ? { code: 'CHILD_AGENT_FAILED', message: 'Child Agent failed.' } : undefined, [{ op: 'setAgent', agentId: child.id, record: nextChild }]);
|
|
2040
|
+
}
|
|
2041
|
+
}
|
|
2042
|
+
completeEffect(effectId, execution, status = 'succeeded', error, additionalMutations = []) {
|
|
2043
|
+
const storedEffect = this.state.effects.get(effectId);
|
|
2044
|
+
if (!storedEffect)
|
|
2045
|
+
return false;
|
|
2046
|
+
if (storedEffect.outcome) {
|
|
2047
|
+
this.tryEmit({ type: 'attempt.late_emit', effectId, data: { status: storedEffect.outcome.status } });
|
|
2048
|
+
return false;
|
|
2049
|
+
}
|
|
2050
|
+
// A completion for an Effect that already sits in QuarantineScope is a reconciliation:
|
|
2051
|
+
// it must clear the quarantine entry and the owner Lane's `unresolvedEffectIds`,
|
|
2052
|
+
// otherwise the persisted snapshot becomes invalid (quarantine entry without a
|
|
2053
|
+
// `reconcile_required` Effect) and the session cannot be restored.
|
|
2054
|
+
const wasQuarantined = storedEffect.state === 'reconcile_required';
|
|
2055
|
+
const effect = structuredClone(storedEffect);
|
|
2056
|
+
const running = this.executions.get(effectId);
|
|
2057
|
+
if (running) {
|
|
2058
|
+
running.controller.abort();
|
|
2059
|
+
this.executions.delete(effectId);
|
|
2060
|
+
}
|
|
2061
|
+
if (execution.executionState === 'remote_unknown') {
|
|
2062
|
+
this.markRemoteUnknown(effectId, execution.sideEffectState ?? 'none');
|
|
2063
|
+
return false;
|
|
2064
|
+
}
|
|
2065
|
+
let effectiveExecution = execution;
|
|
2066
|
+
let outputError = error ?? execution.error;
|
|
2067
|
+
let resultDerivedFrom = [...(effect.derivedFrom ?? [])];
|
|
2068
|
+
let publishedArtifact;
|
|
2069
|
+
const rawInput = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2070
|
+
if (execution.artifact !== undefined) {
|
|
2071
|
+
try {
|
|
2072
|
+
const artifact = prepareArtifactPublication(this.state, { ...execution.artifact, laneId: effect.ownerLaneId, derivedFrom: [...(effect.derivedFrom ?? []), ...(execution.artifact.derivedFrom ?? [])] });
|
|
2073
|
+
publishedArtifact = artifact;
|
|
2074
|
+
resultDerivedFrom = [...resultDerivedFrom, { kind: 'artifact', ref: artifact.ref }];
|
|
2075
|
+
effectiveExecution = { ...effectiveExecution, value: { artifactRef: artifact.ref }, privacy: artifact.privacy, ...(artifact.privacyTaints === undefined ? {} : { privacyTaints: artifact.privacyTaints }) };
|
|
2076
|
+
}
|
|
2077
|
+
catch (cause) {
|
|
2078
|
+
effectiveExecution = { ...effectiveExecution, status: 'failed', executionState: 'failed', error: runtimeErrorFromCause(cause, 'ARTIFACT_PUBLICATION_FAILED') };
|
|
2079
|
+
outputError = effectiveExecution.error;
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
const outputSchema = rawInput.outputSchema;
|
|
2083
|
+
if ((execution.status ?? status) === 'succeeded' && effect.kind === 'llm' && outputSchema !== undefined && !validateJsonSchema(execution.value, outputSchema)) {
|
|
2084
|
+
effectiveExecution = { ...execution, status: 'failed', executionState: 'failed', rejectedOutput: { value: structuredClone(execution.value), ...(execution.privacy === undefined ? {} : { privacy: execution.privacy }), derivedFrom: [...(effect.derivedFrom ?? [])] } };
|
|
2085
|
+
outputError = { code: 'OUTPUT_SCHEMA_VIOLATION', message: 'LLM output did not satisfy the declared output schema.' };
|
|
2086
|
+
}
|
|
2087
|
+
const taintError = validatePrivacyTaints(effectiveExecution.privacyTaints) ?? validatePrivacyTaints(effectiveExecution.rejectedOutput?.privacyTaints);
|
|
2088
|
+
if (taintError) {
|
|
2089
|
+
effectiveExecution = { ...effectiveExecution, status: 'failed', executionState: 'failed', error: { code: taintError, message: 'Effect output contains invalid privacy taints.' } };
|
|
2090
|
+
outputError = effectiveExecution.error;
|
|
2091
|
+
}
|
|
2092
|
+
const effectiveStatus = effect.cancelRequested && (effectiveExecution.status ?? status) === 'succeeded' ? 'cancelled' : (effectiveExecution.status ?? status);
|
|
2093
|
+
effect.state = effectiveStatus;
|
|
2094
|
+
effect.executionState = effectiveStatus === 'succeeded' ? 'succeeded' : effectiveStatus === 'cancelled' ? 'failed' : 'failed';
|
|
2095
|
+
// An in-doubt side effect stays `unknown` unless the completion explicitly reports what happened.
|
|
2096
|
+
effect.sideEffectState = effectiveExecution.sideEffectState ?? (wasQuarantined && effect.sideEffectState === 'unknown' ? 'unknown' : 'none');
|
|
2097
|
+
if (effectiveExecution.executionRef !== undefined)
|
|
2098
|
+
effect.executionRef = structuredClone(effectiveExecution.executionRef);
|
|
2099
|
+
const attempt = effect.attempts?.at(-1);
|
|
2100
|
+
if (attempt) {
|
|
2101
|
+
attempt.executionState = effect.executionState;
|
|
2102
|
+
attempt.sideEffectState = effect.sideEffectState;
|
|
2103
|
+
if (effectiveExecution.executionRef !== undefined)
|
|
2104
|
+
attempt.sideEffectRef = structuredClone(effectiveExecution.executionRef);
|
|
2105
|
+
attempt.settledAt = this.state.now;
|
|
2106
|
+
if (outputError)
|
|
2107
|
+
attempt.error = outputError;
|
|
2108
|
+
}
|
|
2109
|
+
const settledAttemptId = effect.attemptId;
|
|
2110
|
+
if (effect.kind === 'llm' && effect.retryPolicy === undefined && effectiveStatus === 'failed') {
|
|
2111
|
+
const metadata = execution.metadata;
|
|
2112
|
+
const routes = metadata && typeof metadata === 'object' && !Array.isArray(metadata) && Array.isArray(metadata.routes) ? metadata.routes : [];
|
|
2113
|
+
const eligible = routes.filter((route) => route && typeof route === 'object' && !Array.isArray(route) && route.accepted === true).length;
|
|
2114
|
+
effect.retryPolicy = { maxAttempts: Math.max(1, eligible), initialBackoffMs: 0, maxBackoffMs: 0, jitter: false };
|
|
2115
|
+
}
|
|
2116
|
+
const attemptMetadata = execution.metadata && typeof execution.metadata === 'object' && !Array.isArray(execution.metadata) ? execution.metadata : undefined;
|
|
2117
|
+
const selectedModel = attemptMetadata?.selected && typeof attemptMetadata.selected === 'object' && !Array.isArray(attemptMetadata.selected) ? attemptMetadata.selected : undefined;
|
|
2118
|
+
if (attempt && typeof selectedModel?.id === 'string')
|
|
2119
|
+
attempt.modelId = selectedModel.id;
|
|
2120
|
+
if (attempt && typeof selectedModel?.providerId === 'string')
|
|
2121
|
+
attempt.providerId = selectedModel.providerId;
|
|
2122
|
+
if (effectiveStatus === 'failed' && this.scheduleRetry(effect, outputError)) {
|
|
2123
|
+
Object.assign(storedEffect, effect);
|
|
2124
|
+
this.releaseEffectLocks(effectId);
|
|
2125
|
+
this.outbox.ack(`${effect.id}:${settledAttemptId}`);
|
|
2126
|
+
this.refreshWaits();
|
|
2127
|
+
this.schedulePersistence();
|
|
2128
|
+
return false;
|
|
2129
|
+
}
|
|
2130
|
+
let resultSequence = this.state.nextIds.result;
|
|
2131
|
+
while (this.state.results.has(`result-${resultSequence}`))
|
|
2132
|
+
resultSequence++;
|
|
2133
|
+
const resultId = `result-${resultSequence}`;
|
|
2134
|
+
const rejectedOutputId = effectiveStatus !== 'succeeded' && effectiveExecution.rejectedOutput ? resultId : undefined;
|
|
2135
|
+
const outcome = effectiveStatus === 'succeeded' ? { status: effectiveStatus, resultRef: resultId } : { status: effectiveStatus, ...(outputError ? { error: outputError } : {}), ...(effectiveStatus === 'cancelled' ? { reason: outputError?.message ?? outputError?.code ?? 'CANCELLED' } : {}), ...(rejectedOutputId ? { rejectedOutputRefs: [rejectedOutputId] } : {}) };
|
|
2136
|
+
effect.outcome = outcome;
|
|
2137
|
+
const ownerLane = this.state.lanes.get(effect.ownerLaneId);
|
|
2138
|
+
const sourcePrivacy = effect.derivedFrom?.flatMap((ref) => {
|
|
2139
|
+
const result = provenanceRefKind(ref) === 'artifact' ? undefined : this.state.results.get(provenanceRefId(ref));
|
|
2140
|
+
if (result)
|
|
2141
|
+
return [effectivePrivacy(result.privacy, result.privacyTaints)];
|
|
2142
|
+
const source = ownerLane ? privacyMetadataForDerivedRef(this.state, ownerLane, ref) : undefined;
|
|
2143
|
+
return source ? [effectivePrivacy(source.privacy, source.privacyTaints)] : [];
|
|
2144
|
+
}) ?? [];
|
|
2145
|
+
const sourceTaints = ownerLane ? privacyTaintsForDerivedRefs(this.state, ownerLane, effect.derivedFrom ?? []) : [];
|
|
2146
|
+
const outputTaints = [...sourceTaints, ...(effectiveExecution.privacyTaints ?? [])];
|
|
2147
|
+
const rejectedTaints = [...sourceTaints, ...(effectiveExecution.rejectedOutput?.privacyTaints ?? [])];
|
|
2148
|
+
const summaryAllowed = effectiveExecution.summary === undefined || Buffer.byteLength(JSON.stringify(effectiveExecution.summary), 'utf8') <= this.state.maxResultSummaryBytes;
|
|
2149
|
+
const result = effectiveStatus === 'succeeded' && !taintError ? { id: resultId, effectId, producer: { kind: 'effect', id: effect.id }, value: effectiveExecution.value, ...resultMetadata(effectiveExecution.value), storageState: 'memory', pinCount: 0, privacy: effectivePrivacy(strictestPrivacy([effectiveExecution.privacy ?? 'public', ...sourcePrivacy]), outputTaints), ...(outputTaints.length ? { privacyTaints: outputTaints } : {}), derivedFrom: resultDerivedFrom, ...(effectiveExecution.normalized === undefined ? {} : { normalized: effectiveExecution.normalized }), ...(summaryAllowed && effectiveExecution.summary !== undefined ? { summary: effectiveExecution.summary } : {}) } : rejectedOutputId && effectiveExecution.rejectedOutput && !taintError ? { id: rejectedOutputId, effectId, producer: { kind: 'effect', id: effect.id }, kind: 'rejected_output', value: effectiveExecution.rejectedOutput.value, ...resultMetadata(effectiveExecution.rejectedOutput.value), storageState: 'memory', pinCount: 0, privacy: effectivePrivacy(strictestPrivacy([effectiveExecution.rejectedOutput.privacy ?? effectiveExecution.privacy ?? 'public', ...sourcePrivacy]), rejectedTaints), ...(rejectedTaints.length ? { privacyTaints: rejectedTaints } : {}), derivedFrom: [...(effectiveExecution.rejectedOutput.derivedFrom ?? effect.derivedFrom ?? [])] } : undefined;
|
|
2150
|
+
let journalLane;
|
|
2151
|
+
if (ownerLane && result) {
|
|
2152
|
+
journalLane = structuredClone(ownerLane);
|
|
2153
|
+
if (journalLane.visibleResultRefs)
|
|
2154
|
+
journalLane.visibleResultRefs.add(result.id);
|
|
2155
|
+
else
|
|
2156
|
+
journalLane.visibleResultRefs = new Set([result.id]);
|
|
2157
|
+
if (effect.kind === 'llm' && effectiveStatus === 'succeeded') {
|
|
2158
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2159
|
+
const request = input.request && typeof input.request === 'object' && !Array.isArray(input.request) ? input.request : undefined;
|
|
2160
|
+
const contextSpec = request?.contextSpec && typeof request.contextSpec === 'object' && !Array.isArray(request.contextSpec) ? request.contextSpec : undefined;
|
|
2161
|
+
const refs = Array.isArray(contextSpec?.resultRefs) ? contextSpec.resultRefs.filter((ref) => typeof ref === 'string') : (effect.derivedFrom ?? []).filter((ref) => typeof ref === 'string');
|
|
2162
|
+
const instruction = typeof contextSpec?.instruction === 'string' ? contextSpec.instruction : typeof input.instruction === 'string' ? input.instruction : typeof input.task === 'string' ? input.task : effect.key;
|
|
2163
|
+
const selectedRefs = [...new Set(refs)];
|
|
2164
|
+
const resultSelection = selectedRefs.map((ref) => ({ ref, rule: 'explicit-context-result', hash: contentHash(this.state.results.get(ref)?.value ?? null) }));
|
|
2165
|
+
const findings = selectedRefs.filter((ref) => this.state.results.get(ref)?.kind === 'finding');
|
|
2166
|
+
journalLane = appendHistory(journalLane, { effectId: effect.id, instruction, resultRefs: selectedRefs, resultSelection, result: result.id, ...(findings.length ? { findings } : {}), output: structuredClone(effectiveExecution.value), privacy: result.privacy, ...(result.privacyTaints === undefined ? {} : { privacyTaints: structuredClone(result.privacyTaints) }) });
|
|
2167
|
+
journalLane.visibleResultRefs.add(result.id);
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2170
|
+
if (wasQuarantined && ownerLane?.unresolvedEffectIds?.includes(effectId)) {
|
|
2171
|
+
journalLane ??= structuredClone(ownerLane);
|
|
2172
|
+
journalLane.unresolvedEffectIds = journalLane.unresolvedEffectIds.filter((id) => id !== effectId);
|
|
2173
|
+
if (journalLane.unresolvedEffectIds.length === 0)
|
|
2174
|
+
delete journalLane.unresolvedEffectIds;
|
|
2175
|
+
}
|
|
2176
|
+
let correlation;
|
|
2177
|
+
if (effect.kind === 'tool' && effect.toolCallId && result) {
|
|
2178
|
+
const existing = this.state.toolCallCorrelations.get(effect.toolCallId);
|
|
2179
|
+
if (existing) {
|
|
2180
|
+
correlation = { ...existing, toolEffectId: effect.id, resultRef: result.id };
|
|
2181
|
+
}
|
|
2182
|
+
}
|
|
2183
|
+
const publicationMutations = [{ op: 'setEffect', effectId: effect.id, record: structuredClone(effect) }, ...additionalMutations.map((mutation) => structuredClone(mutation))];
|
|
2184
|
+
if (publishedArtifact)
|
|
2185
|
+
publicationMutations.push({ op: 'publishArtifact', record: structuredClone(publishedArtifact) });
|
|
2186
|
+
if (result)
|
|
2187
|
+
publicationMutations.push({ op: 'publishResult', record: structuredClone(result) });
|
|
2188
|
+
if (journalLane)
|
|
2189
|
+
publicationMutations.push({ op: 'setLane', laneId: journalLane.id, record: structuredClone(journalLane) });
|
|
2190
|
+
if (correlation)
|
|
2191
|
+
publicationMutations.push({ op: 'setToolCallCorrelation', record: structuredClone(correlation) });
|
|
2192
|
+
if (effectiveExecution.summary !== undefined && !summaryAllowed)
|
|
2193
|
+
publicationMutations.push({ op: 'appendEvent', event: { type: 'result.summary_rejected', effectId, data: { maxBytes: this.state.maxResultSummaryBytes, actualBytes: Buffer.byteLength(JSON.stringify(effectiveExecution.summary), 'utf8') } } });
|
|
2194
|
+
publicationMutations.push({ op: 'appendEvent', event: { type: 'effect.settled', effectId, data: outcome } });
|
|
2195
|
+
if (execution.metadata !== undefined)
|
|
2196
|
+
publicationMutations.push({ op: 'appendEvent', event: { type: 'effect.execution_metadata', effectId, data: execution.metadata } });
|
|
2197
|
+
try {
|
|
2198
|
+
this.assertStorageAdmission(publicationMutations);
|
|
2199
|
+
}
|
|
2200
|
+
catch (cause) {
|
|
2201
|
+
const storageError = { code: 'SESSION_STORAGE_LIMIT_EXCEEDED', message: cause instanceof Error ? cause.message : String(cause) };
|
|
2202
|
+
effect.state = 'failed';
|
|
2203
|
+
effect.executionState = 'failed';
|
|
2204
|
+
effect.outcome = { status: 'failed', error: storageError };
|
|
2205
|
+
const failedAttempt = effect.attempts?.at(-1);
|
|
2206
|
+
if (failedAttempt)
|
|
2207
|
+
failedAttempt.error = storageError;
|
|
2208
|
+
const failureEvent = { type: 'effect.settled', effectId, data: effect.outcome };
|
|
2209
|
+
const commandAckMutations = additionalMutations.filter((mutation) => mutation.op === 'appendEvent' && mutation.event.type === 'command.applied').map((mutation) => structuredClone(mutation));
|
|
2210
|
+
const failureMutations = [{ op: 'setEffect', effectId, record: structuredClone(effect) }, ...commandAckMutations];
|
|
2211
|
+
try {
|
|
2212
|
+
this.assertStorageAdmission([...failureMutations, { op: 'appendEvent', event: failureEvent }]);
|
|
2213
|
+
failureMutations.push({ op: 'appendEvent', event: failureEvent });
|
|
2214
|
+
}
|
|
2215
|
+
catch {
|
|
2216
|
+
this.assertStorageAdmission(failureMutations);
|
|
2217
|
+
}
|
|
2218
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${settledAttemptId}:storage-rejected`, failureMutations, this.state.now, this.sessionId);
|
|
2219
|
+
Object.assign(storedEffect, effect);
|
|
2220
|
+
this.state.effects.set(effectId, storedEffect);
|
|
2221
|
+
if (wasQuarantined)
|
|
2222
|
+
this.quarantine.reconcile(effectId);
|
|
2223
|
+
this.releaseEffectLocks(effectId);
|
|
2224
|
+
this.outbox.ack(`${effect.id}:${effect.attemptId}`);
|
|
2225
|
+
this.syncStoragePolicy();
|
|
2226
|
+
this.refreshWaits();
|
|
2227
|
+
this.schedulePersistence();
|
|
2228
|
+
return commandAckMutations.length > 0;
|
|
2229
|
+
}
|
|
2230
|
+
this.releaseEffectLocks(effectId);
|
|
2231
|
+
this.outbox.ack(`${effect.id}:${effect.attemptId}`);
|
|
2232
|
+
for (const observation of effectiveExecution.observations ?? [])
|
|
2233
|
+
this.observationInbox.enqueue({ ...observation, agentId: effect.agentId, laneId: effect.ownerLaneId, timestamp: this.state.now });
|
|
2234
|
+
const settlementTransactionId = `effect:${effect.id}:${settledAttemptId}:settled`;
|
|
2235
|
+
const settlementMutations = [...publicationMutations];
|
|
2236
|
+
commitMutationTransaction(this.state, this.mutationLog, settlementTransactionId, settlementMutations, this.state.now, this.sessionId);
|
|
2237
|
+
Object.assign(storedEffect, effect);
|
|
2238
|
+
this.state.effects.set(effectId, storedEffect);
|
|
2239
|
+
if (wasQuarantined)
|
|
2240
|
+
this.quarantine.reconcile(effectId);
|
|
2241
|
+
if (journalLane) {
|
|
2242
|
+
const liveLane = this.state.lanes.get(journalLane.id);
|
|
2243
|
+
if (liveLane) {
|
|
2244
|
+
if (journalLane.unresolvedEffectIds === undefined)
|
|
2245
|
+
delete liveLane.unresolvedEffectIds;
|
|
2246
|
+
Object.assign(liveLane, journalLane);
|
|
2247
|
+
this.state.lanes.set(journalLane.id, liveLane);
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
this.recordBudgetMetadata(execution.metadata);
|
|
2251
|
+
this.syncStoragePolicy();
|
|
2252
|
+
this.refreshWaits();
|
|
2253
|
+
this.dispatchQueuedEffects();
|
|
2254
|
+
this.schedulePersistence();
|
|
2255
|
+
return true;
|
|
2256
|
+
}
|
|
2257
|
+
markRemoteUnknown(effectId, sideEffectState) {
|
|
2258
|
+
const effect = this.state.effects.get(effectId);
|
|
2259
|
+
if (!effect || effect.outcome)
|
|
2260
|
+
return;
|
|
2261
|
+
const candidate = structuredClone(effect);
|
|
2262
|
+
candidate.executionState = 'remote_unknown';
|
|
2263
|
+
candidate.sideEffectState = sideEffectState;
|
|
2264
|
+
const attempt = candidate.attempts?.at(-1);
|
|
2265
|
+
if (attempt) {
|
|
2266
|
+
attempt.executionState = 'remote_unknown';
|
|
2267
|
+
attempt.sideEffectState = sideEffectState;
|
|
2268
|
+
attempt.settledAt = this.state.now;
|
|
2269
|
+
}
|
|
2270
|
+
const lane = this.state.lanes.get(effect.ownerLaneId);
|
|
2271
|
+
const candidateLane = lane === undefined ? undefined : structuredClone(lane);
|
|
2272
|
+
if (sideEffectState === 'unknown') {
|
|
2273
|
+
candidate.state = 'reconcile_required';
|
|
2274
|
+
if (candidateLane)
|
|
2275
|
+
candidateLane.unresolvedEffectIds = [...new Set([...(candidateLane.unresolvedEffectIds ?? []), effectId])];
|
|
2276
|
+
}
|
|
2277
|
+
else {
|
|
2278
|
+
const unknownAttempts = candidate.attempts?.filter((item) => item.executionState === 'remote_unknown').length ?? 0;
|
|
2279
|
+
const canRetry = candidate.duplicateExecutionPolicy === 'allow' && candidate.maxUnknownAttempts !== undefined && unknownAttempts <= candidate.maxUnknownAttempts;
|
|
2280
|
+
if (canRetry) {
|
|
2281
|
+
const settledAttemptId = candidate.attemptId;
|
|
2282
|
+
const running = this.executions.get(effectId);
|
|
2283
|
+
if (running) {
|
|
2284
|
+
running.controller.abort();
|
|
2285
|
+
this.executions.delete(effectId);
|
|
2286
|
+
}
|
|
2287
|
+
if (this.scheduleRetry(candidate, { code: 'REMOTE_EXECUTION_UNKNOWN', message: 'Remote execution outcome is unknown.', details: { unknownAttempts } })) {
|
|
2288
|
+
const retried = this.state.effects.get(effectId);
|
|
2289
|
+
if (retried) {
|
|
2290
|
+
Object.assign(effect, retried);
|
|
2291
|
+
this.state.effects.set(effectId, effect);
|
|
2292
|
+
}
|
|
2293
|
+
this.releaseEffectLocks(effectId);
|
|
2294
|
+
this.outbox.ack(`${effect.id}:${settledAttemptId}`);
|
|
2295
|
+
this.refreshWaits();
|
|
2296
|
+
this.schedulePersistence();
|
|
2297
|
+
return;
|
|
2298
|
+
}
|
|
2299
|
+
}
|
|
2300
|
+
candidate.state = 'failed';
|
|
2301
|
+
candidate.outcome = { status: 'failed', error: { code: 'REMOTE_UNKNOWN', message: 'Remote execution outcome is unknown but no side effect was recorded.' } };
|
|
2302
|
+
}
|
|
2303
|
+
const remoteEvent = { type: 'effect.remote_unknown', effectId, data: { executionState: 'remote_unknown', sideEffectState } };
|
|
2304
|
+
const admission = [{ op: 'setEffect', effectId, record: candidate }];
|
|
2305
|
+
if (candidateLane)
|
|
2306
|
+
admission.push({ op: 'setLane', laneId: candidateLane.id, record: candidateLane });
|
|
2307
|
+
admission.push({ op: 'appendEvent', event: remoteEvent });
|
|
2308
|
+
this.assertStorageAdmission(admission);
|
|
2309
|
+
const running = this.executions.get(effectId);
|
|
2310
|
+
if (running) {
|
|
2311
|
+
running.controller.abort();
|
|
2312
|
+
this.executions.delete(effectId);
|
|
2313
|
+
}
|
|
2314
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:remote-unknown`, admission, this.state.now, this.sessionId);
|
|
2315
|
+
Object.assign(effect, candidate);
|
|
2316
|
+
this.state.effects.set(effectId, effect);
|
|
2317
|
+
if (candidateLane && lane) {
|
|
2318
|
+
Object.assign(lane, candidateLane);
|
|
2319
|
+
this.state.lanes.set(candidateLane.id, lane);
|
|
2320
|
+
}
|
|
2321
|
+
if (sideEffectState === 'unknown')
|
|
2322
|
+
this.quarantine.add(effect.id, this.state.now, 'in_doubt');
|
|
2323
|
+
else {
|
|
2324
|
+
this.releaseEffectLocks(effectId);
|
|
2325
|
+
}
|
|
2326
|
+
this.refreshWaits();
|
|
2327
|
+
this.schedulePersistence();
|
|
2328
|
+
}
|
|
2329
|
+
reconcileEffect(effectId, value, status = 'succeeded', error) {
|
|
2330
|
+
const effect = this.state.effects.get(effectId);
|
|
2331
|
+
if (!effect || effect.state !== 'reconcile_required')
|
|
2332
|
+
return;
|
|
2333
|
+
const safeValue = strictJsonValue(value);
|
|
2334
|
+
if (!['succeeded', 'failed', 'cancelled'].includes(status))
|
|
2335
|
+
throw new Error('INVALID_RECONCILE_STATUS');
|
|
2336
|
+
this.quarantine.reconcile(effectId);
|
|
2337
|
+
this.completeEffect(effectId, { value: safeValue, sideEffectState: 'known' }, status, error);
|
|
2338
|
+
}
|
|
2339
|
+
async reconcileEffectWith(effectId, resolver, signal = new AbortController().signal) {
|
|
2340
|
+
const effect = this.state.effects.get(effectId);
|
|
2341
|
+
if (!effect || effect.state !== 'reconcile_required')
|
|
2342
|
+
return { status: 'unknown', error: { code: 'RECONCILE_NOT_REQUIRED', message: 'Effect is not waiting for reconciliation.' } };
|
|
2343
|
+
let result;
|
|
2344
|
+
try {
|
|
2345
|
+
result = await resolver(effect.executionRef, effect, signal);
|
|
2346
|
+
}
|
|
2347
|
+
catch (cause) {
|
|
2348
|
+
return { status: 'unknown', error: runtimeErrorFromCause(cause, 'RECONCILE_FAILED') };
|
|
2349
|
+
}
|
|
2350
|
+
if (!result || typeof result !== 'object' || !['succeeded', 'failed', 'cancelled', 'unknown'].includes(result.status))
|
|
2351
|
+
return { status: 'unknown', error: { code: 'INVALID_RECONCILE_RESULT', message: 'Reconcile resolver returned an invalid status.', retryable: false } };
|
|
2352
|
+
let output;
|
|
2353
|
+
try {
|
|
2354
|
+
output = result.output === undefined ? undefined : strictJsonValue(result.output);
|
|
2355
|
+
}
|
|
2356
|
+
catch {
|
|
2357
|
+
return { status: 'unknown', error: { code: 'INVALID_RECONCILE_OUTPUT', message: 'Reconcile resolver returned a non-JSON output.', retryable: false } };
|
|
2358
|
+
}
|
|
2359
|
+
if (result.error !== undefined && (typeof result.error !== 'object' || result.error === null || typeof result.error.code !== 'string' || typeof result.error.message !== 'string'))
|
|
2360
|
+
return { status: 'unknown', error: { code: 'INVALID_RECONCILE_ERROR', message: 'Reconcile resolver returned an invalid error.', retryable: false } };
|
|
2361
|
+
if (result.status === 'succeeded' || result.status === 'failed' || result.status === 'cancelled')
|
|
2362
|
+
this.enqueueEffectReconcile(effect, output ?? null, result.status, result.error);
|
|
2363
|
+
return { status: result.status, ...(output === undefined ? {} : { output }), ...(result.error === undefined ? {} : { error: result.error }) };
|
|
2364
|
+
}
|
|
2365
|
+
async reconcileRegisteredEffect(effectId, signal = new AbortController().signal) {
|
|
2366
|
+
return this.reconcileEffectWith(effectId, async (executionRef, effect, resolverSignal) => {
|
|
2367
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2368
|
+
const name = input.name;
|
|
2369
|
+
if (typeof name !== 'string')
|
|
2370
|
+
return { status: 'unknown', error: { code: 'INVALID_TOOL_EFFECT_INPUT', message: 'Tool reconciliation requires a registered tool name.' } };
|
|
2371
|
+
if (executionRef === undefined)
|
|
2372
|
+
return { status: 'unknown', error: { code: 'MISSING_TOOL_EXECUTION_REF', message: 'Tool reconciliation requires an execution reference.' } };
|
|
2373
|
+
try {
|
|
2374
|
+
const result = await this.tools.reconcileDetailed(name, executionRef, { toolCallId: effect.toolCallId ?? '', effectId: effect.id, attemptId: effect.attemptId, agentId: effect.agentId, laneId: effect.ownerLaneId, signal: resolverSignal });
|
|
2375
|
+
return { status: result.status, ...(result.output === undefined ? {} : { output: asJsonValue(result.output) }), ...(result.error === undefined ? {} : { error: result.error }) };
|
|
2376
|
+
}
|
|
2377
|
+
catch (cause) {
|
|
2378
|
+
const error = runtimeErrorFromCause(cause, 'TOOL_RECONCILE_FAILED');
|
|
2379
|
+
return { status: 'unknown', error };
|
|
2380
|
+
}
|
|
2381
|
+
}, signal);
|
|
2382
|
+
}
|
|
2383
|
+
abandonEffect(effectId) {
|
|
2384
|
+
const effect = this.state.effects.get(effectId);
|
|
2385
|
+
if (!effect || effect.state !== 'reconcile_required')
|
|
2386
|
+
return;
|
|
2387
|
+
if (!this.quarantine.has(effectId))
|
|
2388
|
+
return;
|
|
2389
|
+
const candidate = structuredClone(effect);
|
|
2390
|
+
candidate.state = 'failed';
|
|
2391
|
+
candidate.executionState = 'local_closed';
|
|
2392
|
+
candidate.sideEffectState = 'unknown';
|
|
2393
|
+
candidate.outcome = { status: 'failed', error: { code: 'RESOURCE_ABANDONED', message: 'Host abandoned reconciliation for an unknown side effect.' } };
|
|
2394
|
+
const lane = this.state.lanes.get(effect.ownerLaneId);
|
|
2395
|
+
const candidateLane = lane === undefined ? undefined : structuredClone(lane);
|
|
2396
|
+
if (candidateLane?.unresolvedEffectIds)
|
|
2397
|
+
candidateLane.unresolvedEffectIds = candidateLane.unresolvedEffectIds.filter((id) => id !== effectId);
|
|
2398
|
+
const abandonedEvent = { type: 'resource.abandoned', effectId, data: { code: 'RESOURCE_ABANDONED' } };
|
|
2399
|
+
const admission = [{ op: 'setEffect', effectId, record: candidate }];
|
|
2400
|
+
if (candidateLane)
|
|
2401
|
+
admission.push({ op: 'setLane', laneId: candidateLane.id, record: candidateLane });
|
|
2402
|
+
admission.push({ op: 'appendEvent', event: abandonedEvent });
|
|
2403
|
+
this.assertStorageAdmission(admission);
|
|
2404
|
+
if (!this.quarantine.abandon(effectId))
|
|
2405
|
+
return;
|
|
2406
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:abandoned`, admission, this.state.now, this.sessionId);
|
|
2407
|
+
Object.assign(effect, candidate);
|
|
2408
|
+
this.state.effects.set(effectId, effect);
|
|
2409
|
+
if (candidateLane && lane) {
|
|
2410
|
+
Object.assign(lane, candidateLane);
|
|
2411
|
+
this.state.lanes.set(candidateLane.id, lane);
|
|
2412
|
+
}
|
|
2413
|
+
this.releaseEffectLocks(effectId);
|
|
2414
|
+
this.refreshWaits();
|
|
2415
|
+
this.schedulePersistence();
|
|
2416
|
+
}
|
|
2417
|
+
cancelEffect(effectId, graceMs = 0, reason = 'USER_REQUESTED', additionalMutations = []) {
|
|
2418
|
+
return this.requestEffectCancellation(effectId, reason, graceMs, additionalMutations);
|
|
2419
|
+
}
|
|
2420
|
+
publishArtifact(publication) {
|
|
2421
|
+
const record = prepareArtifactPublication(this.state, publication);
|
|
2422
|
+
this.assertStorageAdmission([{ op: 'publishArtifact', record }]);
|
|
2423
|
+
commitMutationTransaction(this.state, this.mutationLog, `artifact:${record.ref}`, [{ op: 'publishArtifact', record }], this.state.now, this.sessionId);
|
|
2424
|
+
this.syncStoragePolicy();
|
|
2425
|
+
this.schedulePersistence();
|
|
2426
|
+
return record;
|
|
2427
|
+
}
|
|
2428
|
+
publishFinding(publication) {
|
|
2429
|
+
const record = prepareFindingPublication(this.state, publication);
|
|
2430
|
+
this.assertStorageAdmission([{ op: 'publishFinding', record }]);
|
|
2431
|
+
commitMutationTransaction(this.state, this.mutationLog, `finding:${record.id}`, [{ op: 'publishFinding', record }], this.state.now, this.sessionId);
|
|
2432
|
+
this.syncStoragePolicy();
|
|
2433
|
+
this.schedulePersistence();
|
|
2434
|
+
return record;
|
|
2435
|
+
}
|
|
2436
|
+
readArtifact(ref) { return readArtifact(this.state, ref); }
|
|
2437
|
+
pinArtifact(ref) { pinArtifact(this.state, ref); this.syncStoragePolicy(); this.schedulePersistence(); }
|
|
2438
|
+
unpinArtifact(ref) { unpinArtifact(this.state, ref); this.syncStoragePolicy(); this.schedulePersistence(); }
|
|
2439
|
+
markArtifactPersisted(ref) { markArtifactPersisted(this.state, ref); this.syncStoragePolicy(); this.schedulePersistence(); }
|
|
2440
|
+
cancelAgent(agentId, reason = 'USER_REQUESTED', additionalMutations = [], commandTransactionId) {
|
|
2441
|
+
const agent = this.state.agents.get(agentId);
|
|
2442
|
+
if (!agent || ['succeeded', 'failed', 'cancelled'].includes(agent.state ?? ''))
|
|
2443
|
+
return false;
|
|
2444
|
+
const targetAgentIds = [];
|
|
2445
|
+
const collect = (currentAgentId) => {
|
|
2446
|
+
if (targetAgentIds.includes(currentAgentId))
|
|
2447
|
+
return;
|
|
2448
|
+
const current = this.state.agents.get(currentAgentId);
|
|
2449
|
+
if (!current || current.detached === true || ['succeeded', 'failed', 'cancelled'].includes(current.state ?? ''))
|
|
2450
|
+
return;
|
|
2451
|
+
targetAgentIds.push(currentAgentId);
|
|
2452
|
+
for (const effect of this.state.effects.values())
|
|
2453
|
+
if (effect.agentId === currentAgentId && effect.childAgentId !== undefined)
|
|
2454
|
+
collect(effect.childAgentId);
|
|
2455
|
+
};
|
|
2456
|
+
collect(agentId);
|
|
2457
|
+
const targetAgentSet = new Set(targetAgentIds);
|
|
2458
|
+
const targetLanes = [...this.state.lanes.values()].filter((lane) => targetAgentSet.has(lane.agentId) && !['succeeded', 'failed', 'cancelled'].includes(lane.status));
|
|
2459
|
+
const targetEffects = [...this.state.effects.values()].filter((effect) => targetAgentSet.has(effect.agentId) && !effect.outcome);
|
|
2460
|
+
const cancellableEffects = targetEffects.filter((effect) => effect.childAgentId === undefined || this.state.agents.get(effect.childAgentId)?.detached !== true);
|
|
2461
|
+
const cancellationEvents = [
|
|
2462
|
+
...targetAgentIds.map((targetId) => ({ type: 'agent.cancelling', agentId: targetId, data: reason })),
|
|
2463
|
+
...targetLanes.map((lane) => ({ type: 'lane.cancelling', laneId: lane.id, data: reason })),
|
|
2464
|
+
...cancellableEffects.map((effect) => ({ type: 'effect.cancel_requested', effectId: effect.id, data: { reason } })),
|
|
2465
|
+
...cancellableEffects.flatMap((effect) => {
|
|
2466
|
+
if (this.executions.has(effect.id) && (effect.cancelGraceMs ?? 0) === 0) {
|
|
2467
|
+
const state = isSideEffectful(effect.sideEffectPolicy) ? 'reconcile_required' : 'cancelled';
|
|
2468
|
+
return [{ type: 'effect.quarantined', effectId: effect.id, data: { reason, state } }];
|
|
2469
|
+
}
|
|
2470
|
+
if (!this.executions.has(effect.id))
|
|
2471
|
+
return [{ type: 'effect.settled', effectId: effect.id, data: { status: 'cancelled', error: { code: 'CANCELLED', message: reason } } }];
|
|
2472
|
+
return [];
|
|
2473
|
+
}),
|
|
2474
|
+
...targetAgentIds.map((targetId) => ({ type: 'agent.cancelled', agentId: targetId, data: reason })),
|
|
2475
|
+
...targetLanes.map((lane) => ({ type: 'lane.cancelled', laneId: lane.id, data: reason })),
|
|
2476
|
+
];
|
|
2477
|
+
const cancellationPreflight = [
|
|
2478
|
+
...additionalMutations.map((mutation) => structuredClone(mutation)),
|
|
2479
|
+
...cancellationEvents.map((event) => ({ op: 'appendEvent', event })),
|
|
2480
|
+
...targetAgentIds.flatMap((targetId) => {
|
|
2481
|
+
const current = this.state.agents.get(targetId);
|
|
2482
|
+
if (!current)
|
|
2483
|
+
return [];
|
|
2484
|
+
const candidate = structuredClone(current);
|
|
2485
|
+
candidate.state = 'cancelling';
|
|
2486
|
+
return [{ op: 'setAgent', agentId: targetId, record: candidate }];
|
|
2487
|
+
}),
|
|
2488
|
+
...targetLanes.flatMap((lane) => {
|
|
2489
|
+
const candidate = structuredClone(lane);
|
|
2490
|
+
if ((lane.status === 'closing' || lane.status === 'waiting') && lane.closingResult !== undefined)
|
|
2491
|
+
candidate.pendingOutcome = { status: 'succeeded', result: structuredClone(lane.closingResult.value) };
|
|
2492
|
+
candidate.status = 'cancelling';
|
|
2493
|
+
candidate.cancelReason = reason;
|
|
2494
|
+
candidate.version++;
|
|
2495
|
+
candidate.unresolvedEffectIds = [...new Set([...(candidate.unresolvedEffectIds ?? []), ...cancellableEffects.filter((effect) => effect.ownerLaneId === lane.id && isSideEffectful(effect.sideEffectPolicy)).map((effect) => effect.id)])];
|
|
2496
|
+
return [{ op: 'setLane', laneId: lane.id, record: candidate }];
|
|
2497
|
+
}),
|
|
2498
|
+
...cancellableEffects.flatMap((effect) => {
|
|
2499
|
+
const candidate = structuredClone(effect);
|
|
2500
|
+
candidate.cancelRequested = { reason, at: this.state.now };
|
|
2501
|
+
if (this.executions.has(effect.id) && (effect.cancelGraceMs ?? 0) === 0) {
|
|
2502
|
+
candidate.executionState = 'remote_unknown';
|
|
2503
|
+
candidate.sideEffectState = isSideEffectful(candidate.sideEffectPolicy) ? 'unknown' : 'none';
|
|
2504
|
+
candidate.state = candidate.sideEffectState === 'unknown' ? 'reconcile_required' : 'cancelled';
|
|
2505
|
+
if (candidate.state === 'cancelled')
|
|
2506
|
+
candidate.outcome = { status: 'cancelled', reason, error: { code: reason, message: reason } };
|
|
2507
|
+
}
|
|
2508
|
+
else if (!this.executions.has(effect.id)) {
|
|
2509
|
+
candidate.state = 'cancelled';
|
|
2510
|
+
candidate.executionState = 'failed';
|
|
2511
|
+
candidate.sideEffectState = 'none';
|
|
2512
|
+
candidate.outcome = { status: 'cancelled', reason, error: { code: 'CANCELLED', message: reason } };
|
|
2513
|
+
}
|
|
2514
|
+
return [{ op: 'setEffect', effectId: effect.id, record: candidate }];
|
|
2515
|
+
}),
|
|
2516
|
+
];
|
|
2517
|
+
this.assertStorageAdmission(cancellationPreflight);
|
|
2518
|
+
let commandApplied = additionalMutations.length === 0;
|
|
2519
|
+
for (const [index, targetId] of targetAgentIds.entries()) {
|
|
2520
|
+
const initialMutations = [{ op: 'appendEvent', event: { type: 'agent.cancelling', agentId: targetId, data: reason } }];
|
|
2521
|
+
if (index === 0)
|
|
2522
|
+
initialMutations.push(...additionalMutations);
|
|
2523
|
+
const committed = this.commitAgentState(targetId, 'cancelling', index === 0 && commandTransactionId ? commandTransactionId : `agent:${targetId}:cancelling:${this.state.now}`, initialMutations);
|
|
2524
|
+
if (index === 0 && additionalMutations.length > 0)
|
|
2525
|
+
commandApplied = committed;
|
|
2526
|
+
}
|
|
2527
|
+
for (const lane of targetLanes) {
|
|
2528
|
+
const nextLane = structuredClone(lane);
|
|
2529
|
+
if ((lane.status === 'closing' || lane.status === 'waiting') && lane.closingResult !== undefined)
|
|
2530
|
+
nextLane.pendingOutcome = { status: 'succeeded', result: structuredClone(lane.closingResult.value) };
|
|
2531
|
+
nextLane.status = 'cancelling';
|
|
2532
|
+
nextLane.cancelReason = reason;
|
|
2533
|
+
nextLane.version++;
|
|
2534
|
+
const event = { type: 'lane.cancelling', laneId: lane.id, data: reason };
|
|
2535
|
+
const mutations = [{ op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event }];
|
|
2536
|
+
this.assertStorageAdmission(mutations);
|
|
2537
|
+
commitMutationTransaction(this.state, this.mutationLog, `lane:${lane.id}:cancelling:${nextLane.version}`, mutations, this.state.now, this.sessionId);
|
|
2538
|
+
Object.assign(lane, nextLane);
|
|
2539
|
+
this.state.lanes.set(lane.id, lane);
|
|
2540
|
+
}
|
|
2541
|
+
for (const effect of targetEffects) {
|
|
2542
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2543
|
+
break;
|
|
2544
|
+
const childAgent = effect.childAgentId === undefined ? undefined : this.state.agents.get(effect.childAgentId);
|
|
2545
|
+
if (childAgent?.detached === true)
|
|
2546
|
+
continue;
|
|
2547
|
+
this.requestEffectCancellation(effect.id, reason, effect.cancelGraceMs ?? 0);
|
|
2548
|
+
this.tickBudget?.consume();
|
|
2549
|
+
}
|
|
2550
|
+
this.finalizeCancellations();
|
|
2551
|
+
this.schedulePersistence();
|
|
2552
|
+
return commandApplied;
|
|
2553
|
+
}
|
|
2554
|
+
finalizeCancellations() {
|
|
2555
|
+
let changed = true;
|
|
2556
|
+
while (changed) {
|
|
2557
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2558
|
+
return;
|
|
2559
|
+
changed = false;
|
|
2560
|
+
for (const lane of [...this.state.lanes.values()]) {
|
|
2561
|
+
if (lane.status !== 'cancelling')
|
|
2562
|
+
continue;
|
|
2563
|
+
const wait = lane.activeWaitId === undefined ? undefined : this.state.waits.get(lane.activeWaitId);
|
|
2564
|
+
if (lane.pendingOutcome?.status === 'succeeded' && wait?.state === 'pending')
|
|
2565
|
+
continue;
|
|
2566
|
+
const nextLane = structuredClone(lane);
|
|
2567
|
+
const mutations = [];
|
|
2568
|
+
if (wait?.state === 'pending') {
|
|
2569
|
+
const dependencies = {};
|
|
2570
|
+
for (const dependency of wait.spec.dependencies) {
|
|
2571
|
+
const target = dependency.target;
|
|
2572
|
+
const outcome = target.kind === 'lane' ? outcomeForSeriesMember(this.state, this.state.lanes.get(target.id), dependency.key) : this.state.effects.get(target.id)?.outcome;
|
|
2573
|
+
dependencies[dependency.key] = outcome === undefined ? { state: 'pending', target } : { state: 'settled', target, outcome };
|
|
2574
|
+
}
|
|
2575
|
+
const resolution = { waitId: wait.id, status: 'unsatisfied', dependencies, error: { code: 'CANCELLED', message: 'Lane cancellation interrupted the wait.' } };
|
|
2576
|
+
const nextWait = structuredClone(wait);
|
|
2577
|
+
nextWait.state = 'unsatisfied';
|
|
2578
|
+
nextWait.resolution = resolution;
|
|
2579
|
+
delete nextLane.activeWaitId;
|
|
2580
|
+
mutations.push({ op: 'setWait', waitId: wait.id, record: nextWait });
|
|
2581
|
+
}
|
|
2582
|
+
nextLane.status = 'cancelled';
|
|
2583
|
+
nextLane.cancelReason = nextLane.cancelReason ?? 'USER_REQUESTED';
|
|
2584
|
+
nextLane.version++;
|
|
2585
|
+
mutations.push({ op: 'setLane', laneId: nextLane.id, record: nextLane }, { op: 'appendEvent', event: { type: 'lane.cancelled', laneId: nextLane.id, data: nextLane.cancelReason } });
|
|
2586
|
+
this.assertStorageAdmission(mutations);
|
|
2587
|
+
commitMutationTransaction(this.state, this.mutationLog, `lane:${nextLane.id}:cancelled:${nextLane.version}`, mutations, this.state.now, this.sessionId);
|
|
2588
|
+
Object.assign(lane, nextLane);
|
|
2589
|
+
this.state.lanes.set(lane.id, lane);
|
|
2590
|
+
this.tickBudget?.consume();
|
|
2591
|
+
changed = true;
|
|
2592
|
+
}
|
|
2593
|
+
if (changed)
|
|
2594
|
+
this.refreshWaits();
|
|
2595
|
+
}
|
|
2596
|
+
for (const agent of [...this.state.agents.values()]) {
|
|
2597
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2598
|
+
return;
|
|
2599
|
+
if (agent.state !== 'cancelling')
|
|
2600
|
+
continue;
|
|
2601
|
+
const lanes = [...this.state.lanes.values()].filter((lane) => lane.agentId === agent.id);
|
|
2602
|
+
if (lanes.some((lane) => !['succeeded', 'failed', 'cancelled'].includes(lane.status)))
|
|
2603
|
+
continue;
|
|
2604
|
+
const effects = [...this.state.effects.values()].filter((effect) => effect.agentId === agent.id);
|
|
2605
|
+
if (effects.some((effect) => !effect.outcome && !['cancelled', 'reconcile_required', 'succeeded', 'failed'].includes(effect.state)))
|
|
2606
|
+
continue;
|
|
2607
|
+
const root = this.state.lanes.get(agent.rootLaneId);
|
|
2608
|
+
const finalState = root?.status === 'succeeded' && root.cancelReason !== undefined ? 'succeeded' : root?.status === 'failed' && root.cancelReason === undefined ? 'failed' : 'cancelled';
|
|
2609
|
+
this.commitAgentState(agent.id, finalState, `agent:${agent.id}:${finalState}:${this.state.now}`, [{ op: 'appendEvent', event: { type: `agent.${finalState}`, agentId: agent.id, data: root?.cancelReason ?? 'USER_REQUESTED' } }]);
|
|
2610
|
+
this.tickBudget?.consume();
|
|
2611
|
+
}
|
|
2612
|
+
}
|
|
2613
|
+
explain(laneId) {
|
|
2614
|
+
const readyItems = this.ready.snapshot(this.state.now);
|
|
2615
|
+
const lastEvent = (kind, id) => { const matching = this.state.events.filter((event) => (kind === 'lane' ? event.laneId === id : event.effectId === id)); return matching.at(-1)?.seq ?? null; };
|
|
2616
|
+
const latestEffectMetadata = (effectId) => { const event = [...this.state.events].reverse().find((candidate) => candidate.type === 'effect.execution_metadata' && candidate.effectId === effectId); return event?.data ?? event?.payload ?? null; };
|
|
2617
|
+
const lanes = [...this.state.lanes.values()].filter((lane) => laneId === undefined || lane.id === laneId).map((lane) => {
|
|
2618
|
+
const ready = readyItems.find((item) => item.laneId === lane.id);
|
|
2619
|
+
const blockedBy = lane.activeWaitId ? `wait:${lane.activeWaitId}` : [...lane.ownedEffectIds].some((effectId) => this.lockBlocked.has(effectId)) ? 'resource_lock' : null;
|
|
2620
|
+
return { id: lane.id, agentId: lane.agentId, status: lane.status, cancelReason: lane.cancelReason ?? null, failure: lane.failure?.error ?? null, goal: lane.goal, basePriority: lane.priority, effectivePriority: ready?.effectivePriority ?? lane.priority, queueWaitMs: ready ? Math.max(0, this.state.now - lane.readySince) : 0, blockedBy, activeWaitId: lane.activeWaitId ?? null, lastEventSeq: lastEvent('lane', lane.id), watchdog: lane.progressWatchdog ?? null, consecutiveControlErrors: lane.consecutiveControlErrors ?? 0, lastInterventionReason: lane.progressWatchdog?.lastReason ?? null, unresolvedEffectIds: lane.unresolvedEffectIds ?? [] };
|
|
2621
|
+
});
|
|
2622
|
+
const effects = [...this.state.effects.values()].filter((effect) => laneId === undefined || effect.ownerLaneId === laneId).map((effect) => ({ id: effect.id, state: effect.state, executionState: effect.executionState, sideEffectState: effect.sideEffectState, attemptId: effect.attemptId, inheritedFloor: effect.inheritedFloor ?? null, deadlineAt: effect.deadlineAt ?? null, lastEventSeq: lastEvent('effect', effect.id), preparation: effect.preparation ?? null, metadata: latestEffectMetadata(effect.id) }));
|
|
2623
|
+
return { now: this.state.now, lanes, effects, preparation: { preparing: this.preparingLLMs.size, prepared: [...this.state.effects.values()].filter((effect) => effect.state === 'queued' && effect.preparation?.state === 'prepared').length, maxPreparing: this.maxPreparingLLMs, maxPrepared: this.maxPreparedLLMs }, quarantine: this.quarantine.unresolvedEffectIds };
|
|
2624
|
+
}
|
|
2625
|
+
retryEffect(effectId, delayMs) {
|
|
2626
|
+
const effect = this.state.effects.get(effectId);
|
|
2627
|
+
if (!effect || effect.outcome)
|
|
2628
|
+
return;
|
|
2629
|
+
this.scheduleRetry(effect, undefined, delayMs);
|
|
2630
|
+
this.schedulePersistence();
|
|
2631
|
+
}
|
|
2632
|
+
readyRetryEffect(effectId, attemptId) {
|
|
2633
|
+
const current = this.state.effects.get(effectId);
|
|
2634
|
+
if (!current || current.outcome || current.state !== 'retry_wait' || current.attemptId !== attemptId)
|
|
2635
|
+
return;
|
|
2636
|
+
const ready = structuredClone(current);
|
|
2637
|
+
ready.state = 'queued';
|
|
2638
|
+
delete ready.retryAt;
|
|
2639
|
+
const readyEvent = { type: 'effect.retry_ready', effectId: current.id, data: current.attemptId };
|
|
2640
|
+
this.assertStorageAdmission([{ op: 'setEffect', effectId: current.id, record: ready }, { op: 'appendEvent', event: readyEvent }]);
|
|
2641
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${current.id}:${current.attemptId}:retry-ready`, [{ op: 'setEffect', effectId: current.id, record: ready }, { op: 'appendEvent', event: readyEvent }], this.state.now, this.sessionId);
|
|
2642
|
+
Object.assign(current, ready);
|
|
2643
|
+
delete current.retryAt;
|
|
2644
|
+
this.state.effects.set(current.id, current);
|
|
2645
|
+
this.dispatchQueuedEffects();
|
|
2646
|
+
}
|
|
2647
|
+
scheduleRetry(effect, error, forcedDelayMs) {
|
|
2648
|
+
if (effect.cancelRequested || (!forcedDelayMs && !effect.retryPolicy))
|
|
2649
|
+
return false;
|
|
2650
|
+
if (forcedDelayMs === undefined && error?.retryable === false)
|
|
2651
|
+
return false;
|
|
2652
|
+
if (forcedDelayMs === undefined && effect.retryPolicy) {
|
|
2653
|
+
if (effect.attemptNo >= effect.retryPolicy.maxAttempts)
|
|
2654
|
+
return false;
|
|
2655
|
+
if (effect.sideEffectState === 'unknown')
|
|
2656
|
+
return false;
|
|
2657
|
+
if (effect.sideEffectState === 'applied' && effect.duplicateExecutionPolicy !== 'allow')
|
|
2658
|
+
return false;
|
|
2659
|
+
}
|
|
2660
|
+
const policy = effect.retryPolicy;
|
|
2661
|
+
const baseDelay = forcedDelayMs ?? Math.min(policy.maxBackoffMs, policy.initialBackoffMs * (2 ** Math.max(0, effect.attemptNo - 1)));
|
|
2662
|
+
const jitter = forcedDelayMs === undefined && policy?.jitter ? Math.floor(baseDelay / 2) : 0;
|
|
2663
|
+
const delayMs = baseDelay + jitter;
|
|
2664
|
+
const previousAttemptId = effect.attemptId;
|
|
2665
|
+
const candidate = structuredClone(effect);
|
|
2666
|
+
candidate.state = 'retry_wait';
|
|
2667
|
+
candidate.executionState = 'local';
|
|
2668
|
+
candidate.attemptNo += 1;
|
|
2669
|
+
candidate.attemptId = `${candidate.id}-attempt-${candidate.attemptNo}`;
|
|
2670
|
+
candidate.retryAt = this.state.now + delayMs;
|
|
2671
|
+
if (candidate.kind === 'llm')
|
|
2672
|
+
candidate.preparation = { state: 'stale', generation: (candidate.preparation?.generation ?? 0) + 1 };
|
|
2673
|
+
const retryEvent = { type: 'effect.retry_scheduled', effectId: effect.id, data: { previousAttemptId, nextAttemptId: candidate.attemptId, delayMs, ...(error ? { error } : {}) } };
|
|
2674
|
+
this.assertStorageAdmission([{ op: 'setEffect', effectId: effect.id, record: candidate }, { op: 'appendEvent', event: retryEvent }]);
|
|
2675
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${previousAttemptId}:retry-scheduled`, [{ op: 'setEffect', effectId: effect.id, record: candidate }, { op: 'appendEvent', event: retryEvent }], this.state.now, this.sessionId);
|
|
2676
|
+
Object.assign(effect, candidate);
|
|
2677
|
+
this.state.effects.set(effect.id, effect);
|
|
2678
|
+
this.scheduleRuntimeTimer(candidate.retryAt, () => this.readyRetryEffect(effect.id, candidate.attemptId));
|
|
2679
|
+
return true;
|
|
2680
|
+
}
|
|
2681
|
+
dispatchQueuedEffects() {
|
|
2682
|
+
if (![...this.state.effects.values()].some((effect) => effect.state === 'queued' && !this.executions.has(effect.id)))
|
|
2683
|
+
return;
|
|
2684
|
+
if (this.persistenceBackend) {
|
|
2685
|
+
if (this.dispatchPersistenceReady) {
|
|
2686
|
+
this.dispatchPersistenceReady = false;
|
|
2687
|
+
this.dispatchQueuedEffectsNow();
|
|
2688
|
+
return;
|
|
2689
|
+
}
|
|
2690
|
+
this.schedulePersistence();
|
|
2691
|
+
if (this.dispatchPersistencePending)
|
|
2692
|
+
return;
|
|
2693
|
+
this.dispatchPersistencePending = true;
|
|
2694
|
+
void this.flushPersistence().then(() => {
|
|
2695
|
+
this.dispatchPersistencePending = false;
|
|
2696
|
+
this.dispatchPersistenceReady = true;
|
|
2697
|
+
this.scheduleWake(true);
|
|
2698
|
+
}).catch((cause) => {
|
|
2699
|
+
// Dispatch is gated on durability; a failed flush must not silently stall the queue.
|
|
2700
|
+
this.dispatchPersistencePending = false;
|
|
2701
|
+
this.tryEmit({ type: 'effect.dispatch_blocked', data: { reason: 'PERSISTENCE_FAILED', error: cause instanceof Error ? cause.message : String(cause) } });
|
|
2702
|
+
this.scheduleRuntimeDelay(Math.min(30_000, 250 * 2 ** Math.min(10, this.persistenceFailures)), () => this.scheduleWake(true));
|
|
2703
|
+
});
|
|
2704
|
+
return;
|
|
2705
|
+
}
|
|
2706
|
+
this.dispatchQueuedEffectsNow();
|
|
2707
|
+
}
|
|
2708
|
+
dispatchQueuedEffectsNow() {
|
|
2709
|
+
// Higher effective priority (max of own priority and inherited floor) dispatches first, matching ReadyQueue semantics.
|
|
2710
|
+
const effectivePriority = (effect) => Math.max(effect.schedulePriority ?? 0, effect.inheritedFloor ?? Number.NEGATIVE_INFINITY);
|
|
2711
|
+
const queued = [...this.state.effects.values()].filter((effect) => effect.state === 'queued' && !this.executions.has(effect.id)).sort((a, b) => (effectivePriority(b) - effectivePriority(a)) || a.id.localeCompare(b.id));
|
|
2712
|
+
for (const effect of queued) {
|
|
2713
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2714
|
+
break;
|
|
2715
|
+
if (effect.state !== 'queued' || this.executions.has(effect.id))
|
|
2716
|
+
continue;
|
|
2717
|
+
if (effect.kind === 'llm' && !this.prepareLLMEffect(effect))
|
|
2718
|
+
continue;
|
|
2719
|
+
if (effect.concurrencyClass !== 'none' && this.runningCount(effect.concurrencyClass) >= this.state.maxRunning[effect.concurrencyClass])
|
|
2720
|
+
continue;
|
|
2721
|
+
const budgetError = this.budgetRejection(effect);
|
|
2722
|
+
if (budgetError) {
|
|
2723
|
+
this.completeEffect(effect.id, { value: null, executionState: 'failed', sideEffectState: 'none' }, 'failed', budgetError);
|
|
2724
|
+
continue;
|
|
2725
|
+
}
|
|
2726
|
+
const outboxEntry = this.outbox.enqueue(effect, this.state.now);
|
|
2727
|
+
if (outboxEntry.state === 'claimed')
|
|
2728
|
+
continue;
|
|
2729
|
+
if (!this.acquireEffectLocks(effect))
|
|
2730
|
+
continue;
|
|
2731
|
+
const running = structuredClone(effect);
|
|
2732
|
+
running.state = 'running';
|
|
2733
|
+
running.executionState = 'running';
|
|
2734
|
+
const attempt = { id: effect.attemptId, effectId: effect.id, executionState: 'running', sideEffectState: effect.sideEffectState, startedAt: this.state.now };
|
|
2735
|
+
running.attempts = [...(running.attempts ?? []), attempt];
|
|
2736
|
+
const dispatchMutations = [{ op: 'setEffect', effectId: effect.id, record: running }];
|
|
2737
|
+
try {
|
|
2738
|
+
this.assertStorageAdmission(dispatchMutations);
|
|
2739
|
+
}
|
|
2740
|
+
catch {
|
|
2741
|
+
this.releaseEffectLocks(effect.id);
|
|
2742
|
+
continue;
|
|
2743
|
+
}
|
|
2744
|
+
if (!this.outbox.claim(outboxEntry.id)) {
|
|
2745
|
+
this.releaseEffectLocks(effect.id);
|
|
2746
|
+
continue;
|
|
2747
|
+
}
|
|
2748
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:dispatched`, dispatchMutations, this.state.now, this.sessionId);
|
|
2749
|
+
Object.assign(effect, running);
|
|
2750
|
+
this.state.effects.set(effect.id, effect);
|
|
2751
|
+
this.tickBudget?.consume();
|
|
2752
|
+
const controller = new AbortController();
|
|
2753
|
+
// Human Effects can use Runtime's pending/reply protocol when the host
|
|
2754
|
+
// opts in, while legacy custom executors retain control by default.
|
|
2755
|
+
if (effect.kind === 'human' && (this.builtinHumanEffects || !this.customExecutor)) {
|
|
2756
|
+
this.emit({ type: 'human.requested', effectId: effect.id, data: effect.input });
|
|
2757
|
+
if (effect.attemptTimeoutMs !== undefined)
|
|
2758
|
+
this.scheduleRuntimeDelay(effect.attemptTimeoutMs, () => { if (!effect.outcome)
|
|
2759
|
+
this.completeEffect(effect.id, { value: null }, 'failed', { code: 'ATTEMPT_TIMEOUT', message: 'Human response timed out.' }); });
|
|
2760
|
+
if (effect.deadlineAt !== undefined)
|
|
2761
|
+
this.scheduleRuntimeTimer(effect.deadlineAt, () => { if (!effect.outcome)
|
|
2762
|
+
this.completeEffect(effect.id, { value: null }, 'failed', { code: 'TIMEOUT', message: 'Human response deadline exceeded.' }); });
|
|
2763
|
+
continue;
|
|
2764
|
+
}
|
|
2765
|
+
const executionRecord = { controller, promise: Promise.resolve() };
|
|
2766
|
+
if (effect.kind === 'timer' && !this.customExecutor) {
|
|
2767
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2768
|
+
const delayMs = input.delayMs;
|
|
2769
|
+
if (typeof delayMs !== 'number' || !Number.isFinite(delayMs) || delayMs < 0) {
|
|
2770
|
+
this.completeEffect(effect.id, { value: null }, 'failed', { code: 'INVALID_TIMER', message: 'Timer Effect requires a non-negative delayMs.' });
|
|
2771
|
+
continue;
|
|
2772
|
+
}
|
|
2773
|
+
let resolveTimer;
|
|
2774
|
+
executionRecord.promise = new Promise((resolve) => { resolveTimer = resolve; });
|
|
2775
|
+
this.executions.set(effect.id, executionRecord);
|
|
2776
|
+
this.scheduleRuntimeDelay(delayMs, () => { if (!effect.outcome)
|
|
2777
|
+
this.completeEffect(effect.id, { value: { firedAt: this.clock.now() } }); resolveTimer(); });
|
|
2778
|
+
if (effect.attemptTimeoutMs !== undefined)
|
|
2779
|
+
executionRecord.timeoutTimer = this.scheduleRuntimeDelay(effect.attemptTimeoutMs, () => this.expireEffect(effect.id, 'ATTEMPT_TIMEOUT'));
|
|
2780
|
+
if (effect.deadlineAt !== undefined)
|
|
2781
|
+
executionRecord.deadlineTimer = this.scheduleRuntimeTimer(effect.deadlineAt, () => this.expireEffect(effect.id, 'TIMEOUT'));
|
|
2782
|
+
continue;
|
|
2783
|
+
}
|
|
2784
|
+
if (effect.kind === 'agent' && !this.customExecutor) {
|
|
2785
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2786
|
+
const programId = input.programId;
|
|
2787
|
+
const programVersion = input.programVersion;
|
|
2788
|
+
const goal = input.goal;
|
|
2789
|
+
const childProgram = typeof programId === 'string' && typeof programVersion === 'string' ? this.programs.get(`${programId}@${programVersion}`) : undefined;
|
|
2790
|
+
if (!childProgram || typeof goal !== 'string') {
|
|
2791
|
+
this.completeEffect(effect.id, { value: null }, 'failed', { code: 'INVALID_AGENT_EFFECT_INPUT', message: 'Agent Effect requires a registered program and goal.' });
|
|
2792
|
+
continue;
|
|
2793
|
+
}
|
|
2794
|
+
const parent = this.state.agents.get(effect.agentId);
|
|
2795
|
+
if ((parent?.depth ?? 0) >= this.maxAgentDepth) {
|
|
2796
|
+
this.completeEffect(effect.id, { value: null }, 'failed', { code: 'MAX_AGENT_DEPTH', message: 'Child Agent depth limit exceeded.' });
|
|
2797
|
+
continue;
|
|
2798
|
+
}
|
|
2799
|
+
const parentLane = this.state.lanes.get(effect.ownerLaneId);
|
|
2800
|
+
const parentScore = parentLane === undefined ? 0 : this.ready.snapshot(this.state.now).find((item) => item.laneId === parentLane.id)?.effectivePriority ?? parentLane.priority;
|
|
2801
|
+
try {
|
|
2802
|
+
const child = this.createAgent({ goal, program: childProgram, parentAgentId: effect.agentId, inheritedFloor: parentScore });
|
|
2803
|
+
const linked = structuredClone(effect);
|
|
2804
|
+
linked.childAgentId = child.agentId;
|
|
2805
|
+
const linkMutations = [{ op: 'setEffect', effectId: effect.id, record: linked }];
|
|
2806
|
+
this.assertStorageAdmission(linkMutations);
|
|
2807
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:child-agent`, linkMutations, this.state.now, this.sessionId);
|
|
2808
|
+
Object.assign(effect, linked);
|
|
2809
|
+
this.state.effects.set(effect.id, effect);
|
|
2810
|
+
this.emit({ type: 'agent.effect_started', effectId: effect.id, data: child.agentId });
|
|
2811
|
+
}
|
|
2812
|
+
catch (cause) {
|
|
2813
|
+
this.completeEffect(effect.id, { value: null, executionState: 'failed', sideEffectState: 'none' }, 'failed', runtimeErrorFromCause(cause, 'CHILD_AGENT_CREATE_FAILED'));
|
|
2814
|
+
}
|
|
2815
|
+
continue;
|
|
2816
|
+
}
|
|
2817
|
+
const emitObservation = (observation) => {
|
|
2818
|
+
const liveEffect = this.state.effects.get(effect.id);
|
|
2819
|
+
if (!liveEffect)
|
|
2820
|
+
return;
|
|
2821
|
+
if (liveEffect.outcome || liveEffect.state !== 'running') {
|
|
2822
|
+
this.tryEmit({ type: 'attempt.late_emit', effectId: effect.id, attemptId: effect.attemptId, data: { kind: 'observation', status: liveEffect.outcome?.status ?? liveEffect.state } });
|
|
2823
|
+
return;
|
|
2824
|
+
}
|
|
2825
|
+
this.observationInbox.enqueue({ ...observation, agentId: effect.agentId, laneId: effect.ownerLaneId, timestamp: this.state.now });
|
|
2826
|
+
};
|
|
2827
|
+
const attemptId = effect.attemptId;
|
|
2828
|
+
const promise = this.executor(effect, controller.signal, emitObservation).then((execution) => {
|
|
2829
|
+
this.enqueueEffectCompletion(effect.id, attemptId, execution);
|
|
2830
|
+
}).catch((cause) => {
|
|
2831
|
+
const runtimeError = runtimeErrorFromCause(cause);
|
|
2832
|
+
this.enqueueEffectCompletion(effect.id, attemptId, { value: null, sideEffectState: 'none' }, 'failed', runtimeError, runtimeError);
|
|
2833
|
+
});
|
|
2834
|
+
executionRecord.promise = promise;
|
|
2835
|
+
if (effect.attemptTimeoutMs !== undefined)
|
|
2836
|
+
executionRecord.timeoutTimer = this.scheduleRuntimeDelay(effect.attemptTimeoutMs, () => this.expireEffect(effect.id, 'ATTEMPT_TIMEOUT'));
|
|
2837
|
+
if (effect.deadlineAt !== undefined)
|
|
2838
|
+
executionRecord.deadlineTimer = this.scheduleRuntimeTimer(effect.deadlineAt, () => this.expireEffect(effect.id, 'TIMEOUT'));
|
|
2839
|
+
this.executions.set(effect.id, executionRecord);
|
|
2840
|
+
if (effect.kind === 'tool') {
|
|
2841
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2842
|
+
if (typeof input.name === 'string' && this.tools.get(input.name) !== undefined)
|
|
2843
|
+
this.executionYieldPending.add(effect.id);
|
|
2844
|
+
}
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
prepareLLMEffect(effect) {
|
|
2848
|
+
if (effect.preparation?.state === 'prepared')
|
|
2849
|
+
return true;
|
|
2850
|
+
if (effect.preparation?.state === 'preparing')
|
|
2851
|
+
return false;
|
|
2852
|
+
if (this.preparingLLMs.size >= this.maxPreparingLLMs)
|
|
2853
|
+
return false;
|
|
2854
|
+
const preparedCount = [...this.state.effects.values()].filter((candidate) => candidate.state === 'queued' && candidate.preparation?.state === 'prepared').length;
|
|
2855
|
+
if (preparedCount >= this.maxPreparedLLMs)
|
|
2856
|
+
return false;
|
|
2857
|
+
const generation = (effect.preparation?.generation ?? 0) + 1;
|
|
2858
|
+
const input = effect.input && typeof effect.input === 'object' && !Array.isArray(effect.input) ? effect.input : {};
|
|
2859
|
+
const request = input.request && typeof input.request === 'object' && !Array.isArray(input.request) ? input.request : undefined;
|
|
2860
|
+
effect.preparation = { state: 'preparing', generation, ...(typeof request?.projectionHash === 'string' ? { projectionRef: request.projectionHash } : {}) };
|
|
2861
|
+
this.preparingLLMs.add(effect.id);
|
|
2862
|
+
Promise.resolve().then(() => {
|
|
2863
|
+
const current = this.state.effects.get(effect.id);
|
|
2864
|
+
const stale = current === undefined || current.outcome !== undefined || current.state !== 'queued' || current.preparation?.generation !== generation || current.cancelRequested !== undefined;
|
|
2865
|
+
this.enqueueLLMPreparation(effect.id, generation, stale ? 'stale' : 'prepared', current?.preparation?.projectionRef);
|
|
2866
|
+
});
|
|
2867
|
+
return false;
|
|
2868
|
+
}
|
|
2869
|
+
expireEffect(effectId, reason) {
|
|
2870
|
+
const effect = this.state.effects.get(effectId);
|
|
2871
|
+
const execution = this.executions.get(effectId);
|
|
2872
|
+
if (!effect || effect.outcome || !execution)
|
|
2873
|
+
return;
|
|
2874
|
+
this.state.now = this.clock.now();
|
|
2875
|
+
execution.controller.abort();
|
|
2876
|
+
this.quarantineEffect(effectId, reason, effect.cancelGraceMs ?? 0, { type: 'limit.rejected', effectId, data: { code: reason } });
|
|
2877
|
+
this.schedulePersistence();
|
|
2878
|
+
}
|
|
2879
|
+
requestEffectCancellation(effectId, reason, graceMs, additionalMutations = []) {
|
|
2880
|
+
const effect = this.state.effects.get(effectId);
|
|
2881
|
+
// `reconcile_required` is owned by QuarantineScope: only the host may settle it (reconcile/abandon).
|
|
2882
|
+
if (!effect || effect.outcome || effect.state === 'reconcile_required')
|
|
2883
|
+
return false;
|
|
2884
|
+
const cancelEvent = { type: 'effect.cancel_requested', effectId, data: { reason } };
|
|
2885
|
+
if (this.executions.has(effectId) && graceMs === 0) {
|
|
2886
|
+
return this.quarantineEffect(effectId, reason, 0, cancelEvent, additionalMutations);
|
|
2887
|
+
}
|
|
2888
|
+
const admitted = structuredClone(effect);
|
|
2889
|
+
admitted.cancelRequested = { reason, at: this.state.now };
|
|
2890
|
+
const cancellationMutations = [{ op: 'setEffect', effectId, record: admitted }, { op: 'appendEvent', event: cancelEvent }];
|
|
2891
|
+
if (this.executions.has(effectId) && graceMs > 0)
|
|
2892
|
+
cancellationMutations.push(...additionalMutations.map((mutation) => structuredClone(mutation)));
|
|
2893
|
+
this.assertStorageAdmission(cancellationMutations);
|
|
2894
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:cancel-requested`, cancellationMutations, this.state.now, this.sessionId);
|
|
2895
|
+
Object.assign(effect, admitted);
|
|
2896
|
+
this.state.effects.set(effectId, effect);
|
|
2897
|
+
if (!this.executions.has(effectId))
|
|
2898
|
+
return this.completeEffect(effectId, { value: null }, 'cancelled', { code: 'CANCELLED', message: reason }, additionalMutations);
|
|
2899
|
+
this.executions.get(effectId).controller.abort();
|
|
2900
|
+
if (graceMs === 0)
|
|
2901
|
+
this.quarantineEffect(effectId, reason, 0);
|
|
2902
|
+
else
|
|
2903
|
+
this.executions.get(effectId).cancelTimer = this.scheduleRuntimeDelay(graceMs, () => this.quarantineEffect(effectId, reason, 0));
|
|
2904
|
+
return this.executions.has(effectId) && graceMs > 0;
|
|
2905
|
+
}
|
|
2906
|
+
quarantineEffect(effectId, reason, _graceMs, precedingEvent, additionalMutations = []) {
|
|
2907
|
+
const effect = this.state.effects.get(effectId);
|
|
2908
|
+
const execution = this.executions.get(effectId);
|
|
2909
|
+
if (!effect || effect.outcome)
|
|
2910
|
+
return false;
|
|
2911
|
+
const candidate = structuredClone(effect);
|
|
2912
|
+
if (precedingEvent?.type === 'effect.cancel_requested' || precedingEvent?.type === 'limit.rejected')
|
|
2913
|
+
candidate.cancelRequested = { reason, at: this.state.now };
|
|
2914
|
+
candidate.executionState = 'remote_unknown';
|
|
2915
|
+
candidate.sideEffectState = isSideEffectful(candidate.sideEffectPolicy) ? 'unknown' : 'none';
|
|
2916
|
+
candidate.state = candidate.sideEffectState === 'unknown' ? 'reconcile_required' : 'cancelled';
|
|
2917
|
+
if (candidate.state === 'cancelled')
|
|
2918
|
+
candidate.outcome = { status: 'cancelled', reason, error: { code: reason, message: reason } };
|
|
2919
|
+
const lane = this.state.lanes.get(effect.ownerLaneId);
|
|
2920
|
+
const candidateLane = lane === undefined ? undefined : structuredClone(lane);
|
|
2921
|
+
if (candidateLane)
|
|
2922
|
+
candidateLane.unresolvedEffectIds = [...new Set([...(candidateLane.unresolvedEffectIds ?? []), effectId])];
|
|
2923
|
+
const quarantineEvent = { type: 'effect.quarantined', effectId, data: { reason, state: candidate.state } };
|
|
2924
|
+
const admission = [{ op: 'setEffect', effectId, record: candidate }];
|
|
2925
|
+
if (candidateLane)
|
|
2926
|
+
admission.push({ op: 'setLane', laneId: candidateLane.id, record: candidateLane });
|
|
2927
|
+
if (precedingEvent)
|
|
2928
|
+
admission.push({ op: 'appendEvent', event: precedingEvent });
|
|
2929
|
+
admission.push(...additionalMutations.map((mutation) => structuredClone(mutation)));
|
|
2930
|
+
admission.push({ op: 'appendEvent', event: quarantineEvent });
|
|
2931
|
+
this.assertStorageAdmission(admission);
|
|
2932
|
+
if (execution) {
|
|
2933
|
+
execution.controller.abort();
|
|
2934
|
+
this.executions.delete(effectId);
|
|
2935
|
+
}
|
|
2936
|
+
this.releaseEffectLocks(effectId);
|
|
2937
|
+
commitMutationTransaction(this.state, this.mutationLog, `effect:${effect.id}:${effect.attemptId}:quarantined`, admission, this.state.now, this.sessionId);
|
|
2938
|
+
Object.assign(effect, candidate);
|
|
2939
|
+
this.state.effects.set(effectId, effect);
|
|
2940
|
+
if (candidateLane && lane) {
|
|
2941
|
+
Object.assign(lane, candidateLane);
|
|
2942
|
+
this.state.lanes.set(candidateLane.id, lane);
|
|
2943
|
+
}
|
|
2944
|
+
this.quarantine.add(effectId, this.state.now, reason);
|
|
2945
|
+
this.refreshWaits();
|
|
2946
|
+
this.schedulePersistence();
|
|
2947
|
+
return true;
|
|
2948
|
+
}
|
|
2949
|
+
propagateCancelledLanes() {
|
|
2950
|
+
for (const lane of this.state.lanes.values())
|
|
2951
|
+
if (lane.status === 'cancelled' || lane.status === 'cancelling')
|
|
2952
|
+
for (const effectId of lane.ownedEffectIds) {
|
|
2953
|
+
const effect = this.state.effects.get(effectId);
|
|
2954
|
+
// Settled, quarantined or already-cancelling Effects need no work; skip them before touching the tick budget.
|
|
2955
|
+
if (!effect || effect.outcome || effect.state === 'reconcile_required' || effect.cancelRequested !== undefined)
|
|
2956
|
+
continue;
|
|
2957
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
2958
|
+
return;
|
|
2959
|
+
const childAgent = effect.childAgentId === undefined ? undefined : this.state.agents.get(effect.childAgentId);
|
|
2960
|
+
if (childAgent?.detached === true)
|
|
2961
|
+
continue;
|
|
2962
|
+
this.requestEffectCancellation(effectId, 'LANE_CANCELLED', effect?.cancelGraceMs ?? 0);
|
|
2963
|
+
this.tickBudget?.consume();
|
|
2964
|
+
}
|
|
2965
|
+
}
|
|
2966
|
+
runningCount(concurrencyClass) { return [...this.state.effects.values()].filter((effect) => effect.concurrencyClass === concurrencyClass && effect.state === 'running').length; }
|
|
2967
|
+
budgetUsage() {
|
|
2968
|
+
return { attempts: [...this.state.effects.values()].reduce((total, effect) => total + (effect.attempts?.length ?? 0), 0), costByCurrency: Object.fromEntries(this.budgetCost.entries()) };
|
|
2969
|
+
}
|
|
2970
|
+
budgetRejection(effect) {
|
|
2971
|
+
const attempts = [...this.state.effects.values()].reduce((total, candidate) => total + (candidate.attempts?.length ?? 0), 0);
|
|
2972
|
+
if (this.budget.maxTotalAttempts !== undefined && attempts >= this.budget.maxTotalAttempts)
|
|
2973
|
+
return { code: 'BUDGET_EXCEEDED', message: 'Runtime attempt budget exceeded.', details: { budget: 'maxTotalAttempts', limit: this.budget.maxTotalAttempts } };
|
|
2974
|
+
const kindLimit = effect.kind === 'llm' ? this.budget.maxLLMAttempts : effect.kind === 'tool' ? this.budget.maxToolAttempts : undefined;
|
|
2975
|
+
const kindAttempts = [...this.state.effects.values()].filter((candidate) => candidate.kind === effect.kind).reduce((total, candidate) => total + (candidate.attempts?.length ?? 0), 0);
|
|
2976
|
+
if (kindLimit !== undefined && kindAttempts >= kindLimit)
|
|
2977
|
+
return { code: 'BUDGET_EXCEEDED', message: `${effect.kind} attempt budget exceeded.`, details: { budget: effect.kind === 'llm' ? 'maxLLMAttempts' : 'maxToolAttempts', limit: kindLimit } };
|
|
2978
|
+
for (const [currency, limit] of Object.entries(this.budget.maxCostByCurrency ?? {}))
|
|
2979
|
+
if ((this.budgetCost.get(currency) ?? 0) >= limit)
|
|
2980
|
+
return { code: 'BUDGET_EXCEEDED', message: `Runtime cost budget exceeded for ${currency}.`, details: { budget: 'maxCostByCurrency', currency, limit } };
|
|
2981
|
+
return undefined;
|
|
2982
|
+
}
|
|
2983
|
+
recordBudgetMetadata(metadata) {
|
|
2984
|
+
if (!metadata || typeof metadata !== 'object' || Array.isArray(metadata))
|
|
2985
|
+
return;
|
|
2986
|
+
const attempts = metadata.attempts;
|
|
2987
|
+
if (!Array.isArray(attempts))
|
|
2988
|
+
return;
|
|
2989
|
+
for (const attempt of attempts) {
|
|
2990
|
+
if (!attempt || typeof attempt !== 'object' || Array.isArray(attempt))
|
|
2991
|
+
continue;
|
|
2992
|
+
const usage = attempt.usage;
|
|
2993
|
+
if (!usage || typeof usage !== 'object' || Array.isArray(usage))
|
|
2994
|
+
continue;
|
|
2995
|
+
const cost = usage.cost;
|
|
2996
|
+
if (!cost || typeof cost !== 'object' || Array.isArray(cost))
|
|
2997
|
+
continue;
|
|
2998
|
+
const currency = cost.currency;
|
|
2999
|
+
const amount = cost.amount;
|
|
3000
|
+
if (typeof currency === 'string' && typeof amount === 'number' && Number.isFinite(amount) && amount >= 0)
|
|
3001
|
+
this.budgetCost.set(currency, (this.budgetCost.get(currency) ?? 0) + amount);
|
|
3002
|
+
}
|
|
3003
|
+
}
|
|
3004
|
+
acquireEffectLocks(effect) {
|
|
3005
|
+
const specs = [...(effect.locks ?? [])].sort((a, b) => a.resource.localeCompare(b.resource) || a.mode.localeCompare(b.mode));
|
|
3006
|
+
const releases = [];
|
|
3007
|
+
for (const [index, spec] of specs.entries()) {
|
|
3008
|
+
const requestId = `${effect.id}:${effect.attemptId}:${index}`;
|
|
3009
|
+
const issued = this.lockRequests.get(effect.id) ?? new Map();
|
|
3010
|
+
issued.set(requestId, spec.resource);
|
|
3011
|
+
this.lockRequests.set(effect.id, issued);
|
|
3012
|
+
let release = this.grantedLockReleases.get(requestId);
|
|
3013
|
+
if (release !== undefined)
|
|
3014
|
+
this.grantedLockReleases.delete(requestId);
|
|
3015
|
+
else
|
|
3016
|
+
release = this.resourceLocks.tryAcquire(spec.resource, spec.mode, requestId);
|
|
3017
|
+
if (!release) {
|
|
3018
|
+
for (const held of releases.reverse())
|
|
3019
|
+
held();
|
|
3020
|
+
this.resourceLocks.wait(spec.resource, spec.mode, requestId, (granted) => {
|
|
3021
|
+
const existing = this.grantedLockReleases.get(requestId);
|
|
3022
|
+
existing?.();
|
|
3023
|
+
this.grantedLockReleases.set(requestId, granted);
|
|
3024
|
+
this.scheduleWake(true);
|
|
3025
|
+
});
|
|
3026
|
+
if (!this.lockBlocked.has(effect.id)) {
|
|
3027
|
+
this.lockBlocked.add(effect.id);
|
|
3028
|
+
this.emit({ type: 'effect.lock_blocked', effectId: effect.id, data: { resource: spec.resource, mode: spec.mode } });
|
|
3029
|
+
}
|
|
3030
|
+
return false;
|
|
3031
|
+
}
|
|
3032
|
+
releases.push(release);
|
|
3033
|
+
}
|
|
3034
|
+
if (releases.length)
|
|
3035
|
+
this.lockReleases.set(effect.id, releases);
|
|
3036
|
+
this.lockBlocked.delete(effect.id);
|
|
3037
|
+
return true;
|
|
3038
|
+
}
|
|
3039
|
+
/**
|
|
3040
|
+
* Release everything an Effect may hold on the lock manager: granted locks,
|
|
3041
|
+
* grants that arrived asynchronously, and requests still waiting in a queue.
|
|
3042
|
+
* This must run unconditionally on every terminal path. An Effect that was
|
|
3043
|
+
* cancelled while still *waiting* for a lock never had an entry in
|
|
3044
|
+
* `lockReleases`; skipping the cleanup in that case leaves a ghost request
|
|
3045
|
+
* that later receives the grant and holds the resource forever.
|
|
3046
|
+
*/
|
|
3047
|
+
releaseEffectLocks(effectId) {
|
|
3048
|
+
const releases = this.lockReleases.get(effectId);
|
|
3049
|
+
this.lockReleases.delete(effectId);
|
|
3050
|
+
this.lockBlocked.delete(effectId);
|
|
3051
|
+
if (releases)
|
|
3052
|
+
for (const release of releases.reverse())
|
|
3053
|
+
release();
|
|
3054
|
+
const effect = this.state.effects.get(effectId);
|
|
3055
|
+
const issued = new Map(this.lockRequests.get(effectId) ?? []);
|
|
3056
|
+
this.lockRequests.delete(effectId);
|
|
3057
|
+
for (const [index, spec] of [...(effect?.locks ?? [])].sort((a, b) => a.resource.localeCompare(b.resource) || a.mode.localeCompare(b.mode)).entries())
|
|
3058
|
+
issued.set(`${effectId}:${effect?.attemptId ?? ''}:${index}`, spec.resource);
|
|
3059
|
+
for (const [requestId, resource] of issued) {
|
|
3060
|
+
const granted = this.grantedLockReleases.get(requestId);
|
|
3061
|
+
if (granted !== undefined) {
|
|
3062
|
+
this.grantedLockReleases.delete(requestId);
|
|
3063
|
+
granted();
|
|
3064
|
+
}
|
|
3065
|
+
this.resourceLocks.cancelWait(resource, requestId);
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
enqueueNewReadyLanes() {
|
|
3069
|
+
for (const lane of this.state.lanes.values())
|
|
3070
|
+
if (lane.status === 'ready' && !this.ready.has(lane.id))
|
|
3071
|
+
this.enqueueLane(lane.id);
|
|
3072
|
+
}
|
|
3073
|
+
commitAgentState(agentId, state, transactionId, additionalMutations = []) {
|
|
3074
|
+
const agent = this.state.agents.get(agentId);
|
|
3075
|
+
if (!agent || agent.state === state)
|
|
3076
|
+
return false;
|
|
3077
|
+
const nextAgent = structuredClone(agent);
|
|
3078
|
+
nextAgent.state = state;
|
|
3079
|
+
const mutations = [{ op: 'setAgent', agentId, record: nextAgent }, ...additionalMutations.map((mutation) => structuredClone(mutation))];
|
|
3080
|
+
try {
|
|
3081
|
+
this.assertStorageAdmission(mutations);
|
|
3082
|
+
}
|
|
3083
|
+
catch (cause) {
|
|
3084
|
+
throw cause instanceof Error ? cause : new Error(String(cause));
|
|
3085
|
+
}
|
|
3086
|
+
commitMutationTransaction(this.state, this.mutationLog, transactionId, mutations, this.state.now, this.sessionId);
|
|
3087
|
+
this.schedulePersistence();
|
|
3088
|
+
return true;
|
|
3089
|
+
}
|
|
3090
|
+
commitLaneControlInput(lane, input, event, patch = {}) {
|
|
3091
|
+
const nextLane = structuredClone(lane);
|
|
3092
|
+
replaceResumeInput(nextLane, structuredClone(input));
|
|
3093
|
+
Object.assign(nextLane, structuredClone(patch));
|
|
3094
|
+
nextLane.version = lane.version + 1;
|
|
3095
|
+
const mutations = [{ op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event }];
|
|
3096
|
+
try {
|
|
3097
|
+
this.assertStorageAdmission(mutations);
|
|
3098
|
+
}
|
|
3099
|
+
catch (cause) {
|
|
3100
|
+
this.failLane(lane, { code: 'SESSION_STORAGE_LIMIT_EXCEEDED', message: cause instanceof Error ? cause.message : String(cause) }, patch);
|
|
3101
|
+
return false;
|
|
3102
|
+
}
|
|
3103
|
+
commitMutationTransaction(this.state, this.mutationLog, `lane:${lane.id}:control-error:${nextLane.version}`, mutations, this.state.now, this.sessionId);
|
|
3104
|
+
Object.assign(lane, nextLane);
|
|
3105
|
+
this.state.lanes.set(lane.id, lane);
|
|
3106
|
+
this.enqueueLane(lane.id);
|
|
3107
|
+
this.schedulePersistence();
|
|
3108
|
+
return true;
|
|
3109
|
+
}
|
|
3110
|
+
failLane(lane, failure, patch = {}) {
|
|
3111
|
+
const nextLane = structuredClone(lane);
|
|
3112
|
+
Object.assign(nextLane, structuredClone(patch));
|
|
3113
|
+
nextLane.status = 'failed';
|
|
3114
|
+
nextLane.failure = { error: structuredClone(failure), privacy: 'public' };
|
|
3115
|
+
nextLane.version++;
|
|
3116
|
+
const event = { type: 'lane.failed', laneId: lane.id, data: failure };
|
|
3117
|
+
try {
|
|
3118
|
+
this.assertStorageAdmission([{ op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event }]);
|
|
3119
|
+
commitMutationTransaction(this.state, this.mutationLog, `lane:${lane.id}:failed:${nextLane.version}`, [{ op: 'setLane', laneId: lane.id, record: nextLane }, { op: 'appendEvent', event }], this.state.now, this.sessionId);
|
|
3120
|
+
}
|
|
3121
|
+
catch {
|
|
3122
|
+
try {
|
|
3123
|
+
this.assertStorageAdmission([{ op: 'setLane', laneId: lane.id, record: nextLane }]);
|
|
3124
|
+
commitMutationTransaction(this.state, this.mutationLog, `lane:${lane.id}:failed:${nextLane.version}`, [{ op: 'setLane', laneId: lane.id, record: nextLane }], this.state.now, this.sessionId);
|
|
3125
|
+
}
|
|
3126
|
+
catch {
|
|
3127
|
+
return;
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
this.schedulePersistence();
|
|
3131
|
+
this.refreshWaits();
|
|
3132
|
+
}
|
|
3133
|
+
refreshWaits() {
|
|
3134
|
+
let changed = true;
|
|
3135
|
+
while (changed) {
|
|
3136
|
+
if (this.tickBudget && !this.tickBudget.canStart())
|
|
3137
|
+
return;
|
|
3138
|
+
changed = false;
|
|
3139
|
+
for (const wait of [...this.state.waits.values()]) {
|
|
3140
|
+
if (wait.state !== 'pending')
|
|
3141
|
+
continue;
|
|
3142
|
+
let budgetExhausted = false;
|
|
3143
|
+
const commitResolution = (nextWait, nextLane, extra = []) => {
|
|
3144
|
+
if (this.tickBudget && !this.tickBudget.canStart()) {
|
|
3145
|
+
budgetExhausted = true;
|
|
3146
|
+
return false;
|
|
3147
|
+
}
|
|
3148
|
+
const mutations = [{ op: 'setWait', waitId: nextWait.id, record: nextWait }];
|
|
3149
|
+
if (nextLane)
|
|
3150
|
+
mutations.push({ op: 'setLane', laneId: nextLane.id, record: nextLane });
|
|
3151
|
+
mutations.push(...extra);
|
|
3152
|
+
try {
|
|
3153
|
+
this.assertStorageAdmission(mutations);
|
|
3154
|
+
}
|
|
3155
|
+
catch {
|
|
3156
|
+
return false;
|
|
3157
|
+
}
|
|
3158
|
+
commitMutationTransaction(this.state, this.mutationLog, `wait:${nextWait.id}:${nextWait.state}:${this.state.now}`, mutations, this.state.now, this.sessionId);
|
|
3159
|
+
this.tickBudget?.consume();
|
|
3160
|
+
this.cancelWaitDeadline(nextWait.id);
|
|
3161
|
+
if (nextLane?.status === 'ready')
|
|
3162
|
+
this.enqueueLane(nextLane.id);
|
|
3163
|
+
this.schedulePersistence();
|
|
3164
|
+
changed = true;
|
|
3165
|
+
return true;
|
|
3166
|
+
};
|
|
3167
|
+
const exposeResolutionResults = (nextLane, observations) => {
|
|
3168
|
+
if (!nextLane)
|
|
3169
|
+
return;
|
|
3170
|
+
const refs = Object.values(observations).flatMap((observation) => observation.state === 'pending' ? [] : [
|
|
3171
|
+
...(observation.outcome?.resultRef === undefined ? [] : [observation.outcome.resultRef]),
|
|
3172
|
+
...(observation.outcome?.rejectedOutputRefs ?? []),
|
|
3173
|
+
]);
|
|
3174
|
+
if (!refs.length)
|
|
3175
|
+
return;
|
|
3176
|
+
if (nextLane.visibleResultRefs)
|
|
3177
|
+
for (const ref of refs)
|
|
3178
|
+
nextLane.visibleResultRefs.add(ref);
|
|
3179
|
+
else
|
|
3180
|
+
nextLane.visibleResultRefs = new Set(refs);
|
|
3181
|
+
};
|
|
3182
|
+
const observations = {};
|
|
3183
|
+
let pending = false;
|
|
3184
|
+
let unsatisfied;
|
|
3185
|
+
let satisfied = 0;
|
|
3186
|
+
let ignored = 0;
|
|
3187
|
+
let pendingCount = 0;
|
|
3188
|
+
for (const dependency of wait.spec.dependencies) {
|
|
3189
|
+
const target = dependency.target;
|
|
3190
|
+
const outcome = target.kind === 'lane' ? outcomeForSeriesMember(this.state, this.state.lanes.get(target.id), dependency.key) : this.state.effects.get(target.id)?.outcome;
|
|
3191
|
+
if (!outcome) {
|
|
3192
|
+
observations[dependency.key] = { state: 'pending', target };
|
|
3193
|
+
pending = true;
|
|
3194
|
+
pendingCount++;
|
|
3195
|
+
continue;
|
|
3196
|
+
}
|
|
3197
|
+
if (outcome.status === 'cancelled' && wait.spec.onCancelled === 'ignore') {
|
|
3198
|
+
observations[dependency.key] = { state: 'ignored', target, outcome };
|
|
3199
|
+
ignored++;
|
|
3200
|
+
}
|
|
3201
|
+
else if (dependency.condition === 'success' && outcome.status !== 'succeeded') {
|
|
3202
|
+
observations[dependency.key] = { state: 'settled', target, outcome };
|
|
3203
|
+
unsatisfied = { code: 'DEPENDENCY_FAILED', message: `${dependency.key} did not succeed` };
|
|
3204
|
+
}
|
|
3205
|
+
else {
|
|
3206
|
+
observations[dependency.key] = { state: 'settled', target, outcome };
|
|
3207
|
+
satisfied++;
|
|
3208
|
+
}
|
|
3209
|
+
}
|
|
3210
|
+
const required = wait.spec.mode === 'all' ? wait.spec.dependencies.length - ignored : wait.spec.mode === 'any' ? 1 : wait.spec.quorum;
|
|
3211
|
+
const modeSatisfied = satisfied >= required;
|
|
3212
|
+
const impossible = wait.spec.mode === 'all' ? Boolean(unsatisfied && !pending) : satisfied + pendingCount < required;
|
|
3213
|
+
const modeUnsatisfied = !modeSatisfied && (impossible || (!pending && satisfied < required));
|
|
3214
|
+
const hardFailure = wait.spec.mode === 'all' && unsatisfied !== undefined;
|
|
3215
|
+
if ((hardFailure || modeUnsatisfied) && wait.spec.onUnsatisfied === 'fail_lane') {
|
|
3216
|
+
const error = unsatisfied ?? { code: 'WAIT_QUORUM_UNREACHABLE', message: 'Wait can no longer satisfy its quorum.' };
|
|
3217
|
+
const resolution = { waitId: wait.id, status: 'unsatisfied', dependencies: observations, error };
|
|
3218
|
+
const nextWait = structuredClone(wait);
|
|
3219
|
+
nextWait.state = 'unsatisfied';
|
|
3220
|
+
nextWait.resolution = resolution;
|
|
3221
|
+
const lane = this.state.lanes.get(wait.laneId);
|
|
3222
|
+
const nextLane = lane === undefined || ['succeeded', 'failed', 'cancelled'].includes(lane.status) ? undefined : structuredClone(lane);
|
|
3223
|
+
if (nextLane) {
|
|
3224
|
+
exposeResolutionResults(nextLane, observations);
|
|
3225
|
+
const cancelled = nextLane.status === 'cancelling';
|
|
3226
|
+
nextLane.status = cancelled ? 'cancelled' : 'failed';
|
|
3227
|
+
if (cancelled)
|
|
3228
|
+
nextLane.cancelReason = nextLane.cancelReason ?? 'USER_REQUESTED';
|
|
3229
|
+
else
|
|
3230
|
+
nextLane.failure = { error: structuredClone(error), privacy: 'public' };
|
|
3231
|
+
delete nextLane.activeWaitId;
|
|
3232
|
+
nextLane.version++;
|
|
3233
|
+
}
|
|
3234
|
+
commitResolution(nextWait, nextLane, nextLane === undefined ? [] : [{ op: 'appendEvent', event: { type: nextLane.status === 'cancelled' ? 'lane.cancelled' : 'lane.failed', laneId: nextLane.id, data: nextLane.status === 'cancelled' ? nextLane.cancelReason : error } }]);
|
|
3235
|
+
}
|
|
3236
|
+
else if (modeUnsatisfied || (hardFailure && !pending)) {
|
|
3237
|
+
const resolution = { waitId: wait.id, status: 'unsatisfied', dependencies: observations, error: unsatisfied ?? { code: 'WAIT_QUORUM_UNREACHABLE', message: 'Wait can no longer satisfy its quorum.' } };
|
|
3238
|
+
const nextWait = structuredClone(wait);
|
|
3239
|
+
nextWait.state = 'unsatisfied';
|
|
3240
|
+
nextWait.resolution = resolution;
|
|
3241
|
+
const lane = this.state.lanes.get(wait.laneId);
|
|
3242
|
+
const nextLane = lane === undefined || ['succeeded', 'failed', 'cancelled'].includes(lane.status) ? undefined : structuredClone(lane);
|
|
3243
|
+
if (nextLane) {
|
|
3244
|
+
exposeResolutionResults(nextLane, observations);
|
|
3245
|
+
const cancelled = nextLane.status === 'cancelling';
|
|
3246
|
+
nextLane.status = cancelled ? 'cancelled' : 'ready';
|
|
3247
|
+
if (cancelled)
|
|
3248
|
+
nextLane.cancelReason = nextLane.cancelReason ?? 'USER_REQUESTED';
|
|
3249
|
+
delete nextLane.activeWaitId;
|
|
3250
|
+
if (!cancelled)
|
|
3251
|
+
replaceResumeInput(nextLane, { type: 'wait', resolution });
|
|
3252
|
+
}
|
|
3253
|
+
commitResolution(nextWait, nextLane, nextLane?.status === 'cancelled' ? [{ op: 'appendEvent', event: { type: 'lane.cancelled', laneId: nextLane.id, data: nextLane.cancelReason ?? 'USER_REQUESTED' } }] : []);
|
|
3254
|
+
}
|
|
3255
|
+
else if (modeSatisfied || (!pending && !unsatisfied && wait.spec.mode === 'all')) {
|
|
3256
|
+
const resolution = { waitId: wait.id, status: 'satisfied', dependencies: observations };
|
|
3257
|
+
const nextWait = structuredClone(wait);
|
|
3258
|
+
nextWait.state = 'satisfied';
|
|
3259
|
+
nextWait.resolution = resolution;
|
|
3260
|
+
const lane = this.state.lanes.get(wait.laneId);
|
|
3261
|
+
const nextLane = lane === undefined || ['succeeded', 'failed', 'cancelled'].includes(lane.status) ? undefined : structuredClone(lane);
|
|
3262
|
+
if (nextLane) {
|
|
3263
|
+
exposeResolutionResults(nextLane, observations);
|
|
3264
|
+
if (nextLane.closingResult) {
|
|
3265
|
+
let resultSequence = this.state.nextIds.result;
|
|
3266
|
+
while (this.state.results.has(`result-${resultSequence}`))
|
|
3267
|
+
resultSequence++;
|
|
3268
|
+
const resultId = `result-${resultSequence}`;
|
|
3269
|
+
const result = { id: resultId, producer: { kind: 'lane', id: nextLane.id }, value: nextLane.closingResult.value, ...resultMetadata(nextLane.closingResult.value), storageState: 'memory', pinCount: 0, privacy: nextLane.closingResult.privacy, ...(nextLane.closingResult.privacyTaints === undefined ? {} : { privacyTaints: structuredClone(nextLane.closingResult.privacyTaints) }), derivedFrom: [...(nextLane.closingResult.derivedFrom ?? [])] };
|
|
3270
|
+
delete nextLane.activeWaitId;
|
|
3271
|
+
nextLane.status = 'succeeded';
|
|
3272
|
+
delete nextLane.pendingOutcome;
|
|
3273
|
+
nextLane.resultRef = resultId;
|
|
3274
|
+
delete nextLane.closingResult;
|
|
3275
|
+
if (nextLane.visibleResultRefs)
|
|
3276
|
+
nextLane.visibleResultRefs.add(resultId);
|
|
3277
|
+
else
|
|
3278
|
+
nextLane.visibleResultRefs = new Set([resultId]);
|
|
3279
|
+
if (!commitResolution(nextWait, nextLane, [{ op: 'publishResult', record: result }, { op: 'appendEvent', event: { type: 'lane.succeeded', laneId: nextLane.id, data: resultId } }]) && !budgetExhausted) {
|
|
3280
|
+
const storageError = { code: 'SESSION_STORAGE_LIMIT_EXCEEDED', message: 'Session storage limit exceeded while committing a closing Lane result.' };
|
|
3281
|
+
const failedWait = structuredClone(wait);
|
|
3282
|
+
failedWait.state = 'unsatisfied';
|
|
3283
|
+
failedWait.resolution = { waitId: wait.id, status: 'unsatisfied', dependencies: observations, error: storageError };
|
|
3284
|
+
const failedLane = structuredClone(lane);
|
|
3285
|
+
delete failedLane.activeWaitId;
|
|
3286
|
+
failedLane.status = 'failed';
|
|
3287
|
+
failedLane.failure = { error: storageError, privacy: 'public' };
|
|
3288
|
+
failedLane.version++;
|
|
3289
|
+
commitResolution(failedWait, failedLane, [{ op: 'appendEvent', event: { type: 'lane.failed', laneId: failedLane.id, data: storageError } }]);
|
|
3290
|
+
}
|
|
3291
|
+
}
|
|
3292
|
+
else {
|
|
3293
|
+
delete nextLane.activeWaitId;
|
|
3294
|
+
if (nextLane.status === 'cancelling') {
|
|
3295
|
+
nextLane.status = 'cancelled';
|
|
3296
|
+
nextLane.cancelReason = nextLane.cancelReason ?? 'USER_REQUESTED';
|
|
3297
|
+
commitResolution(nextWait, nextLane, [{ op: 'appendEvent', event: { type: 'lane.cancelled', laneId: nextLane.id, data: nextLane.cancelReason } }]);
|
|
3298
|
+
}
|
|
3299
|
+
else {
|
|
3300
|
+
nextLane.status = 'ready';
|
|
3301
|
+
replaceResumeInput(nextLane, { type: 'wait', resolution });
|
|
3302
|
+
commitResolution(nextWait, nextLane);
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
}
|
|
3306
|
+
else {
|
|
3307
|
+
commitResolution(nextWait);
|
|
3308
|
+
}
|
|
3309
|
+
}
|
|
3310
|
+
if (budgetExhausted)
|
|
3311
|
+
return;
|
|
3312
|
+
}
|
|
3313
|
+
}
|
|
3314
|
+
this.recomputePriorityInheritance();
|
|
3315
|
+
}
|
|
3316
|
+
scheduleWaitDeadline(wait) {
|
|
3317
|
+
if (wait.state !== 'pending' || wait.spec.deadlineAt === undefined || this.waitDeadlineTimers.has(wait.id))
|
|
3318
|
+
return;
|
|
3319
|
+
const timerId = this.scheduleRuntimeTimer(wait.spec.deadlineAt, () => this.expireWait(wait.id));
|
|
3320
|
+
this.waitDeadlineTimers.set(wait.id, timerId);
|
|
3321
|
+
}
|
|
3322
|
+
cancelWaitDeadline(waitId) {
|
|
3323
|
+
const timerId = this.waitDeadlineTimers.get(waitId);
|
|
3324
|
+
if (timerId !== undefined) {
|
|
3325
|
+
this.clock.timers.cancel(timerId);
|
|
3326
|
+
this.waitDeadlineTimers.delete(waitId);
|
|
3327
|
+
}
|
|
3328
|
+
}
|
|
3329
|
+
expireWait(waitId) {
|
|
3330
|
+
const wait = this.state.waits.get(waitId);
|
|
3331
|
+
if (!wait || wait.state !== 'pending')
|
|
3332
|
+
return;
|
|
3333
|
+
const observations = {};
|
|
3334
|
+
for (const dependency of wait.spec.dependencies) {
|
|
3335
|
+
const target = dependency.target;
|
|
3336
|
+
const outcome = target.kind === 'lane' ? outcomeForSeriesMember(this.state, this.state.lanes.get(target.id), dependency.key) : this.state.effects.get(target.id)?.outcome;
|
|
3337
|
+
observations[dependency.key] = outcome === undefined ? { state: 'pending', target } : outcome.status === 'cancelled' && wait.spec.onCancelled === 'ignore' ? { state: 'ignored', target, outcome } : { state: 'settled', target, outcome };
|
|
3338
|
+
}
|
|
3339
|
+
const error = { code: 'WAIT_DEADLINE_EXCEEDED', message: 'Wait deadline exceeded.', details: { deadlineAt: wait.spec.deadlineAt ?? this.state.now } };
|
|
3340
|
+
const resolution = { waitId: wait.id, status: 'unsatisfied', dependencies: observations, error };
|
|
3341
|
+
const candidateWait = structuredClone(wait);
|
|
3342
|
+
candidateWait.state = 'unsatisfied';
|
|
3343
|
+
candidateWait.resolution = resolution;
|
|
3344
|
+
const lane = this.state.lanes.get(wait.laneId);
|
|
3345
|
+
const candidateLane = lane === undefined ? undefined : structuredClone(lane);
|
|
3346
|
+
const events = [];
|
|
3347
|
+
if (candidateLane && !['succeeded', 'failed', 'cancelled'].includes(candidateLane.status)) {
|
|
3348
|
+
delete candidateLane.activeWaitId;
|
|
3349
|
+
if (candidateLane.status === 'cancelling') {
|
|
3350
|
+
candidateLane.status = 'cancelled';
|
|
3351
|
+
candidateLane.cancelReason = candidateLane.cancelReason ?? 'USER_REQUESTED';
|
|
3352
|
+
candidateLane.version++;
|
|
3353
|
+
events.push({ type: 'lane.cancelled', laneId: candidateLane.id, data: candidateLane.cancelReason });
|
|
3354
|
+
}
|
|
3355
|
+
else if (wait.spec.onUnsatisfied === 'fail_lane') {
|
|
3356
|
+
candidateLane.status = 'failed';
|
|
3357
|
+
candidateLane.version++;
|
|
3358
|
+
events.push({ type: 'lane.failed', laneId: candidateLane.id, data: error });
|
|
3359
|
+
}
|
|
3360
|
+
else {
|
|
3361
|
+
candidateLane.status = 'ready';
|
|
3362
|
+
replaceResumeInput(candidateLane, { type: 'wait', resolution });
|
|
3363
|
+
}
|
|
3364
|
+
}
|
|
3365
|
+
events.push({ type: 'wait.deadline_exceeded', laneId: wait.laneId, data: { waitId: wait.id, deadlineAt: wait.spec.deadlineAt ?? this.state.now } });
|
|
3366
|
+
const mutations = [{ op: 'setWait', waitId: wait.id, record: candidateWait }];
|
|
3367
|
+
if (candidateLane)
|
|
3368
|
+
mutations.push({ op: 'setLane', laneId: candidateLane.id, record: candidateLane });
|
|
3369
|
+
for (const event of events)
|
|
3370
|
+
mutations.push({ op: 'appendEvent', event });
|
|
3371
|
+
try {
|
|
3372
|
+
this.assertStorageAdmission(mutations);
|
|
3373
|
+
}
|
|
3374
|
+
catch (cause) {
|
|
3375
|
+
const timerId = this.scheduleRuntimeTimer(this.clock.now(), () => this.expireWait(waitId));
|
|
3376
|
+
this.waitDeadlineTimers.set(waitId, timerId);
|
|
3377
|
+
throw cause;
|
|
3378
|
+
}
|
|
3379
|
+
this.waitDeadlineTimers.delete(waitId);
|
|
3380
|
+
commitMutationTransaction(this.state, this.mutationLog, `wait:${wait.id}:deadline:${wait.spec.deadlineAt ?? this.state.now}`, mutations, this.state.now, this.sessionId);
|
|
3381
|
+
if (candidateLane && candidateLane.status === 'ready')
|
|
3382
|
+
this.enqueueLane(candidateLane.id);
|
|
3383
|
+
this.refreshWaits();
|
|
3384
|
+
}
|
|
3385
|
+
recomputePriorityInheritance() {
|
|
3386
|
+
this.priorityInheritance.clear();
|
|
3387
|
+
for (const effect of this.state.effects.values())
|
|
3388
|
+
delete effect.inheritedFloor;
|
|
3389
|
+
const childEffectsWithPendingWait = new Set();
|
|
3390
|
+
for (const wait of this.state.waits.values())
|
|
3391
|
+
if (wait.state === 'pending')
|
|
3392
|
+
for (const dependency of wait.spec.dependencies)
|
|
3393
|
+
if (dependency.target.kind === 'effect') {
|
|
3394
|
+
const effect = this.state.effects.get(dependency.target.id);
|
|
3395
|
+
if (effect?.childAgentId)
|
|
3396
|
+
childEffectsWithPendingWait.add(effect.id);
|
|
3397
|
+
}
|
|
3398
|
+
for (const agent of this.state.agents.values()) {
|
|
3399
|
+
const root = this.state.lanes.get(agent.rootLaneId);
|
|
3400
|
+
const ownedAgentEffect = [...this.state.effects.values()].find((effect) => effect.childAgentId === agent.id);
|
|
3401
|
+
if (root && (!ownedAgentEffect || !childEffectsWithPendingWait.has(ownedAgentEffect.id)))
|
|
3402
|
+
delete root.inheritedFloor;
|
|
3403
|
+
}
|
|
3404
|
+
for (const wait of this.state.waits.values()) {
|
|
3405
|
+
if (wait.state !== 'pending')
|
|
3406
|
+
continue;
|
|
3407
|
+
const consumer = this.state.lanes.get(wait.laneId);
|
|
3408
|
+
if (!consumer)
|
|
3409
|
+
continue;
|
|
3410
|
+
for (const dependency of wait.spec.dependencies) {
|
|
3411
|
+
const target = dependency.target;
|
|
3412
|
+
if (target.kind === 'lane') {
|
|
3413
|
+
const lane = this.state.lanes.get(target.id);
|
|
3414
|
+
if (lane && lane.status === 'ready') {
|
|
3415
|
+
this.priorityInheritance.raise(lane.id, consumer.id, consumer.priority);
|
|
3416
|
+
const inheritedFloor = this.priorityInheritance.floor(lane.id);
|
|
3417
|
+
this.enqueueReadyItem({ ...readyItemFromLane(lane), ...(inheritedFloor === undefined ? {} : { inheritedFloor }) });
|
|
3418
|
+
}
|
|
3419
|
+
}
|
|
3420
|
+
else {
|
|
3421
|
+
const effect = this.state.effects.get(target.id);
|
|
3422
|
+
if (effect && effect.state === 'queued') {
|
|
3423
|
+
this.priorityInheritance.raise(effect.id, consumer.id, consumer.priority);
|
|
3424
|
+
const inheritedFloor = this.priorityInheritance.floor(effect.id);
|
|
3425
|
+
if (inheritedFloor === undefined)
|
|
3426
|
+
delete effect.inheritedFloor;
|
|
3427
|
+
else
|
|
3428
|
+
effect.inheritedFloor = inheritedFloor;
|
|
3429
|
+
if (effect.childAgentId) {
|
|
3430
|
+
const childRoot = this.state.agents.get(effect.childAgentId)?.rootLaneId;
|
|
3431
|
+
const childLane = childRoot === undefined ? undefined : this.state.lanes.get(childRoot);
|
|
3432
|
+
if (childLane && childLane.status === 'ready') {
|
|
3433
|
+
if (inheritedFloor === undefined)
|
|
3434
|
+
delete childLane.inheritedFloor;
|
|
3435
|
+
else
|
|
3436
|
+
childLane.inheritedFloor = inheritedFloor;
|
|
3437
|
+
this.enqueueReadyItem(readyItemFromLane(childLane));
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
}
|
|
3441
|
+
}
|
|
3442
|
+
}
|
|
3443
|
+
}
|
|
3444
|
+
}
|
|
3445
|
+
}
|