@dsh-enhanced/assistant-automations 0.1.7 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -7
- package/lib/coordinator.d.ts +45 -5
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +418 -86
- package/lib/coordinator.js.map +1 -1
- package/lib/growth.d.ts +120 -0
- package/lib/growth.d.ts.map +1 -0
- package/lib/growth.js +383 -0
- package/lib/growth.js.map +1 -0
- package/lib/host-executors.d.ts +25 -0
- package/lib/host-executors.d.ts.map +1 -0
- package/lib/host-executors.js +181 -0
- package/lib/host-executors.js.map +1 -0
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/operator.d.ts +2 -0
- package/lib/operator.d.ts.map +1 -1
- package/lib/operator.js +6 -2
- package/lib/operator.js.map +1 -1
- package/lib/proposals.d.ts +2 -2
- package/lib/proposals.d.ts.map +1 -1
- package/lib/proposals.js +34 -22
- package/lib/proposals.js.map +1 -1
- package/lib/runner.d.ts +28 -2
- package/lib/runner.d.ts.map +1 -1
- package/lib/runner.js +524 -73
- package/lib/runner.js.map +1 -1
- package/lib/service.d.ts +109 -4
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +987 -16
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +369 -41
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +175 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +1889 -71
- package/lib/store.js.map +1 -1
- package/lib/tools.js +1 -1
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +354 -11
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +13 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.d.ts.map +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/package.json +10 -8
package/lib/coordinator.js
CHANGED
|
@@ -1,8 +1,20 @@
|
|
|
1
|
+
import { HostAutomationExecutorRegistry } from './host-executors.js';
|
|
1
2
|
import { AutomationStoreError } from './store.js';
|
|
3
|
+
import { isHostAutomationDefinition, legacyAutomationExecutionDiagnostic } from './types.js';
|
|
2
4
|
export class AutomationRunnerAmbiguousError extends Error {
|
|
5
|
+
diagnostic;
|
|
3
6
|
constructor(message, options) {
|
|
4
7
|
super(message, options);
|
|
5
8
|
this.name = 'AutomationRunnerAmbiguousError';
|
|
9
|
+
this.diagnostic = options?.diagnostic ?? legacyAutomationExecutionDiagnostic;
|
|
10
|
+
}
|
|
11
|
+
}
|
|
12
|
+
export class AutomationRunnerFailureError extends Error {
|
|
13
|
+
diagnostic;
|
|
14
|
+
constructor(message, diagnostic, options) {
|
|
15
|
+
super(message, options);
|
|
16
|
+
this.diagnostic = diagnostic;
|
|
17
|
+
this.name = 'AutomationRunnerFailureError';
|
|
6
18
|
}
|
|
7
19
|
}
|
|
8
20
|
function sessionId(task) {
|
|
@@ -22,6 +34,17 @@ function boundedText(value, maximumBytes) {
|
|
|
22
34
|
return undefined;
|
|
23
35
|
return normalized;
|
|
24
36
|
}
|
|
37
|
+
function incidentAlertText(incident) {
|
|
38
|
+
return [
|
|
39
|
+
`Automation incident ${incident.id}`,
|
|
40
|
+
`automation: ${incident.automationId}`,
|
|
41
|
+
`state: ${incident.state}`,
|
|
42
|
+
`stage: ${incident.stage}`,
|
|
43
|
+
`failure: ${incident.failureClass}/${incident.failurePhase}/${incident.failureCode}`,
|
|
44
|
+
`side effects: ${incident.sideEffectState}`,
|
|
45
|
+
`retryability: ${incident.retryability}`,
|
|
46
|
+
].join('\n');
|
|
47
|
+
}
|
|
25
48
|
const exposureReceiptTimeoutMs = 2_000;
|
|
26
49
|
const evaluationMaxAttempts = 8;
|
|
27
50
|
const evaluationRetryBaseMs = 1_000;
|
|
@@ -49,6 +72,7 @@ export class AutomationCoordinator {
|
|
|
49
72
|
maxCatchUp;
|
|
50
73
|
maxConcurrency;
|
|
51
74
|
tickIntervalMs;
|
|
75
|
+
hostExecutors;
|
|
52
76
|
active = new Map();
|
|
53
77
|
activeEvaluations = new Map();
|
|
54
78
|
delivery;
|
|
@@ -72,6 +96,7 @@ export class AutomationCoordinator {
|
|
|
72
96
|
this.maxCatchUp = options.maxCatchUp;
|
|
73
97
|
this.maxConcurrency = options.maxConcurrency;
|
|
74
98
|
this.tickIntervalMs = options.tickIntervalMs ?? Math.max(1_000, Math.floor(options.dutyLeaseMs / 3));
|
|
99
|
+
this.hostExecutors = options.hostExecutors ?? new HostAutomationExecutorRegistry();
|
|
75
100
|
}
|
|
76
101
|
start() {
|
|
77
102
|
if (this.stopped)
|
|
@@ -88,17 +113,27 @@ export class AutomationCoordinator {
|
|
|
88
113
|
const now = this.now();
|
|
89
114
|
if (!this.ensureDuty(now))
|
|
90
115
|
return;
|
|
116
|
+
this.store.recoverExpiredCircuitProbes({ now });
|
|
91
117
|
this.store.recoverExpiredTasks({ now, limit: this.maxCatchUp });
|
|
92
118
|
this.dispatchPendingEvaluations();
|
|
93
119
|
this.dispatchPendingEvidence();
|
|
94
120
|
this.dispatchPendingDeliveries();
|
|
95
|
-
this.
|
|
121
|
+
this.dispatchPendingIncidentAlerts();
|
|
122
|
+
const materializeAvailability = this.hostAvailability(this.store.listDueHostExecutionRequirements({ now }), 'materialize');
|
|
123
|
+
this.store.materializeDue({
|
|
124
|
+
now,
|
|
125
|
+
misfireGraceMs: this.misfireGraceMs,
|
|
126
|
+
maxCatchUp: this.maxCatchUp,
|
|
127
|
+
hostAvailability: materializeAvailability,
|
|
128
|
+
});
|
|
96
129
|
while (this.active.size < this.maxConcurrency) {
|
|
130
|
+
const claimAvailability = this.hostAvailability(this.store.listClaimableHostExecutionRequirements(), 'claim');
|
|
97
131
|
const claimed = this.store.claimNextTask({
|
|
98
132
|
ownerId: this.ownerId,
|
|
99
133
|
fencingToken: this.fencingToken,
|
|
100
134
|
now,
|
|
101
135
|
leaseMs: this.taskLeaseMs,
|
|
136
|
+
hostAvailability: claimAvailability,
|
|
102
137
|
});
|
|
103
138
|
if (claimed === undefined)
|
|
104
139
|
break;
|
|
@@ -107,6 +142,7 @@ export class AutomationCoordinator {
|
|
|
107
142
|
this.abortRequestedActive();
|
|
108
143
|
this.launch(claimed);
|
|
109
144
|
}
|
|
145
|
+
this.dispatchPendingIncidentAlerts();
|
|
110
146
|
}
|
|
111
147
|
setDeliveryDispatcher(delivery) {
|
|
112
148
|
this.delivery = delivery;
|
|
@@ -150,6 +186,32 @@ export class AutomationCoordinator {
|
|
|
150
186
|
for (const value of this.active.values())
|
|
151
187
|
value.controller.abort('coordinator-stop');
|
|
152
188
|
await this.whenIdle();
|
|
189
|
+
if (this.fencingToken !== undefined) {
|
|
190
|
+
try {
|
|
191
|
+
this.store.releaseDuty({ ownerId: this.ownerId, fencingToken: this.fencingToken, now: this.now() });
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
// A successor that already advanced the fence owns the row. The
|
|
195
|
+
// stopped coordinator must not interfere with it.
|
|
196
|
+
if (!(error instanceof AutomationStoreError) || error.code !== 'stale-fence')
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
finally {
|
|
200
|
+
this.fencingToken = undefined;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
hostAvailability(requirements, stage) {
|
|
205
|
+
return requirements.map(requirement => {
|
|
206
|
+
const proof = this.hostExecutors.prove(requirement.execution);
|
|
207
|
+
return Object.freeze({
|
|
208
|
+
automationId: requirement.automationId,
|
|
209
|
+
definitionHash: requirement.definitionHash,
|
|
210
|
+
stage,
|
|
211
|
+
available: proof.available,
|
|
212
|
+
reasonCode: proof.available ? 'host-executor-available' : proof.reasonCode,
|
|
213
|
+
});
|
|
214
|
+
});
|
|
153
215
|
}
|
|
154
216
|
ensureDuty(now) {
|
|
155
217
|
if (this.fencingToken !== undefined) {
|
|
@@ -186,89 +248,221 @@ export class AutomationCoordinator {
|
|
|
186
248
|
}
|
|
187
249
|
}
|
|
188
250
|
async execute(claimed, controller) {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
if (automation === undefined || occurrence === undefined || this.fencingToken === undefined) {
|
|
192
|
-
throw new Error('assistant-automations: claimed task snapshot is missing');
|
|
193
|
-
}
|
|
251
|
+
if (this.fencingToken === undefined)
|
|
252
|
+
throw new Error('assistant-automations: duty fence is missing');
|
|
194
253
|
const fence = this.fencingToken;
|
|
195
254
|
const freshSessionId = sessionId(claimed);
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
ownerId: this.ownerId,
|
|
199
|
-
fencingToken: fence,
|
|
200
|
-
now: this.now(),
|
|
201
|
-
leaseMs: this.taskLeaseMs,
|
|
202
|
-
sessionId: freshSessionId,
|
|
203
|
-
});
|
|
255
|
+
let timeout;
|
|
256
|
+
let heartbeat;
|
|
204
257
|
let timedOut = false;
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
258
|
+
try {
|
|
259
|
+
const started = this.store.startTask({
|
|
260
|
+
taskId: claimed.id,
|
|
261
|
+
ownerId: this.ownerId,
|
|
262
|
+
fencingToken: fence,
|
|
263
|
+
now: this.now(),
|
|
264
|
+
leaseMs: this.taskLeaseMs,
|
|
265
|
+
sessionId: freshSessionId,
|
|
266
|
+
});
|
|
267
|
+
let automation;
|
|
268
|
+
let occurrence;
|
|
211
269
|
try {
|
|
212
|
-
|
|
213
|
-
this.store.
|
|
270
|
+
automation = this.store.getTaskExecutionSnapshot(claimed.id);
|
|
271
|
+
occurrence = this.store.getOccurrence(claimed.occurrenceId);
|
|
272
|
+
}
|
|
273
|
+
catch {
|
|
274
|
+
this.store.quarantineInvalidExecutionSnapshot({
|
|
275
|
+
taskId: started.id,
|
|
214
276
|
ownerId: this.ownerId,
|
|
215
277
|
fencingToken: fence,
|
|
216
|
-
now,
|
|
217
|
-
leaseMs: this.dutyLeaseMs,
|
|
278
|
+
now: this.now(),
|
|
218
279
|
});
|
|
219
|
-
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
if (automation === undefined || occurrence === undefined) {
|
|
283
|
+
this.store.quarantineInvalidExecutionSnapshot({
|
|
220
284
|
taskId: started.id,
|
|
221
285
|
ownerId: this.ownerId,
|
|
222
286
|
fencingToken: fence,
|
|
223
|
-
now,
|
|
224
|
-
leaseMs: this.taskLeaseMs,
|
|
287
|
+
now: this.now(),
|
|
225
288
|
});
|
|
289
|
+
return;
|
|
290
|
+
}
|
|
291
|
+
const hostExecution = isHostAutomationDefinition(automation.definition);
|
|
292
|
+
timeout = setTimeout(() => {
|
|
293
|
+
timedOut = true;
|
|
294
|
+
controller.abort('timeout');
|
|
295
|
+
}, automation.definition.timeoutMs);
|
|
296
|
+
timeout.unref?.();
|
|
297
|
+
heartbeat = setInterval(() => {
|
|
298
|
+
try {
|
|
299
|
+
const now = this.now();
|
|
300
|
+
this.store.renewDuty({
|
|
301
|
+
ownerId: this.ownerId,
|
|
302
|
+
fencingToken: fence,
|
|
303
|
+
now,
|
|
304
|
+
leaseMs: this.dutyLeaseMs,
|
|
305
|
+
});
|
|
306
|
+
this.store.heartbeatTask({
|
|
307
|
+
taskId: started.id,
|
|
308
|
+
ownerId: this.ownerId,
|
|
309
|
+
fencingToken: fence,
|
|
310
|
+
now,
|
|
311
|
+
leaseMs: this.taskLeaseMs,
|
|
312
|
+
});
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
controller.abort('lease-lost');
|
|
316
|
+
}
|
|
317
|
+
}, Math.max(250, Math.floor(Math.min(this.dutyLeaseMs, this.taskLeaseMs) / 3)));
|
|
318
|
+
heartbeat.unref?.();
|
|
319
|
+
let result;
|
|
320
|
+
try {
|
|
321
|
+
const admission = this.store.acquireCircuitExecutionForTask({
|
|
322
|
+
taskId: started.id,
|
|
323
|
+
now: this.now(),
|
|
324
|
+
leaseMs: Math.min(86_400_000, Math.max(this.taskLeaseMs, automation.definition.timeoutMs)),
|
|
325
|
+
});
|
|
326
|
+
if (admission.kind === 'blocked') {
|
|
327
|
+
result = {
|
|
328
|
+
outcome: 'failed',
|
|
329
|
+
output: `execution circuit is ${admission.circuit.state} for immutable definition ${admission.circuit.definitionHash}`,
|
|
330
|
+
usage: {},
|
|
331
|
+
diagnostic: Object.freeze({
|
|
332
|
+
schemaVersion: 1,
|
|
333
|
+
failureClass: admission.circuit.failureClass,
|
|
334
|
+
failurePhase: 'preflight',
|
|
335
|
+
failureCode: 'circuit-open',
|
|
336
|
+
promptSubmissionState: hostExecution ? 'not-applicable' : 'not-submitted',
|
|
337
|
+
sideEffectState: 'none',
|
|
338
|
+
retryability: 'after-intervention',
|
|
339
|
+
budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'not-reserved',
|
|
340
|
+
}),
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
else {
|
|
344
|
+
const runnerPromise = Promise.resolve().then(() => this.runner.run({
|
|
345
|
+
automation,
|
|
346
|
+
occurrence,
|
|
347
|
+
task: started,
|
|
348
|
+
sessionId: freshSessionId,
|
|
349
|
+
signal: controller.signal,
|
|
350
|
+
}));
|
|
351
|
+
// The runner owns Agent flush/disposal. It may violate cancellation,
|
|
352
|
+
// so its late settlement is deliberately detached from terminal I/O.
|
|
353
|
+
const settledRunner = runnerPromise.then(value => ({ kind: 'result', value }), error => ({ kind: 'error', error }));
|
|
354
|
+
let removeAbort = () => { };
|
|
355
|
+
const aborted = new Promise(resolve => {
|
|
356
|
+
const finish = () => resolve({ kind: 'aborted' });
|
|
357
|
+
if (controller.signal.aborted)
|
|
358
|
+
finish();
|
|
359
|
+
else {
|
|
360
|
+
controller.signal.addEventListener('abort', finish, { once: true });
|
|
361
|
+
removeAbort = () => controller.signal.removeEventListener('abort', finish);
|
|
362
|
+
}
|
|
363
|
+
});
|
|
364
|
+
const settled = await Promise.race([settledRunner, aborted]);
|
|
365
|
+
removeAbort();
|
|
366
|
+
void settledRunner.then(() => { });
|
|
367
|
+
if (settled.kind === 'aborted' || controller.signal.aborted) {
|
|
368
|
+
const isTimeout = timedOut;
|
|
369
|
+
result = {
|
|
370
|
+
outcome: isTimeout ? 'timed_out' : 'cancelled',
|
|
371
|
+
output: isTimeout ? 'automation execution timed out' : 'automation execution was cancelled',
|
|
372
|
+
usage: {},
|
|
373
|
+
diagnostic: Object.freeze({
|
|
374
|
+
schemaVersion: 1,
|
|
375
|
+
failureClass: isTimeout ? 'timeout' : 'cancelled',
|
|
376
|
+
failurePhase: 'unknown',
|
|
377
|
+
failureCode: isTimeout ? 'execution-timeout' : 'execution-cancelled',
|
|
378
|
+
promptSubmissionState: hostExecution ? 'not-applicable' : 'unknown',
|
|
379
|
+
sideEffectState: 'unknown',
|
|
380
|
+
retryability: 'unsafe',
|
|
381
|
+
budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'unknown',
|
|
382
|
+
}),
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
else if (settled.kind === 'error') {
|
|
386
|
+
const error = settled.error;
|
|
387
|
+
const diagnostic = error instanceof AutomationRunnerFailureError
|
|
388
|
+
|| error instanceof AutomationRunnerAmbiguousError
|
|
389
|
+
? error.diagnostic
|
|
390
|
+
: legacyAutomationExecutionDiagnostic;
|
|
391
|
+
result = {
|
|
392
|
+
outcome: error instanceof AutomationRunnerAmbiguousError ? 'unknown' : 'failed',
|
|
393
|
+
output: error instanceof Error ? error.message : String(error),
|
|
394
|
+
usage: {},
|
|
395
|
+
diagnostic,
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
result = settled.value;
|
|
400
|
+
}
|
|
401
|
+
}
|
|
226
402
|
}
|
|
227
403
|
catch {
|
|
228
|
-
|
|
404
|
+
result = {
|
|
405
|
+
outcome: 'unknown',
|
|
406
|
+
output: 'automation circuit admission could not be proven',
|
|
407
|
+
usage: {},
|
|
408
|
+
diagnostic: Object.freeze({
|
|
409
|
+
schemaVersion: 1,
|
|
410
|
+
failureClass: 'infrastructure',
|
|
411
|
+
failurePhase: 'preflight',
|
|
412
|
+
failureCode: 'circuit-admission-failed',
|
|
413
|
+
promptSubmissionState: hostExecution ? 'not-applicable' : 'not-submitted',
|
|
414
|
+
sideEffectState: 'none',
|
|
415
|
+
retryability: 'after-intervention',
|
|
416
|
+
budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'not-reserved',
|
|
417
|
+
}),
|
|
418
|
+
};
|
|
229
419
|
}
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
automation,
|
|
236
|
-
occurrence,
|
|
237
|
-
task: started,
|
|
238
|
-
sessionId: freshSessionId,
|
|
239
|
-
signal: controller.signal,
|
|
240
|
-
});
|
|
241
|
-
}
|
|
242
|
-
catch (error) {
|
|
243
|
-
result = {
|
|
244
|
-
outcome: controller.signal.aborted
|
|
245
|
-
? 'cancelled'
|
|
246
|
-
: error instanceof AutomationRunnerAmbiguousError ? 'unknown' : 'failed',
|
|
247
|
-
output: error instanceof Error ? error.message : String(error),
|
|
248
|
-
usage: {},
|
|
249
|
-
};
|
|
250
|
-
}
|
|
251
|
-
finally {
|
|
252
|
-
clearTimeout(timeout);
|
|
253
|
-
}
|
|
254
|
-
try {
|
|
255
|
-
const evidenceSessionId = result.sessionId === freshSessionId ? freshSessionId : undefined;
|
|
420
|
+
if (timeout !== undefined)
|
|
421
|
+
clearTimeout(timeout);
|
|
422
|
+
const evidenceSessionId = occurrence.dryRun === false && result.sessionId === freshSessionId
|
|
423
|
+
? freshSessionId
|
|
424
|
+
: undefined;
|
|
256
425
|
const exposure = evidenceSessionId === undefined
|
|
257
426
|
? undefined
|
|
258
427
|
: await this.getExposure(automation, evidenceSessionId);
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
: controller.signal.aborted ? 'cancelled' : result.outcome;
|
|
428
|
+
let outcome = result.outcome;
|
|
429
|
+
let diagnostic = result.diagnostic ?? legacyAutomationExecutionDiagnostic;
|
|
262
430
|
const outputPreview = preview(result.output);
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
431
|
+
let artifactRef;
|
|
432
|
+
try {
|
|
433
|
+
artifactRef = this.artifacts.write(occurrence.id, {
|
|
434
|
+
occurrenceId: occurrence.id,
|
|
435
|
+
automationId: automation.id,
|
|
436
|
+
taskId: started.id,
|
|
437
|
+
outcome,
|
|
438
|
+
sessionId: result.sessionId ?? freshSessionId,
|
|
439
|
+
output: result.output,
|
|
440
|
+
usage: result.usage,
|
|
441
|
+
diagnostic,
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
catch {
|
|
445
|
+
// The execution may already have crossed prompt/tool boundaries. Commit
|
|
446
|
+
// a content-free unknown recovery receipt instead of waiting for lease
|
|
447
|
+
// expiry or pretending the missing artifact did not matter.
|
|
448
|
+
outcome = 'unknown';
|
|
449
|
+
diagnostic = Object.freeze({
|
|
450
|
+
schemaVersion: 1,
|
|
451
|
+
failureClass: 'infrastructure',
|
|
452
|
+
failurePhase: 'artifact-write',
|
|
453
|
+
failureCode: 'artifact-write-failed',
|
|
454
|
+
promptSubmissionState: diagnostic.promptSubmissionState,
|
|
455
|
+
sideEffectState: diagnostic.sideEffectState,
|
|
456
|
+
retryability: diagnostic.promptSubmissionState === 'not-submitted'
|
|
457
|
+
&& diagnostic.sideEffectState === 'none'
|
|
458
|
+
&& (diagnostic.budgetSettlementState === 'not-required'
|
|
459
|
+
|| diagnostic.budgetSettlementState === 'not-reserved'
|
|
460
|
+
|| diagnostic.budgetSettlementState === 'released')
|
|
461
|
+
? 'safe'
|
|
462
|
+
: 'unsafe',
|
|
463
|
+
budgetSettlementState: diagnostic.budgetSettlementState,
|
|
464
|
+
});
|
|
465
|
+
}
|
|
272
466
|
const run = this.store.completeTask({
|
|
273
467
|
taskId: started.id,
|
|
274
468
|
ownerId: this.ownerId,
|
|
@@ -276,10 +470,11 @@ export class AutomationCoordinator {
|
|
|
276
470
|
now: this.now(),
|
|
277
471
|
outcome,
|
|
278
472
|
sessionId: result.sessionId ?? freshSessionId,
|
|
279
|
-
artifactRef,
|
|
280
|
-
outputPreview,
|
|
281
|
-
usage: result.usage,
|
|
282
|
-
|
|
473
|
+
...(artifactRef === undefined ? {} : { artifactRef }),
|
|
474
|
+
outputPreview: artifactRef === undefined ? 'artifact persistence failed' : outputPreview,
|
|
475
|
+
usage: artifactRef === undefined ? {} : result.usage,
|
|
476
|
+
diagnostic,
|
|
477
|
+
...(artifactRef === undefined || evidenceSessionId === undefined
|
|
283
478
|
? {}
|
|
284
479
|
: {
|
|
285
480
|
evidenceAttribution: {
|
|
@@ -293,7 +488,10 @@ export class AutomationCoordinator {
|
|
|
293
488
|
this.dispatchRunEvaluation(run.id);
|
|
294
489
|
}
|
|
295
490
|
finally {
|
|
296
|
-
|
|
491
|
+
if (timeout !== undefined)
|
|
492
|
+
clearTimeout(timeout);
|
|
493
|
+
if (heartbeat !== undefined)
|
|
494
|
+
clearInterval(heartbeat);
|
|
297
495
|
}
|
|
298
496
|
}
|
|
299
497
|
async getExposure(automation, actualSessionId) {
|
|
@@ -335,16 +533,36 @@ export class AutomationCoordinator {
|
|
|
335
533
|
dispatchPendingEvidence() {
|
|
336
534
|
if (this.outcomeRecorder === undefined)
|
|
337
535
|
return;
|
|
338
|
-
for (const run of this.store.listPendingEvidence(100))
|
|
339
|
-
|
|
536
|
+
for (const run of this.store.listPendingEvidence(100)) {
|
|
537
|
+
try {
|
|
538
|
+
this.dispatchRunEvidence(run);
|
|
539
|
+
}
|
|
540
|
+
catch {
|
|
541
|
+
// One corrupt/concurrently-settled row must not starve its peers.
|
|
542
|
+
}
|
|
543
|
+
}
|
|
340
544
|
}
|
|
341
545
|
dispatchRunEvidence(run) {
|
|
342
546
|
const recorder = this.outcomeRecorder;
|
|
343
547
|
if (recorder === undefined || run.evidenceStatus !== 'pending' || run.evidence === undefined)
|
|
344
548
|
return;
|
|
549
|
+
let proven;
|
|
345
550
|
try {
|
|
346
|
-
|
|
347
|
-
|
|
551
|
+
proven = this.store.getProvenProductionRun(run.id);
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
this.store.suppressRunEvidence({ runId: run.id, expectedStatus: 'pending', now: this.now() });
|
|
555
|
+
return;
|
|
556
|
+
}
|
|
557
|
+
if (proven === undefined
|
|
558
|
+
|| (proven.run.status !== 'succeeded' && proven.run.status !== 'failed' && proven.run.status !== 'timed_out')
|
|
559
|
+
|| proven.run.evidenceStatus !== 'pending' || proven.run.evidence === undefined) {
|
|
560
|
+
this.store.suppressRunEvidence({ runId: run.id, expectedStatus: 'pending', now: this.now() });
|
|
561
|
+
return;
|
|
562
|
+
}
|
|
563
|
+
try {
|
|
564
|
+
recorder.recordAutomationOutcome(proven.run.evidence);
|
|
565
|
+
this.store.completeRunEvidence({ runId: proven.run.id, expectedStatus: 'pending', now: this.now() });
|
|
348
566
|
}
|
|
349
567
|
catch {
|
|
350
568
|
// Evidence is an outbox lane: the run remains terminal and a later leader
|
|
@@ -376,7 +594,20 @@ export class AutomationCoordinator {
|
|
|
376
594
|
const recorder = this.evaluationRecorder;
|
|
377
595
|
if (recorder === undefined || entry.status !== 'pending' || this.activeEvaluations.has(entry.id))
|
|
378
596
|
return;
|
|
379
|
-
const
|
|
597
|
+
const authoritativeMode = this.store.getRunExecutionMode(entry.runId);
|
|
598
|
+
const payloadMode = entry.payload.executionMode;
|
|
599
|
+
if (authoritativeMode !== 'production' || (payloadMode !== undefined && payloadMode !== 'production')) {
|
|
600
|
+
// Repair old preview rows without ever presenting them to a trusted sink.
|
|
601
|
+
// `recorded` here means the durable outbox item was conclusively handled;
|
|
602
|
+
// no Evaluation outcome is invented.
|
|
603
|
+
this.store.completeEvaluation({ id: entry.id, expectedStatus: 'pending', now: this.now() });
|
|
604
|
+
return;
|
|
605
|
+
}
|
|
606
|
+
const normalizedEntry = payloadMode === 'production'
|
|
607
|
+
? entry
|
|
608
|
+
: Object.freeze({ ...entry, payload: Object.freeze({ ...entry.payload, executionMode: 'production' }) });
|
|
609
|
+
const promise = this.recordEvaluation(recorder, normalizedEntry)
|
|
610
|
+
.finally(() => this.activeEvaluations.delete(entry.id));
|
|
380
611
|
this.activeEvaluations.set(entry.id, promise);
|
|
381
612
|
void promise.catch(() => { });
|
|
382
613
|
}
|
|
@@ -420,31 +651,132 @@ export class AutomationCoordinator {
|
|
|
420
651
|
}
|
|
421
652
|
}
|
|
422
653
|
dispatchPendingDeliveries() {
|
|
423
|
-
for (const run of this.store.listPendingDeliveries(100))
|
|
424
|
-
|
|
654
|
+
for (const run of this.store.listPendingDeliveries(100)) {
|
|
655
|
+
try {
|
|
656
|
+
this.dispatchRunDelivery(run);
|
|
657
|
+
}
|
|
658
|
+
catch {
|
|
659
|
+
// Per-row isolation: the next pending delivery remains dispatchable.
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
dispatchPendingIncidentAlerts() {
|
|
664
|
+
const delivery = this.delivery;
|
|
665
|
+
if (delivery === undefined)
|
|
666
|
+
return;
|
|
667
|
+
for (const incident of this.store.listPendingIncidentAlerts(100)) {
|
|
668
|
+
try {
|
|
669
|
+
const target = this.store.getIncidentNotificationTarget(incident.id);
|
|
670
|
+
if (target === undefined)
|
|
671
|
+
continue;
|
|
672
|
+
const lifecycleKey = `automation-incident:${incident.id}:g${incident.lifecycleGeneration}`;
|
|
673
|
+
let outbox;
|
|
674
|
+
if (target.kind === 'owner-route') {
|
|
675
|
+
const enqueue = delivery.enqueueBackgroundRoute;
|
|
676
|
+
const publish = delivery.publishDeliveryPresentation;
|
|
677
|
+
if (enqueue === undefined || publish === undefined)
|
|
678
|
+
continue;
|
|
679
|
+
outbox = incident.alertRef === undefined
|
|
680
|
+
? enqueue.call(delivery, {
|
|
681
|
+
sourceId: 'assistant-automations-incidents',
|
|
682
|
+
authorityId: target.authorityId,
|
|
683
|
+
idempotencyKey: lifecycleKey,
|
|
684
|
+
// Immutable bootstrap text. The typed presentation below owns all
|
|
685
|
+
// mutable details and eventually replaces this exact message.
|
|
686
|
+
text: `Automation incident ${incident.id}`,
|
|
687
|
+
format: 'plain',
|
|
688
|
+
})
|
|
689
|
+
: { id: incident.alertRef, status: 'enqueued' };
|
|
690
|
+
publish.call(delivery, {
|
|
691
|
+
presentationKey: lifecycleKey,
|
|
692
|
+
originalOutboxIdempotencyKey: lifecycleKey,
|
|
693
|
+
revision: incident.presentationRevision,
|
|
694
|
+
presentation: {
|
|
695
|
+
kind: 'automation-incident',
|
|
696
|
+
incidentId: incident.id,
|
|
697
|
+
automationId: incident.automationId,
|
|
698
|
+
definitionHash: incident.definitionHash,
|
|
699
|
+
stage: incident.stage,
|
|
700
|
+
state: incident.state,
|
|
701
|
+
failureClass: incident.failureClass,
|
|
702
|
+
failurePhase: incident.failurePhase,
|
|
703
|
+
failureCode: incident.failureCode,
|
|
704
|
+
sideEffectState: incident.sideEffectState,
|
|
705
|
+
retryability: incident.retryability,
|
|
706
|
+
lifecycleGeneration: incident.lifecycleGeneration,
|
|
707
|
+
incidentRevision: incident.presentationRevision,
|
|
708
|
+
openedAt: incident.openedAt,
|
|
709
|
+
updatedAt: incident.updatedAt,
|
|
710
|
+
...(incident.resolvedAt === undefined ? {} : { resolvedAt: incident.resolvedAt }),
|
|
711
|
+
},
|
|
712
|
+
});
|
|
713
|
+
}
|
|
714
|
+
else {
|
|
715
|
+
// Agent approval bindings are Conversation bindings, not Host owner-
|
|
716
|
+
// route authorities. Emit one idempotent content-free update per
|
|
717
|
+
// durable revision; Delivery intentionally reserves mutable incident
|
|
718
|
+
// presentations for stable Host owner routes.
|
|
719
|
+
outbox = delivery.enqueueBackground({
|
|
720
|
+
sourceId: 'assistant-automations-incidents',
|
|
721
|
+
workspace: target.workspace,
|
|
722
|
+
bindingId: target.bindingId,
|
|
723
|
+
idempotencyKey: `${lifecycleKey}:r${incident.presentationRevision}`,
|
|
724
|
+
text: incidentAlertText(incident),
|
|
725
|
+
format: 'plain',
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
this.store.completeIncidentAlert({
|
|
729
|
+
incidentId: incident.id,
|
|
730
|
+
expectedStatus: 'pending',
|
|
731
|
+
expectedLifecycleGeneration: incident.lifecycleGeneration,
|
|
732
|
+
expectedPresentationRevision: incident.presentationRevision,
|
|
733
|
+
expectedVersion: incident.version,
|
|
734
|
+
alertRef: outbox.id,
|
|
735
|
+
now: this.now(),
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
catch {
|
|
739
|
+
// Emergency stop / Policy refusal keeps the durable incident and its
|
|
740
|
+
// pending alert. A later authorized tick uses the same idempotency key.
|
|
741
|
+
}
|
|
742
|
+
}
|
|
425
743
|
}
|
|
426
744
|
dispatchRunDelivery(run) {
|
|
427
745
|
if (run.deliveryStatus !== 'pending')
|
|
428
746
|
return;
|
|
429
|
-
|
|
747
|
+
let proof;
|
|
748
|
+
try {
|
|
749
|
+
proof = this.store.getProvenProductionRun(run.id);
|
|
750
|
+
}
|
|
751
|
+
catch {
|
|
752
|
+
this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
const automation = proof?.automation;
|
|
756
|
+
const evidence = proof?.run.evidence;
|
|
430
757
|
const bindingId = automation?.definition.deliveryBindingId;
|
|
431
|
-
if (
|
|
758
|
+
if (proof === undefined || proof.run.status !== 'succeeded'
|
|
759
|
+
|| evidence === undefined || automation === undefined || bindingId === undefined) {
|
|
760
|
+
// Unknown/legacy mode, missing immutable snapshot, hash mismatch, and a
|
|
761
|
+
// missing binding all reduce monotonically to a terminal no-send state.
|
|
762
|
+
this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
|
|
432
763
|
return;
|
|
764
|
+
}
|
|
765
|
+
run = proof.run;
|
|
433
766
|
const normalizedOutput = run.outputPreview.normalize('NFC').trim();
|
|
434
767
|
if (normalizedOutput === '' || automation.definition.deliverySuppressExact?.includes(normalizedOutput) === true) {
|
|
435
768
|
this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
|
|
436
769
|
return;
|
|
437
770
|
}
|
|
438
|
-
if (this.delivery === undefined)
|
|
771
|
+
if (this.delivery?.enqueueAutomationResult === undefined)
|
|
439
772
|
return;
|
|
440
773
|
try {
|
|
441
|
-
const outbox = this.delivery.
|
|
442
|
-
|
|
774
|
+
const outbox = this.delivery.enqueueAutomationResult({
|
|
775
|
+
automationId: automation.id,
|
|
776
|
+
runId: run.id,
|
|
443
777
|
workspace: automation.definition.workspace,
|
|
444
778
|
bindingId,
|
|
445
|
-
|
|
446
|
-
text: run.outputPreview,
|
|
447
|
-
format: 'markdown',
|
|
779
|
+
outputPreview: run.outputPreview,
|
|
448
780
|
});
|
|
449
781
|
this.store.completeRunDelivery({
|
|
450
782
|
runId: run.id,
|