@dsh-enhanced/assistant-automations 0.1.7 → 0.1.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. package/README.md +37 -7
  2. package/lib/coordinator.d.ts +45 -5
  3. package/lib/coordinator.d.ts.map +1 -1
  4. package/lib/coordinator.js +418 -86
  5. package/lib/coordinator.js.map +1 -1
  6. package/lib/growth.d.ts +120 -0
  7. package/lib/growth.d.ts.map +1 -0
  8. package/lib/growth.js +383 -0
  9. package/lib/growth.js.map +1 -0
  10. package/lib/host-executors.d.ts +25 -0
  11. package/lib/host-executors.d.ts.map +1 -0
  12. package/lib/host-executors.js +181 -0
  13. package/lib/host-executors.js.map +1 -0
  14. package/lib/index.d.ts +1 -1
  15. package/lib/index.d.ts.map +1 -1
  16. package/lib/index.js +1 -1
  17. package/lib/index.js.map +1 -1
  18. package/lib/operator.d.ts +2 -0
  19. package/lib/operator.d.ts.map +1 -1
  20. package/lib/operator.js +6 -2
  21. package/lib/operator.js.map +1 -1
  22. package/lib/proposals.d.ts +2 -2
  23. package/lib/proposals.d.ts.map +1 -1
  24. package/lib/proposals.js +34 -22
  25. package/lib/proposals.js.map +1 -1
  26. package/lib/runner.d.ts +28 -2
  27. package/lib/runner.d.ts.map +1 -1
  28. package/lib/runner.js +524 -73
  29. package/lib/runner.js.map +1 -1
  30. package/lib/schedule.d.ts.map +1 -1
  31. package/lib/schedule.js +37 -9
  32. package/lib/schedule.js.map +1 -1
  33. package/lib/service.d.ts +109 -4
  34. package/lib/service.d.ts.map +1 -1
  35. package/lib/service.js +987 -16
  36. package/lib/service.js.map +1 -1
  37. package/lib/sqlite.d.ts +1 -1
  38. package/lib/sqlite.d.ts.map +1 -1
  39. package/lib/sqlite.js +369 -41
  40. package/lib/sqlite.js.map +1 -1
  41. package/lib/store.d.ts +175 -1
  42. package/lib/store.d.ts.map +1 -1
  43. package/lib/store.js +1889 -71
  44. package/lib/store.js.map +1 -1
  45. package/lib/tools.js +1 -1
  46. package/lib/tools.js.map +1 -1
  47. package/lib/types.d.ts +354 -11
  48. package/lib/types.d.ts.map +1 -1
  49. package/lib/types.js +13 -1
  50. package/lib/types.js.map +1 -1
  51. package/lib/version.d.ts +1 -1
  52. package/lib/version.d.ts.map +1 -1
  53. package/lib/version.js +1 -1
  54. package/lib/version.js.map +1 -1
  55. package/package.json +10 -8
@@ -1,8 +1,20 @@
1
+ import { HostAutomationExecutorRegistry } from './host-executors.js';
1
2
  import { AutomationStoreError } from './store.js';
3
+ import { isHostAutomationDefinition, legacyAutomationExecutionDiagnostic } from './types.js';
2
4
  export class AutomationRunnerAmbiguousError extends Error {
5
+ diagnostic;
3
6
  constructor(message, options) {
4
7
  super(message, options);
5
8
  this.name = 'AutomationRunnerAmbiguousError';
9
+ this.diagnostic = options?.diagnostic ?? legacyAutomationExecutionDiagnostic;
10
+ }
11
+ }
12
+ export class AutomationRunnerFailureError extends Error {
13
+ diagnostic;
14
+ constructor(message, diagnostic, options) {
15
+ super(message, options);
16
+ this.diagnostic = diagnostic;
17
+ this.name = 'AutomationRunnerFailureError';
6
18
  }
7
19
  }
8
20
  function sessionId(task) {
@@ -22,6 +34,17 @@ function boundedText(value, maximumBytes) {
22
34
  return undefined;
23
35
  return normalized;
24
36
  }
37
+ function incidentAlertText(incident) {
38
+ return [
39
+ `Automation incident ${incident.id}`,
40
+ `automation: ${incident.automationId}`,
41
+ `state: ${incident.state}`,
42
+ `stage: ${incident.stage}`,
43
+ `failure: ${incident.failureClass}/${incident.failurePhase}/${incident.failureCode}`,
44
+ `side effects: ${incident.sideEffectState}`,
45
+ `retryability: ${incident.retryability}`,
46
+ ].join('\n');
47
+ }
25
48
  const exposureReceiptTimeoutMs = 2_000;
26
49
  const evaluationMaxAttempts = 8;
27
50
  const evaluationRetryBaseMs = 1_000;
@@ -49,6 +72,7 @@ export class AutomationCoordinator {
49
72
  maxCatchUp;
50
73
  maxConcurrency;
51
74
  tickIntervalMs;
75
+ hostExecutors;
52
76
  active = new Map();
53
77
  activeEvaluations = new Map();
54
78
  delivery;
@@ -72,6 +96,7 @@ export class AutomationCoordinator {
72
96
  this.maxCatchUp = options.maxCatchUp;
73
97
  this.maxConcurrency = options.maxConcurrency;
74
98
  this.tickIntervalMs = options.tickIntervalMs ?? Math.max(1_000, Math.floor(options.dutyLeaseMs / 3));
99
+ this.hostExecutors = options.hostExecutors ?? new HostAutomationExecutorRegistry();
75
100
  }
76
101
  start() {
77
102
  if (this.stopped)
@@ -88,17 +113,27 @@ export class AutomationCoordinator {
88
113
  const now = this.now();
89
114
  if (!this.ensureDuty(now))
90
115
  return;
116
+ this.store.recoverExpiredCircuitProbes({ now });
91
117
  this.store.recoverExpiredTasks({ now, limit: this.maxCatchUp });
92
118
  this.dispatchPendingEvaluations();
93
119
  this.dispatchPendingEvidence();
94
120
  this.dispatchPendingDeliveries();
95
- this.store.materializeDue({ now, misfireGraceMs: this.misfireGraceMs, maxCatchUp: this.maxCatchUp });
121
+ this.dispatchPendingIncidentAlerts();
122
+ const materializeAvailability = this.hostAvailability(this.store.listDueHostExecutionRequirements({ now }), 'materialize');
123
+ this.store.materializeDue({
124
+ now,
125
+ misfireGraceMs: this.misfireGraceMs,
126
+ maxCatchUp: this.maxCatchUp,
127
+ hostAvailability: materializeAvailability,
128
+ });
96
129
  while (this.active.size < this.maxConcurrency) {
130
+ const claimAvailability = this.hostAvailability(this.store.listClaimableHostExecutionRequirements(), 'claim');
97
131
  const claimed = this.store.claimNextTask({
98
132
  ownerId: this.ownerId,
99
133
  fencingToken: this.fencingToken,
100
134
  now,
101
135
  leaseMs: this.taskLeaseMs,
136
+ hostAvailability: claimAvailability,
102
137
  });
103
138
  if (claimed === undefined)
104
139
  break;
@@ -107,6 +142,7 @@ export class AutomationCoordinator {
107
142
  this.abortRequestedActive();
108
143
  this.launch(claimed);
109
144
  }
145
+ this.dispatchPendingIncidentAlerts();
110
146
  }
111
147
  setDeliveryDispatcher(delivery) {
112
148
  this.delivery = delivery;
@@ -150,6 +186,32 @@ export class AutomationCoordinator {
150
186
  for (const value of this.active.values())
151
187
  value.controller.abort('coordinator-stop');
152
188
  await this.whenIdle();
189
+ if (this.fencingToken !== undefined) {
190
+ try {
191
+ this.store.releaseDuty({ ownerId: this.ownerId, fencingToken: this.fencingToken, now: this.now() });
192
+ }
193
+ catch (error) {
194
+ // A successor that already advanced the fence owns the row. The
195
+ // stopped coordinator must not interfere with it.
196
+ if (!(error instanceof AutomationStoreError) || error.code !== 'stale-fence')
197
+ throw error;
198
+ }
199
+ finally {
200
+ this.fencingToken = undefined;
201
+ }
202
+ }
203
+ }
204
+ hostAvailability(requirements, stage) {
205
+ return requirements.map(requirement => {
206
+ const proof = this.hostExecutors.prove(requirement.execution);
207
+ return Object.freeze({
208
+ automationId: requirement.automationId,
209
+ definitionHash: requirement.definitionHash,
210
+ stage,
211
+ available: proof.available,
212
+ reasonCode: proof.available ? 'host-executor-available' : proof.reasonCode,
213
+ });
214
+ });
153
215
  }
154
216
  ensureDuty(now) {
155
217
  if (this.fencingToken !== undefined) {
@@ -186,89 +248,221 @@ export class AutomationCoordinator {
186
248
  }
187
249
  }
188
250
  async execute(claimed, controller) {
189
- const automation = this.store.getTaskExecutionSnapshot(claimed.id);
190
- const occurrence = this.store.getOccurrence(claimed.occurrenceId);
191
- if (automation === undefined || occurrence === undefined || this.fencingToken === undefined) {
192
- throw new Error('assistant-automations: claimed task snapshot is missing');
193
- }
251
+ if (this.fencingToken === undefined)
252
+ throw new Error('assistant-automations: duty fence is missing');
194
253
  const fence = this.fencingToken;
195
254
  const freshSessionId = sessionId(claimed);
196
- const started = this.store.startTask({
197
- taskId: claimed.id,
198
- ownerId: this.ownerId,
199
- fencingToken: fence,
200
- now: this.now(),
201
- leaseMs: this.taskLeaseMs,
202
- sessionId: freshSessionId,
203
- });
255
+ let timeout;
256
+ let heartbeat;
204
257
  let timedOut = false;
205
- const timeout = setTimeout(() => {
206
- timedOut = true;
207
- controller.abort('timeout');
208
- }, automation.definition.timeoutMs);
209
- timeout.unref?.();
210
- const heartbeat = setInterval(() => {
258
+ try {
259
+ const started = this.store.startTask({
260
+ taskId: claimed.id,
261
+ ownerId: this.ownerId,
262
+ fencingToken: fence,
263
+ now: this.now(),
264
+ leaseMs: this.taskLeaseMs,
265
+ sessionId: freshSessionId,
266
+ });
267
+ let automation;
268
+ let occurrence;
211
269
  try {
212
- const now = this.now();
213
- this.store.renewDuty({
270
+ automation = this.store.getTaskExecutionSnapshot(claimed.id);
271
+ occurrence = this.store.getOccurrence(claimed.occurrenceId);
272
+ }
273
+ catch {
274
+ this.store.quarantineInvalidExecutionSnapshot({
275
+ taskId: started.id,
214
276
  ownerId: this.ownerId,
215
277
  fencingToken: fence,
216
- now,
217
- leaseMs: this.dutyLeaseMs,
278
+ now: this.now(),
218
279
  });
219
- this.store.heartbeatTask({
280
+ return;
281
+ }
282
+ if (automation === undefined || occurrence === undefined) {
283
+ this.store.quarantineInvalidExecutionSnapshot({
220
284
  taskId: started.id,
221
285
  ownerId: this.ownerId,
222
286
  fencingToken: fence,
223
- now,
224
- leaseMs: this.taskLeaseMs,
287
+ now: this.now(),
225
288
  });
289
+ return;
290
+ }
291
+ const hostExecution = isHostAutomationDefinition(automation.definition);
292
+ timeout = setTimeout(() => {
293
+ timedOut = true;
294
+ controller.abort('timeout');
295
+ }, automation.definition.timeoutMs);
296
+ timeout.unref?.();
297
+ heartbeat = setInterval(() => {
298
+ try {
299
+ const now = this.now();
300
+ this.store.renewDuty({
301
+ ownerId: this.ownerId,
302
+ fencingToken: fence,
303
+ now,
304
+ leaseMs: this.dutyLeaseMs,
305
+ });
306
+ this.store.heartbeatTask({
307
+ taskId: started.id,
308
+ ownerId: this.ownerId,
309
+ fencingToken: fence,
310
+ now,
311
+ leaseMs: this.taskLeaseMs,
312
+ });
313
+ }
314
+ catch {
315
+ controller.abort('lease-lost');
316
+ }
317
+ }, Math.max(250, Math.floor(Math.min(this.dutyLeaseMs, this.taskLeaseMs) / 3)));
318
+ heartbeat.unref?.();
319
+ let result;
320
+ try {
321
+ const admission = this.store.acquireCircuitExecutionForTask({
322
+ taskId: started.id,
323
+ now: this.now(),
324
+ leaseMs: Math.min(86_400_000, Math.max(this.taskLeaseMs, automation.definition.timeoutMs)),
325
+ });
326
+ if (admission.kind === 'blocked') {
327
+ result = {
328
+ outcome: 'failed',
329
+ output: `execution circuit is ${admission.circuit.state} for immutable definition ${admission.circuit.definitionHash}`,
330
+ usage: {},
331
+ diagnostic: Object.freeze({
332
+ schemaVersion: 1,
333
+ failureClass: admission.circuit.failureClass,
334
+ failurePhase: 'preflight',
335
+ failureCode: 'circuit-open',
336
+ promptSubmissionState: hostExecution ? 'not-applicable' : 'not-submitted',
337
+ sideEffectState: 'none',
338
+ retryability: 'after-intervention',
339
+ budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'not-reserved',
340
+ }),
341
+ };
342
+ }
343
+ else {
344
+ const runnerPromise = Promise.resolve().then(() => this.runner.run({
345
+ automation,
346
+ occurrence,
347
+ task: started,
348
+ sessionId: freshSessionId,
349
+ signal: controller.signal,
350
+ }));
351
+ // The runner owns Agent flush/disposal. It may violate cancellation,
352
+ // so its late settlement is deliberately detached from terminal I/O.
353
+ const settledRunner = runnerPromise.then(value => ({ kind: 'result', value }), error => ({ kind: 'error', error }));
354
+ let removeAbort = () => { };
355
+ const aborted = new Promise(resolve => {
356
+ const finish = () => resolve({ kind: 'aborted' });
357
+ if (controller.signal.aborted)
358
+ finish();
359
+ else {
360
+ controller.signal.addEventListener('abort', finish, { once: true });
361
+ removeAbort = () => controller.signal.removeEventListener('abort', finish);
362
+ }
363
+ });
364
+ const settled = await Promise.race([settledRunner, aborted]);
365
+ removeAbort();
366
+ void settledRunner.then(() => { });
367
+ if (settled.kind === 'aborted' || controller.signal.aborted) {
368
+ const isTimeout = timedOut;
369
+ result = {
370
+ outcome: isTimeout ? 'timed_out' : 'cancelled',
371
+ output: isTimeout ? 'automation execution timed out' : 'automation execution was cancelled',
372
+ usage: {},
373
+ diagnostic: Object.freeze({
374
+ schemaVersion: 1,
375
+ failureClass: isTimeout ? 'timeout' : 'cancelled',
376
+ failurePhase: 'unknown',
377
+ failureCode: isTimeout ? 'execution-timeout' : 'execution-cancelled',
378
+ promptSubmissionState: hostExecution ? 'not-applicable' : 'unknown',
379
+ sideEffectState: 'unknown',
380
+ retryability: 'unsafe',
381
+ budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'unknown',
382
+ }),
383
+ };
384
+ }
385
+ else if (settled.kind === 'error') {
386
+ const error = settled.error;
387
+ const diagnostic = error instanceof AutomationRunnerFailureError
388
+ || error instanceof AutomationRunnerAmbiguousError
389
+ ? error.diagnostic
390
+ : legacyAutomationExecutionDiagnostic;
391
+ result = {
392
+ outcome: error instanceof AutomationRunnerAmbiguousError ? 'unknown' : 'failed',
393
+ output: error instanceof Error ? error.message : String(error),
394
+ usage: {},
395
+ diagnostic,
396
+ };
397
+ }
398
+ else {
399
+ result = settled.value;
400
+ }
401
+ }
226
402
  }
227
403
  catch {
228
- controller.abort('lease-lost');
404
+ result = {
405
+ outcome: 'unknown',
406
+ output: 'automation circuit admission could not be proven',
407
+ usage: {},
408
+ diagnostic: Object.freeze({
409
+ schemaVersion: 1,
410
+ failureClass: 'infrastructure',
411
+ failurePhase: 'preflight',
412
+ failureCode: 'circuit-admission-failed',
413
+ promptSubmissionState: hostExecution ? 'not-applicable' : 'not-submitted',
414
+ sideEffectState: 'none',
415
+ retryability: 'after-intervention',
416
+ budgetSettlementState: automation.definition.budgetId === undefined ? 'not-required' : 'not-reserved',
417
+ }),
418
+ };
229
419
  }
230
- }, Math.max(250, Math.floor(Math.min(this.dutyLeaseMs, this.taskLeaseMs) / 3)));
231
- heartbeat.unref?.();
232
- let result;
233
- try {
234
- result = await this.runner.run({
235
- automation,
236
- occurrence,
237
- task: started,
238
- sessionId: freshSessionId,
239
- signal: controller.signal,
240
- });
241
- }
242
- catch (error) {
243
- result = {
244
- outcome: controller.signal.aborted
245
- ? 'cancelled'
246
- : error instanceof AutomationRunnerAmbiguousError ? 'unknown' : 'failed',
247
- output: error instanceof Error ? error.message : String(error),
248
- usage: {},
249
- };
250
- }
251
- finally {
252
- clearTimeout(timeout);
253
- }
254
- try {
255
- const evidenceSessionId = result.sessionId === freshSessionId ? freshSessionId : undefined;
420
+ if (timeout !== undefined)
421
+ clearTimeout(timeout);
422
+ const evidenceSessionId = occurrence.dryRun === false && result.sessionId === freshSessionId
423
+ ? freshSessionId
424
+ : undefined;
256
425
  const exposure = evidenceSessionId === undefined
257
426
  ? undefined
258
427
  : await this.getExposure(automation, evidenceSessionId);
259
- const outcome = timedOut
260
- ? 'timed_out'
261
- : controller.signal.aborted ? 'cancelled' : result.outcome;
428
+ let outcome = result.outcome;
429
+ let diagnostic = result.diagnostic ?? legacyAutomationExecutionDiagnostic;
262
430
  const outputPreview = preview(result.output);
263
- const artifactRef = this.artifacts.write(occurrence.id, {
264
- occurrenceId: occurrence.id,
265
- automationId: automation.id,
266
- taskId: started.id,
267
- outcome,
268
- sessionId: result.sessionId ?? freshSessionId,
269
- output: result.output,
270
- usage: result.usage,
271
- });
431
+ let artifactRef;
432
+ try {
433
+ artifactRef = this.artifacts.write(occurrence.id, {
434
+ occurrenceId: occurrence.id,
435
+ automationId: automation.id,
436
+ taskId: started.id,
437
+ outcome,
438
+ sessionId: result.sessionId ?? freshSessionId,
439
+ output: result.output,
440
+ usage: result.usage,
441
+ diagnostic,
442
+ });
443
+ }
444
+ catch {
445
+ // The execution may already have crossed prompt/tool boundaries. Commit
446
+ // a content-free unknown recovery receipt instead of waiting for lease
447
+ // expiry or pretending the missing artifact did not matter.
448
+ outcome = 'unknown';
449
+ diagnostic = Object.freeze({
450
+ schemaVersion: 1,
451
+ failureClass: 'infrastructure',
452
+ failurePhase: 'artifact-write',
453
+ failureCode: 'artifact-write-failed',
454
+ promptSubmissionState: diagnostic.promptSubmissionState,
455
+ sideEffectState: diagnostic.sideEffectState,
456
+ retryability: diagnostic.promptSubmissionState === 'not-submitted'
457
+ && diagnostic.sideEffectState === 'none'
458
+ && (diagnostic.budgetSettlementState === 'not-required'
459
+ || diagnostic.budgetSettlementState === 'not-reserved'
460
+ || diagnostic.budgetSettlementState === 'released')
461
+ ? 'safe'
462
+ : 'unsafe',
463
+ budgetSettlementState: diagnostic.budgetSettlementState,
464
+ });
465
+ }
272
466
  const run = this.store.completeTask({
273
467
  taskId: started.id,
274
468
  ownerId: this.ownerId,
@@ -276,10 +470,11 @@ export class AutomationCoordinator {
276
470
  now: this.now(),
277
471
  outcome,
278
472
  sessionId: result.sessionId ?? freshSessionId,
279
- artifactRef,
280
- outputPreview,
281
- usage: result.usage,
282
- ...(evidenceSessionId === undefined
473
+ ...(artifactRef === undefined ? {} : { artifactRef }),
474
+ outputPreview: artifactRef === undefined ? 'artifact persistence failed' : outputPreview,
475
+ usage: artifactRef === undefined ? {} : result.usage,
476
+ diagnostic,
477
+ ...(artifactRef === undefined || evidenceSessionId === undefined
283
478
  ? {}
284
479
  : {
285
480
  evidenceAttribution: {
@@ -293,7 +488,10 @@ export class AutomationCoordinator {
293
488
  this.dispatchRunEvaluation(run.id);
294
489
  }
295
490
  finally {
296
- clearInterval(heartbeat);
491
+ if (timeout !== undefined)
492
+ clearTimeout(timeout);
493
+ if (heartbeat !== undefined)
494
+ clearInterval(heartbeat);
297
495
  }
298
496
  }
299
497
  async getExposure(automation, actualSessionId) {
@@ -335,16 +533,36 @@ export class AutomationCoordinator {
335
533
  dispatchPendingEvidence() {
336
534
  if (this.outcomeRecorder === undefined)
337
535
  return;
338
- for (const run of this.store.listPendingEvidence(100))
339
- this.dispatchRunEvidence(run);
536
+ for (const run of this.store.listPendingEvidence(100)) {
537
+ try {
538
+ this.dispatchRunEvidence(run);
539
+ }
540
+ catch {
541
+ // One corrupt/concurrently-settled row must not starve its peers.
542
+ }
543
+ }
340
544
  }
341
545
  dispatchRunEvidence(run) {
342
546
  const recorder = this.outcomeRecorder;
343
547
  if (recorder === undefined || run.evidenceStatus !== 'pending' || run.evidence === undefined)
344
548
  return;
549
+ let proven;
345
550
  try {
346
- recorder.recordAutomationOutcome(run.evidence);
347
- this.store.completeRunEvidence({ runId: run.id, expectedStatus: 'pending', now: this.now() });
551
+ proven = this.store.getProvenProductionRun(run.id);
552
+ }
553
+ catch {
554
+ this.store.suppressRunEvidence({ runId: run.id, expectedStatus: 'pending', now: this.now() });
555
+ return;
556
+ }
557
+ if (proven === undefined
558
+ || (proven.run.status !== 'succeeded' && proven.run.status !== 'failed' && proven.run.status !== 'timed_out')
559
+ || proven.run.evidenceStatus !== 'pending' || proven.run.evidence === undefined) {
560
+ this.store.suppressRunEvidence({ runId: run.id, expectedStatus: 'pending', now: this.now() });
561
+ return;
562
+ }
563
+ try {
564
+ recorder.recordAutomationOutcome(proven.run.evidence);
565
+ this.store.completeRunEvidence({ runId: proven.run.id, expectedStatus: 'pending', now: this.now() });
348
566
  }
349
567
  catch {
350
568
  // Evidence is an outbox lane: the run remains terminal and a later leader
@@ -376,7 +594,20 @@ export class AutomationCoordinator {
376
594
  const recorder = this.evaluationRecorder;
377
595
  if (recorder === undefined || entry.status !== 'pending' || this.activeEvaluations.has(entry.id))
378
596
  return;
379
- const promise = this.recordEvaluation(recorder, entry).finally(() => this.activeEvaluations.delete(entry.id));
597
+ const authoritativeMode = this.store.getRunExecutionMode(entry.runId);
598
+ const payloadMode = entry.payload.executionMode;
599
+ if (authoritativeMode !== 'production' || (payloadMode !== undefined && payloadMode !== 'production')) {
600
+ // Repair old preview rows without ever presenting them to a trusted sink.
601
+ // `recorded` here means the durable outbox item was conclusively handled;
602
+ // no Evaluation outcome is invented.
603
+ this.store.completeEvaluation({ id: entry.id, expectedStatus: 'pending', now: this.now() });
604
+ return;
605
+ }
606
+ const normalizedEntry = payloadMode === 'production'
607
+ ? entry
608
+ : Object.freeze({ ...entry, payload: Object.freeze({ ...entry.payload, executionMode: 'production' }) });
609
+ const promise = this.recordEvaluation(recorder, normalizedEntry)
610
+ .finally(() => this.activeEvaluations.delete(entry.id));
380
611
  this.activeEvaluations.set(entry.id, promise);
381
612
  void promise.catch(() => { });
382
613
  }
@@ -420,31 +651,132 @@ export class AutomationCoordinator {
420
651
  }
421
652
  }
422
653
  dispatchPendingDeliveries() {
423
- for (const run of this.store.listPendingDeliveries(100))
424
- this.dispatchRunDelivery(run);
654
+ for (const run of this.store.listPendingDeliveries(100)) {
655
+ try {
656
+ this.dispatchRunDelivery(run);
657
+ }
658
+ catch {
659
+ // Per-row isolation: the next pending delivery remains dispatchable.
660
+ }
661
+ }
662
+ }
663
+ dispatchPendingIncidentAlerts() {
664
+ const delivery = this.delivery;
665
+ if (delivery === undefined)
666
+ return;
667
+ for (const incident of this.store.listPendingIncidentAlerts(100)) {
668
+ try {
669
+ const target = this.store.getIncidentNotificationTarget(incident.id);
670
+ if (target === undefined)
671
+ continue;
672
+ const lifecycleKey = `automation-incident:${incident.id}:g${incident.lifecycleGeneration}`;
673
+ let outbox;
674
+ if (target.kind === 'owner-route') {
675
+ const enqueue = delivery.enqueueBackgroundRoute;
676
+ const publish = delivery.publishDeliveryPresentation;
677
+ if (enqueue === undefined || publish === undefined)
678
+ continue;
679
+ outbox = incident.alertRef === undefined
680
+ ? enqueue.call(delivery, {
681
+ sourceId: 'assistant-automations-incidents',
682
+ authorityId: target.authorityId,
683
+ idempotencyKey: lifecycleKey,
684
+ // Immutable bootstrap text. The typed presentation below owns all
685
+ // mutable details and eventually replaces this exact message.
686
+ text: `Automation incident ${incident.id}`,
687
+ format: 'plain',
688
+ })
689
+ : { id: incident.alertRef, status: 'enqueued' };
690
+ publish.call(delivery, {
691
+ presentationKey: lifecycleKey,
692
+ originalOutboxIdempotencyKey: lifecycleKey,
693
+ revision: incident.presentationRevision,
694
+ presentation: {
695
+ kind: 'automation-incident',
696
+ incidentId: incident.id,
697
+ automationId: incident.automationId,
698
+ definitionHash: incident.definitionHash,
699
+ stage: incident.stage,
700
+ state: incident.state,
701
+ failureClass: incident.failureClass,
702
+ failurePhase: incident.failurePhase,
703
+ failureCode: incident.failureCode,
704
+ sideEffectState: incident.sideEffectState,
705
+ retryability: incident.retryability,
706
+ lifecycleGeneration: incident.lifecycleGeneration,
707
+ incidentRevision: incident.presentationRevision,
708
+ openedAt: incident.openedAt,
709
+ updatedAt: incident.updatedAt,
710
+ ...(incident.resolvedAt === undefined ? {} : { resolvedAt: incident.resolvedAt }),
711
+ },
712
+ });
713
+ }
714
+ else {
715
+ // Agent approval bindings are Conversation bindings, not Host owner-
716
+ // route authorities. Emit one idempotent content-free update per
717
+ // durable revision; Delivery intentionally reserves mutable incident
718
+ // presentations for stable Host owner routes.
719
+ outbox = delivery.enqueueBackground({
720
+ sourceId: 'assistant-automations-incidents',
721
+ workspace: target.workspace,
722
+ bindingId: target.bindingId,
723
+ idempotencyKey: `${lifecycleKey}:r${incident.presentationRevision}`,
724
+ text: incidentAlertText(incident),
725
+ format: 'plain',
726
+ });
727
+ }
728
+ this.store.completeIncidentAlert({
729
+ incidentId: incident.id,
730
+ expectedStatus: 'pending',
731
+ expectedLifecycleGeneration: incident.lifecycleGeneration,
732
+ expectedPresentationRevision: incident.presentationRevision,
733
+ expectedVersion: incident.version,
734
+ alertRef: outbox.id,
735
+ now: this.now(),
736
+ });
737
+ }
738
+ catch {
739
+ // Emergency stop / Policy refusal keeps the durable incident and its
740
+ // pending alert. A later authorized tick uses the same idempotency key.
741
+ }
742
+ }
425
743
  }
426
744
  dispatchRunDelivery(run) {
427
745
  if (run.deliveryStatus !== 'pending')
428
746
  return;
429
- const automation = this.store.getRunExecutionSnapshot(run.id);
747
+ let proof;
748
+ try {
749
+ proof = this.store.getProvenProductionRun(run.id);
750
+ }
751
+ catch {
752
+ this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
753
+ return;
754
+ }
755
+ const automation = proof?.automation;
756
+ const evidence = proof?.run.evidence;
430
757
  const bindingId = automation?.definition.deliveryBindingId;
431
- if (automation === undefined || bindingId === undefined)
758
+ if (proof === undefined || proof.run.status !== 'succeeded'
759
+ || evidence === undefined || automation === undefined || bindingId === undefined) {
760
+ // Unknown/legacy mode, missing immutable snapshot, hash mismatch, and a
761
+ // missing binding all reduce monotonically to a terminal no-send state.
762
+ this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
432
763
  return;
764
+ }
765
+ run = proof.run;
433
766
  const normalizedOutput = run.outputPreview.normalize('NFC').trim();
434
767
  if (normalizedOutput === '' || automation.definition.deliverySuppressExact?.includes(normalizedOutput) === true) {
435
768
  this.store.suppressRunDelivery({ runId: run.id, expectedStatus: 'pending', now: this.now() });
436
769
  return;
437
770
  }
438
- if (this.delivery === undefined)
771
+ if (this.delivery?.enqueueAutomationResult === undefined)
439
772
  return;
440
773
  try {
441
- const outbox = this.delivery.enqueueBackground({
442
- sourceId: automation.id,
774
+ const outbox = this.delivery.enqueueAutomationResult({
775
+ automationId: automation.id,
776
+ runId: run.id,
443
777
  workspace: automation.definition.workspace,
444
778
  bindingId,
445
- idempotencyKey: `automation:${run.occurrenceId}:${bindingId}`,
446
- text: run.outputPreview,
447
- format: 'markdown',
779
+ outputPreview: run.outputPreview,
448
780
  });
449
781
  this.store.completeRunDelivery({
450
782
  runId: run.id,