@dsh-enhanced/assistant-automations 0.1.7 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -7
- package/lib/coordinator.d.ts +45 -5
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +418 -86
- package/lib/coordinator.js.map +1 -1
- package/lib/growth.d.ts +120 -0
- package/lib/growth.d.ts.map +1 -0
- package/lib/growth.js +383 -0
- package/lib/growth.js.map +1 -0
- package/lib/host-executors.d.ts +25 -0
- package/lib/host-executors.d.ts.map +1 -0
- package/lib/host-executors.js +181 -0
- package/lib/host-executors.js.map +1 -0
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/operator.d.ts +2 -0
- package/lib/operator.d.ts.map +1 -1
- package/lib/operator.js +6 -2
- package/lib/operator.js.map +1 -1
- package/lib/proposals.d.ts +2 -2
- package/lib/proposals.d.ts.map +1 -1
- package/lib/proposals.js +34 -22
- package/lib/proposals.js.map +1 -1
- package/lib/runner.d.ts +28 -2
- package/lib/runner.d.ts.map +1 -1
- package/lib/runner.js +524 -73
- package/lib/runner.js.map +1 -1
- package/lib/service.d.ts +109 -4
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +987 -16
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +369 -41
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +175 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +1889 -71
- package/lib/store.js.map +1 -1
- package/lib/tools.js +1 -1
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +354 -11
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +13 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.d.ts.map +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/package.json +10 -8
package/lib/store.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
2
|
import { isAbsolute, resolve } from 'node:path';
|
|
3
3
|
import { dueOccurrences, nextOccurrence, previousOccurrence, validateSchedule, } from './schedule.js';
|
|
4
4
|
import { AutomationDatabaseError, openAutomationDatabase } from './sqlite.js';
|
|
5
|
+
import { isHostAutomationDefinition, legacyAutomationExecutionDiagnostic } from './types.js';
|
|
5
6
|
export class AutomationStoreError extends Error {
|
|
6
7
|
code;
|
|
7
8
|
constructor(code, message) {
|
|
@@ -10,6 +11,55 @@ export class AutomationStoreError extends Error {
|
|
|
10
11
|
this.name = 'AutomationStoreError';
|
|
11
12
|
}
|
|
12
13
|
}
|
|
14
|
+
// Growth's owner is deliberately spelled out at the Store admission boundary
|
|
15
|
+
// instead of treating every paused system-owned definition as runnable. Keep
|
|
16
|
+
// this coupled to GrowthAutomationStore.createExecutionTask(), which is the
|
|
17
|
+
// only writer of the task/occurrence/operation tuple below.
|
|
18
|
+
const growthAutomationOwner = 'assistant-growth-experiments';
|
|
19
|
+
/**
|
|
20
|
+
* SQL-only admission proof for the one paused-definition exception.
|
|
21
|
+
*
|
|
22
|
+
* A queued task is eligible only when it is the exact task created for a
|
|
23
|
+
* pending Growth shadow or canary operation. The artifact must still bind to
|
|
24
|
+
* the current definition revision, and the immutable occurrence key must name
|
|
25
|
+
* the same operation. This intentionally cannot match ordinary paused work.
|
|
26
|
+
*/
|
|
27
|
+
function pausedGrowthTaskAdmission(taskAlias, definitionAlias) {
|
|
28
|
+
return `
|
|
29
|
+
EXISTS (
|
|
30
|
+
SELECT 1
|
|
31
|
+
FROM automation_growth_artifacts artifact
|
|
32
|
+
JOIN automation_occurrences occurrence
|
|
33
|
+
ON occurrence.id = ${taskAlias}.occurrence_id
|
|
34
|
+
AND occurrence.automation_id = artifact.automation_id
|
|
35
|
+
JOIN automation_growth_operations operation
|
|
36
|
+
ON occurrence.trigger_key = CASE operation.operation_kind
|
|
37
|
+
WHEN 'shadow' THEN 'growth-shadow:' || operation.operation_id
|
|
38
|
+
WHEN 'canary' THEN 'growth-canary:' || operation.operation_id
|
|
39
|
+
ELSE ''
|
|
40
|
+
END
|
|
41
|
+
WHERE artifact.automation_id = ${definitionAlias}.id
|
|
42
|
+
AND artifact.definition_hash = ${definitionAlias}.definition_hash
|
|
43
|
+
AND artifact.definition_version = ${definitionAlias}.version
|
|
44
|
+
AND occurrence.trigger_kind = 'manual'
|
|
45
|
+
AND operation.status = 'pending'
|
|
46
|
+
AND (
|
|
47
|
+
(
|
|
48
|
+
artifact.shadow_task_id = ${taskAlias}.id
|
|
49
|
+
AND artifact.state = 'paused'
|
|
50
|
+
AND occurrence.dry_run = 1
|
|
51
|
+
AND operation.operation_kind = 'shadow'
|
|
52
|
+
)
|
|
53
|
+
OR (
|
|
54
|
+
artifact.canary_task_id = ${taskAlias}.id
|
|
55
|
+
AND artifact.state = 'canary-pending'
|
|
56
|
+
AND occurrence.dry_run = 0
|
|
57
|
+
AND operation.operation_kind = 'canary'
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
`;
|
|
62
|
+
}
|
|
13
63
|
function hash(value) {
|
|
14
64
|
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
15
65
|
}
|
|
@@ -56,24 +106,27 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
56
106
|
throw new AutomationStoreError('invalid-definition', 'automation definition must be an object');
|
|
57
107
|
}
|
|
58
108
|
const input = value;
|
|
59
|
-
const
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
109
|
+
const host = typeof input['execution'] === 'object' && input['execution'] !== null
|
|
110
|
+
&& !Array.isArray(input['execution'])
|
|
111
|
+
&& input['execution']['kind'] === 'host';
|
|
112
|
+
const commonAllowed = [
|
|
113
|
+
'agentPreset', 'budgetAmount', 'budgetId', 'maxRetries', 'misfire', 'name', 'overlap',
|
|
114
|
+
'principal', 'retrySafety', 'schedule', 'timeoutMs', 'workspace',
|
|
115
|
+
];
|
|
116
|
+
const allowed = new Set(host
|
|
117
|
+
? [...commonAllowed, 'execution']
|
|
118
|
+
: [
|
|
119
|
+
...commonAllowed, 'allowedTools', 'approvalBindingId', 'deliveryBindingId', 'deliverySuppressExact',
|
|
120
|
+
'maxOutputTokens', 'maxToolCalls', 'model', 'prompt', 'provider',
|
|
121
|
+
]);
|
|
64
122
|
if (Object.keys(input).some(key => !allowed.has(key))) {
|
|
65
123
|
throw new AutomationStoreError('invalid-definition', 'automation definition contains an unknown field');
|
|
66
124
|
}
|
|
67
|
-
|
|
125
|
+
const workspace = text(input['workspace'], 'workspace', 4_096);
|
|
126
|
+
const agentPreset = text(input['agentPreset'], 'agentPreset', 200);
|
|
127
|
+
if (!isAbsolute(workspace)) {
|
|
68
128
|
throw new AutomationStoreError('invalid-definition', 'automation workspace must be absolute');
|
|
69
129
|
}
|
|
70
|
-
if (!Array.isArray(input['allowedTools']) || input['allowedTools'].length > maxAllowedTools) {
|
|
71
|
-
throw new AutomationStoreError('invalid-definition', 'allowedTools must be a bounded array');
|
|
72
|
-
}
|
|
73
|
-
const allowedTools = input['allowedTools'].map(value => text(value, 'allowedTools item', 200));
|
|
74
|
-
if (new Set(allowedTools).size !== allowedTools.length) {
|
|
75
|
-
throw new AutomationStoreError('invalid-definition', 'allowedTools contains a duplicate');
|
|
76
|
-
}
|
|
77
130
|
const overlap = input['overlap'];
|
|
78
131
|
if (overlap !== 'skip' && overlap !== 'queue-one' && overlap !== 'cancel-previous') {
|
|
79
132
|
throw new AutomationStoreError('invalid-definition', 'invalid overlap policy');
|
|
@@ -89,6 +142,66 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
89
142
|
if ((budgetId === undefined) !== (budgetAmount === undefined)) {
|
|
90
143
|
throw new AutomationStoreError('invalid-definition', 'budgetId and budgetAmount must be supplied together');
|
|
91
144
|
}
|
|
145
|
+
const control = {
|
|
146
|
+
name: text(input['name'], 'name', 500),
|
|
147
|
+
schedule: validateSchedule(input['schedule']),
|
|
148
|
+
workspace,
|
|
149
|
+
agentPreset,
|
|
150
|
+
timeoutMs: safeInteger(input['timeoutMs'], 'timeoutMs', 1_000, 86_400_000),
|
|
151
|
+
misfire: misfire(input['misfire']),
|
|
152
|
+
overlap: overlap,
|
|
153
|
+
retrySafety: retrySafety,
|
|
154
|
+
maxRetries: safeInteger(input['maxRetries'], 'maxRetries', 0, 10),
|
|
155
|
+
principal: text(input['principal'], 'principal', 500),
|
|
156
|
+
...(budgetId === undefined ? {} : { budgetId, budgetAmount: budgetAmount }),
|
|
157
|
+
};
|
|
158
|
+
if (host) {
|
|
159
|
+
const rawExecution = input['execution'];
|
|
160
|
+
const executionAllowed = new Set([
|
|
161
|
+
'activationNonce', 'catalogDigest', 'executorContractVersion', 'executorId', 'kind',
|
|
162
|
+
'ownerRouteId', 'runbookId', 'runbookVersion', 'scopeDigest', 'targetScope',
|
|
163
|
+
]);
|
|
164
|
+
if (Object.keys(rawExecution).some(key => !executionAllowed.has(key))) {
|
|
165
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution spec contains an unknown field');
|
|
166
|
+
}
|
|
167
|
+
const rawTarget = rawExecution['targetScope'];
|
|
168
|
+
if (typeof rawTarget !== 'object' || rawTarget === null || Array.isArray(rawTarget)
|
|
169
|
+
|| Object.keys(rawTarget).some(key => key !== 'workspace' && key !== 'preset')) {
|
|
170
|
+
throw new AutomationStoreError('invalid-definition', 'Host targetScope must be an exact object');
|
|
171
|
+
}
|
|
172
|
+
const target = rawTarget;
|
|
173
|
+
const targetWorkspace = text(target['workspace'], 'execution.targetScope.workspace', 4_096);
|
|
174
|
+
const targetPreset = text(target['preset'], 'execution.targetScope.preset', 200);
|
|
175
|
+
if (targetWorkspace !== workspace || targetPreset !== agentPreset) {
|
|
176
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution targetScope must exactly match automation workspace and agentPreset');
|
|
177
|
+
}
|
|
178
|
+
const catalogDigest = text(rawExecution['catalogDigest'], 'execution.catalogDigest', 64).toLowerCase();
|
|
179
|
+
if (!/^[a-f0-9]{64}$/u.test(catalogDigest)) {
|
|
180
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution catalogDigest must be a SHA-256 digest');
|
|
181
|
+
}
|
|
182
|
+
const execution = Object.freeze({
|
|
183
|
+
kind: 'host',
|
|
184
|
+
executorId: text(rawExecution['executorId'], 'execution.executorId', 200),
|
|
185
|
+
executorContractVersion: safeInteger(rawExecution['executorContractVersion'], 'execution.executorContractVersion', 1, 1_000_000),
|
|
186
|
+
runbookId: text(rawExecution['runbookId'], 'execution.runbookId', 200),
|
|
187
|
+
runbookVersion: safeInteger(rawExecution['runbookVersion'], 'execution.runbookVersion', 1, 1_000_000),
|
|
188
|
+
catalogDigest,
|
|
189
|
+
targetScope: Object.freeze({ workspace: targetWorkspace, preset: targetPreset }),
|
|
190
|
+
// Never trust a caller-provided digest. Recompute the canonical tuple
|
|
191
|
+
// that is actually stored in the immutable definition.
|
|
192
|
+
scopeDigest: hash([targetWorkspace, targetPreset]),
|
|
193
|
+
ownerRouteId: text(rawExecution['ownerRouteId'], 'execution.ownerRouteId', 500),
|
|
194
|
+
activationNonce: text(rawExecution['activationNonce'], 'execution.activationNonce', 500),
|
|
195
|
+
});
|
|
196
|
+
return Object.freeze({ ...control, execution });
|
|
197
|
+
}
|
|
198
|
+
if (!Array.isArray(input['allowedTools']) || input['allowedTools'].length > maxAllowedTools) {
|
|
199
|
+
throw new AutomationStoreError('invalid-definition', 'allowedTools must be a bounded array');
|
|
200
|
+
}
|
|
201
|
+
const allowedTools = input['allowedTools'].map(value => text(value, 'allowedTools item', 200));
|
|
202
|
+
if (new Set(allowedTools).size !== allowedTools.length) {
|
|
203
|
+
throw new AutomationStoreError('invalid-definition', 'allowedTools contains a duplicate');
|
|
204
|
+
}
|
|
92
205
|
const deliverySuppressExact = input['deliverySuppressExact'] === undefined
|
|
93
206
|
? undefined
|
|
94
207
|
: (() => {
|
|
@@ -105,23 +218,26 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
105
218
|
throw new AutomationStoreError('invalid-definition', 'deliverySuppressExact requires deliveryBindingId');
|
|
106
219
|
}
|
|
107
220
|
return Object.freeze({
|
|
108
|
-
name:
|
|
221
|
+
name: control.name,
|
|
109
222
|
prompt: text(input['prompt'], 'prompt', maxPromptBytes),
|
|
110
|
-
schedule:
|
|
111
|
-
workspace:
|
|
112
|
-
agentPreset:
|
|
223
|
+
schedule: control.schedule,
|
|
224
|
+
workspace: control.workspace,
|
|
225
|
+
agentPreset: control.agentPreset,
|
|
113
226
|
provider: text(input['provider'], 'provider', 200),
|
|
114
227
|
model: text(input['model'], 'model', 500),
|
|
115
228
|
allowedTools: Object.freeze(allowedTools),
|
|
116
|
-
timeoutMs:
|
|
229
|
+
timeoutMs: control.timeoutMs,
|
|
117
230
|
maxOutputTokens: safeInteger(input['maxOutputTokens'], 'maxOutputTokens', 1, 1_000_000),
|
|
118
231
|
maxToolCalls: safeInteger(input['maxToolCalls'], 'maxToolCalls', 0, 10_000),
|
|
119
|
-
misfire: misfire
|
|
120
|
-
overlap: overlap,
|
|
121
|
-
retrySafety: retrySafety,
|
|
122
|
-
maxRetries:
|
|
123
|
-
principal:
|
|
232
|
+
misfire: control.misfire,
|
|
233
|
+
overlap: control.overlap,
|
|
234
|
+
retrySafety: control.retrySafety,
|
|
235
|
+
maxRetries: control.maxRetries,
|
|
236
|
+
principal: control.principal,
|
|
124
237
|
...(budgetId === undefined ? {} : { budgetId, budgetAmount: budgetAmount }),
|
|
238
|
+
...(input['approvalBindingId'] === undefined
|
|
239
|
+
? {}
|
|
240
|
+
: { approvalBindingId: text(input['approvalBindingId'], 'approvalBindingId', 500) }),
|
|
125
241
|
...(input['deliveryBindingId'] === undefined
|
|
126
242
|
? {}
|
|
127
243
|
: { deliveryBindingId: text(input['deliveryBindingId'], 'deliveryBindingId', 500) }),
|
|
@@ -202,6 +318,24 @@ function record(row) {
|
|
|
202
318
|
version: row.version,
|
|
203
319
|
});
|
|
204
320
|
}
|
|
321
|
+
function incidentNotificationRoute(definition) {
|
|
322
|
+
if (isHostAutomationDefinition(definition)) {
|
|
323
|
+
return {
|
|
324
|
+
id: definition.execution.ownerRouteId,
|
|
325
|
+
target: Object.freeze({ kind: 'owner-route', authorityId: definition.execution.ownerRouteId }),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
// Approval delegation is the narrow owner-facing route for a background
|
|
329
|
+
// Agent. deliveryBindingId is retained only for legacy definitions that
|
|
330
|
+
// predate the separate approval route.
|
|
331
|
+
const bindingId = definition.approvalBindingId ?? definition.deliveryBindingId;
|
|
332
|
+
return bindingId === undefined
|
|
333
|
+
? undefined
|
|
334
|
+
: {
|
|
335
|
+
id: bindingId,
|
|
336
|
+
target: Object.freeze({ kind: 'binding', bindingId, workspace: definition.workspace }),
|
|
337
|
+
};
|
|
338
|
+
}
|
|
205
339
|
function deliveryStatus(value) {
|
|
206
340
|
if (value === null)
|
|
207
341
|
return undefined;
|
|
@@ -214,6 +348,159 @@ function evidenceStatus(value) {
|
|
|
214
348
|
return value;
|
|
215
349
|
throw new AutomationStoreError('invalid-state', 'automation run has an invalid evidence status');
|
|
216
350
|
}
|
|
351
|
+
const failureClasses = new Set([
|
|
352
|
+
'none', 'budget', 'cancelled', 'configuration', 'execution', 'infrastructure',
|
|
353
|
+
'policy', 'provider', 'timeout', 'unknown',
|
|
354
|
+
]);
|
|
355
|
+
const failurePhases = new Set([
|
|
356
|
+
'none', 'artifact-write', 'agent-creation', 'agent-disposal', 'agent-setup', 'budget-reservation', 'budget-settlement',
|
|
357
|
+
'executor-availability', 'host-execution', 'model-execution', 'preflight', 'preset-resolution', 'prompt-submission', 'recovery',
|
|
358
|
+
'session-flush', 'terminal-commit', 'unknown',
|
|
359
|
+
]);
|
|
360
|
+
const promptStates = new Set([
|
|
361
|
+
'not-applicable', 'not-submitted', 'submitted', 'unknown',
|
|
362
|
+
]);
|
|
363
|
+
const sideEffectStates = new Set([
|
|
364
|
+
'none', 'possible', 'unknown',
|
|
365
|
+
]);
|
|
366
|
+
const retryabilities = new Set([
|
|
367
|
+
'safe', 'unsafe', 'after-intervention', 'unknown',
|
|
368
|
+
]);
|
|
369
|
+
const budgetSettlementStates = new Set([
|
|
370
|
+
'not-required', 'not-reserved', 'reserved', 'released', 'finalized', 'unknown',
|
|
371
|
+
]);
|
|
372
|
+
function executionDiagnostic(value, errorCode) {
|
|
373
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
374
|
+
throw new AutomationStoreError(errorCode, 'automation execution diagnostic must be an object');
|
|
375
|
+
}
|
|
376
|
+
const input = value;
|
|
377
|
+
const expected = new Set([
|
|
378
|
+
'schemaVersion', 'failureClass', 'failurePhase', 'failureCode', 'promptSubmissionState',
|
|
379
|
+
'sideEffectState', 'retryability', 'budgetSettlementState',
|
|
380
|
+
]);
|
|
381
|
+
if (Object.keys(input).some(key => !expected.has(key))
|
|
382
|
+
|| input['schemaVersion'] !== 1
|
|
383
|
+
|| !failureClasses.has(input['failureClass'])
|
|
384
|
+
|| !failurePhases.has(input['failurePhase'])
|
|
385
|
+
|| typeof input['failureCode'] !== 'string'
|
|
386
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(input['failureCode'])
|
|
387
|
+
|| !promptStates.has(input['promptSubmissionState'])
|
|
388
|
+
|| !sideEffectStates.has(input['sideEffectState'])
|
|
389
|
+
|| !retryabilities.has(input['retryability'])
|
|
390
|
+
|| !budgetSettlementStates.has(input['budgetSettlementState'])) {
|
|
391
|
+
throw new AutomationStoreError(errorCode, 'automation execution diagnostic is invalid');
|
|
392
|
+
}
|
|
393
|
+
return Object.freeze({
|
|
394
|
+
schemaVersion: 1,
|
|
395
|
+
failureClass: input['failureClass'],
|
|
396
|
+
failurePhase: input['failurePhase'],
|
|
397
|
+
failureCode: input['failureCode'],
|
|
398
|
+
promptSubmissionState: input['promptSubmissionState'],
|
|
399
|
+
sideEffectState: input['sideEffectState'],
|
|
400
|
+
retryability: input['retryability'],
|
|
401
|
+
budgetSettlementState: input['budgetSettlementState'],
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
function assertDiagnosticOutcome(outcome, value, definition) {
|
|
405
|
+
const noneTuple = value.failureClass === 'none'
|
|
406
|
+
&& value.failurePhase === 'none'
|
|
407
|
+
&& value.failureCode === 'none';
|
|
408
|
+
const hasPartialNoneTuple = value.failureClass === 'none'
|
|
409
|
+
|| value.failurePhase === 'none'
|
|
410
|
+
|| value.failureCode === 'none';
|
|
411
|
+
const host = isHostAutomationDefinition(definition);
|
|
412
|
+
const hasBudget = definition.budgetId !== undefined;
|
|
413
|
+
const effectfulSettlement = value.budgetSettlementState === 'reserved'
|
|
414
|
+
|| value.budgetSettlementState === 'finalized'
|
|
415
|
+
|| value.budgetSettlementState === 'unknown';
|
|
416
|
+
const postPromptPhase = value.failurePhase === 'prompt-submission'
|
|
417
|
+
|| value.failurePhase === 'model-execution'
|
|
418
|
+
|| value.failurePhase === 'session-flush'
|
|
419
|
+
|| value.failurePhase === 'agent-disposal';
|
|
420
|
+
if ((!noneTuple && hasPartialNoneTuple)
|
|
421
|
+
|| (outcome === 'succeeded' && !noneTuple)
|
|
422
|
+
|| (outcome !== 'succeeded' && noneTuple)
|
|
423
|
+
|| (value.retryability === 'safe' && value.sideEffectState !== 'none')
|
|
424
|
+
|| (noneTuple && (value.retryability === 'after-intervention' || value.retryability === 'unknown'))
|
|
425
|
+
|| (host && value.promptSubmissionState !== 'not-applicable')
|
|
426
|
+
|| (!host && value.promptSubmissionState === 'not-applicable')
|
|
427
|
+
|| (host && (value.failurePhase === 'agent-creation' || value.failurePhase === 'agent-setup'
|
|
428
|
+
|| value.failurePhase === 'prompt-submission' || value.failurePhase === 'model-execution'
|
|
429
|
+
|| value.failurePhase === 'session-flush' || value.failurePhase === 'agent-disposal'))
|
|
430
|
+
|| (!host && (value.failurePhase === 'executor-availability' || value.failurePhase === 'host-execution'))
|
|
431
|
+
|| (!host && value.promptSubmissionState === 'submitted' && value.sideEffectState === 'none')
|
|
432
|
+
|| (!host && value.promptSubmissionState === 'submitted' && value.retryability !== 'unsafe')
|
|
433
|
+
|| (!host && postPromptPhase && value.promptSubmissionState === 'not-submitted')
|
|
434
|
+
|| (!hasBudget && value.budgetSettlementState !== 'not-required')
|
|
435
|
+
|| (hasBudget && value.budgetSettlementState === 'not-required')
|
|
436
|
+
|| (effectfulSettlement && value.sideEffectState === 'none')
|
|
437
|
+
|| (effectfulSettlement && value.retryability !== 'unsafe')) {
|
|
438
|
+
throw new AutomationStoreError('invalid-definition', 'fresh execution diagnostic contradicts the terminal outcome');
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
function isLegacyExecutionDiagnostic(value) {
|
|
442
|
+
return value.failureClass === legacyAutomationExecutionDiagnostic.failureClass
|
|
443
|
+
&& value.failurePhase === legacyAutomationExecutionDiagnostic.failurePhase
|
|
444
|
+
&& value.failureCode === legacyAutomationExecutionDiagnostic.failureCode
|
|
445
|
+
&& value.promptSubmissionState === legacyAutomationExecutionDiagnostic.promptSubmissionState
|
|
446
|
+
&& value.sideEffectState === legacyAutomationExecutionDiagnostic.sideEffectState
|
|
447
|
+
&& value.retryability === legacyAutomationExecutionDiagnostic.retryability
|
|
448
|
+
&& value.budgetSettlementState === legacyAutomationExecutionDiagnostic.budgetSettlementState;
|
|
449
|
+
}
|
|
450
|
+
function parseExecutionDiagnostic(row) {
|
|
451
|
+
if (Buffer.byteLength(row.diagnostic_json, 'utf8') > 4_096) {
|
|
452
|
+
throw new AutomationStoreError('invalid-state', 'automation execution diagnostic exceeds 4096 bytes');
|
|
453
|
+
}
|
|
454
|
+
try {
|
|
455
|
+
return executionDiagnostic(JSON.parse(row.diagnostic_json), 'invalid-state');
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
if (error instanceof AutomationStoreError)
|
|
459
|
+
throw error;
|
|
460
|
+
throw new AutomationStoreError('invalid-state', 'automation execution diagnostic contains invalid JSON');
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
function incident(row) {
|
|
464
|
+
if (!/^[a-f0-9]{64}$/u.test(row.definition_hash)
|
|
465
|
+
|| (row.stage !== 'materialize' && row.stage !== 'claim' && row.stage !== 'terminal')
|
|
466
|
+
|| (row.state !== 'open' && row.state !== 'recovering' && row.state !== 'resolved')
|
|
467
|
+
|| !failureClasses.has(row.failure_class)
|
|
468
|
+
|| !failurePhases.has(row.failure_phase)
|
|
469
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(row.failure_code)
|
|
470
|
+
|| !sideEffectStates.has(row.side_effect_state)
|
|
471
|
+
|| !retryabilities.has(row.retryability)
|
|
472
|
+
|| (row.alert_status !== 'pending' && row.alert_status !== 'enqueued' && row.alert_status !== 'suppressed')
|
|
473
|
+
|| !Number.isSafeInteger(row.lifecycle_generation) || row.lifecycle_generation < 1
|
|
474
|
+
|| !Number.isSafeInteger(row.presentation_revision) || row.presentation_revision < 1
|
|
475
|
+
|| !Number.isSafeInteger(row.opened_at) || !Number.isSafeInteger(row.updated_at)
|
|
476
|
+
|| !Number.isSafeInteger(row.version) || row.version < 1
|
|
477
|
+
|| (row.state === 'resolved') !== (row.resolved_at !== null)
|
|
478
|
+
|| (row.alert_status === 'enqueued' && row.alert_ref === null)) {
|
|
479
|
+
throw new AutomationStoreError('invalid-state', 'automation incident row is invalid');
|
|
480
|
+
}
|
|
481
|
+
return Object.freeze({
|
|
482
|
+
id: row.id,
|
|
483
|
+
automationId: row.automation_id,
|
|
484
|
+
definitionHash: row.definition_hash,
|
|
485
|
+
stage: row.stage,
|
|
486
|
+
state: row.state,
|
|
487
|
+
failureClass: row.failure_class,
|
|
488
|
+
failurePhase: row.failure_phase,
|
|
489
|
+
failureCode: row.failure_code,
|
|
490
|
+
sideEffectState: row.side_effect_state,
|
|
491
|
+
retryability: row.retryability,
|
|
492
|
+
notificationRouteId: row.notification_route_id,
|
|
493
|
+
lifecycleGeneration: row.lifecycle_generation,
|
|
494
|
+
presentationRevision: row.presentation_revision,
|
|
495
|
+
alertStatus: row.alert_status,
|
|
496
|
+
...(row.alert_ref === null ? {} : { alertRef: row.alert_ref }),
|
|
497
|
+
...(row.run_id === null ? {} : { runId: row.run_id }),
|
|
498
|
+
openedAt: row.opened_at,
|
|
499
|
+
updatedAt: row.updated_at,
|
|
500
|
+
...(row.resolved_at === null ? {} : { resolvedAt: row.resolved_at }),
|
|
501
|
+
version: row.version,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
217
504
|
function parseEvidence(row) {
|
|
218
505
|
const status = evidenceStatus(row.evidence_status);
|
|
219
506
|
if (status === 'suppressed') {
|
|
@@ -225,7 +512,43 @@ function parseEvidence(row) {
|
|
|
225
512
|
if (row.evidence_json === null || Buffer.byteLength(row.evidence_json, 'utf8') > 16_384) {
|
|
226
513
|
throw new AutomationStoreError('invalid-state', 'pending automation evidence must be bounded JSON');
|
|
227
514
|
}
|
|
228
|
-
|
|
515
|
+
let value;
|
|
516
|
+
try {
|
|
517
|
+
value = JSON.parse(row.evidence_json);
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence contains invalid JSON');
|
|
521
|
+
}
|
|
522
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
523
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence must be an object');
|
|
524
|
+
}
|
|
525
|
+
const input = value;
|
|
526
|
+
const allowed = new Set([
|
|
527
|
+
'situation', 'outcome', 'detail', 'idempotencyKey', 'occurredAt', 'workspace',
|
|
528
|
+
'agentPreset', 'automationId', 'runId', 'sessionId', 'ruleId', 'guidanceVersion',
|
|
529
|
+
]);
|
|
530
|
+
const stringWithin = (field, maximum) => typeof input[field] === 'string'
|
|
531
|
+
&& input[field].normalize('NFC').trim() !== ''
|
|
532
|
+
&& Buffer.byteLength(input[field], 'utf8') <= maximum;
|
|
533
|
+
if (Object.keys(input).some(key => !allowed.has(key))
|
|
534
|
+
|| !stringWithin('situation', 200)
|
|
535
|
+
|| (input['outcome'] !== 'succeeded' && input['outcome'] !== 'failed')
|
|
536
|
+
|| !stringWithin('detail', 2_000)
|
|
537
|
+
|| !stringWithin('idempotencyKey', 500)
|
|
538
|
+
|| !Number.isSafeInteger(input['occurredAt'])
|
|
539
|
+
|| !stringWithin('workspace', 4_096) || !isAbsolute(input['workspace'])
|
|
540
|
+
|| !stringWithin('agentPreset', 200)
|
|
541
|
+
|| !stringWithin('automationId', 500)
|
|
542
|
+
|| !stringWithin('runId', 500)
|
|
543
|
+
|| (input['sessionId'] !== undefined && !stringWithin('sessionId', 500))
|
|
544
|
+
|| (input['ruleId'] !== undefined && !stringWithin('ruleId', 200))
|
|
545
|
+
|| (input['guidanceVersion'] !== undefined
|
|
546
|
+
&& (!Number.isSafeInteger(input['guidanceVersion'])
|
|
547
|
+
|| input['guidanceVersion'] < 1
|
|
548
|
+
|| input['guidanceVersion'] > 1_000_000_000))
|
|
549
|
+
|| ((input['ruleId'] === undefined) !== (input['guidanceVersion'] === undefined))) {
|
|
550
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence payload is invalid');
|
|
551
|
+
}
|
|
229
552
|
return Object.freeze(value);
|
|
230
553
|
}
|
|
231
554
|
function occurrence(row) {
|
|
@@ -271,6 +594,9 @@ function run(row) {
|
|
|
271
594
|
...(row.artifact_ref === null ? {} : { artifactRef: row.artifact_ref }),
|
|
272
595
|
outputPreview: row.output_preview,
|
|
273
596
|
usage: Object.freeze(JSON.parse(row.usage_json)),
|
|
597
|
+
executionMode: row.execution_mode,
|
|
598
|
+
...(row.definition_hash === null ? {} : { definitionHash: row.definition_hash }),
|
|
599
|
+
diagnostic: parseExecutionDiagnostic(row),
|
|
274
600
|
...(status === undefined ? {} : { deliveryStatus: status }),
|
|
275
601
|
...(row.delivery_ref === null ? {} : { deliveryRef: row.delivery_ref }),
|
|
276
602
|
evidenceStatus: evidenceStatus(row.evidence_status),
|
|
@@ -279,6 +605,141 @@ function run(row) {
|
|
|
279
605
|
updatedAt: row.updated_at,
|
|
280
606
|
});
|
|
281
607
|
}
|
|
608
|
+
function circuit(row) {
|
|
609
|
+
const durableProbe = row.state === 'half-open' || row.state === 'probing';
|
|
610
|
+
const probeTupleValid = durableProbe
|
|
611
|
+
? typeof row.probe_token === 'string'
|
|
612
|
+
&& /^probe-[a-f0-9-]{36}$/u.test(row.probe_token)
|
|
613
|
+
&& Number.isSafeInteger(row.probe_lease_until)
|
|
614
|
+
&& (row.state === 'probing'
|
|
615
|
+
? typeof row.probe_task_id === 'string' && row.probe_task_id !== ''
|
|
616
|
+
: row.probe_task_id === null)
|
|
617
|
+
: row.probe_token === null && row.probe_lease_until === null && row.probe_task_id === null;
|
|
618
|
+
if (!Number.isSafeInteger(row.opened_at) || !Number.isSafeInteger(row.updated_at)
|
|
619
|
+
|| !Number.isSafeInteger(row.version) || row.version < 1
|
|
620
|
+
|| !/^[a-f0-9]{64}$/u.test(row.definition_hash)
|
|
621
|
+
|| !['closed', 'half-open', 'open', 'probing'].includes(row.state)
|
|
622
|
+
|| !['budget', 'configuration', 'policy'].includes(row.failure_class)
|
|
623
|
+
|| !probeTupleValid
|
|
624
|
+
|| !failurePhases.has(row.failure_phase)
|
|
625
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(row.failure_code)) {
|
|
626
|
+
throw new AutomationStoreError('invalid-state', 'automation circuit row is invalid');
|
|
627
|
+
}
|
|
628
|
+
return Object.freeze({
|
|
629
|
+
automationId: row.automation_id,
|
|
630
|
+
definitionHash: row.definition_hash,
|
|
631
|
+
state: row.state,
|
|
632
|
+
failureClass: row.failure_class,
|
|
633
|
+
failurePhase: row.failure_phase,
|
|
634
|
+
failureCode: row.failure_code,
|
|
635
|
+
openedAt: row.opened_at,
|
|
636
|
+
updatedAt: row.updated_at,
|
|
637
|
+
...(row.probe_token === null ? {} : { probeToken: row.probe_token }),
|
|
638
|
+
...(row.probe_lease_until === null ? {} : { probeLeaseUntil: row.probe_lease_until }),
|
|
639
|
+
...(row.probe_task_id === null ? {} : { probeTaskId: row.probe_task_id }),
|
|
640
|
+
version: row.version,
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
function parseCircuitResult(json) {
|
|
644
|
+
if (Buffer.byteLength(json, 'utf8') > 4_096) {
|
|
645
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result exceeds its bound');
|
|
646
|
+
}
|
|
647
|
+
let value;
|
|
648
|
+
try {
|
|
649
|
+
value = JSON.parse(json);
|
|
650
|
+
}
|
|
651
|
+
catch {
|
|
652
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result is invalid JSON');
|
|
653
|
+
}
|
|
654
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
655
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result is invalid');
|
|
656
|
+
}
|
|
657
|
+
const input = value;
|
|
658
|
+
return circuit({
|
|
659
|
+
automation_id: input['automationId'],
|
|
660
|
+
definition_hash: input['definitionHash'],
|
|
661
|
+
state: input['state'],
|
|
662
|
+
failure_class: input['failureClass'],
|
|
663
|
+
failure_phase: input['failurePhase'],
|
|
664
|
+
failure_code: input['failureCode'],
|
|
665
|
+
opened_at: input['openedAt'],
|
|
666
|
+
updated_at: input['updatedAt'],
|
|
667
|
+
probe_token: (input['probeToken'] ?? null),
|
|
668
|
+
probe_lease_until: (input['probeLeaseUntil'] ?? null),
|
|
669
|
+
probe_task_id: (input['probeTaskId'] ?? null),
|
|
670
|
+
version: input['version'],
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
function parseCircuitCanaryResult(json) {
|
|
674
|
+
if (Buffer.byteLength(json, 'utf8') > 8_192) {
|
|
675
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result exceeds its bound');
|
|
676
|
+
}
|
|
677
|
+
let value;
|
|
678
|
+
try {
|
|
679
|
+
value = JSON.parse(json);
|
|
680
|
+
}
|
|
681
|
+
catch {
|
|
682
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result is invalid JSON');
|
|
683
|
+
}
|
|
684
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)
|
|
685
|
+
|| Object.keys(value).sort().join(',') !== 'circuit,executionMode,occurrenceId,taskId') {
|
|
686
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result is invalid');
|
|
687
|
+
}
|
|
688
|
+
const input = value;
|
|
689
|
+
const occurrenceId = input['occurrenceId'];
|
|
690
|
+
const taskId = input['taskId'];
|
|
691
|
+
if (typeof occurrenceId !== 'string' || !/^occ-[a-f0-9]{64}$/u.test(occurrenceId)
|
|
692
|
+
|| typeof taskId !== 'string' || taskId !== `task-${occurrenceId}`
|
|
693
|
+
|| input['executionMode'] !== 'production') {
|
|
694
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result tuple is invalid');
|
|
695
|
+
}
|
|
696
|
+
return Object.freeze({
|
|
697
|
+
circuit: parseCircuitResult(JSON.stringify(input['circuit'])),
|
|
698
|
+
occurrenceId,
|
|
699
|
+
taskId,
|
|
700
|
+
executionMode: 'production',
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
function parseSystemPauseResult(json) {
|
|
704
|
+
if (Buffer.byteLength(json, 'utf8') > 4_096) {
|
|
705
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result exceeds its bound');
|
|
706
|
+
}
|
|
707
|
+
let value;
|
|
708
|
+
try {
|
|
709
|
+
value = JSON.parse(json);
|
|
710
|
+
}
|
|
711
|
+
catch {
|
|
712
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result is invalid JSON');
|
|
713
|
+
}
|
|
714
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)
|
|
715
|
+
|| Object.keys(value).sort().join(',')
|
|
716
|
+
!== 'automationId,automationStatus,definitionHash,definitionVersion,expectedVersion,owner') {
|
|
717
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result is invalid');
|
|
718
|
+
}
|
|
719
|
+
const input = value;
|
|
720
|
+
const owner = input['owner'];
|
|
721
|
+
const automationId = input['automationId'];
|
|
722
|
+
const expectedVersion = input['expectedVersion'];
|
|
723
|
+
const definitionVersion = input['definitionVersion'];
|
|
724
|
+
if (typeof owner !== 'string' || owner.normalize('NFC').trim() !== owner
|
|
725
|
+
|| owner === '' || Buffer.byteLength(owner, 'utf8') > 200
|
|
726
|
+
|| typeof automationId !== 'string' || automationId.normalize('NFC').trim() !== automationId
|
|
727
|
+
|| automationId === '' || Buffer.byteLength(automationId, 'utf8') > 500
|
|
728
|
+
|| typeof input['definitionHash'] !== 'string' || !/^[a-f0-9]{64}$/u.test(input['definitionHash'])
|
|
729
|
+
|| !Number.isSafeInteger(expectedVersion) || expectedVersion < 1
|
|
730
|
+
|| !Number.isSafeInteger(definitionVersion) || definitionVersion !== expectedVersion + 1
|
|
731
|
+
|| input['automationStatus'] !== 'paused') {
|
|
732
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result tuple is invalid');
|
|
733
|
+
}
|
|
734
|
+
return Object.freeze({
|
|
735
|
+
owner,
|
|
736
|
+
automationId,
|
|
737
|
+
definitionHash: input['definitionHash'],
|
|
738
|
+
expectedVersion: expectedVersion,
|
|
739
|
+
definitionVersion: definitionVersion,
|
|
740
|
+
automationStatus: 'paused',
|
|
741
|
+
});
|
|
742
|
+
}
|
|
282
743
|
function deepFreezeJson(value) {
|
|
283
744
|
if (typeof value !== 'object' || value === null || Object.isFrozen(value))
|
|
284
745
|
return value;
|
|
@@ -341,12 +802,23 @@ function normalizeEvidenceAttribution(value) {
|
|
|
341
802
|
...(ruleId === undefined ? {} : { ruleId, guidanceVersion: guidanceVersion }),
|
|
342
803
|
});
|
|
343
804
|
}
|
|
805
|
+
function automationSituation(automationId) {
|
|
806
|
+
const raw = `automation:${automationId}`;
|
|
807
|
+
return Buffer.byteLength(raw, 'utf8') <= 200
|
|
808
|
+
? raw
|
|
809
|
+
: `automation:${createHash('sha256').update(automationId).digest('hex')}`;
|
|
810
|
+
}
|
|
344
811
|
function buildOutcomeEvidence(input) {
|
|
345
|
-
|
|
812
|
+
// Preview is an operator inspection mode, not a production exposure. Feeding
|
|
813
|
+
// it into Evolution would let a side-effect-free rehearsal change live
|
|
814
|
+
// guidance even though the production behaviour was never exercised.
|
|
815
|
+
if (input.executionMode !== 'production'
|
|
816
|
+
|| input.outcome === 'cancelled'
|
|
817
|
+
|| input.outcome === 'unknown') {
|
|
346
818
|
return { status: 'suppressed', json: null };
|
|
347
819
|
}
|
|
348
820
|
const evidence = {
|
|
349
|
-
situation:
|
|
821
|
+
situation: automationSituation(input.automation.id),
|
|
350
822
|
outcome: input.outcome === 'succeeded' ? 'succeeded' : 'failed',
|
|
351
823
|
detail: `automation "${input.automation.definition.name}": run ${input.outcome}`,
|
|
352
824
|
idempotencyKey: `automation-run:${input.runId}`,
|
|
@@ -371,6 +843,12 @@ function nonNegativeUsageMetric(usage, key) {
|
|
|
371
843
|
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
372
844
|
}
|
|
373
845
|
function buildTerminalEvaluation(input) {
|
|
846
|
+
// Preview is validation evidence for its owning Host workflow, never a
|
|
847
|
+
// production outcome. Keeping it out of this durable trusted outbox avoids
|
|
848
|
+
// silently turning a dry-run into learning truth when Evaluation is added
|
|
849
|
+
// later or after a restart.
|
|
850
|
+
if (input.executionMode !== 'production')
|
|
851
|
+
return undefined;
|
|
374
852
|
const inputTokens = nonNegativeUsageMetric(input.usage, 'inputTokens');
|
|
375
853
|
const outputTokens = nonNegativeUsageMetric(input.usage, 'outputTokens');
|
|
376
854
|
const toolCalls = nonNegativeUsageMetric(input.usage, 'toolCalls');
|
|
@@ -383,18 +861,22 @@ function buildTerminalEvaluation(input) {
|
|
|
383
861
|
...(latencyMs === undefined ? {} : { latencyMs }),
|
|
384
862
|
...(retries === 0 ? {} : { retries }),
|
|
385
863
|
});
|
|
386
|
-
const
|
|
387
|
-
const
|
|
388
|
-
|
|
389
|
-
|
|
864
|
+
const situation = automationSituation(input.automation.id);
|
|
865
|
+
const fixedHostObjective = input.executionMode === 'production'
|
|
866
|
+
&& isHostAutomationDefinition(input.automation.definition)
|
|
867
|
+
&& (input.outcome === 'succeeded' || input.outcome === 'failed' || input.outcome === 'timed_out');
|
|
868
|
+
const objectiveStatus = !fixedHostObjective
|
|
869
|
+
? 'unknown'
|
|
870
|
+
: input.outcome === 'succeeded' ? 'achieved' : 'not-achieved';
|
|
390
871
|
return Object.freeze({
|
|
872
|
+
executionMode: 'production',
|
|
391
873
|
scope: Object.freeze({
|
|
392
874
|
workspace: input.automation.definition.workspace,
|
|
393
875
|
preset: input.automation.definition.agentPreset,
|
|
394
876
|
}),
|
|
395
877
|
situation,
|
|
396
878
|
executionStatus: evaluationExecutionStatus(input.outcome),
|
|
397
|
-
objectiveStatus
|
|
879
|
+
objectiveStatus,
|
|
398
880
|
deliveryStatus: input.deliveryStatus,
|
|
399
881
|
source: Object.freeze({ kind: 'automation', id: 'assistant-automations' }),
|
|
400
882
|
trust: 'trusted',
|
|
@@ -402,7 +884,10 @@ function buildTerminalEvaluation(input) {
|
|
|
402
884
|
metrics,
|
|
403
885
|
occurredAt: input.occurredAt,
|
|
404
886
|
idempotencyKey: `assistant-automations:terminal:${input.runId}:v1`,
|
|
405
|
-
evaluator: Object.freeze({
|
|
887
|
+
evaluator: Object.freeze({
|
|
888
|
+
id: 'assistant-automations',
|
|
889
|
+
version: fixedHostObjective ? 'host-runbook-v1' : 'terminal-v1',
|
|
890
|
+
}),
|
|
406
891
|
});
|
|
407
892
|
}
|
|
408
893
|
export function stableOccurrenceId(automationId, triggerKind, triggerKey) {
|
|
@@ -536,6 +1021,610 @@ export class AutomationStore {
|
|
|
536
1021
|
return this.database.prepare('SELECT * FROM automation_definitions ORDER BY created_at, id').all()
|
|
537
1022
|
.map(record);
|
|
538
1023
|
}
|
|
1024
|
+
getDefinitionHash(automationId) {
|
|
1025
|
+
const row = this.database.prepare('SELECT definition_hash FROM automation_definitions WHERE id = ?')
|
|
1026
|
+
.get(automationId);
|
|
1027
|
+
return row?.definition_hash;
|
|
1028
|
+
}
|
|
1029
|
+
getCircuit(automationId, definitionHash) {
|
|
1030
|
+
if (!/^[a-f0-9]{64}$/u.test(definitionHash)) {
|
|
1031
|
+
throw new AutomationStoreError('invalid-definition', 'circuit definition hash is invalid');
|
|
1032
|
+
}
|
|
1033
|
+
const row = this.database.prepare(`
|
|
1034
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1035
|
+
`).get(automationId, definitionHash);
|
|
1036
|
+
return row === undefined ? undefined : circuit(row);
|
|
1037
|
+
}
|
|
1038
|
+
/** Read-only compatibility view; execution admission uses the transactional seam below. */
|
|
1039
|
+
getOpenCircuitForTask(taskId) {
|
|
1040
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
1041
|
+
if (taskRow === undefined || taskRow.attempt_count < 1)
|
|
1042
|
+
return undefined;
|
|
1043
|
+
const occurrenceRow = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
1044
|
+
.get(taskRow.occurrence_id);
|
|
1045
|
+
// Preview is an observation-only execution lane. It may exercise the
|
|
1046
|
+
// runner, but it can neither observe nor mutate the production circuit.
|
|
1047
|
+
if (occurrenceRow?.dry_run !== 0)
|
|
1048
|
+
return undefined;
|
|
1049
|
+
const attempt = this.database.prepare(`
|
|
1050
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
1051
|
+
`).get(taskId, taskRow.attempt_count);
|
|
1052
|
+
if (attempt === undefined)
|
|
1053
|
+
return undefined;
|
|
1054
|
+
const snapshot = automationSnapshot(attempt, taskRow.automation_id);
|
|
1055
|
+
if (snapshot === undefined)
|
|
1056
|
+
return undefined;
|
|
1057
|
+
const definitionHash = hash(snapshot.definition);
|
|
1058
|
+
const value = this.getCircuit(taskRow.automation_id, definitionHash);
|
|
1059
|
+
return value !== undefined && value.state !== 'closed' ? value : undefined;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Host-only exact CAS. Arming never closes the circuit: it creates one
|
|
1063
|
+
* durable, expiring half-open capability that a task must atomically acquire.
|
|
1064
|
+
*/
|
|
1065
|
+
armCircuitProbe(input) {
|
|
1066
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1067
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1068
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1069
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1070
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1071
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1072
|
+
throw new AutomationStoreError('invalid-definition', 'circuit probe tuple is invalid');
|
|
1073
|
+
}
|
|
1074
|
+
const inputHash = hash({
|
|
1075
|
+
owner,
|
|
1076
|
+
automationId,
|
|
1077
|
+
definitionHash: input.definitionHash,
|
|
1078
|
+
expectedVersion: input.expectedVersion,
|
|
1079
|
+
leaseMs: input.leaseMs,
|
|
1080
|
+
});
|
|
1081
|
+
return this.transaction(() => {
|
|
1082
|
+
const prior = this.database.prepare(`
|
|
1083
|
+
SELECT input_hash, result_json FROM automation_circuit_operations WHERE operation_id = ?
|
|
1084
|
+
`).get(operationId);
|
|
1085
|
+
if (prior !== undefined) {
|
|
1086
|
+
if (prior.input_hash !== inputHash) {
|
|
1087
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit probe operation id was reused with different input');
|
|
1088
|
+
}
|
|
1089
|
+
return Object.freeze({
|
|
1090
|
+
operationId,
|
|
1091
|
+
circuit: parseCircuitResult(prior.result_json),
|
|
1092
|
+
replayed: true,
|
|
1093
|
+
});
|
|
1094
|
+
}
|
|
1095
|
+
const definition = this.database.prepare(`
|
|
1096
|
+
SELECT system_owner, definition_hash FROM automation_definitions WHERE id = ?
|
|
1097
|
+
`).get(automationId);
|
|
1098
|
+
if (definition === undefined || definition.system_owner !== owner
|
|
1099
|
+
|| definition.definition_hash !== input.definitionHash) {
|
|
1100
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1101
|
+
}
|
|
1102
|
+
const probeToken = `probe-${randomUUID()}`;
|
|
1103
|
+
const changed = this.database.prepare(`
|
|
1104
|
+
UPDATE automation_circuits
|
|
1105
|
+
SET state = 'half-open', probe_token = ?, probe_lease_until = ?, probe_task_id = NULL,
|
|
1106
|
+
updated_at = ?, version = version + 1
|
|
1107
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'open' AND version = ?
|
|
1108
|
+
`).run(probeToken, input.now + input.leaseMs, input.now, automationId, input.definitionHash, input.expectedVersion);
|
|
1109
|
+
const row = this.database.prepare(`
|
|
1110
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1111
|
+
`).get(automationId, input.definitionHash);
|
|
1112
|
+
if (row === undefined)
|
|
1113
|
+
throw new AutomationStoreError('not-found', 'automation circuit was not found');
|
|
1114
|
+
const current = circuit(row);
|
|
1115
|
+
if (changed.changes !== 1) {
|
|
1116
|
+
throw new AutomationStoreError('version-conflict', 'automation circuit changed before exact probe arm');
|
|
1117
|
+
}
|
|
1118
|
+
this.markIncidentRecoveringInTransaction(automationId, input.definitionHash, input.now);
|
|
1119
|
+
this.database.prepare(`
|
|
1120
|
+
INSERT INTO automation_circuit_operations(
|
|
1121
|
+
operation_id, system_owner, automation_id, definition_hash,
|
|
1122
|
+
expected_circuit_version, lease_ms, input_hash, result_json, created_at
|
|
1123
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1124
|
+
`).run(operationId, owner, automationId, input.definitionHash, input.expectedVersion, input.leaseMs, inputHash, JSON.stringify(current), input.now);
|
|
1125
|
+
return Object.freeze({ operationId, circuit: current, replayed: false });
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* Atomically arms one exact production probe and schedules the only task that
|
|
1130
|
+
* may consume it. The operation ledger, circuit transition, occurrence and
|
|
1131
|
+
* task commit or roll back together.
|
|
1132
|
+
*/
|
|
1133
|
+
probeCircuitAndScheduleCanary(input) {
|
|
1134
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1135
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1136
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1137
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1138
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1139
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1140
|
+
throw new AutomationStoreError('invalid-definition', 'circuit canary tuple is invalid');
|
|
1141
|
+
}
|
|
1142
|
+
const inputHash = hash({
|
|
1143
|
+
lane: 'probe-and-schedule-canary-v1',
|
|
1144
|
+
owner,
|
|
1145
|
+
automationId,
|
|
1146
|
+
definitionHash: input.definitionHash,
|
|
1147
|
+
expectedVersion: input.expectedVersion,
|
|
1148
|
+
leaseMs: input.leaseMs,
|
|
1149
|
+
});
|
|
1150
|
+
return this.transaction(() => {
|
|
1151
|
+
const prior = this.database.prepare(`
|
|
1152
|
+
SELECT input_hash, result_json FROM automation_circuit_operations WHERE operation_id = ?
|
|
1153
|
+
`).get(operationId);
|
|
1154
|
+
if (prior !== undefined) {
|
|
1155
|
+
if (prior.input_hash !== inputHash) {
|
|
1156
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit canary operation id was reused with different input');
|
|
1157
|
+
}
|
|
1158
|
+
const replay = parseCircuitCanaryResult(prior.result_json);
|
|
1159
|
+
return Object.freeze({ operationId, ...replay, replayed: true });
|
|
1160
|
+
}
|
|
1161
|
+
const definition = this.database.prepare(`
|
|
1162
|
+
SELECT system_owner, definition_hash, status
|
|
1163
|
+
FROM automation_definitions WHERE id = ?
|
|
1164
|
+
`).get(automationId);
|
|
1165
|
+
if (definition === undefined || definition.system_owner !== owner
|
|
1166
|
+
|| definition.definition_hash !== input.definitionHash) {
|
|
1167
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1168
|
+
}
|
|
1169
|
+
if (definition.status !== 'active') {
|
|
1170
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary automation is not active');
|
|
1171
|
+
}
|
|
1172
|
+
const probeToken = `probe-${randomUUID()}`;
|
|
1173
|
+
const changed = this.database.prepare(`
|
|
1174
|
+
UPDATE automation_circuits
|
|
1175
|
+
SET state = 'half-open', probe_token = ?, probe_lease_until = ?, probe_task_id = NULL,
|
|
1176
|
+
updated_at = ?, version = version + 1
|
|
1177
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'open' AND version = ?
|
|
1178
|
+
`).run(probeToken, input.now + input.leaseMs, input.now, automationId, input.definitionHash, input.expectedVersion);
|
|
1179
|
+
const row = this.database.prepare(`
|
|
1180
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1181
|
+
`).get(automationId, input.definitionHash);
|
|
1182
|
+
if (row === undefined)
|
|
1183
|
+
throw new AutomationStoreError('not-found', 'automation circuit was not found');
|
|
1184
|
+
const current = circuit(row);
|
|
1185
|
+
if (changed.changes !== 1) {
|
|
1186
|
+
throw new AutomationStoreError('version-conflict', 'automation circuit changed before exact canary arm');
|
|
1187
|
+
}
|
|
1188
|
+
this.markIncidentRecoveringInTransaction(automationId, input.definitionHash, input.now);
|
|
1189
|
+
const inserted = this.insertOccurrence({
|
|
1190
|
+
automationId,
|
|
1191
|
+
triggerKind: 'manual',
|
|
1192
|
+
triggerKey: `circuit-canary:${operationId}`,
|
|
1193
|
+
scheduledAt: input.now,
|
|
1194
|
+
status: 'pending',
|
|
1195
|
+
dryRun: false,
|
|
1196
|
+
});
|
|
1197
|
+
if (!inserted.created || inserted.value.dryRun || inserted.value.status !== 'pending') {
|
|
1198
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit canary occurrence already exists outside the atomic operation');
|
|
1199
|
+
}
|
|
1200
|
+
const occurrenceId = inserted.value.id;
|
|
1201
|
+
const taskId = `task-${occurrenceId}`;
|
|
1202
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?')
|
|
1203
|
+
.get(taskId);
|
|
1204
|
+
if (taskRow === undefined || taskRow.occurrence_id !== occurrenceId
|
|
1205
|
+
|| taskRow.automation_id !== automationId || taskRow.status !== 'scheduled'
|
|
1206
|
+
|| taskRow.attempt_count !== 0) {
|
|
1207
|
+
throw new AutomationStoreError('invalid-state', 'atomic circuit canary task was not created');
|
|
1208
|
+
}
|
|
1209
|
+
const stored = Object.freeze({
|
|
1210
|
+
circuit: current,
|
|
1211
|
+
occurrenceId,
|
|
1212
|
+
taskId,
|
|
1213
|
+
executionMode: 'production',
|
|
1214
|
+
});
|
|
1215
|
+
this.database.prepare(`
|
|
1216
|
+
INSERT INTO automation_circuit_operations(
|
|
1217
|
+
operation_id, system_owner, automation_id, definition_hash,
|
|
1218
|
+
expected_circuit_version, lease_ms, input_hash, result_json, created_at
|
|
1219
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1220
|
+
`).run(operationId, owner, automationId, input.definitionHash, input.expectedVersion, input.leaseMs, inputHash, JSON.stringify(stored), input.now);
|
|
1221
|
+
return Object.freeze({ operationId, ...stored, replayed: false });
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
/**
|
|
1225
|
+
* Atomically admits an ordinary execution or consumes the one half-open
|
|
1226
|
+
* capability. A probing row is idempotent only for its exact task id.
|
|
1227
|
+
*/
|
|
1228
|
+
acquireCircuitExecutionForTask(input) {
|
|
1229
|
+
const taskId = text(input.taskId, 'taskId', 1_000);
|
|
1230
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1231
|
+
return this.transaction(() => {
|
|
1232
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
1233
|
+
if (taskRow === undefined)
|
|
1234
|
+
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
1235
|
+
if (taskRow.status !== 'running' || taskRow.attempt_count < 1) {
|
|
1236
|
+
throw new AutomationStoreError('invalid-state', 'only a running task may acquire circuit execution');
|
|
1237
|
+
}
|
|
1238
|
+
const occurrenceRow = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
1239
|
+
.get(taskRow.occurrence_id);
|
|
1240
|
+
if (occurrenceRow === undefined || (occurrenceRow.dry_run !== 0 && occurrenceRow.dry_run !== 1)) {
|
|
1241
|
+
throw new AutomationStoreError('invalid-state', 'running task has no valid occurrence execution mode');
|
|
1242
|
+
}
|
|
1243
|
+
// A preview never consumes a production probe and is never blocked by a
|
|
1244
|
+
// production circuit. Its terminal receipt is still persisted normally.
|
|
1245
|
+
if (occurrenceRow.dry_run === 1)
|
|
1246
|
+
return { kind: 'normal' };
|
|
1247
|
+
const attempt = this.database.prepare(`
|
|
1248
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
1249
|
+
`).get(taskId, taskRow.attempt_count);
|
|
1250
|
+
if (attempt === undefined)
|
|
1251
|
+
throw new AutomationStoreError('invalid-state', 'running task has no immutable attempt');
|
|
1252
|
+
const snapshot = automationSnapshot(attempt, taskRow.automation_id);
|
|
1253
|
+
if (snapshot === undefined)
|
|
1254
|
+
throw new AutomationStoreError('invalid-state', 'running task has no immutable snapshot');
|
|
1255
|
+
const definitionHash = hash(snapshot.definition);
|
|
1256
|
+
const selected = this.database.prepare(`
|
|
1257
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1258
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1259
|
+
if (selected === undefined || selected.state === 'closed')
|
|
1260
|
+
return { kind: 'normal' };
|
|
1261
|
+
if ((selected.state === 'half-open' || selected.state === 'probing')
|
|
1262
|
+
&& selected.probe_lease_until !== null && selected.probe_lease_until <= input.now) {
|
|
1263
|
+
this.database.prepare(`
|
|
1264
|
+
UPDATE automation_circuits
|
|
1265
|
+
SET state = 'open', probe_token = NULL, probe_lease_until = NULL, probe_task_id = NULL,
|
|
1266
|
+
updated_at = ?, version = version + 1
|
|
1267
|
+
WHERE automation_id = ? AND definition_hash = ? AND version = ?
|
|
1268
|
+
`).run(input.now, taskRow.automation_id, definitionHash, selected.version);
|
|
1269
|
+
const expired = this.database.prepare(`
|
|
1270
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1271
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1272
|
+
return { kind: 'blocked', circuit: circuit(expired) };
|
|
1273
|
+
}
|
|
1274
|
+
if (selected.state === 'open')
|
|
1275
|
+
return { kind: 'blocked', circuit: circuit(selected) };
|
|
1276
|
+
if (selected.state === 'probing') {
|
|
1277
|
+
return selected.probe_task_id === taskId
|
|
1278
|
+
? { kind: 'probe', circuit: circuit(selected) }
|
|
1279
|
+
: { kind: 'blocked', circuit: circuit(selected) };
|
|
1280
|
+
}
|
|
1281
|
+
// The atomic canary lane binds a half-open transition to one durable
|
|
1282
|
+
// task. Legacy armCircuitProbe rows have no taskId and retain their
|
|
1283
|
+
// existing first-claim-wins behaviour.
|
|
1284
|
+
const designated = this.database.prepare(`
|
|
1285
|
+
SELECT json_extract(result_json, '$.taskId') AS task_id
|
|
1286
|
+
FROM automation_circuit_operations
|
|
1287
|
+
WHERE automation_id = ? AND definition_hash = ?
|
|
1288
|
+
AND expected_circuit_version = ?
|
|
1289
|
+
AND json_extract(result_json, '$.taskId') IS NOT NULL
|
|
1290
|
+
ORDER BY created_at DESC, operation_id DESC LIMIT 1
|
|
1291
|
+
`).get(taskRow.automation_id, definitionHash, selected.version - 1);
|
|
1292
|
+
if (designated !== undefined && designated.task_id !== taskId) {
|
|
1293
|
+
return { kind: 'blocked', circuit: circuit(selected) };
|
|
1294
|
+
}
|
|
1295
|
+
const probeLeaseUntil = Math.min(selected.probe_lease_until, input.now + input.leaseMs);
|
|
1296
|
+
const changed = this.database.prepare(`
|
|
1297
|
+
UPDATE automation_circuits
|
|
1298
|
+
SET state = 'probing', probe_task_id = ?, probe_lease_until = ?,
|
|
1299
|
+
updated_at = ?, version = version + 1
|
|
1300
|
+
WHERE automation_id = ? AND definition_hash = ?
|
|
1301
|
+
AND state = 'half-open' AND version = ? AND probe_token = ?
|
|
1302
|
+
`).run(taskId, probeLeaseUntil, input.now, taskRow.automation_id, definitionHash, selected.version, selected.probe_token);
|
|
1303
|
+
const acquired = this.database.prepare(`
|
|
1304
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1305
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1306
|
+
if (changed.changes !== 1)
|
|
1307
|
+
return { kind: 'blocked', circuit: circuit(acquired) };
|
|
1308
|
+
return { kind: 'probe', circuit: circuit(acquired) };
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
recoverExpiredCircuitProbes(input) {
|
|
1312
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
1313
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1314
|
+
}
|
|
1315
|
+
return this.transaction(() => {
|
|
1316
|
+
const expired = this.database.prepare(`
|
|
1317
|
+
SELECT automation_id, definition_hash FROM automation_circuits
|
|
1318
|
+
WHERE state IN ('half-open', 'probing') AND probe_lease_until <= ?
|
|
1319
|
+
`).all(input.now);
|
|
1320
|
+
const changed = Number(this.database.prepare(`
|
|
1321
|
+
UPDATE automation_circuits
|
|
1322
|
+
SET state = 'open', probe_token = NULL, probe_lease_until = NULL, probe_task_id = NULL,
|
|
1323
|
+
updated_at = ?, version = version + 1
|
|
1324
|
+
WHERE state IN ('half-open', 'probing') AND probe_lease_until <= ?
|
|
1325
|
+
`).run(input.now, input.now).changes);
|
|
1326
|
+
for (const row of expired) {
|
|
1327
|
+
this.markRecoveringIncidentOpenInTransaction(row.automation_id, row.definition_hash, input.now);
|
|
1328
|
+
}
|
|
1329
|
+
return changed;
|
|
1330
|
+
});
|
|
1331
|
+
}
|
|
1332
|
+
/** Content-free bounded inventory for one exact system owner. */
|
|
1333
|
+
listSystemOwned(input) {
|
|
1334
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1335
|
+
const limit = input.limit ?? 1_000;
|
|
1336
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1337
|
+
throw new AutomationStoreError('invalid-definition', 'system-owned inventory limit must be between 1 and 1000');
|
|
1338
|
+
}
|
|
1339
|
+
return this.transaction(() => Object.freeze(this.database.prepare(`
|
|
1340
|
+
SELECT id, system_owner, definition_hash, status, version
|
|
1341
|
+
FROM automation_definitions
|
|
1342
|
+
WHERE system_owner = ?
|
|
1343
|
+
ORDER BY id
|
|
1344
|
+
LIMIT ?
|
|
1345
|
+
`).all(owner, limit).map(row => Object.freeze({
|
|
1346
|
+
owner,
|
|
1347
|
+
automationId: row.id,
|
|
1348
|
+
automationStatus: row.status,
|
|
1349
|
+
definitionHash: row.definition_hash,
|
|
1350
|
+
definitionVersion: row.version,
|
|
1351
|
+
}))));
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* Pauses one exact system-owned row without accepting or rewriting a
|
|
1355
|
+
* definition. The lifecycle CAS and durable receipt share one transaction.
|
|
1356
|
+
*/
|
|
1357
|
+
pauseSystemOwned(input) {
|
|
1358
|
+
const owner = text(input.owner, 'owner', 200);
|
|
1359
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1360
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1361
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1362
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1363
|
+
throw new AutomationStoreError('invalid-definition', 'system pause tuple is invalid');
|
|
1364
|
+
}
|
|
1365
|
+
const inputHash = hash({
|
|
1366
|
+
lane: 'pause-system-owned-v1',
|
|
1367
|
+
owner,
|
|
1368
|
+
automationId,
|
|
1369
|
+
definitionHash: input.definitionHash,
|
|
1370
|
+
expectedVersion: input.expectedVersion,
|
|
1371
|
+
});
|
|
1372
|
+
return this.transaction(() => {
|
|
1373
|
+
const prior = this.database.prepare(`
|
|
1374
|
+
SELECT input_hash, result_json
|
|
1375
|
+
FROM automation_system_reconciles WHERE idempotency_key = ?
|
|
1376
|
+
`).get(operationId);
|
|
1377
|
+
if (prior !== undefined) {
|
|
1378
|
+
if (prior.input_hash !== inputHash) {
|
|
1379
|
+
throw new AutomationStoreError('idempotency-conflict', 'system pause operation id was reused with different input');
|
|
1380
|
+
}
|
|
1381
|
+
const replay = parseSystemPauseResult(prior.result_json);
|
|
1382
|
+
if (replay.owner !== owner || replay.automationId !== automationId
|
|
1383
|
+
|| replay.definitionHash !== input.definitionHash
|
|
1384
|
+
|| replay.expectedVersion !== input.expectedVersion) {
|
|
1385
|
+
throw new AutomationStoreError('invalid-state', 'system pause receipt does not match its operation');
|
|
1386
|
+
}
|
|
1387
|
+
return Object.freeze({ operationId, ...replay, replayed: true });
|
|
1388
|
+
}
|
|
1389
|
+
const current = this.database.prepare(`
|
|
1390
|
+
SELECT * FROM automation_definitions WHERE id = ?
|
|
1391
|
+
`).get(automationId);
|
|
1392
|
+
if (current === undefined || current.system_owner !== owner
|
|
1393
|
+
|| current.definition_hash !== input.definitionHash) {
|
|
1394
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1395
|
+
}
|
|
1396
|
+
if (current.version !== input.expectedVersion) {
|
|
1397
|
+
throw new AutomationStoreError('version-conflict', 'system-owned automation version changed before pause');
|
|
1398
|
+
}
|
|
1399
|
+
if (current.status !== 'active') {
|
|
1400
|
+
throw new AutomationStoreError('invalid-state', 'only an active system-owned automation can be paused');
|
|
1401
|
+
}
|
|
1402
|
+
const now = this.now();
|
|
1403
|
+
const changed = this.database.prepare(`
|
|
1404
|
+
UPDATE automation_definitions
|
|
1405
|
+
SET status = 'paused', next_run_at = NULL, updated_at = ?, version = version + 1
|
|
1406
|
+
WHERE id = ? AND system_owner = ? AND definition_hash = ?
|
|
1407
|
+
AND version = ? AND status = 'active'
|
|
1408
|
+
`).run(now, automationId, owner, input.definitionHash, input.expectedVersion);
|
|
1409
|
+
if (changed.changes !== 1) {
|
|
1410
|
+
throw new AutomationStoreError('version-conflict', 'system-owned automation changed before pause commit');
|
|
1411
|
+
}
|
|
1412
|
+
const stored = Object.freeze({
|
|
1413
|
+
owner,
|
|
1414
|
+
automationId,
|
|
1415
|
+
definitionHash: input.definitionHash,
|
|
1416
|
+
expectedVersion: input.expectedVersion,
|
|
1417
|
+
definitionVersion: input.expectedVersion + 1,
|
|
1418
|
+
automationStatus: 'paused',
|
|
1419
|
+
});
|
|
1420
|
+
this.database.prepare(`
|
|
1421
|
+
INSERT INTO automation_system_reconciles(
|
|
1422
|
+
idempotency_key, system_owner, automation_id, input_hash, result_json, created_at
|
|
1423
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
1424
|
+
`).run(operationId, owner, automationId, inputHash, JSON.stringify(stored), now);
|
|
1425
|
+
return Object.freeze({ operationId, ...stored, replayed: false });
|
|
1426
|
+
});
|
|
1427
|
+
}
|
|
1428
|
+
inspectSystemOwned(input) {
|
|
1429
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1430
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1431
|
+
return this.transaction(() => {
|
|
1432
|
+
const definitionRow = this.database.prepare('SELECT * FROM automation_definitions WHERE id = ?')
|
|
1433
|
+
.get(automationId);
|
|
1434
|
+
if (definitionRow === undefined || definitionRow.system_owner !== owner) {
|
|
1435
|
+
throw new AutomationStoreError('not-found', 'system-owned automation was not found for exact owner');
|
|
1436
|
+
}
|
|
1437
|
+
const automation = record(definitionRow);
|
|
1438
|
+
const projectRun = (executionMode) => {
|
|
1439
|
+
const latestRow = this.database.prepare(`
|
|
1440
|
+
SELECT * FROM automation_runs WHERE automation_id = ? AND execution_mode = ?
|
|
1441
|
+
ORDER BY created_at DESC, id DESC LIMIT 1
|
|
1442
|
+
`).get(automationId, executionMode);
|
|
1443
|
+
if (latestRow === undefined)
|
|
1444
|
+
return undefined;
|
|
1445
|
+
const latest = run(latestRow);
|
|
1446
|
+
let snapshot;
|
|
1447
|
+
try {
|
|
1448
|
+
snapshot = this.getRunExecutionSnapshot(latest.id);
|
|
1449
|
+
}
|
|
1450
|
+
catch {
|
|
1451
|
+
snapshot = undefined;
|
|
1452
|
+
}
|
|
1453
|
+
const immutableContext = snapshot !== undefined
|
|
1454
|
+
&& latest.definitionHash !== undefined
|
|
1455
|
+
&& hash(snapshot.definition) === latest.definitionHash
|
|
1456
|
+
? Object.freeze({
|
|
1457
|
+
state: 'verified',
|
|
1458
|
+
definitionHash: latest.definitionHash,
|
|
1459
|
+
definitionVersion: snapshot.version,
|
|
1460
|
+
scope: Object.freeze({
|
|
1461
|
+
workspace: snapshot.definition.workspace,
|
|
1462
|
+
agentPreset: snapshot.definition.agentPreset,
|
|
1463
|
+
}),
|
|
1464
|
+
})
|
|
1465
|
+
: Object.freeze({ state: 'unknown' });
|
|
1466
|
+
return Object.freeze({
|
|
1467
|
+
runId: latest.id,
|
|
1468
|
+
status: latest.status,
|
|
1469
|
+
executionMode,
|
|
1470
|
+
diagnostic: latest.diagnostic,
|
|
1471
|
+
createdAt: latest.createdAt,
|
|
1472
|
+
immutableContext,
|
|
1473
|
+
});
|
|
1474
|
+
};
|
|
1475
|
+
const latestProductionRun = projectRun('production');
|
|
1476
|
+
const latestPreviewRun = projectRun('preview');
|
|
1477
|
+
const latestTerminalRuns = Object.freeze({
|
|
1478
|
+
...(latestProductionRun === undefined ? {} : { production: latestProductionRun }),
|
|
1479
|
+
...(latestPreviewRun === undefined ? {} : { preview: latestPreviewRun }),
|
|
1480
|
+
});
|
|
1481
|
+
const current = this.getCircuit(automationId, definitionRow.definition_hash);
|
|
1482
|
+
const currentCircuit = current === undefined
|
|
1483
|
+
? undefined
|
|
1484
|
+
: Object.freeze({
|
|
1485
|
+
definitionHash: current.definitionHash,
|
|
1486
|
+
state: current.state,
|
|
1487
|
+
failureClass: current.failureClass,
|
|
1488
|
+
failurePhase: current.failurePhase,
|
|
1489
|
+
failureCode: current.failureCode,
|
|
1490
|
+
openedAt: current.openedAt,
|
|
1491
|
+
updatedAt: current.updatedAt,
|
|
1492
|
+
...(current.probeLeaseUntil === undefined ? {} : { probeLeaseUntil: current.probeLeaseUntil }),
|
|
1493
|
+
version: current.version,
|
|
1494
|
+
});
|
|
1495
|
+
const activeIncident = this.getCurrentIncident(automationId, definitionRow.definition_hash);
|
|
1496
|
+
const currentIncident = activeIncident === undefined
|
|
1497
|
+
? undefined
|
|
1498
|
+
: Object.freeze({
|
|
1499
|
+
definitionHash: activeIncident.definitionHash,
|
|
1500
|
+
stage: activeIncident.stage,
|
|
1501
|
+
failureClass: activeIncident.failureClass,
|
|
1502
|
+
failurePhase: activeIncident.failurePhase,
|
|
1503
|
+
failureCode: activeIncident.failureCode,
|
|
1504
|
+
state: activeIncident.state,
|
|
1505
|
+
lifecycleGeneration: activeIncident.lifecycleGeneration,
|
|
1506
|
+
presentationRevision: activeIncident.presentationRevision,
|
|
1507
|
+
alertStatus: activeIncident.alertStatus,
|
|
1508
|
+
openedAt: activeIncident.openedAt,
|
|
1509
|
+
updatedAt: activeIncident.updatedAt,
|
|
1510
|
+
version: activeIncident.version,
|
|
1511
|
+
});
|
|
1512
|
+
return Object.freeze({
|
|
1513
|
+
owner,
|
|
1514
|
+
automationId,
|
|
1515
|
+
automationStatus: automation.status,
|
|
1516
|
+
definitionHash: definitionRow.definition_hash,
|
|
1517
|
+
definitionVersion: automation.version,
|
|
1518
|
+
...(latestProductionRun === undefined ? {} : { latestTerminalRun: latestProductionRun }),
|
|
1519
|
+
latestTerminalRuns,
|
|
1520
|
+
...(currentCircuit === undefined ? {} : { currentCircuit }),
|
|
1521
|
+
...(currentIncident === undefined ? {} : { currentIncident }),
|
|
1522
|
+
});
|
|
1523
|
+
});
|
|
1524
|
+
}
|
|
1525
|
+
listIncidents(input) {
|
|
1526
|
+
if (!Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 1_000) {
|
|
1527
|
+
throw new AutomationStoreError('invalid-definition', 'incident limit must be between 1 and 1000');
|
|
1528
|
+
}
|
|
1529
|
+
const rows = input.automationId === undefined
|
|
1530
|
+
? this.database.prepare(`
|
|
1531
|
+
SELECT * FROM automation_incidents ORDER BY opened_at DESC, id DESC LIMIT ?
|
|
1532
|
+
`).all(input.limit)
|
|
1533
|
+
: this.database.prepare(`
|
|
1534
|
+
SELECT * FROM automation_incidents WHERE automation_id = ?
|
|
1535
|
+
ORDER BY opened_at DESC, id DESC LIMIT ?
|
|
1536
|
+
`).all(input.automationId, input.limit);
|
|
1537
|
+
return rows.map(incident);
|
|
1538
|
+
}
|
|
1539
|
+
listPendingIncidentAlerts(limit) {
|
|
1540
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1541
|
+
throw new AutomationStoreError('invalid-definition', 'incident alert limit must be between 1 and 1000');
|
|
1542
|
+
}
|
|
1543
|
+
const output = [];
|
|
1544
|
+
const rows = this.database.prepare(`
|
|
1545
|
+
SELECT * FROM automation_incidents
|
|
1546
|
+
WHERE alert_status = 'pending'
|
|
1547
|
+
ORDER BY updated_at, id LIMIT 1000
|
|
1548
|
+
`).all();
|
|
1549
|
+
for (const row of rows) {
|
|
1550
|
+
if (output.length === limit)
|
|
1551
|
+
break;
|
|
1552
|
+
try {
|
|
1553
|
+
output.push(incident(row));
|
|
1554
|
+
}
|
|
1555
|
+
catch {
|
|
1556
|
+
// A malformed poison row is terminally suppressed so it cannot starve
|
|
1557
|
+
// later incident alerts or leak unvalidated content into Delivery.
|
|
1558
|
+
this.database.prepare(`
|
|
1559
|
+
UPDATE automation_incidents
|
|
1560
|
+
SET alert_status = 'suppressed',
|
|
1561
|
+
updated_at = ?, version = version + 1
|
|
1562
|
+
WHERE id = ? AND alert_status = 'pending'
|
|
1563
|
+
`).run(this.now(), row.id);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
return output;
|
|
1567
|
+
}
|
|
1568
|
+
completeIncidentAlert(input) {
|
|
1569
|
+
if (!Number.isSafeInteger(input.now)
|
|
1570
|
+
|| !Number.isSafeInteger(input.expectedLifecycleGeneration) || input.expectedLifecycleGeneration < 1
|
|
1571
|
+
|| !Number.isSafeInteger(input.expectedPresentationRevision) || input.expectedPresentationRevision < 1
|
|
1572
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1573
|
+
throw new AutomationStoreError('invalid-definition', 'incident alert time is invalid');
|
|
1574
|
+
}
|
|
1575
|
+
const alertRef = text(input.alertRef, 'alertRef', 500);
|
|
1576
|
+
const changed = this.database.prepare(`
|
|
1577
|
+
UPDATE automation_incidents
|
|
1578
|
+
SET alert_status = 'enqueued', alert_ref = ?, updated_at = ?, version = version + 1
|
|
1579
|
+
WHERE id = ? AND alert_status = ? AND lifecycle_generation = ?
|
|
1580
|
+
AND presentation_revision = ? AND version = ?
|
|
1581
|
+
`).run(alertRef, input.now, input.incidentId, input.expectedStatus, input.expectedLifecycleGeneration, input.expectedPresentationRevision, input.expectedVersion);
|
|
1582
|
+
const row = this.database.prepare('SELECT * FROM automation_incidents WHERE id = ?')
|
|
1583
|
+
.get(input.incidentId);
|
|
1584
|
+
if (row === undefined || (changed.changes !== 1
|
|
1585
|
+
&& !(row.alert_status === 'enqueued' && row.alert_ref === alertRef
|
|
1586
|
+
&& row.lifecycle_generation === input.expectedLifecycleGeneration
|
|
1587
|
+
&& row.presentation_revision === input.expectedPresentationRevision))) {
|
|
1588
|
+
throw new AutomationStoreError('version-conflict', 'incident alert state changed before completion');
|
|
1589
|
+
}
|
|
1590
|
+
return incident(row);
|
|
1591
|
+
}
|
|
1592
|
+
getCurrentIncident(automationId, definitionHash) {
|
|
1593
|
+
const row = this.database.prepare(`
|
|
1594
|
+
SELECT * FROM automation_incidents
|
|
1595
|
+
WHERE automation_id = ? AND definition_hash = ? AND state <> 'resolved'
|
|
1596
|
+
ORDER BY updated_at DESC, id DESC LIMIT 1
|
|
1597
|
+
`).get(automationId, definitionHash);
|
|
1598
|
+
return row === undefined ? undefined : incident(row);
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* Re-prove the exact transport target before an incident leaves this store.
|
|
1602
|
+
* Terminal targets come from the winning claim snapshot rather than the
|
|
1603
|
+
* mutable current definition, so an edit/restart cannot redirect an alert.
|
|
1604
|
+
*/
|
|
1605
|
+
getIncidentNotificationTarget(incidentId) {
|
|
1606
|
+
const id = text(incidentId, 'incidentId', 1_000);
|
|
1607
|
+
const row = this.database.prepare('SELECT * FROM automation_incidents WHERE id = ?')
|
|
1608
|
+
.get(id);
|
|
1609
|
+
if (row === undefined)
|
|
1610
|
+
return undefined;
|
|
1611
|
+
const value = incident(row);
|
|
1612
|
+
if (value.stage !== 'terminal') {
|
|
1613
|
+
// Availability incidents can only be opened for a Host definition. The
|
|
1614
|
+
// route itself was copied into the incident in the opening transaction.
|
|
1615
|
+
return Object.freeze({ kind: 'owner-route', authorityId: value.notificationRouteId });
|
|
1616
|
+
}
|
|
1617
|
+
if (value.runId === undefined)
|
|
1618
|
+
return undefined;
|
|
1619
|
+
const proof = this.getProvenProductionRun(value.runId);
|
|
1620
|
+
if (proof === undefined || proof.run.automationId !== value.automationId
|
|
1621
|
+
|| proof.run.definitionHash !== value.definitionHash)
|
|
1622
|
+
return undefined;
|
|
1623
|
+
const route = incidentNotificationRoute(proof.automation.definition);
|
|
1624
|
+
if (route === undefined || route.id !== value.notificationRouteId)
|
|
1625
|
+
return undefined;
|
|
1626
|
+
return route.target;
|
|
1627
|
+
}
|
|
539
1628
|
health() {
|
|
540
1629
|
const scalar = (sql) => this.database.prepare(sql).get().count;
|
|
541
1630
|
return {
|
|
@@ -552,6 +1641,9 @@ export class AutomationStore {
|
|
|
552
1641
|
failedEvaluationAttempts: scalar('SELECT COALESCE(SUM(attempt_count), 0) AS count FROM automation_evaluation_outbox'),
|
|
553
1642
|
deadLetterEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'dead-letter'"),
|
|
554
1643
|
oldestPendingEvaluationAt: scalar("SELECT COALESCE(MIN(created_at), 0) AS count FROM automation_evaluation_outbox WHERE status = 'pending'"),
|
|
1644
|
+
openCircuits: scalar("SELECT COUNT(*) AS count FROM automation_circuits WHERE state <> 'closed'"),
|
|
1645
|
+
openIncidents: scalar("SELECT COUNT(*) AS count FROM automation_incidents WHERE state <> 'resolved'"),
|
|
1646
|
+
pendingIncidentAlerts: scalar("SELECT COUNT(*) AS count FROM automation_incidents WHERE alert_status = 'pending'"),
|
|
555
1647
|
};
|
|
556
1648
|
}
|
|
557
1649
|
changeApproved(input) {
|
|
@@ -597,6 +1689,58 @@ export class AutomationStore {
|
|
|
597
1689
|
return result;
|
|
598
1690
|
});
|
|
599
1691
|
}
|
|
1692
|
+
listDueHostExecutionRequirements(input) {
|
|
1693
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
1694
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement time is invalid');
|
|
1695
|
+
}
|
|
1696
|
+
const limit = input.limit ?? 1_000;
|
|
1697
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1698
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement limit is invalid');
|
|
1699
|
+
}
|
|
1700
|
+
const rows = this.database.prepare(`
|
|
1701
|
+
SELECT * FROM automation_definitions
|
|
1702
|
+
WHERE status = 'active' AND next_run_at IS NOT NULL AND next_run_at <= ?
|
|
1703
|
+
ORDER BY next_run_at, id LIMIT ?
|
|
1704
|
+
`).all(input.now, limit);
|
|
1705
|
+
return rows.flatMap(row => {
|
|
1706
|
+
const automation = record(row);
|
|
1707
|
+
return isHostAutomationDefinition(automation.definition)
|
|
1708
|
+
? [Object.freeze({
|
|
1709
|
+
automationId: automation.id,
|
|
1710
|
+
definitionHash: row.definition_hash,
|
|
1711
|
+
execution: automation.definition.execution,
|
|
1712
|
+
})]
|
|
1713
|
+
: [];
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
listClaimableHostExecutionRequirements(limit = 100) {
|
|
1717
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1718
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement limit is invalid');
|
|
1719
|
+
}
|
|
1720
|
+
const rows = this.database.prepare(`
|
|
1721
|
+
SELECT DISTINCT definition.* FROM automation_tasks AS task
|
|
1722
|
+
JOIN automation_definitions AS definition ON definition.id = task.automation_id
|
|
1723
|
+
WHERE task.status = 'scheduled' AND (
|
|
1724
|
+
definition.status = 'active'
|
|
1725
|
+
OR (
|
|
1726
|
+
definition.status = 'paused'
|
|
1727
|
+
AND definition.system_owner = ?
|
|
1728
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
1729
|
+
)
|
|
1730
|
+
)
|
|
1731
|
+
ORDER BY task.created_at, task.id LIMIT ?
|
|
1732
|
+
`).all(growthAutomationOwner, limit);
|
|
1733
|
+
return rows.flatMap(row => {
|
|
1734
|
+
const automation = record(row);
|
|
1735
|
+
return isHostAutomationDefinition(automation.definition)
|
|
1736
|
+
? [Object.freeze({
|
|
1737
|
+
automationId: automation.id,
|
|
1738
|
+
definitionHash: row.definition_hash,
|
|
1739
|
+
execution: automation.definition.execution,
|
|
1740
|
+
})]
|
|
1741
|
+
: [];
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
600
1744
|
materializeDue(input) {
|
|
601
1745
|
if (!Number.isSafeInteger(input.now) || !Number.isSafeInteger(input.misfireGraceMs)
|
|
602
1746
|
|| input.misfireGraceMs < 0 || !Number.isSafeInteger(input.maxCatchUp)
|
|
@@ -612,6 +1756,20 @@ export class AutomationStore {
|
|
|
612
1756
|
const created = [];
|
|
613
1757
|
for (const row of due) {
|
|
614
1758
|
const automation = record(row);
|
|
1759
|
+
if (isHostAutomationDefinition(automation.definition)) {
|
|
1760
|
+
const availability = this.exactHostAvailability(input.hostAvailability, automation.id, row.definition_hash, 'materialize');
|
|
1761
|
+
if (availability?.available !== true) {
|
|
1762
|
+
this.upsertAvailabilityIncidentInTransaction({
|
|
1763
|
+
automation,
|
|
1764
|
+
definitionHash: row.definition_hash,
|
|
1765
|
+
stage: 'materialize',
|
|
1766
|
+
failureCode: availability?.reasonCode ?? 'host-executor-proof-missing',
|
|
1767
|
+
now: input.now,
|
|
1768
|
+
});
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
this.resolveIncidentInTransaction(automation.id, row.definition_hash, 'materialize', input.now);
|
|
1772
|
+
}
|
|
615
1773
|
const firstDue = automation.nextRunAt;
|
|
616
1774
|
const cutoff = input.now - input.misfireGraceMs;
|
|
617
1775
|
const current = dueOccurrences(automation.definition.schedule, Math.max(firstDue - 1, cutoff), input.now, input.maxCatchUp);
|
|
@@ -765,6 +1923,169 @@ export class AutomationStore {
|
|
|
765
1923
|
`).get(runId);
|
|
766
1924
|
return row === undefined ? undefined : automationSnapshot(row, row.expected_automation_id);
|
|
767
1925
|
}
|
|
1926
|
+
/** Host-derived occurrence mode associated with a durable terminal run. */
|
|
1927
|
+
getRunExecutionMode(runId) {
|
|
1928
|
+
const row = this.database.prepare(`SELECT execution_mode FROM automation_runs WHERE id = ?`)
|
|
1929
|
+
.get(runId);
|
|
1930
|
+
if (row === undefined)
|
|
1931
|
+
return undefined;
|
|
1932
|
+
if (row.execution_mode === 'production' || row.execution_mode === 'preview' || row.execution_mode === 'unknown') {
|
|
1933
|
+
return row.execution_mode;
|
|
1934
|
+
}
|
|
1935
|
+
throw new AutomationStoreError('invalid-state', 'automation run has an invalid execution mode');
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* Re-prove the exact production tuple immediately before an effect sink.
|
|
1939
|
+
* Legacy rows, mutable current definitions, and a mode flag alone are never
|
|
1940
|
+
* sufficient. Corrupt snapshots throw so dispatchers can quarantine the row.
|
|
1941
|
+
*/
|
|
1942
|
+
getProvenProductionRun(runId) {
|
|
1943
|
+
const row = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId);
|
|
1944
|
+
if (row === undefined || row.execution_mode !== 'production' || row.definition_hash === null)
|
|
1945
|
+
return undefined;
|
|
1946
|
+
// Parse all bounded run fields before treating the row as effect-capable.
|
|
1947
|
+
const value = run(row);
|
|
1948
|
+
const occurrenceRow = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
1949
|
+
.get(row.occurrence_id);
|
|
1950
|
+
if (occurrenceRow === undefined || occurrenceRow.dry_run !== 0)
|
|
1951
|
+
return undefined;
|
|
1952
|
+
const attempt = this.database.prepare('SELECT * FROM automation_attempts WHERE id = ?')
|
|
1953
|
+
.get(row.attempt_id);
|
|
1954
|
+
if (attempt === undefined)
|
|
1955
|
+
return undefined;
|
|
1956
|
+
const snapshot = automationSnapshot(attempt, row.automation_id);
|
|
1957
|
+
if (snapshot === undefined || hash(snapshot.definition) !== row.definition_hash)
|
|
1958
|
+
return undefined;
|
|
1959
|
+
const evidence = value.evidence;
|
|
1960
|
+
if (evidence !== undefined) {
|
|
1961
|
+
const expectedOutcome = row.status === 'succeeded' ? 'succeeded' : 'failed';
|
|
1962
|
+
if (evidence.situation !== automationSituation(snapshot.id)
|
|
1963
|
+
|| evidence.outcome !== expectedOutcome
|
|
1964
|
+
|| evidence.detail !== `automation "${snapshot.definition.name}": run ${row.status}`
|
|
1965
|
+
|| evidence.idempotencyKey !== `automation-run:${row.id}`
|
|
1966
|
+
|| evidence.occurredAt !== row.created_at
|
|
1967
|
+
|| evidence.workspace !== snapshot.definition.workspace
|
|
1968
|
+
|| evidence.agentPreset !== snapshot.definition.agentPreset
|
|
1969
|
+
|| evidence.automationId !== snapshot.id
|
|
1970
|
+
|| evidence.runId !== row.id) {
|
|
1971
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence does not match its immutable run');
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
return Object.freeze({ run: value, automation: snapshot });
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Resolve an Evaluation automation-run reference into a content-minimal,
|
|
1978
|
+
* exactly re-provable production receipt. Callers must supply the Evaluation
|
|
1979
|
+
* tuple; this method never guesses scope or situation from mutable state.
|
|
1980
|
+
*/
|
|
1981
|
+
resolveQualityEvidence(input) {
|
|
1982
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1983
|
+
const runId = text(input.runId, 'runId', 1_000);
|
|
1984
|
+
const workspace = text(input.expectedScope.workspace, 'expectedScope.workspace', 4_096);
|
|
1985
|
+
const preset = text(input.expectedScope.preset, 'expectedScope.preset', 200);
|
|
1986
|
+
const situation = text(input.expectedSituation, 'expectedSituation', 1_000);
|
|
1987
|
+
if (!Number.isSafeInteger(input.expectedOccurredAt)
|
|
1988
|
+
|| input.evidenceRef.kind !== 'automation-run' || input.evidenceRef.ref !== runId) {
|
|
1989
|
+
throw new AutomationStoreError('invalid-definition', 'quality evidence expectation is invalid');
|
|
1990
|
+
}
|
|
1991
|
+
const owner = input.owner === undefined ? undefined : text(input.owner, 'owner', 500);
|
|
1992
|
+
const proof = this.getProvenProductionRun(runId);
|
|
1993
|
+
if (proof === undefined || proof.run.automationId !== automationId
|
|
1994
|
+
|| (proof.run.status !== 'succeeded'
|
|
1995
|
+
&& proof.run.status !== 'failed' && proof.run.status !== 'timed_out')
|
|
1996
|
+
|| proof.run.definitionHash === undefined
|
|
1997
|
+
|| proof.run.evidence === undefined
|
|
1998
|
+
|| (proof.run.evidenceStatus !== 'pending' && proof.run.evidenceStatus !== 'recorded')
|
|
1999
|
+
|| proof.automation.definition.workspace !== workspace
|
|
2000
|
+
|| proof.automation.definition.agentPreset !== preset
|
|
2001
|
+
|| proof.run.evidence.situation !== situation
|
|
2002
|
+
|| proof.run.evidence.occurredAt !== input.expectedOccurredAt
|
|
2003
|
+
|| proof.run.evidence.runId !== runId
|
|
2004
|
+
|| proof.run.evidence.automationId !== automationId
|
|
2005
|
+
|| (owner !== undefined && proof.automation.owner !== owner))
|
|
2006
|
+
return undefined;
|
|
2007
|
+
const base = {
|
|
2008
|
+
schemaVersion: 1,
|
|
2009
|
+
source: 'assistant-automations',
|
|
2010
|
+
executionKind: isHostAutomationDefinition(proof.automation.definition) ? 'host' : 'agent',
|
|
2011
|
+
automationId,
|
|
2012
|
+
runId,
|
|
2013
|
+
definitionHash: proof.run.definitionHash,
|
|
2014
|
+
status: proof.run.status,
|
|
2015
|
+
scope: Object.freeze({ workspace, preset }),
|
|
2016
|
+
situation,
|
|
2017
|
+
occurredAt: input.expectedOccurredAt,
|
|
2018
|
+
evidenceRef: Object.freeze({ kind: 'automation-run', ref: runId }),
|
|
2019
|
+
...(proof.run.evidence.sessionId === undefined ? {} : { sessionId: proof.run.evidence.sessionId }),
|
|
2020
|
+
...(proof.run.evidence.ruleId === undefined ? {} : { ruleId: proof.run.evidence.ruleId }),
|
|
2021
|
+
...(proof.run.evidence.guidanceVersion === undefined
|
|
2022
|
+
? {}
|
|
2023
|
+
: { guidanceVersion: proof.run.evidence.guidanceVersion }),
|
|
2024
|
+
};
|
|
2025
|
+
return Object.freeze({ ...base, proofDigest: hash(base) });
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Prove that one exact owner-bound output is the immutable output of an
|
|
2029
|
+
* Agent production run. This is narrower than quality evidence and is used
|
|
2030
|
+
* only by Delivery's reserved learning-metadata lane.
|
|
2031
|
+
*/
|
|
2032
|
+
resolveDeliveryEvidence(input) {
|
|
2033
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
2034
|
+
const runId = text(input.runId, 'runId', 1_000);
|
|
2035
|
+
const workspace = text(input.expectedWorkspace, 'expectedWorkspace', 4_096);
|
|
2036
|
+
const bindingId = text(input.expectedBindingId, 'expectedBindingId', 1_000);
|
|
2037
|
+
if (!/^[a-f0-9]{64}$/u.test(input.expectedOutputDigest)) {
|
|
2038
|
+
throw new AutomationStoreError('invalid-definition', 'delivery output digest is invalid');
|
|
2039
|
+
}
|
|
2040
|
+
const proof = this.getProvenProductionRun(runId);
|
|
2041
|
+
if (proof === undefined || proof.run.automationId !== automationId
|
|
2042
|
+
|| proof.run.status !== 'succeeded'
|
|
2043
|
+
|| (proof.run.deliveryStatus !== 'pending' && proof.run.deliveryStatus !== 'enqueued')
|
|
2044
|
+
|| proof.run.evidence === undefined
|
|
2045
|
+
|| (proof.run.evidenceStatus !== 'pending' && proof.run.evidenceStatus !== 'recorded')
|
|
2046
|
+
|| isHostAutomationDefinition(proof.automation.definition)
|
|
2047
|
+
|| proof.automation.definition.workspace !== workspace
|
|
2048
|
+
|| proof.automation.definition.deliveryBindingId !== bindingId
|
|
2049
|
+
|| proof.run.evidence.automationId !== automationId
|
|
2050
|
+
|| proof.run.evidence.runId !== runId
|
|
2051
|
+
|| proof.run.evidence.situation !== `automation:${automationId}`
|
|
2052
|
+
|| hashText(proof.run.outputPreview) !== input.expectedOutputDigest)
|
|
2053
|
+
return undefined;
|
|
2054
|
+
const base = {
|
|
2055
|
+
schemaVersion: 1,
|
|
2056
|
+
source: 'assistant-automations',
|
|
2057
|
+
executionKind: 'agent',
|
|
2058
|
+
automationId,
|
|
2059
|
+
runId,
|
|
2060
|
+
occurrenceId: proof.run.occurrenceId,
|
|
2061
|
+
workspace,
|
|
2062
|
+
agentPreset: proof.automation.definition.agentPreset,
|
|
2063
|
+
bindingId,
|
|
2064
|
+
situation: proof.run.evidence.situation,
|
|
2065
|
+
occurredAt: proof.run.evidence.occurredAt,
|
|
2066
|
+
executionStatus: 'succeeded',
|
|
2067
|
+
outputDigest: input.expectedOutputDigest,
|
|
2068
|
+
};
|
|
2069
|
+
return Object.freeze({ ...base, proofDigest: hash(base) });
|
|
2070
|
+
}
|
|
2071
|
+
validateQualityEvidence(receipt) {
|
|
2072
|
+
try {
|
|
2073
|
+
if (receipt.schemaVersion !== 1 || receipt.source !== 'assistant-automations')
|
|
2074
|
+
return false;
|
|
2075
|
+
const resolved = this.resolveQualityEvidence({
|
|
2076
|
+
automationId: receipt.automationId,
|
|
2077
|
+
runId: receipt.runId,
|
|
2078
|
+
expectedScope: receipt.scope,
|
|
2079
|
+
expectedSituation: receipt.situation,
|
|
2080
|
+
expectedOccurredAt: receipt.occurredAt,
|
|
2081
|
+
evidenceRef: receipt.evidenceRef,
|
|
2082
|
+
});
|
|
2083
|
+
return resolved !== undefined && JSON.stringify(resolved) === JSON.stringify(receipt);
|
|
2084
|
+
}
|
|
2085
|
+
catch {
|
|
2086
|
+
return false;
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
768
2089
|
getRun(runId) {
|
|
769
2090
|
const row = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId);
|
|
770
2091
|
return row === undefined ? undefined : run(row);
|
|
@@ -804,6 +2125,34 @@ export class AutomationStore {
|
|
|
804
2125
|
return duty({ ...current, lease_until: leaseUntil }, true);
|
|
805
2126
|
});
|
|
806
2127
|
}
|
|
2128
|
+
/**
|
|
2129
|
+
* Relinquish a live coordinator lease without weakening its fencing token.
|
|
2130
|
+
*
|
|
2131
|
+
* A cleanly stopped process may expire only the exact lease it owns. The
|
|
2132
|
+
* row is deliberately retained so the next owner must advance the fencing
|
|
2133
|
+
* token; a late/stale owner can therefore never release its successor.
|
|
2134
|
+
*/
|
|
2135
|
+
releaseDuty(input) {
|
|
2136
|
+
const ownerId = text(input.ownerId, 'ownerId', 500);
|
|
2137
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
2138
|
+
throw new AutomationStoreError('invalid-definition', 'release time is invalid');
|
|
2139
|
+
}
|
|
2140
|
+
return this.transaction(() => {
|
|
2141
|
+
const current = this.database.prepare('SELECT * FROM duty_lease WHERE singleton = 1').get();
|
|
2142
|
+
if (current === undefined || current.owner_id !== ownerId || current.fencing_token !== input.fencingToken) {
|
|
2143
|
+
throw new AutomationStoreError('stale-fence', 'duty ownership is missing or fenced');
|
|
2144
|
+
}
|
|
2145
|
+
const leaseUntil = Math.min(current.lease_until, input.now);
|
|
2146
|
+
const changed = this.database.prepare(`
|
|
2147
|
+
UPDATE duty_lease SET lease_until = ?, updated_at = ?
|
|
2148
|
+
WHERE singleton = 1 AND owner_id = ? AND fencing_token = ?
|
|
2149
|
+
`).run(leaseUntil, input.now, ownerId, input.fencingToken);
|
|
2150
|
+
if (changed.changes !== 1) {
|
|
2151
|
+
throw new AutomationStoreError('stale-fence', 'duty ownership changed while releasing');
|
|
2152
|
+
}
|
|
2153
|
+
return duty({ ...current, lease_until: leaseUntil }, false);
|
|
2154
|
+
});
|
|
2155
|
+
}
|
|
807
2156
|
claimNextTask(input) {
|
|
808
2157
|
this.validateLease(input.now, input.leaseMs);
|
|
809
2158
|
return this.transaction(() => {
|
|
@@ -811,11 +2160,18 @@ export class AutomationStore {
|
|
|
811
2160
|
const candidates = this.database.prepare(`
|
|
812
2161
|
SELECT task.* FROM automation_tasks task
|
|
813
2162
|
JOIN automation_definitions definition ON definition.id = task.automation_id
|
|
814
|
-
WHERE task.status = 'scheduled' AND
|
|
2163
|
+
WHERE task.status = 'scheduled' AND (
|
|
2164
|
+
definition.status = 'active'
|
|
2165
|
+
OR (
|
|
2166
|
+
definition.status = 'paused'
|
|
2167
|
+
AND definition.system_owner = ?
|
|
2168
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
2169
|
+
)
|
|
2170
|
+
)
|
|
815
2171
|
ORDER BY task.created_at, task.id LIMIT 100
|
|
816
|
-
`).all();
|
|
2172
|
+
`).all(growthAutomationOwner);
|
|
817
2173
|
for (const candidate of candidates) {
|
|
818
|
-
const result = this.claimTaskInTransaction(candidate, input);
|
|
2174
|
+
const result = this.claimTaskInTransaction(candidate, input, input.hostAvailability);
|
|
819
2175
|
if (result !== undefined)
|
|
820
2176
|
return result;
|
|
821
2177
|
}
|
|
@@ -831,7 +2187,7 @@ export class AutomationStore {
|
|
|
831
2187
|
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
832
2188
|
if (row.status !== 'scheduled')
|
|
833
2189
|
throw new AutomationStoreError('invalid-state', 'only scheduled tasks can be claimed');
|
|
834
|
-
return this.claimTaskInTransaction(row, input);
|
|
2190
|
+
return this.claimTaskInTransaction(row, input, input.hostAvailability);
|
|
835
2191
|
});
|
|
836
2192
|
}
|
|
837
2193
|
startTask(input) {
|
|
@@ -847,6 +2203,28 @@ export class AutomationStore {
|
|
|
847
2203
|
UPDATE automation_attempts SET status = 'running', session_id = ?, started_at = ?, updated_at = ?
|
|
848
2204
|
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
849
2205
|
`).run(sessionId, input.now, input.now, row.id, row.attempt_count, input.fencingToken);
|
|
2206
|
+
const occurrence = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
2207
|
+
.get(row.occurrence_id);
|
|
2208
|
+
const attempt = this.database.prepare(`
|
|
2209
|
+
SELECT * FROM automation_attempts
|
|
2210
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2211
|
+
`).get(row.id, row.attempt_count, input.fencingToken);
|
|
2212
|
+
let snapshot;
|
|
2213
|
+
try {
|
|
2214
|
+
snapshot = attempt === undefined ? undefined : automationSnapshot(attempt, row.automation_id);
|
|
2215
|
+
}
|
|
2216
|
+
catch {
|
|
2217
|
+
// The coordinator's quarantine path owns malformed snapshots. Starting
|
|
2218
|
+
// the task must still commit so that path can write its terminal receipt.
|
|
2219
|
+
}
|
|
2220
|
+
// Host recovery is explicitly armed by its circuit probe. A routed Agent
|
|
2221
|
+
// has no separate Host probe, so a later production attempt itself is
|
|
2222
|
+
// the durable transition from open to recovering.
|
|
2223
|
+
if (occurrence?.dry_run === 0 && snapshot !== undefined
|
|
2224
|
+
&& !isHostAutomationDefinition(snapshot.definition)
|
|
2225
|
+
&& incidentNotificationRoute(snapshot.definition) !== undefined) {
|
|
2226
|
+
this.markIncidentRecoveringInTransaction(snapshot.id, hash(snapshot.definition), input.now);
|
|
2227
|
+
}
|
|
850
2228
|
return this.getTask(row.id);
|
|
851
2229
|
});
|
|
852
2230
|
}
|
|
@@ -859,6 +2237,62 @@ export class AutomationStore {
|
|
|
859
2237
|
return this.getTask(row.id);
|
|
860
2238
|
});
|
|
861
2239
|
}
|
|
2240
|
+
/**
|
|
2241
|
+
* Terminally quarantine a started task whose immutable execution snapshot
|
|
2242
|
+
* cannot be decoded. No run/evaluation is invented because historical scope
|
|
2243
|
+
* is unavailable; the occurrence and attempt still become durably unknown.
|
|
2244
|
+
*/
|
|
2245
|
+
quarantineInvalidExecutionSnapshot(input) {
|
|
2246
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
2247
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2248
|
+
}
|
|
2249
|
+
return this.transaction(() => {
|
|
2250
|
+
const row = this.requireMutableTask(input, ['running']);
|
|
2251
|
+
const attempt = this.database.prepare(`
|
|
2252
|
+
SELECT * FROM automation_attempts
|
|
2253
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2254
|
+
`).get(row.id, row.attempt_count, input.fencingToken);
|
|
2255
|
+
const occurrence = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2256
|
+
.get(row.occurrence_id);
|
|
2257
|
+
if (attempt === undefined || occurrence === undefined) {
|
|
2258
|
+
throw new AutomationStoreError('invalid-state', 'invalid snapshot task has no exact attempt or occurrence');
|
|
2259
|
+
}
|
|
2260
|
+
const executionMode = occurrence.dry_run === 0
|
|
2261
|
+
? 'production'
|
|
2262
|
+
: occurrence.dry_run === 1 ? 'preview' : 'unknown';
|
|
2263
|
+
const diagnostic = Object.freeze({
|
|
2264
|
+
schemaVersion: 1,
|
|
2265
|
+
failureClass: 'infrastructure',
|
|
2266
|
+
failurePhase: 'recovery',
|
|
2267
|
+
failureCode: 'execution-snapshot-invalid',
|
|
2268
|
+
promptSubmissionState: 'not-submitted',
|
|
2269
|
+
sideEffectState: 'none',
|
|
2270
|
+
retryability: 'after-intervention',
|
|
2271
|
+
budgetSettlementState: 'unknown',
|
|
2272
|
+
});
|
|
2273
|
+
this.database.prepare(`
|
|
2274
|
+
INSERT INTO automation_runs(
|
|
2275
|
+
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
2276
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2277
|
+
delivery_status, delivery_ref, evidence_status, evidence_json, created_at, updated_at
|
|
2278
|
+
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', ?, NULL, ?, NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
2279
|
+
`).run(`run-${row.id}`, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'immutable execution snapshot is invalid', executionMode, JSON.stringify(diagnostic), input.now, input.now);
|
|
2280
|
+
this.database.prepare(`
|
|
2281
|
+
UPDATE automation_attempts
|
|
2282
|
+
SET status = 'unknown', failure_code = 'execution-snapshot-invalid',
|
|
2283
|
+
finished_at = ?, updated_at = ?
|
|
2284
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2285
|
+
`).run(input.now, input.now, row.id, row.attempt_count, input.fencingToken);
|
|
2286
|
+
this.database.prepare(`
|
|
2287
|
+
UPDATE automation_tasks SET status = 'unknown', lease_until = NULL, updated_at = ? WHERE id = ?
|
|
2288
|
+
`).run(input.now, row.id);
|
|
2289
|
+
this.database.prepare(`
|
|
2290
|
+
UPDATE automation_occurrences
|
|
2291
|
+
SET status = 'unknown', reason = 'execution-snapshot-invalid', updated_at = ? WHERE id = ?
|
|
2292
|
+
`).run(input.now, row.occurrence_id);
|
|
2293
|
+
return this.getTask(row.id);
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
862
2296
|
requestCancellation(input) {
|
|
863
2297
|
if (!Number.isSafeInteger(input.now))
|
|
864
2298
|
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
@@ -892,6 +2326,28 @@ export class AutomationStore {
|
|
|
892
2326
|
if (usageJson === undefined || Buffer.byteLength(usageJson, 'utf8') > 16_384) {
|
|
893
2327
|
throw new AutomationStoreError('invalid-definition', 'usage must be bounded JSON');
|
|
894
2328
|
}
|
|
2329
|
+
const diagnostic = executionDiagnostic(input.diagnostic ?? legacyAutomationExecutionDiagnostic, 'invalid-definition');
|
|
2330
|
+
if (input.diagnostic !== undefined && !isLegacyExecutionDiagnostic(diagnostic)) {
|
|
2331
|
+
assertDiagnosticOutcome(input.outcome, diagnostic, (() => {
|
|
2332
|
+
const row = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?')
|
|
2333
|
+
.get(input.taskId);
|
|
2334
|
+
if (row === undefined || row.attempt_count < 1) {
|
|
2335
|
+
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
2336
|
+
}
|
|
2337
|
+
const attempt = this.database.prepare(`
|
|
2338
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
2339
|
+
`).get(row.id, row.attempt_count);
|
|
2340
|
+
const snapshot = attempt === undefined ? undefined : automationSnapshot(attempt, row.automation_id);
|
|
2341
|
+
if (snapshot === undefined) {
|
|
2342
|
+
throw new AutomationStoreError('invalid-state', 'fresh diagnostic has no immutable definition');
|
|
2343
|
+
}
|
|
2344
|
+
return snapshot.definition;
|
|
2345
|
+
})());
|
|
2346
|
+
}
|
|
2347
|
+
const diagnosticJson = JSON.stringify(diagnostic);
|
|
2348
|
+
if (Buffer.byteLength(diagnosticJson, 'utf8') > 4_096) {
|
|
2349
|
+
throw new AutomationStoreError('invalid-definition', 'execution diagnostic exceeds 4096 bytes');
|
|
2350
|
+
}
|
|
895
2351
|
const evidenceAttribution = normalizeEvidenceAttribution(input.evidenceAttribution);
|
|
896
2352
|
return this.transaction(() => {
|
|
897
2353
|
const existing = this.database.prepare('SELECT * FROM automation_runs WHERE task_id = ?').get(input.taskId);
|
|
@@ -904,7 +2360,8 @@ export class AutomationStore {
|
|
|
904
2360
|
&& existing.session_id === (input.sessionId ?? null)
|
|
905
2361
|
&& existing.artifact_ref === (input.artifactRef ?? null)
|
|
906
2362
|
&& existing.output_preview === preview
|
|
907
|
-
&& existing.usage_json === usageJson
|
|
2363
|
+
&& existing.usage_json === usageJson
|
|
2364
|
+
&& existing.diagnostic_json === diagnosticJson;
|
|
908
2365
|
if (!exact)
|
|
909
2366
|
throw new AutomationStoreError('idempotency-conflict', 'task completion was replayed with different output');
|
|
910
2367
|
return run(existing);
|
|
@@ -920,15 +2377,27 @@ export class AutomationStore {
|
|
|
920
2377
|
if (automation === undefined) {
|
|
921
2378
|
throw new AutomationStoreError('invalid-state', 'winning attempt has no immutable automation snapshot');
|
|
922
2379
|
}
|
|
2380
|
+
const occurrenceRow = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2381
|
+
.get(row.occurrence_id);
|
|
2382
|
+
if (occurrenceRow === undefined || (occurrenceRow.dry_run !== 0 && occurrenceRow.dry_run !== 1)) {
|
|
2383
|
+
throw new AutomationStoreError('invalid-state', 'winning task has no valid occurrence execution mode');
|
|
2384
|
+
}
|
|
2385
|
+
// `dryRun` remains the public compatibility field, while terminal sinks
|
|
2386
|
+
// reason about an explicit mode. Future evaluation/shadow modes must stay
|
|
2387
|
+
// non-production here unless they deliberately earn sink capabilities.
|
|
2388
|
+
const executionMode = occurrenceRow.dry_run === 1 ? 'preview' : 'production';
|
|
2389
|
+
const definitionHash = hash(automation.definition);
|
|
923
2390
|
const evidence = buildOutcomeEvidence({
|
|
924
2391
|
automation,
|
|
925
2392
|
runId,
|
|
926
2393
|
outcome: input.outcome,
|
|
2394
|
+
executionMode,
|
|
927
2395
|
occurredAt: input.now,
|
|
928
2396
|
attribution: evidenceAttribution,
|
|
929
2397
|
});
|
|
930
2398
|
const normalizedOutput = preview.normalize('NFC').trim();
|
|
931
2399
|
const runDeliveryStatus = input.outcome === 'succeeded'
|
|
2400
|
+
&& executionMode === 'production'
|
|
932
2401
|
&& automation.definition.deliveryBindingId !== undefined
|
|
933
2402
|
? normalizedOutput === '' || automation.definition.deliverySuppressExact?.includes(normalizedOutput) === true
|
|
934
2403
|
? 'suppressed'
|
|
@@ -938,39 +2407,112 @@ export class AutomationStore {
|
|
|
938
2407
|
automation,
|
|
939
2408
|
runId,
|
|
940
2409
|
outcome: input.outcome,
|
|
2410
|
+
executionMode,
|
|
941
2411
|
occurredAt: input.now,
|
|
942
2412
|
usage: input.usage,
|
|
943
2413
|
startedAt: attempt.started_at,
|
|
944
2414
|
attemptNumber: attempt.attempt_number,
|
|
945
2415
|
deliveryStatus: runDeliveryStatus === 'pending' ? 'unknown' : 'not-required',
|
|
946
2416
|
});
|
|
947
|
-
const evaluationJson = JSON.stringify(evaluation);
|
|
948
|
-
if (Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
2417
|
+
const evaluationJson = evaluation === undefined ? undefined : JSON.stringify(evaluation);
|
|
2418
|
+
if (evaluationJson !== undefined && Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
949
2419
|
throw new AutomationStoreError('invalid-definition', 'automation evaluation payload exceeds 32768 bytes');
|
|
950
2420
|
}
|
|
951
2421
|
this.database.prepare(`
|
|
952
2422
|
INSERT INTO automation_runs(
|
|
953
2423
|
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
954
|
-
output_preview, usage_json,
|
|
2424
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2425
|
+
delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
955
2426
|
created_at, updated_at
|
|
956
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
|
|
957
|
-
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson, runDeliveryStatus, evidence.status, evidence.json, input.now, input.now);
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
2427
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
|
|
2428
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson, executionMode, definitionHash, diagnosticJson, runDeliveryStatus, evidence.status, evidence.json, input.now, input.now);
|
|
2429
|
+
if (evaluationJson !== undefined) {
|
|
2430
|
+
this.database.prepare(`
|
|
2431
|
+
INSERT INTO automation_evaluation_outbox(
|
|
2432
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
2433
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
2434
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
2435
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
2436
|
+
}
|
|
964
2437
|
this.database.prepare(`
|
|
965
2438
|
UPDATE automation_attempts
|
|
966
2439
|
SET status = ?, failure_code = ?, finished_at = ?, updated_at = ? WHERE id = ?
|
|
967
|
-
`).run(input.outcome,
|
|
2440
|
+
`).run(input.outcome, diagnostic.failureClass === 'none' ? null : diagnostic.failureCode, input.now, input.now, attempt.id);
|
|
968
2441
|
this.database.prepare(`
|
|
969
2442
|
UPDATE automation_tasks SET status = ?, lease_until = NULL, updated_at = ? WHERE id = ?
|
|
970
2443
|
`).run(input.outcome, input.now, row.id);
|
|
971
2444
|
this.database.prepare(`
|
|
972
2445
|
UPDATE automation_occurrences SET status = ?, reason = ?, updated_at = ? WHERE id = ?
|
|
973
|
-
`).run(input.outcome,
|
|
2446
|
+
`).run(input.outcome, diagnostic.failureClass === 'none' ? null : diagnostic.failureCode, input.now, row.occurrence_id);
|
|
2447
|
+
const circuitRow = this.database.prepare(`
|
|
2448
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
2449
|
+
`).get(automation.id, definitionHash);
|
|
2450
|
+
const atomicCanary = occurrenceRow.trigger_kind === 'manual'
|
|
2451
|
+
&& occurrenceRow.trigger_key.startsWith('circuit-canary:');
|
|
2452
|
+
let terminalSuccessResolvesIncident = input.outcome === 'succeeded' && !atomicCanary;
|
|
2453
|
+
if (executionMode === 'production'
|
|
2454
|
+
&& circuitRow?.state === 'probing' && circuitRow.probe_task_id === row.id) {
|
|
2455
|
+
const probePassed = input.outcome === 'succeeded'
|
|
2456
|
+
&& diagnostic.failureClass === 'none'
|
|
2457
|
+
&& circuitRow.probe_lease_until !== null
|
|
2458
|
+
&& circuitRow.probe_lease_until > input.now;
|
|
2459
|
+
const classifiedProbeFailure = diagnostic.failureClass === 'configuration'
|
|
2460
|
+
|| diagnostic.failureClass === 'policy'
|
|
2461
|
+
|| diagnostic.failureClass === 'budget';
|
|
2462
|
+
this.database.prepare(`
|
|
2463
|
+
UPDATE automation_circuits
|
|
2464
|
+
SET state = ?, failure_class = ?, failure_phase = ?, failure_code = ?,
|
|
2465
|
+
opened_at = ?, updated_at = ?, probe_token = NULL,
|
|
2466
|
+
probe_lease_until = NULL, probe_task_id = NULL, version = version + 1
|
|
2467
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'probing'
|
|
2468
|
+
AND probe_task_id = ? AND version = ?
|
|
2469
|
+
`).run(probePassed ? 'closed' : 'open', classifiedProbeFailure ? diagnostic.failureClass : circuitRow.failure_class, classifiedProbeFailure ? diagnostic.failurePhase : circuitRow.failure_phase, classifiedProbeFailure ? diagnostic.failureCode : circuitRow.failure_code, probePassed ? circuitRow.opened_at : input.now, input.now, automation.id, definitionHash, row.id, circuitRow.version);
|
|
2470
|
+
terminalSuccessResolvesIncident = probePassed;
|
|
2471
|
+
}
|
|
2472
|
+
if (executionMode === 'production'
|
|
2473
|
+
&& diagnostic.failureCode !== 'circuit-open'
|
|
2474
|
+
&& (diagnostic.failureClass === 'configuration'
|
|
2475
|
+
|| diagnostic.failureClass === 'policy'
|
|
2476
|
+
|| diagnostic.failureClass === 'budget')) {
|
|
2477
|
+
this.database.prepare(`
|
|
2478
|
+
INSERT INTO automation_circuits(
|
|
2479
|
+
automation_id, definition_hash, state, failure_class, failure_phase,
|
|
2480
|
+
failure_code, opened_at, updated_at, version
|
|
2481
|
+
) VALUES (?, ?, 'open', ?, ?, ?, ?, ?, 1)
|
|
2482
|
+
ON CONFLICT(automation_id, definition_hash) DO UPDATE SET
|
|
2483
|
+
state = 'open', failure_class = excluded.failure_class,
|
|
2484
|
+
failure_phase = excluded.failure_phase, failure_code = excluded.failure_code,
|
|
2485
|
+
opened_at = excluded.opened_at, updated_at = excluded.updated_at,
|
|
2486
|
+
version = automation_circuits.version + 1
|
|
2487
|
+
WHERE automation_circuits.state = 'closed'
|
|
2488
|
+
`).run(automation.id, definitionHash, diagnostic.failureClass, diagnostic.failurePhase, diagnostic.failureCode, input.now, input.now);
|
|
2489
|
+
}
|
|
2490
|
+
const notificationRoute = incidentNotificationRoute(automation.definition);
|
|
2491
|
+
if (executionMode === 'production' && notificationRoute !== undefined) {
|
|
2492
|
+
if (terminalSuccessResolvesIncident) {
|
|
2493
|
+
this.resolveIncidentInTransaction(automation.id, definitionHash, 'terminal', input.now);
|
|
2494
|
+
}
|
|
2495
|
+
else if (input.outcome === 'succeeded' && diagnostic.failureClass === 'none') {
|
|
2496
|
+
// A canary that completed after its exact probe lease is evidence of
|
|
2497
|
+
// execution, not evidence that the open circuit recovered.
|
|
2498
|
+
this.markRecoveringIncidentOpenInTransaction(automation.id, definitionHash, input.now);
|
|
2499
|
+
}
|
|
2500
|
+
else if (!isHostAutomationDefinition(automation.definition) && input.outcome === 'cancelled') {
|
|
2501
|
+
// Expected coordinator stop/cancel-previous is not an owner incident.
|
|
2502
|
+
// If it interrupted an Agent recovery attempt, retain the prior open
|
|
2503
|
+
// incident rather than leaving its lifecycle stuck at recovering.
|
|
2504
|
+
this.markRecoveringIncidentOpenInTransaction(automation.id, definitionHash, input.now);
|
|
2505
|
+
}
|
|
2506
|
+
else {
|
|
2507
|
+
this.upsertTerminalIncidentInTransaction({
|
|
2508
|
+
automation,
|
|
2509
|
+
definitionHash,
|
|
2510
|
+
runId,
|
|
2511
|
+
diagnostic,
|
|
2512
|
+
now: input.now,
|
|
2513
|
+
});
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
974
2516
|
return run(this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId));
|
|
975
2517
|
});
|
|
976
2518
|
}
|
|
@@ -978,9 +2520,69 @@ export class AutomationStore {
|
|
|
978
2520
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
979
2521
|
throw new AutomationStoreError('invalid-definition', 'evidence limit must be between 1 and 1000');
|
|
980
2522
|
}
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
2523
|
+
const output = [];
|
|
2524
|
+
const rows = this.database.prepare(`
|
|
2525
|
+
SELECT * FROM automation_runs
|
|
2526
|
+
WHERE evidence_status = 'pending' ORDER BY updated_at, id LIMIT 1000
|
|
2527
|
+
`).all();
|
|
2528
|
+
for (const row of rows) {
|
|
2529
|
+
if (output.length === limit)
|
|
2530
|
+
break;
|
|
2531
|
+
try {
|
|
2532
|
+
const proof = this.getProvenProductionRun(row.id);
|
|
2533
|
+
if (proof === undefined
|
|
2534
|
+
|| (proof.run.status !== 'succeeded' && proof.run.status !== 'failed' && proof.run.status !== 'timed_out')
|
|
2535
|
+
|| proof.run.evidenceStatus !== 'pending' || proof.run.evidence === undefined) {
|
|
2536
|
+
try {
|
|
2537
|
+
this.suppressRunEvidence({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2538
|
+
}
|
|
2539
|
+
catch {
|
|
2540
|
+
// Concurrent settlement already removed it from this lane.
|
|
2541
|
+
}
|
|
2542
|
+
continue;
|
|
2543
|
+
}
|
|
2544
|
+
output.push(proof.run);
|
|
2545
|
+
}
|
|
2546
|
+
catch {
|
|
2547
|
+
// A poison row is monotonically quarantined so it cannot occupy the
|
|
2548
|
+
// head of a bounded batch forever or block independent peers.
|
|
2549
|
+
try {
|
|
2550
|
+
this.suppressRunEvidence({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2551
|
+
}
|
|
2552
|
+
catch {
|
|
2553
|
+
// Concurrent settlement already removed it from this lane.
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
return output;
|
|
2558
|
+
}
|
|
2559
|
+
suppressRunEvidence(input) {
|
|
2560
|
+
if (!Number.isSafeInteger(input.now))
|
|
2561
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2562
|
+
const changed = this.database.prepare(`
|
|
2563
|
+
UPDATE automation_runs
|
|
2564
|
+
SET evidence_status = 'suppressed', evidence_json = NULL, updated_at = ?
|
|
2565
|
+
WHERE id = ? AND evidence_status = ?
|
|
2566
|
+
`).run(input.now, input.runId, input.expectedStatus);
|
|
2567
|
+
if (changed.changes !== 1) {
|
|
2568
|
+
const current = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId);
|
|
2569
|
+
if (current === undefined)
|
|
2570
|
+
return undefined;
|
|
2571
|
+
if (current.evidence_status !== 'suppressed') {
|
|
2572
|
+
throw new AutomationStoreError('version-conflict', 'run evidence state changed before suppression');
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
const current = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId);
|
|
2576
|
+
if (current === undefined)
|
|
2577
|
+
return undefined;
|
|
2578
|
+
try {
|
|
2579
|
+
return run(current);
|
|
2580
|
+
}
|
|
2581
|
+
catch {
|
|
2582
|
+
// The effect intent is already terminal even if unrelated legacy fields
|
|
2583
|
+
// remain corrupt; callers do not need a public projection to stay safe.
|
|
2584
|
+
return undefined;
|
|
2585
|
+
}
|
|
984
2586
|
}
|
|
985
2587
|
deferRunEvidence(input) {
|
|
986
2588
|
if (!Number.isSafeInteger(input.now))
|
|
@@ -1111,9 +2713,32 @@ export class AutomationStore {
|
|
|
1111
2713
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
1112
2714
|
throw new AutomationStoreError('invalid-definition', 'delivery limit must be between 1 and 1000');
|
|
1113
2715
|
}
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
2716
|
+
const output = [];
|
|
2717
|
+
const rows = this.database.prepare(`
|
|
2718
|
+
SELECT * FROM automation_runs WHERE delivery_status = 'pending' ORDER BY created_at, id LIMIT 1000
|
|
2719
|
+
`).all();
|
|
2720
|
+
for (const row of rows) {
|
|
2721
|
+
if (output.length === limit)
|
|
2722
|
+
break;
|
|
2723
|
+
try {
|
|
2724
|
+
const proof = this.getProvenProductionRun(row.id);
|
|
2725
|
+
if (proof === undefined || proof.run.status !== 'succeeded'
|
|
2726
|
+
|| proof.automation.definition.deliveryBindingId === undefined) {
|
|
2727
|
+
this.suppressRunDelivery({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2728
|
+
continue;
|
|
2729
|
+
}
|
|
2730
|
+
output.push(proof.run);
|
|
2731
|
+
}
|
|
2732
|
+
catch {
|
|
2733
|
+
try {
|
|
2734
|
+
this.suppressRunDelivery({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2735
|
+
}
|
|
2736
|
+
catch {
|
|
2737
|
+
// A concurrent terminal settlement owns this row.
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
return output;
|
|
1117
2742
|
}
|
|
1118
2743
|
completeRunDelivery(input) {
|
|
1119
2744
|
if (!Number.isSafeInteger(input.now))
|
|
@@ -1247,33 +2872,61 @@ export class AutomationStore {
|
|
|
1247
2872
|
if (target === 'unknown') {
|
|
1248
2873
|
if (snapshot !== undefined) {
|
|
1249
2874
|
const runId = `run-${row.id}`;
|
|
2875
|
+
const recoveredOccurrence = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2876
|
+
.get(row.occurrence_id);
|
|
2877
|
+
const recoveredMode = recoveredOccurrence?.dry_run === 0
|
|
2878
|
+
? 'production'
|
|
2879
|
+
: recoveredOccurrence?.dry_run === 1 ? 'preview' : 'unknown';
|
|
2880
|
+
const recoveredDiagnostic = Object.freeze({
|
|
2881
|
+
schemaVersion: 1,
|
|
2882
|
+
failureClass: 'infrastructure',
|
|
2883
|
+
failurePhase: 'recovery',
|
|
2884
|
+
failureCode: 'runner-lease-expired',
|
|
2885
|
+
promptSubmissionState: 'unknown',
|
|
2886
|
+
sideEffectState: 'unknown',
|
|
2887
|
+
retryability: 'unsafe',
|
|
2888
|
+
budgetSettlementState: 'unknown',
|
|
2889
|
+
});
|
|
1250
2890
|
const evaluation = buildTerminalEvaluation({
|
|
1251
2891
|
automation: snapshot,
|
|
1252
2892
|
runId,
|
|
1253
2893
|
outcome: 'unknown',
|
|
2894
|
+
executionMode: recoveredMode,
|
|
1254
2895
|
occurredAt: input.now,
|
|
1255
2896
|
usage: {},
|
|
1256
2897
|
startedAt: attempt.started_at,
|
|
1257
2898
|
attemptNumber: attempt.attempt_number,
|
|
1258
2899
|
deliveryStatus: 'not-required',
|
|
1259
2900
|
});
|
|
1260
|
-
const evaluationJson = JSON.stringify(evaluation);
|
|
1261
|
-
if (Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
2901
|
+
const evaluationJson = evaluation === undefined ? undefined : JSON.stringify(evaluation);
|
|
2902
|
+
if (evaluationJson !== undefined && Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
1262
2903
|
throw new AutomationStoreError('invalid-state', 'recovered automation evaluation payload is oversized');
|
|
1263
2904
|
}
|
|
1264
2905
|
this.database.prepare(`
|
|
1265
2906
|
INSERT INTO automation_runs(
|
|
1266
2907
|
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
1267
|
-
output_preview, usage_json,
|
|
2908
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2909
|
+
delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
1268
2910
|
created_at, updated_at
|
|
1269
|
-
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
1270
|
-
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'runner lease expired before terminal receipt', input.now, input.now);
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
2911
|
+
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', ?, ?, ?, NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
2912
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'runner lease expired before terminal receipt', recoveredMode, hash(snapshot.definition), JSON.stringify(recoveredDiagnostic), input.now, input.now);
|
|
2913
|
+
if (evaluationJson !== undefined) {
|
|
2914
|
+
this.database.prepare(`
|
|
2915
|
+
INSERT INTO automation_evaluation_outbox(
|
|
2916
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
2917
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
2918
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
2919
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
2920
|
+
}
|
|
2921
|
+
if (recoveredMode === 'production') {
|
|
2922
|
+
this.upsertTerminalIncidentInTransaction({
|
|
2923
|
+
automation: snapshot,
|
|
2924
|
+
definitionHash: hash(snapshot.definition),
|
|
2925
|
+
runId,
|
|
2926
|
+
diagnostic: recoveredDiagnostic,
|
|
2927
|
+
now: input.now,
|
|
2928
|
+
});
|
|
2929
|
+
}
|
|
1277
2930
|
}
|
|
1278
2931
|
this.database.prepare(`
|
|
1279
2932
|
UPDATE automation_occurrences SET status = 'unknown', reason = 'runner-lease-expired', updated_at = ? WHERE id = ?
|
|
@@ -1314,10 +2967,55 @@ export class AutomationStore {
|
|
|
1314
2967
|
throw new AutomationStoreError('invalid-state', 'task state does not allow this transition');
|
|
1315
2968
|
return row;
|
|
1316
2969
|
}
|
|
1317
|
-
claimTaskInTransaction(row, input) {
|
|
1318
|
-
const
|
|
1319
|
-
if (
|
|
2970
|
+
claimTaskInTransaction(row, input, hostAvailability) {
|
|
2971
|
+
const storedAutomation = this.get(row.automation_id);
|
|
2972
|
+
if (storedAutomation === undefined)
|
|
1320
2973
|
return undefined;
|
|
2974
|
+
// claimTask() bypasses claimNextTask()'s candidate scan, so retain the
|
|
2975
|
+
// same exact paused-Growth proof here as an independent admission fence.
|
|
2976
|
+
const designatedGrowthTask = storedAutomation.status === 'paused'
|
|
2977
|
+
&& this.database.prepare(`
|
|
2978
|
+
SELECT 1 AS present
|
|
2979
|
+
FROM automation_tasks task
|
|
2980
|
+
JOIN automation_definitions definition ON definition.id = task.automation_id
|
|
2981
|
+
WHERE task.id = ?
|
|
2982
|
+
AND definition.status = 'paused'
|
|
2983
|
+
AND definition.system_owner = ?
|
|
2984
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
2985
|
+
`).get(row.id, growthAutomationOwner) !== undefined;
|
|
2986
|
+
if (storedAutomation.status !== 'active' && !designatedGrowthTask)
|
|
2987
|
+
return undefined;
|
|
2988
|
+
// A Growth rehearsal/exposure never activates the mutable definition. Its
|
|
2989
|
+
// one designated task receives an immutable active execution snapshot,
|
|
2990
|
+
// while every scheduler/materialization query still sees the artifact as
|
|
2991
|
+
// paused and therefore cannot create a second exposure.
|
|
2992
|
+
const automation = designatedGrowthTask
|
|
2993
|
+
? Object.freeze({ ...storedAutomation, status: 'active', nextRunAt: undefined })
|
|
2994
|
+
: storedAutomation;
|
|
2995
|
+
if (isHostAutomationDefinition(automation.definition)) {
|
|
2996
|
+
const occurrence = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
2997
|
+
.get(row.occurrence_id);
|
|
2998
|
+
if (occurrence === undefined || (occurrence.dry_run !== 0 && occurrence.dry_run !== 1)) {
|
|
2999
|
+
throw new AutomationStoreError('invalid-state', 'Host task has no exact execution mode');
|
|
3000
|
+
}
|
|
3001
|
+
const preview = occurrence.dry_run === 1;
|
|
3002
|
+
const definitionHash = hash(automation.definition);
|
|
3003
|
+
const availability = this.exactHostAvailability(hostAvailability, automation.id, definitionHash, 'claim');
|
|
3004
|
+
if (availability?.available !== true) {
|
|
3005
|
+
if (!preview) {
|
|
3006
|
+
this.upsertAvailabilityIncidentInTransaction({
|
|
3007
|
+
automation,
|
|
3008
|
+
definitionHash,
|
|
3009
|
+
stage: 'claim',
|
|
3010
|
+
failureCode: availability?.reasonCode ?? 'host-executor-proof-missing',
|
|
3011
|
+
now: input.now,
|
|
3012
|
+
});
|
|
3013
|
+
}
|
|
3014
|
+
return undefined;
|
|
3015
|
+
}
|
|
3016
|
+
if (!preview)
|
|
3017
|
+
this.resolveIncidentInTransaction(automation.id, definitionHash, 'claim', input.now);
|
|
3018
|
+
}
|
|
1321
3019
|
const active = this.database.prepare(`
|
|
1322
3020
|
SELECT * FROM automation_tasks
|
|
1323
3021
|
WHERE automation_id = ? AND id != ? AND status IN ('claimed', 'running')
|
|
@@ -1358,6 +3056,126 @@ export class AutomationStore {
|
|
|
1358
3056
|
`).run(attemptId, row.id, attemptNumber, input.ownerId, input.fencingToken, snapshot.hash, snapshot.json, input.now, input.now);
|
|
1359
3057
|
return this.getTask(row.id);
|
|
1360
3058
|
}
|
|
3059
|
+
exactHostAvailability(decisions, automationId, definitionHash, stage) {
|
|
3060
|
+
const exact = decisions?.filter(decision => decision.automationId === automationId
|
|
3061
|
+
&& decision.definitionHash === definitionHash && decision.stage === stage) ?? [];
|
|
3062
|
+
if (exact.length !== 1)
|
|
3063
|
+
return undefined;
|
|
3064
|
+
const decision = exact[0];
|
|
3065
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(decision.reasonCode))
|
|
3066
|
+
return undefined;
|
|
3067
|
+
return decision;
|
|
3068
|
+
}
|
|
3069
|
+
upsertAvailabilityIncidentInTransaction(input) {
|
|
3070
|
+
if (!isHostAutomationDefinition(input.automation.definition))
|
|
3071
|
+
return;
|
|
3072
|
+
const code = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(input.failureCode)
|
|
3073
|
+
? input.failureCode
|
|
3074
|
+
: 'host-executor-unavailable';
|
|
3075
|
+
const id = `incident-${hash([
|
|
3076
|
+
input.automation.id, input.definitionHash, input.stage,
|
|
3077
|
+
])}`;
|
|
3078
|
+
this.database.prepare(`
|
|
3079
|
+
INSERT INTO automation_incidents(
|
|
3080
|
+
id, automation_id, definition_hash, stage, state, failure_class,
|
|
3081
|
+
failure_phase, failure_code, side_effect_state, retryability,
|
|
3082
|
+
notification_route_id, lifecycle_generation, presentation_revision,
|
|
3083
|
+
alert_status, alert_ref, run_id,
|
|
3084
|
+
opened_at, updated_at, resolved_at, version
|
|
3085
|
+
) VALUES (?, ?, ?, ?, 'open', 'configuration', 'executor-availability', ?,
|
|
3086
|
+
'none', 'after-intervention', ?, 1, 1, 'pending', NULL, NULL, ?, ?, NULL, 1)
|
|
3087
|
+
ON CONFLICT(automation_id, definition_hash, stage) DO UPDATE SET
|
|
3088
|
+
state = 'open', failure_class = 'configuration',
|
|
3089
|
+
failure_phase = 'executor-availability', failure_code = excluded.failure_code,
|
|
3090
|
+
side_effect_state = 'none', retryability = 'after-intervention',
|
|
3091
|
+
notification_route_id = excluded.notification_route_id,
|
|
3092
|
+
lifecycle_generation = CASE
|
|
3093
|
+
WHEN automation_incidents.state = 'resolved'
|
|
3094
|
+
THEN automation_incidents.lifecycle_generation + 1
|
|
3095
|
+
ELSE automation_incidents.lifecycle_generation
|
|
3096
|
+
END,
|
|
3097
|
+
presentation_revision = automation_incidents.presentation_revision + 1,
|
|
3098
|
+
alert_status = 'pending',
|
|
3099
|
+
alert_ref = CASE
|
|
3100
|
+
WHEN automation_incidents.state = 'resolved' THEN NULL
|
|
3101
|
+
ELSE automation_incidents.alert_ref
|
|
3102
|
+
END,
|
|
3103
|
+
opened_at = CASE
|
|
3104
|
+
WHEN automation_incidents.state = 'resolved' THEN excluded.opened_at
|
|
3105
|
+
ELSE automation_incidents.opened_at
|
|
3106
|
+
END,
|
|
3107
|
+
updated_at = excluded.updated_at, resolved_at = NULL,
|
|
3108
|
+
version = automation_incidents.version + 1
|
|
3109
|
+
`).run(id, input.automation.id, input.definitionHash, input.stage, code, input.automation.definition.execution.ownerRouteId, input.now, input.now);
|
|
3110
|
+
}
|
|
3111
|
+
resolveIncidentInTransaction(automationId, definitionHash, stage, now) {
|
|
3112
|
+
this.database.prepare(`
|
|
3113
|
+
UPDATE automation_incidents
|
|
3114
|
+
SET state = 'resolved', resolved_at = ?, updated_at = ?,
|
|
3115
|
+
presentation_revision = presentation_revision + 1,
|
|
3116
|
+
alert_status = 'pending',
|
|
3117
|
+
version = version + 1
|
|
3118
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = ?
|
|
3119
|
+
AND state IN ('open', 'recovering')
|
|
3120
|
+
`).run(now, now, automationId, definitionHash, stage);
|
|
3121
|
+
}
|
|
3122
|
+
markIncidentRecoveringInTransaction(automationId, definitionHash, now) {
|
|
3123
|
+
this.database.prepare(`
|
|
3124
|
+
UPDATE automation_incidents
|
|
3125
|
+
SET state = 'recovering', resolved_at = NULL, updated_at = ?,
|
|
3126
|
+
presentation_revision = presentation_revision + 1,
|
|
3127
|
+
alert_status = 'pending', version = version + 1
|
|
3128
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = 'terminal' AND state = 'open'
|
|
3129
|
+
`).run(now, automationId, definitionHash);
|
|
3130
|
+
}
|
|
3131
|
+
markRecoveringIncidentOpenInTransaction(automationId, definitionHash, now) {
|
|
3132
|
+
this.database.prepare(`
|
|
3133
|
+
UPDATE automation_incidents
|
|
3134
|
+
SET state = 'open', updated_at = ?, presentation_revision = presentation_revision + 1,
|
|
3135
|
+
alert_status = 'pending', version = version + 1
|
|
3136
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = 'terminal' AND state = 'recovering'
|
|
3137
|
+
`).run(now, automationId, definitionHash);
|
|
3138
|
+
}
|
|
3139
|
+
upsertTerminalIncidentInTransaction(input) {
|
|
3140
|
+
const route = incidentNotificationRoute(input.automation.definition);
|
|
3141
|
+
if (route === undefined || input.diagnostic.failureClass === 'none')
|
|
3142
|
+
return;
|
|
3143
|
+
const id = `incident-${hash([input.automation.id, input.definitionHash, 'terminal'])}`;
|
|
3144
|
+
this.database.prepare(`
|
|
3145
|
+
INSERT INTO automation_incidents(
|
|
3146
|
+
id, automation_id, definition_hash, stage, state, failure_class,
|
|
3147
|
+
failure_phase, failure_code, side_effect_state, retryability,
|
|
3148
|
+
notification_route_id, lifecycle_generation, presentation_revision,
|
|
3149
|
+
alert_status, alert_ref, run_id,
|
|
3150
|
+
opened_at, updated_at, resolved_at, version
|
|
3151
|
+
) VALUES (?, ?, ?, 'terminal', 'open', ?, ?, ?, ?, ?, ?,
|
|
3152
|
+
1, 1, 'pending', NULL, ?, ?, ?, NULL, 1)
|
|
3153
|
+
ON CONFLICT(automation_id, definition_hash, stage) DO UPDATE SET
|
|
3154
|
+
state = 'open', failure_class = excluded.failure_class,
|
|
3155
|
+
failure_phase = excluded.failure_phase, failure_code = excluded.failure_code,
|
|
3156
|
+
side_effect_state = excluded.side_effect_state,
|
|
3157
|
+
retryability = excluded.retryability,
|
|
3158
|
+
notification_route_id = excluded.notification_route_id,
|
|
3159
|
+
run_id = excluded.run_id,
|
|
3160
|
+
lifecycle_generation = CASE
|
|
3161
|
+
WHEN automation_incidents.state = 'resolved'
|
|
3162
|
+
THEN automation_incidents.lifecycle_generation + 1
|
|
3163
|
+
ELSE automation_incidents.lifecycle_generation
|
|
3164
|
+
END,
|
|
3165
|
+
presentation_revision = automation_incidents.presentation_revision + 1,
|
|
3166
|
+
alert_status = 'pending',
|
|
3167
|
+
alert_ref = CASE
|
|
3168
|
+
WHEN automation_incidents.state = 'resolved' THEN NULL
|
|
3169
|
+
ELSE automation_incidents.alert_ref
|
|
3170
|
+
END,
|
|
3171
|
+
opened_at = CASE
|
|
3172
|
+
WHEN automation_incidents.state = 'resolved' THEN excluded.opened_at
|
|
3173
|
+
ELSE automation_incidents.opened_at
|
|
3174
|
+
END,
|
|
3175
|
+
updated_at = excluded.updated_at, resolved_at = NULL,
|
|
3176
|
+
version = automation_incidents.version + 1
|
|
3177
|
+
`).run(id, input.automation.id, input.definitionHash, input.diagnostic.failureClass, input.diagnostic.failurePhase, input.diagnostic.failureCode, input.diagnostic.sideEffectState, input.diagnostic.retryability, route.id, input.runId, input.now, input.now);
|
|
3178
|
+
}
|
|
1361
3179
|
insertOccurrence(input) {
|
|
1362
3180
|
const id = stableOccurrenceId(input.automationId, input.triggerKind, input.triggerKey);
|
|
1363
3181
|
const existing = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?').get(id);
|