@dsh-enhanced/assistant-automations 0.1.6 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -7
- package/lib/coordinator.d.ts +57 -5
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +508 -88
- package/lib/coordinator.js.map +1 -1
- package/lib/growth.d.ts +120 -0
- package/lib/growth.d.ts.map +1 -0
- package/lib/growth.js +383 -0
- package/lib/growth.js.map +1 -0
- package/lib/host-executors.d.ts +25 -0
- package/lib/host-executors.d.ts.map +1 -0
- package/lib/host-executors.js +181 -0
- package/lib/host-executors.js.map +1 -0
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/operator.d.ts +2 -0
- package/lib/operator.d.ts.map +1 -1
- package/lib/operator.js +6 -2
- package/lib/operator.js.map +1 -1
- package/lib/proposals.d.ts +2 -2
- package/lib/proposals.d.ts.map +1 -1
- package/lib/proposals.js +34 -22
- package/lib/proposals.js.map +1 -1
- package/lib/runner.d.ts +28 -2
- package/lib/runner.d.ts.map +1 -1
- package/lib/runner.js +525 -73
- package/lib/runner.js.map +1 -1
- package/lib/service.d.ts +110 -4
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +1021 -16
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +450 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +211 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +2260 -58
- package/lib/store.js.map +1 -1
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +18 -6
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +399 -9
- package/lib/types.d.ts.map +1 -1
- package/lib/types.js +13 -1
- package/lib/types.js.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.d.ts.map +1 -1
- package/lib/version.js +1 -1
- package/lib/version.js.map +1 -1
- package/package.json +16 -5
package/lib/store.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import { isAbsolute } from 'node:path';
|
|
1
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
3
3
|
import { dueOccurrences, nextOccurrence, previousOccurrence, validateSchedule, } from './schedule.js';
|
|
4
4
|
import { AutomationDatabaseError, openAutomationDatabase } from './sqlite.js';
|
|
5
|
+
import { isHostAutomationDefinition, legacyAutomationExecutionDiagnostic } from './types.js';
|
|
5
6
|
export class AutomationStoreError extends Error {
|
|
6
7
|
code;
|
|
7
8
|
constructor(code, message) {
|
|
@@ -10,9 +11,61 @@ export class AutomationStoreError extends Error {
|
|
|
10
11
|
this.name = 'AutomationStoreError';
|
|
11
12
|
}
|
|
12
13
|
}
|
|
14
|
+
// Growth's owner is deliberately spelled out at the Store admission boundary
|
|
15
|
+
// instead of treating every paused system-owned definition as runnable. Keep
|
|
16
|
+
// this coupled to GrowthAutomationStore.createExecutionTask(), which is the
|
|
17
|
+
// only writer of the task/occurrence/operation tuple below.
|
|
18
|
+
const growthAutomationOwner = 'assistant-growth-experiments';
|
|
19
|
+
/**
|
|
20
|
+
* SQL-only admission proof for the one paused-definition exception.
|
|
21
|
+
*
|
|
22
|
+
* A queued task is eligible only when it is the exact task created for a
|
|
23
|
+
* pending Growth shadow or canary operation. The artifact must still bind to
|
|
24
|
+
* the current definition revision, and the immutable occurrence key must name
|
|
25
|
+
* the same operation. This intentionally cannot match ordinary paused work.
|
|
26
|
+
*/
|
|
27
|
+
function pausedGrowthTaskAdmission(taskAlias, definitionAlias) {
|
|
28
|
+
return `
|
|
29
|
+
EXISTS (
|
|
30
|
+
SELECT 1
|
|
31
|
+
FROM automation_growth_artifacts artifact
|
|
32
|
+
JOIN automation_occurrences occurrence
|
|
33
|
+
ON occurrence.id = ${taskAlias}.occurrence_id
|
|
34
|
+
AND occurrence.automation_id = artifact.automation_id
|
|
35
|
+
JOIN automation_growth_operations operation
|
|
36
|
+
ON occurrence.trigger_key = CASE operation.operation_kind
|
|
37
|
+
WHEN 'shadow' THEN 'growth-shadow:' || operation.operation_id
|
|
38
|
+
WHEN 'canary' THEN 'growth-canary:' || operation.operation_id
|
|
39
|
+
ELSE ''
|
|
40
|
+
END
|
|
41
|
+
WHERE artifact.automation_id = ${definitionAlias}.id
|
|
42
|
+
AND artifact.definition_hash = ${definitionAlias}.definition_hash
|
|
43
|
+
AND artifact.definition_version = ${definitionAlias}.version
|
|
44
|
+
AND occurrence.trigger_kind = 'manual'
|
|
45
|
+
AND operation.status = 'pending'
|
|
46
|
+
AND (
|
|
47
|
+
(
|
|
48
|
+
artifact.shadow_task_id = ${taskAlias}.id
|
|
49
|
+
AND artifact.state = 'paused'
|
|
50
|
+
AND occurrence.dry_run = 1
|
|
51
|
+
AND operation.operation_kind = 'shadow'
|
|
52
|
+
)
|
|
53
|
+
OR (
|
|
54
|
+
artifact.canary_task_id = ${taskAlias}.id
|
|
55
|
+
AND artifact.state = 'canary-pending'
|
|
56
|
+
AND occurrence.dry_run = 0
|
|
57
|
+
AND operation.operation_kind = 'canary'
|
|
58
|
+
)
|
|
59
|
+
)
|
|
60
|
+
)
|
|
61
|
+
`;
|
|
62
|
+
}
|
|
13
63
|
function hash(value) {
|
|
14
64
|
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
15
65
|
}
|
|
66
|
+
function hashText(value) {
|
|
67
|
+
return createHash('sha256').update(value).digest('hex');
|
|
68
|
+
}
|
|
16
69
|
function text(value, field, maximum = 1_000) {
|
|
17
70
|
if (typeof value !== 'string' || value.trim() === '') {
|
|
18
71
|
throw new AutomationStoreError('invalid-definition', `${field} must be a non-empty string`);
|
|
@@ -53,24 +106,27 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
53
106
|
throw new AutomationStoreError('invalid-definition', 'automation definition must be an object');
|
|
54
107
|
}
|
|
55
108
|
const input = value;
|
|
56
|
-
const
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
109
|
+
const host = typeof input['execution'] === 'object' && input['execution'] !== null
|
|
110
|
+
&& !Array.isArray(input['execution'])
|
|
111
|
+
&& input['execution']['kind'] === 'host';
|
|
112
|
+
const commonAllowed = [
|
|
113
|
+
'agentPreset', 'budgetAmount', 'budgetId', 'maxRetries', 'misfire', 'name', 'overlap',
|
|
114
|
+
'principal', 'retrySafety', 'schedule', 'timeoutMs', 'workspace',
|
|
115
|
+
];
|
|
116
|
+
const allowed = new Set(host
|
|
117
|
+
? [...commonAllowed, 'execution']
|
|
118
|
+
: [
|
|
119
|
+
...commonAllowed, 'allowedTools', 'approvalBindingId', 'deliveryBindingId', 'deliverySuppressExact',
|
|
120
|
+
'maxOutputTokens', 'maxToolCalls', 'model', 'prompt', 'provider',
|
|
121
|
+
]);
|
|
61
122
|
if (Object.keys(input).some(key => !allowed.has(key))) {
|
|
62
123
|
throw new AutomationStoreError('invalid-definition', 'automation definition contains an unknown field');
|
|
63
124
|
}
|
|
64
|
-
|
|
125
|
+
const workspace = text(input['workspace'], 'workspace', 4_096);
|
|
126
|
+
const agentPreset = text(input['agentPreset'], 'agentPreset', 200);
|
|
127
|
+
if (!isAbsolute(workspace)) {
|
|
65
128
|
throw new AutomationStoreError('invalid-definition', 'automation workspace must be absolute');
|
|
66
129
|
}
|
|
67
|
-
if (!Array.isArray(input['allowedTools']) || input['allowedTools'].length > maxAllowedTools) {
|
|
68
|
-
throw new AutomationStoreError('invalid-definition', 'allowedTools must be a bounded array');
|
|
69
|
-
}
|
|
70
|
-
const allowedTools = input['allowedTools'].map(value => text(value, 'allowedTools item', 200));
|
|
71
|
-
if (new Set(allowedTools).size !== allowedTools.length) {
|
|
72
|
-
throw new AutomationStoreError('invalid-definition', 'allowedTools contains a duplicate');
|
|
73
|
-
}
|
|
74
130
|
const overlap = input['overlap'];
|
|
75
131
|
if (overlap !== 'skip' && overlap !== 'queue-one' && overlap !== 'cancel-previous') {
|
|
76
132
|
throw new AutomationStoreError('invalid-definition', 'invalid overlap policy');
|
|
@@ -86,6 +142,66 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
86
142
|
if ((budgetId === undefined) !== (budgetAmount === undefined)) {
|
|
87
143
|
throw new AutomationStoreError('invalid-definition', 'budgetId and budgetAmount must be supplied together');
|
|
88
144
|
}
|
|
145
|
+
const control = {
|
|
146
|
+
name: text(input['name'], 'name', 500),
|
|
147
|
+
schedule: validateSchedule(input['schedule']),
|
|
148
|
+
workspace,
|
|
149
|
+
agentPreset,
|
|
150
|
+
timeoutMs: safeInteger(input['timeoutMs'], 'timeoutMs', 1_000, 86_400_000),
|
|
151
|
+
misfire: misfire(input['misfire']),
|
|
152
|
+
overlap: overlap,
|
|
153
|
+
retrySafety: retrySafety,
|
|
154
|
+
maxRetries: safeInteger(input['maxRetries'], 'maxRetries', 0, 10),
|
|
155
|
+
principal: text(input['principal'], 'principal', 500),
|
|
156
|
+
...(budgetId === undefined ? {} : { budgetId, budgetAmount: budgetAmount }),
|
|
157
|
+
};
|
|
158
|
+
if (host) {
|
|
159
|
+
const rawExecution = input['execution'];
|
|
160
|
+
const executionAllowed = new Set([
|
|
161
|
+
'activationNonce', 'catalogDigest', 'executorContractVersion', 'executorId', 'kind',
|
|
162
|
+
'ownerRouteId', 'runbookId', 'runbookVersion', 'scopeDigest', 'targetScope',
|
|
163
|
+
]);
|
|
164
|
+
if (Object.keys(rawExecution).some(key => !executionAllowed.has(key))) {
|
|
165
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution spec contains an unknown field');
|
|
166
|
+
}
|
|
167
|
+
const rawTarget = rawExecution['targetScope'];
|
|
168
|
+
if (typeof rawTarget !== 'object' || rawTarget === null || Array.isArray(rawTarget)
|
|
169
|
+
|| Object.keys(rawTarget).some(key => key !== 'workspace' && key !== 'preset')) {
|
|
170
|
+
throw new AutomationStoreError('invalid-definition', 'Host targetScope must be an exact object');
|
|
171
|
+
}
|
|
172
|
+
const target = rawTarget;
|
|
173
|
+
const targetWorkspace = text(target['workspace'], 'execution.targetScope.workspace', 4_096);
|
|
174
|
+
const targetPreset = text(target['preset'], 'execution.targetScope.preset', 200);
|
|
175
|
+
if (targetWorkspace !== workspace || targetPreset !== agentPreset) {
|
|
176
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution targetScope must exactly match automation workspace and agentPreset');
|
|
177
|
+
}
|
|
178
|
+
const catalogDigest = text(rawExecution['catalogDigest'], 'execution.catalogDigest', 64).toLowerCase();
|
|
179
|
+
if (!/^[a-f0-9]{64}$/u.test(catalogDigest)) {
|
|
180
|
+
throw new AutomationStoreError('invalid-definition', 'Host execution catalogDigest must be a SHA-256 digest');
|
|
181
|
+
}
|
|
182
|
+
const execution = Object.freeze({
|
|
183
|
+
kind: 'host',
|
|
184
|
+
executorId: text(rawExecution['executorId'], 'execution.executorId', 200),
|
|
185
|
+
executorContractVersion: safeInteger(rawExecution['executorContractVersion'], 'execution.executorContractVersion', 1, 1_000_000),
|
|
186
|
+
runbookId: text(rawExecution['runbookId'], 'execution.runbookId', 200),
|
|
187
|
+
runbookVersion: safeInteger(rawExecution['runbookVersion'], 'execution.runbookVersion', 1, 1_000_000),
|
|
188
|
+
catalogDigest,
|
|
189
|
+
targetScope: Object.freeze({ workspace: targetWorkspace, preset: targetPreset }),
|
|
190
|
+
// Never trust a caller-provided digest. Recompute the canonical tuple
|
|
191
|
+
// that is actually stored in the immutable definition.
|
|
192
|
+
scopeDigest: hash([targetWorkspace, targetPreset]),
|
|
193
|
+
ownerRouteId: text(rawExecution['ownerRouteId'], 'execution.ownerRouteId', 500),
|
|
194
|
+
activationNonce: text(rawExecution['activationNonce'], 'execution.activationNonce', 500),
|
|
195
|
+
});
|
|
196
|
+
return Object.freeze({ ...control, execution });
|
|
197
|
+
}
|
|
198
|
+
if (!Array.isArray(input['allowedTools']) || input['allowedTools'].length > maxAllowedTools) {
|
|
199
|
+
throw new AutomationStoreError('invalid-definition', 'allowedTools must be a bounded array');
|
|
200
|
+
}
|
|
201
|
+
const allowedTools = input['allowedTools'].map(value => text(value, 'allowedTools item', 200));
|
|
202
|
+
if (new Set(allowedTools).size !== allowedTools.length) {
|
|
203
|
+
throw new AutomationStoreError('invalid-definition', 'allowedTools contains a duplicate');
|
|
204
|
+
}
|
|
89
205
|
const deliverySuppressExact = input['deliverySuppressExact'] === undefined
|
|
90
206
|
? undefined
|
|
91
207
|
: (() => {
|
|
@@ -102,29 +218,94 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
102
218
|
throw new AutomationStoreError('invalid-definition', 'deliverySuppressExact requires deliveryBindingId');
|
|
103
219
|
}
|
|
104
220
|
return Object.freeze({
|
|
105
|
-
name:
|
|
221
|
+
name: control.name,
|
|
106
222
|
prompt: text(input['prompt'], 'prompt', maxPromptBytes),
|
|
107
|
-
schedule:
|
|
108
|
-
workspace:
|
|
109
|
-
agentPreset:
|
|
223
|
+
schedule: control.schedule,
|
|
224
|
+
workspace: control.workspace,
|
|
225
|
+
agentPreset: control.agentPreset,
|
|
110
226
|
provider: text(input['provider'], 'provider', 200),
|
|
111
227
|
model: text(input['model'], 'model', 500),
|
|
112
228
|
allowedTools: Object.freeze(allowedTools),
|
|
113
|
-
timeoutMs:
|
|
229
|
+
timeoutMs: control.timeoutMs,
|
|
114
230
|
maxOutputTokens: safeInteger(input['maxOutputTokens'], 'maxOutputTokens', 1, 1_000_000),
|
|
115
231
|
maxToolCalls: safeInteger(input['maxToolCalls'], 'maxToolCalls', 0, 10_000),
|
|
116
|
-
misfire: misfire
|
|
117
|
-
overlap: overlap,
|
|
118
|
-
retrySafety: retrySafety,
|
|
119
|
-
maxRetries:
|
|
120
|
-
principal:
|
|
232
|
+
misfire: control.misfire,
|
|
233
|
+
overlap: control.overlap,
|
|
234
|
+
retrySafety: control.retrySafety,
|
|
235
|
+
maxRetries: control.maxRetries,
|
|
236
|
+
principal: control.principal,
|
|
121
237
|
...(budgetId === undefined ? {} : { budgetId, budgetAmount: budgetAmount }),
|
|
238
|
+
...(input['approvalBindingId'] === undefined
|
|
239
|
+
? {}
|
|
240
|
+
: { approvalBindingId: text(input['approvalBindingId'], 'approvalBindingId', 500) }),
|
|
122
241
|
...(input['deliveryBindingId'] === undefined
|
|
123
242
|
? {}
|
|
124
243
|
: { deliveryBindingId: text(input['deliveryBindingId'], 'deliveryBindingId', 500) }),
|
|
125
244
|
...(deliverySuppressExact === undefined ? {} : { deliverySuppressExact }),
|
|
126
245
|
});
|
|
127
246
|
}
|
|
247
|
+
function freezeAutomationSnapshot(value, expectedId) {
|
|
248
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
249
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot must be an object');
|
|
250
|
+
}
|
|
251
|
+
const input = value;
|
|
252
|
+
if (typeof input['id'] !== 'string' || (expectedId !== undefined && input['id'] !== expectedId)) {
|
|
253
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot id does not match its task');
|
|
254
|
+
}
|
|
255
|
+
if (input['status'] !== 'active' || !Number.isSafeInteger(input['version']) || input['version'] < 1
|
|
256
|
+
|| !Number.isSafeInteger(input['createdAt']) || !Number.isSafeInteger(input['updatedAt'])) {
|
|
257
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot metadata is invalid');
|
|
258
|
+
}
|
|
259
|
+
const nextRunAt = input['nextRunAt'];
|
|
260
|
+
if (nextRunAt !== undefined && !Number.isSafeInteger(nextRunAt)) {
|
|
261
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot nextRunAt is invalid');
|
|
262
|
+
}
|
|
263
|
+
const owner = input['owner'];
|
|
264
|
+
if (owner !== undefined && typeof owner !== 'string') {
|
|
265
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot owner is invalid');
|
|
266
|
+
}
|
|
267
|
+
const definition = normalizeAutomationDefinition(input['definition'], 16 * 1024 * 1024, 10_000);
|
|
268
|
+
return Object.freeze({
|
|
269
|
+
id: input['id'],
|
|
270
|
+
...(owner === undefined ? {} : { owner }),
|
|
271
|
+
definition,
|
|
272
|
+
status: 'active',
|
|
273
|
+
nextRunAt: nextRunAt,
|
|
274
|
+
createdAt: input['createdAt'],
|
|
275
|
+
updatedAt: input['updatedAt'],
|
|
276
|
+
version: input['version'],
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
function encodeAutomationSnapshot(value) {
|
|
280
|
+
const json = JSON.stringify(value);
|
|
281
|
+
if (Buffer.byteLength(json, 'utf8') > 16 * 1024 * 1024) {
|
|
282
|
+
throw new AutomationStoreError('invalid-definition', 'automation execution snapshot exceeds 16777216 bytes');
|
|
283
|
+
}
|
|
284
|
+
return { hash: hashText(json), json };
|
|
285
|
+
}
|
|
286
|
+
function decodeAutomationSnapshot(snapshotHash, snapshotJson, expectedId) {
|
|
287
|
+
if (snapshotHash === null || snapshotJson === null)
|
|
288
|
+
return undefined;
|
|
289
|
+
if (Buffer.byteLength(snapshotJson, 'utf8') > 16 * 1024 * 1024
|
|
290
|
+
|| hashText(snapshotJson) !== snapshotHash) {
|
|
291
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot integrity check failed');
|
|
292
|
+
}
|
|
293
|
+
try {
|
|
294
|
+
return freezeAutomationSnapshot(JSON.parse(snapshotJson), expectedId);
|
|
295
|
+
}
|
|
296
|
+
catch (error) {
|
|
297
|
+
if (error instanceof AutomationStoreError)
|
|
298
|
+
throw error;
|
|
299
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot contains invalid JSON');
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
function automationSnapshot(row, expectedId) {
|
|
303
|
+
return decodeAutomationSnapshot(row.automation_snapshot_hash, row.automation_snapshot_json, expectedId);
|
|
304
|
+
}
|
|
305
|
+
function executionScopeMatches(snapshot, scope) {
|
|
306
|
+
return resolve(snapshot.definition.workspace.normalize('NFC')) === resolve(scope.workspace.normalize('NFC'))
|
|
307
|
+
&& snapshot.definition.agentPreset.normalize('NFC').trim() === scope.agentPreset.normalize('NFC').trim();
|
|
308
|
+
}
|
|
128
309
|
function record(row) {
|
|
129
310
|
return Object.freeze({
|
|
130
311
|
id: row.id,
|
|
@@ -137,6 +318,24 @@ function record(row) {
|
|
|
137
318
|
version: row.version,
|
|
138
319
|
});
|
|
139
320
|
}
|
|
321
|
+
function incidentNotificationRoute(definition) {
|
|
322
|
+
if (isHostAutomationDefinition(definition)) {
|
|
323
|
+
return {
|
|
324
|
+
id: definition.execution.ownerRouteId,
|
|
325
|
+
target: Object.freeze({ kind: 'owner-route', authorityId: definition.execution.ownerRouteId }),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
// Approval delegation is the narrow owner-facing route for a background
|
|
329
|
+
// Agent. deliveryBindingId is retained only for legacy definitions that
|
|
330
|
+
// predate the separate approval route.
|
|
331
|
+
const bindingId = definition.approvalBindingId ?? definition.deliveryBindingId;
|
|
332
|
+
return bindingId === undefined
|
|
333
|
+
? undefined
|
|
334
|
+
: {
|
|
335
|
+
id: bindingId,
|
|
336
|
+
target: Object.freeze({ kind: 'binding', bindingId, workspace: definition.workspace }),
|
|
337
|
+
};
|
|
338
|
+
}
|
|
140
339
|
function deliveryStatus(value) {
|
|
141
340
|
if (value === null)
|
|
142
341
|
return undefined;
|
|
@@ -149,6 +348,159 @@ function evidenceStatus(value) {
|
|
|
149
348
|
return value;
|
|
150
349
|
throw new AutomationStoreError('invalid-state', 'automation run has an invalid evidence status');
|
|
151
350
|
}
|
|
351
|
+
const failureClasses = new Set([
|
|
352
|
+
'none', 'budget', 'cancelled', 'configuration', 'execution', 'infrastructure',
|
|
353
|
+
'policy', 'provider', 'timeout', 'unknown',
|
|
354
|
+
]);
|
|
355
|
+
const failurePhases = new Set([
|
|
356
|
+
'none', 'artifact-write', 'agent-creation', 'agent-disposal', 'agent-setup', 'budget-reservation', 'budget-settlement',
|
|
357
|
+
'executor-availability', 'host-execution', 'model-execution', 'preflight', 'preset-resolution', 'prompt-submission', 'recovery',
|
|
358
|
+
'session-flush', 'terminal-commit', 'unknown',
|
|
359
|
+
]);
|
|
360
|
+
const promptStates = new Set([
|
|
361
|
+
'not-applicable', 'not-submitted', 'submitted', 'unknown',
|
|
362
|
+
]);
|
|
363
|
+
const sideEffectStates = new Set([
|
|
364
|
+
'none', 'possible', 'unknown',
|
|
365
|
+
]);
|
|
366
|
+
const retryabilities = new Set([
|
|
367
|
+
'safe', 'unsafe', 'after-intervention', 'unknown',
|
|
368
|
+
]);
|
|
369
|
+
const budgetSettlementStates = new Set([
|
|
370
|
+
'not-required', 'not-reserved', 'reserved', 'released', 'finalized', 'unknown',
|
|
371
|
+
]);
|
|
372
|
+
function executionDiagnostic(value, errorCode) {
|
|
373
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
374
|
+
throw new AutomationStoreError(errorCode, 'automation execution diagnostic must be an object');
|
|
375
|
+
}
|
|
376
|
+
const input = value;
|
|
377
|
+
const expected = new Set([
|
|
378
|
+
'schemaVersion', 'failureClass', 'failurePhase', 'failureCode', 'promptSubmissionState',
|
|
379
|
+
'sideEffectState', 'retryability', 'budgetSettlementState',
|
|
380
|
+
]);
|
|
381
|
+
if (Object.keys(input).some(key => !expected.has(key))
|
|
382
|
+
|| input['schemaVersion'] !== 1
|
|
383
|
+
|| !failureClasses.has(input['failureClass'])
|
|
384
|
+
|| !failurePhases.has(input['failurePhase'])
|
|
385
|
+
|| typeof input['failureCode'] !== 'string'
|
|
386
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(input['failureCode'])
|
|
387
|
+
|| !promptStates.has(input['promptSubmissionState'])
|
|
388
|
+
|| !sideEffectStates.has(input['sideEffectState'])
|
|
389
|
+
|| !retryabilities.has(input['retryability'])
|
|
390
|
+
|| !budgetSettlementStates.has(input['budgetSettlementState'])) {
|
|
391
|
+
throw new AutomationStoreError(errorCode, 'automation execution diagnostic is invalid');
|
|
392
|
+
}
|
|
393
|
+
return Object.freeze({
|
|
394
|
+
schemaVersion: 1,
|
|
395
|
+
failureClass: input['failureClass'],
|
|
396
|
+
failurePhase: input['failurePhase'],
|
|
397
|
+
failureCode: input['failureCode'],
|
|
398
|
+
promptSubmissionState: input['promptSubmissionState'],
|
|
399
|
+
sideEffectState: input['sideEffectState'],
|
|
400
|
+
retryability: input['retryability'],
|
|
401
|
+
budgetSettlementState: input['budgetSettlementState'],
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
function assertDiagnosticOutcome(outcome, value, definition) {
|
|
405
|
+
const noneTuple = value.failureClass === 'none'
|
|
406
|
+
&& value.failurePhase === 'none'
|
|
407
|
+
&& value.failureCode === 'none';
|
|
408
|
+
const hasPartialNoneTuple = value.failureClass === 'none'
|
|
409
|
+
|| value.failurePhase === 'none'
|
|
410
|
+
|| value.failureCode === 'none';
|
|
411
|
+
const host = isHostAutomationDefinition(definition);
|
|
412
|
+
const hasBudget = definition.budgetId !== undefined;
|
|
413
|
+
const effectfulSettlement = value.budgetSettlementState === 'reserved'
|
|
414
|
+
|| value.budgetSettlementState === 'finalized'
|
|
415
|
+
|| value.budgetSettlementState === 'unknown';
|
|
416
|
+
const postPromptPhase = value.failurePhase === 'prompt-submission'
|
|
417
|
+
|| value.failurePhase === 'model-execution'
|
|
418
|
+
|| value.failurePhase === 'session-flush'
|
|
419
|
+
|| value.failurePhase === 'agent-disposal';
|
|
420
|
+
if ((!noneTuple && hasPartialNoneTuple)
|
|
421
|
+
|| (outcome === 'succeeded' && !noneTuple)
|
|
422
|
+
|| (outcome !== 'succeeded' && noneTuple)
|
|
423
|
+
|| (value.retryability === 'safe' && value.sideEffectState !== 'none')
|
|
424
|
+
|| (noneTuple && (value.retryability === 'after-intervention' || value.retryability === 'unknown'))
|
|
425
|
+
|| (host && value.promptSubmissionState !== 'not-applicable')
|
|
426
|
+
|| (!host && value.promptSubmissionState === 'not-applicable')
|
|
427
|
+
|| (host && (value.failurePhase === 'agent-creation' || value.failurePhase === 'agent-setup'
|
|
428
|
+
|| value.failurePhase === 'prompt-submission' || value.failurePhase === 'model-execution'
|
|
429
|
+
|| value.failurePhase === 'session-flush' || value.failurePhase === 'agent-disposal'))
|
|
430
|
+
|| (!host && (value.failurePhase === 'executor-availability' || value.failurePhase === 'host-execution'))
|
|
431
|
+
|| (!host && value.promptSubmissionState === 'submitted' && value.sideEffectState === 'none')
|
|
432
|
+
|| (!host && value.promptSubmissionState === 'submitted' && value.retryability !== 'unsafe')
|
|
433
|
+
|| (!host && postPromptPhase && value.promptSubmissionState === 'not-submitted')
|
|
434
|
+
|| (!hasBudget && value.budgetSettlementState !== 'not-required')
|
|
435
|
+
|| (hasBudget && value.budgetSettlementState === 'not-required')
|
|
436
|
+
|| (effectfulSettlement && value.sideEffectState === 'none')
|
|
437
|
+
|| (effectfulSettlement && value.retryability !== 'unsafe')) {
|
|
438
|
+
throw new AutomationStoreError('invalid-definition', 'fresh execution diagnostic contradicts the terminal outcome');
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
function isLegacyExecutionDiagnostic(value) {
|
|
442
|
+
return value.failureClass === legacyAutomationExecutionDiagnostic.failureClass
|
|
443
|
+
&& value.failurePhase === legacyAutomationExecutionDiagnostic.failurePhase
|
|
444
|
+
&& value.failureCode === legacyAutomationExecutionDiagnostic.failureCode
|
|
445
|
+
&& value.promptSubmissionState === legacyAutomationExecutionDiagnostic.promptSubmissionState
|
|
446
|
+
&& value.sideEffectState === legacyAutomationExecutionDiagnostic.sideEffectState
|
|
447
|
+
&& value.retryability === legacyAutomationExecutionDiagnostic.retryability
|
|
448
|
+
&& value.budgetSettlementState === legacyAutomationExecutionDiagnostic.budgetSettlementState;
|
|
449
|
+
}
|
|
450
|
+
function parseExecutionDiagnostic(row) {
|
|
451
|
+
if (Buffer.byteLength(row.diagnostic_json, 'utf8') > 4_096) {
|
|
452
|
+
throw new AutomationStoreError('invalid-state', 'automation execution diagnostic exceeds 4096 bytes');
|
|
453
|
+
}
|
|
454
|
+
try {
|
|
455
|
+
return executionDiagnostic(JSON.parse(row.diagnostic_json), 'invalid-state');
|
|
456
|
+
}
|
|
457
|
+
catch (error) {
|
|
458
|
+
if (error instanceof AutomationStoreError)
|
|
459
|
+
throw error;
|
|
460
|
+
throw new AutomationStoreError('invalid-state', 'automation execution diagnostic contains invalid JSON');
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
function incident(row) {
|
|
464
|
+
if (!/^[a-f0-9]{64}$/u.test(row.definition_hash)
|
|
465
|
+
|| (row.stage !== 'materialize' && row.stage !== 'claim' && row.stage !== 'terminal')
|
|
466
|
+
|| (row.state !== 'open' && row.state !== 'recovering' && row.state !== 'resolved')
|
|
467
|
+
|| !failureClasses.has(row.failure_class)
|
|
468
|
+
|| !failurePhases.has(row.failure_phase)
|
|
469
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(row.failure_code)
|
|
470
|
+
|| !sideEffectStates.has(row.side_effect_state)
|
|
471
|
+
|| !retryabilities.has(row.retryability)
|
|
472
|
+
|| (row.alert_status !== 'pending' && row.alert_status !== 'enqueued' && row.alert_status !== 'suppressed')
|
|
473
|
+
|| !Number.isSafeInteger(row.lifecycle_generation) || row.lifecycle_generation < 1
|
|
474
|
+
|| !Number.isSafeInteger(row.presentation_revision) || row.presentation_revision < 1
|
|
475
|
+
|| !Number.isSafeInteger(row.opened_at) || !Number.isSafeInteger(row.updated_at)
|
|
476
|
+
|| !Number.isSafeInteger(row.version) || row.version < 1
|
|
477
|
+
|| (row.state === 'resolved') !== (row.resolved_at !== null)
|
|
478
|
+
|| (row.alert_status === 'enqueued' && row.alert_ref === null)) {
|
|
479
|
+
throw new AutomationStoreError('invalid-state', 'automation incident row is invalid');
|
|
480
|
+
}
|
|
481
|
+
return Object.freeze({
|
|
482
|
+
id: row.id,
|
|
483
|
+
automationId: row.automation_id,
|
|
484
|
+
definitionHash: row.definition_hash,
|
|
485
|
+
stage: row.stage,
|
|
486
|
+
state: row.state,
|
|
487
|
+
failureClass: row.failure_class,
|
|
488
|
+
failurePhase: row.failure_phase,
|
|
489
|
+
failureCode: row.failure_code,
|
|
490
|
+
sideEffectState: row.side_effect_state,
|
|
491
|
+
retryability: row.retryability,
|
|
492
|
+
notificationRouteId: row.notification_route_id,
|
|
493
|
+
lifecycleGeneration: row.lifecycle_generation,
|
|
494
|
+
presentationRevision: row.presentation_revision,
|
|
495
|
+
alertStatus: row.alert_status,
|
|
496
|
+
...(row.alert_ref === null ? {} : { alertRef: row.alert_ref }),
|
|
497
|
+
...(row.run_id === null ? {} : { runId: row.run_id }),
|
|
498
|
+
openedAt: row.opened_at,
|
|
499
|
+
updatedAt: row.updated_at,
|
|
500
|
+
...(row.resolved_at === null ? {} : { resolvedAt: row.resolved_at }),
|
|
501
|
+
version: row.version,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
152
504
|
function parseEvidence(row) {
|
|
153
505
|
const status = evidenceStatus(row.evidence_status);
|
|
154
506
|
if (status === 'suppressed') {
|
|
@@ -160,7 +512,43 @@ function parseEvidence(row) {
|
|
|
160
512
|
if (row.evidence_json === null || Buffer.byteLength(row.evidence_json, 'utf8') > 16_384) {
|
|
161
513
|
throw new AutomationStoreError('invalid-state', 'pending automation evidence must be bounded JSON');
|
|
162
514
|
}
|
|
163
|
-
|
|
515
|
+
let value;
|
|
516
|
+
try {
|
|
517
|
+
value = JSON.parse(row.evidence_json);
|
|
518
|
+
}
|
|
519
|
+
catch {
|
|
520
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence contains invalid JSON');
|
|
521
|
+
}
|
|
522
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
523
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence must be an object');
|
|
524
|
+
}
|
|
525
|
+
const input = value;
|
|
526
|
+
const allowed = new Set([
|
|
527
|
+
'situation', 'outcome', 'detail', 'idempotencyKey', 'occurredAt', 'workspace',
|
|
528
|
+
'agentPreset', 'automationId', 'runId', 'sessionId', 'ruleId', 'guidanceVersion',
|
|
529
|
+
]);
|
|
530
|
+
const stringWithin = (field, maximum) => typeof input[field] === 'string'
|
|
531
|
+
&& input[field].normalize('NFC').trim() !== ''
|
|
532
|
+
&& Buffer.byteLength(input[field], 'utf8') <= maximum;
|
|
533
|
+
if (Object.keys(input).some(key => !allowed.has(key))
|
|
534
|
+
|| !stringWithin('situation', 200)
|
|
535
|
+
|| (input['outcome'] !== 'succeeded' && input['outcome'] !== 'failed')
|
|
536
|
+
|| !stringWithin('detail', 2_000)
|
|
537
|
+
|| !stringWithin('idempotencyKey', 500)
|
|
538
|
+
|| !Number.isSafeInteger(input['occurredAt'])
|
|
539
|
+
|| !stringWithin('workspace', 4_096) || !isAbsolute(input['workspace'])
|
|
540
|
+
|| !stringWithin('agentPreset', 200)
|
|
541
|
+
|| !stringWithin('automationId', 500)
|
|
542
|
+
|| !stringWithin('runId', 500)
|
|
543
|
+
|| (input['sessionId'] !== undefined && !stringWithin('sessionId', 500))
|
|
544
|
+
|| (input['ruleId'] !== undefined && !stringWithin('ruleId', 200))
|
|
545
|
+
|| (input['guidanceVersion'] !== undefined
|
|
546
|
+
&& (!Number.isSafeInteger(input['guidanceVersion'])
|
|
547
|
+
|| input['guidanceVersion'] < 1
|
|
548
|
+
|| input['guidanceVersion'] > 1_000_000_000))
|
|
549
|
+
|| ((input['ruleId'] === undefined) !== (input['guidanceVersion'] === undefined))) {
|
|
550
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence payload is invalid');
|
|
551
|
+
}
|
|
164
552
|
return Object.freeze(value);
|
|
165
553
|
}
|
|
166
554
|
function occurrence(row) {
|
|
@@ -206,6 +594,9 @@ function run(row) {
|
|
|
206
594
|
...(row.artifact_ref === null ? {} : { artifactRef: row.artifact_ref }),
|
|
207
595
|
outputPreview: row.output_preview,
|
|
208
596
|
usage: Object.freeze(JSON.parse(row.usage_json)),
|
|
597
|
+
executionMode: row.execution_mode,
|
|
598
|
+
...(row.definition_hash === null ? {} : { definitionHash: row.definition_hash }),
|
|
599
|
+
diagnostic: parseExecutionDiagnostic(row),
|
|
209
600
|
...(status === undefined ? {} : { deliveryStatus: status }),
|
|
210
601
|
...(row.delivery_ref === null ? {} : { deliveryRef: row.delivery_ref }),
|
|
211
602
|
evidenceStatus: evidenceStatus(row.evidence_status),
|
|
@@ -214,6 +605,179 @@ function run(row) {
|
|
|
214
605
|
updatedAt: row.updated_at,
|
|
215
606
|
});
|
|
216
607
|
}
|
|
608
|
+
function circuit(row) {
|
|
609
|
+
const durableProbe = row.state === 'half-open' || row.state === 'probing';
|
|
610
|
+
const probeTupleValid = durableProbe
|
|
611
|
+
? typeof row.probe_token === 'string'
|
|
612
|
+
&& /^probe-[a-f0-9-]{36}$/u.test(row.probe_token)
|
|
613
|
+
&& Number.isSafeInteger(row.probe_lease_until)
|
|
614
|
+
&& (row.state === 'probing'
|
|
615
|
+
? typeof row.probe_task_id === 'string' && row.probe_task_id !== ''
|
|
616
|
+
: row.probe_task_id === null)
|
|
617
|
+
: row.probe_token === null && row.probe_lease_until === null && row.probe_task_id === null;
|
|
618
|
+
if (!Number.isSafeInteger(row.opened_at) || !Number.isSafeInteger(row.updated_at)
|
|
619
|
+
|| !Number.isSafeInteger(row.version) || row.version < 1
|
|
620
|
+
|| !/^[a-f0-9]{64}$/u.test(row.definition_hash)
|
|
621
|
+
|| !['closed', 'half-open', 'open', 'probing'].includes(row.state)
|
|
622
|
+
|| !['budget', 'configuration', 'policy'].includes(row.failure_class)
|
|
623
|
+
|| !probeTupleValid
|
|
624
|
+
|| !failurePhases.has(row.failure_phase)
|
|
625
|
+
|| !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(row.failure_code)) {
|
|
626
|
+
throw new AutomationStoreError('invalid-state', 'automation circuit row is invalid');
|
|
627
|
+
}
|
|
628
|
+
return Object.freeze({
|
|
629
|
+
automationId: row.automation_id,
|
|
630
|
+
definitionHash: row.definition_hash,
|
|
631
|
+
state: row.state,
|
|
632
|
+
failureClass: row.failure_class,
|
|
633
|
+
failurePhase: row.failure_phase,
|
|
634
|
+
failureCode: row.failure_code,
|
|
635
|
+
openedAt: row.opened_at,
|
|
636
|
+
updatedAt: row.updated_at,
|
|
637
|
+
...(row.probe_token === null ? {} : { probeToken: row.probe_token }),
|
|
638
|
+
...(row.probe_lease_until === null ? {} : { probeLeaseUntil: row.probe_lease_until }),
|
|
639
|
+
...(row.probe_task_id === null ? {} : { probeTaskId: row.probe_task_id }),
|
|
640
|
+
version: row.version,
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
function parseCircuitResult(json) {
|
|
644
|
+
if (Buffer.byteLength(json, 'utf8') > 4_096) {
|
|
645
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result exceeds its bound');
|
|
646
|
+
}
|
|
647
|
+
let value;
|
|
648
|
+
try {
|
|
649
|
+
value = JSON.parse(json);
|
|
650
|
+
}
|
|
651
|
+
catch {
|
|
652
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result is invalid JSON');
|
|
653
|
+
}
|
|
654
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
655
|
+
throw new AutomationStoreError('invalid-state', 'circuit operation result is invalid');
|
|
656
|
+
}
|
|
657
|
+
const input = value;
|
|
658
|
+
return circuit({
|
|
659
|
+
automation_id: input['automationId'],
|
|
660
|
+
definition_hash: input['definitionHash'],
|
|
661
|
+
state: input['state'],
|
|
662
|
+
failure_class: input['failureClass'],
|
|
663
|
+
failure_phase: input['failurePhase'],
|
|
664
|
+
failure_code: input['failureCode'],
|
|
665
|
+
opened_at: input['openedAt'],
|
|
666
|
+
updated_at: input['updatedAt'],
|
|
667
|
+
probe_token: (input['probeToken'] ?? null),
|
|
668
|
+
probe_lease_until: (input['probeLeaseUntil'] ?? null),
|
|
669
|
+
probe_task_id: (input['probeTaskId'] ?? null),
|
|
670
|
+
version: input['version'],
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
function parseCircuitCanaryResult(json) {
|
|
674
|
+
if (Buffer.byteLength(json, 'utf8') > 8_192) {
|
|
675
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result exceeds its bound');
|
|
676
|
+
}
|
|
677
|
+
let value;
|
|
678
|
+
try {
|
|
679
|
+
value = JSON.parse(json);
|
|
680
|
+
}
|
|
681
|
+
catch {
|
|
682
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result is invalid JSON');
|
|
683
|
+
}
|
|
684
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)
|
|
685
|
+
|| Object.keys(value).sort().join(',') !== 'circuit,executionMode,occurrenceId,taskId') {
|
|
686
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result is invalid');
|
|
687
|
+
}
|
|
688
|
+
const input = value;
|
|
689
|
+
const occurrenceId = input['occurrenceId'];
|
|
690
|
+
const taskId = input['taskId'];
|
|
691
|
+
if (typeof occurrenceId !== 'string' || !/^occ-[a-f0-9]{64}$/u.test(occurrenceId)
|
|
692
|
+
|| typeof taskId !== 'string' || taskId !== `task-${occurrenceId}`
|
|
693
|
+
|| input['executionMode'] !== 'production') {
|
|
694
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary operation result tuple is invalid');
|
|
695
|
+
}
|
|
696
|
+
return Object.freeze({
|
|
697
|
+
circuit: parseCircuitResult(JSON.stringify(input['circuit'])),
|
|
698
|
+
occurrenceId,
|
|
699
|
+
taskId,
|
|
700
|
+
executionMode: 'production',
|
|
701
|
+
});
|
|
702
|
+
}
|
|
703
|
+
function parseSystemPauseResult(json) {
|
|
704
|
+
if (Buffer.byteLength(json, 'utf8') > 4_096) {
|
|
705
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result exceeds its bound');
|
|
706
|
+
}
|
|
707
|
+
let value;
|
|
708
|
+
try {
|
|
709
|
+
value = JSON.parse(json);
|
|
710
|
+
}
|
|
711
|
+
catch {
|
|
712
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result is invalid JSON');
|
|
713
|
+
}
|
|
714
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)
|
|
715
|
+
|| Object.keys(value).sort().join(',')
|
|
716
|
+
!== 'automationId,automationStatus,definitionHash,definitionVersion,expectedVersion,owner') {
|
|
717
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result is invalid');
|
|
718
|
+
}
|
|
719
|
+
const input = value;
|
|
720
|
+
const owner = input['owner'];
|
|
721
|
+
const automationId = input['automationId'];
|
|
722
|
+
const expectedVersion = input['expectedVersion'];
|
|
723
|
+
const definitionVersion = input['definitionVersion'];
|
|
724
|
+
if (typeof owner !== 'string' || owner.normalize('NFC').trim() !== owner
|
|
725
|
+
|| owner === '' || Buffer.byteLength(owner, 'utf8') > 200
|
|
726
|
+
|| typeof automationId !== 'string' || automationId.normalize('NFC').trim() !== automationId
|
|
727
|
+
|| automationId === '' || Buffer.byteLength(automationId, 'utf8') > 500
|
|
728
|
+
|| typeof input['definitionHash'] !== 'string' || !/^[a-f0-9]{64}$/u.test(input['definitionHash'])
|
|
729
|
+
|| !Number.isSafeInteger(expectedVersion) || expectedVersion < 1
|
|
730
|
+
|| !Number.isSafeInteger(definitionVersion) || definitionVersion !== expectedVersion + 1
|
|
731
|
+
|| input['automationStatus'] !== 'paused') {
|
|
732
|
+
throw new AutomationStoreError('invalid-state', 'system pause operation result tuple is invalid');
|
|
733
|
+
}
|
|
734
|
+
return Object.freeze({
|
|
735
|
+
owner,
|
|
736
|
+
automationId,
|
|
737
|
+
definitionHash: input['definitionHash'],
|
|
738
|
+
expectedVersion: expectedVersion,
|
|
739
|
+
definitionVersion: definitionVersion,
|
|
740
|
+
automationStatus: 'paused',
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
function deepFreezeJson(value) {
|
|
744
|
+
if (typeof value !== 'object' || value === null || Object.isFrozen(value))
|
|
745
|
+
return value;
|
|
746
|
+
for (const child of Object.values(value))
|
|
747
|
+
deepFreezeJson(child);
|
|
748
|
+
return Object.freeze(value);
|
|
749
|
+
}
|
|
750
|
+
function evaluationOutboxEntry(row) {
|
|
751
|
+
if (row.observation_kind !== 'terminal'
|
|
752
|
+
|| (row.status !== 'pending' && row.status !== 'recorded' && row.status !== 'dead-letter')
|
|
753
|
+
|| !Number.isSafeInteger(row.attempt_count) || row.attempt_count < 0
|
|
754
|
+
|| !Number.isSafeInteger(row.next_attempt_at)) {
|
|
755
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation outbox row has an invalid state');
|
|
756
|
+
}
|
|
757
|
+
if (Buffer.byteLength(row.payload_json, 'utf8') > 32_768) {
|
|
758
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation payload exceeds 32768 bytes');
|
|
759
|
+
}
|
|
760
|
+
let payload;
|
|
761
|
+
try {
|
|
762
|
+
payload = deepFreezeJson(JSON.parse(row.payload_json));
|
|
763
|
+
}
|
|
764
|
+
catch {
|
|
765
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation payload contains invalid JSON');
|
|
766
|
+
}
|
|
767
|
+
return Object.freeze({
|
|
768
|
+
id: row.id,
|
|
769
|
+
runId: row.run_id,
|
|
770
|
+
kind: row.observation_kind,
|
|
771
|
+
status: row.status,
|
|
772
|
+
payload,
|
|
773
|
+
attemptCount: row.attempt_count,
|
|
774
|
+
nextAttemptAt: row.next_attempt_at,
|
|
775
|
+
...(row.last_failure_at === null ? {} : { lastFailureAt: row.last_failure_at }),
|
|
776
|
+
...(row.last_error_code === null ? {} : { lastErrorCode: row.last_error_code }),
|
|
777
|
+
createdAt: row.created_at,
|
|
778
|
+
updatedAt: row.updated_at,
|
|
779
|
+
});
|
|
780
|
+
}
|
|
217
781
|
function duty(row, acquired) {
|
|
218
782
|
return Object.freeze({
|
|
219
783
|
acquired,
|
|
@@ -238,12 +802,23 @@ function normalizeEvidenceAttribution(value) {
|
|
|
238
802
|
...(ruleId === undefined ? {} : { ruleId, guidanceVersion: guidanceVersion }),
|
|
239
803
|
});
|
|
240
804
|
}
|
|
805
|
+
function automationSituation(automationId) {
|
|
806
|
+
const raw = `automation:${automationId}`;
|
|
807
|
+
return Buffer.byteLength(raw, 'utf8') <= 200
|
|
808
|
+
? raw
|
|
809
|
+
: `automation:${createHash('sha256').update(automationId).digest('hex')}`;
|
|
810
|
+
}
|
|
241
811
|
function buildOutcomeEvidence(input) {
|
|
242
|
-
|
|
812
|
+
// Preview is an operator inspection mode, not a production exposure. Feeding
|
|
813
|
+
// it into Evolution would let a side-effect-free rehearsal change live
|
|
814
|
+
// guidance even though the production behaviour was never exercised.
|
|
815
|
+
if (input.executionMode !== 'production'
|
|
816
|
+
|| input.outcome === 'cancelled'
|
|
817
|
+
|| input.outcome === 'unknown') {
|
|
243
818
|
return { status: 'suppressed', json: null };
|
|
244
819
|
}
|
|
245
820
|
const evidence = {
|
|
246
|
-
situation:
|
|
821
|
+
situation: automationSituation(input.automation.id),
|
|
247
822
|
outcome: input.outcome === 'succeeded' ? 'succeeded' : 'failed',
|
|
248
823
|
detail: `automation "${input.automation.definition.name}": run ${input.outcome}`,
|
|
249
824
|
idempotencyKey: `automation-run:${input.runId}`,
|
|
@@ -260,6 +835,61 @@ function buildOutcomeEvidence(input) {
|
|
|
260
835
|
}
|
|
261
836
|
return { status: 'pending', json };
|
|
262
837
|
}
|
|
838
|
+
function evaluationExecutionStatus(status) {
|
|
839
|
+
return status === 'timed_out' ? 'timed-out' : status;
|
|
840
|
+
}
|
|
841
|
+
function nonNegativeUsageMetric(usage, key) {
|
|
842
|
+
const value = usage[key];
|
|
843
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
844
|
+
}
|
|
845
|
+
function buildTerminalEvaluation(input) {
|
|
846
|
+
// Preview is validation evidence for its owning Host workflow, never a
|
|
847
|
+
// production outcome. Keeping it out of this durable trusted outbox avoids
|
|
848
|
+
// silently turning a dry-run into learning truth when Evaluation is added
|
|
849
|
+
// later or after a restart.
|
|
850
|
+
if (input.executionMode !== 'production')
|
|
851
|
+
return undefined;
|
|
852
|
+
const inputTokens = nonNegativeUsageMetric(input.usage, 'inputTokens');
|
|
853
|
+
const outputTokens = nonNegativeUsageMetric(input.usage, 'outputTokens');
|
|
854
|
+
const toolCalls = nonNegativeUsageMetric(input.usage, 'toolCalls');
|
|
855
|
+
const latencyMs = input.startedAt === null ? undefined : Math.max(0, input.occurredAt - input.startedAt);
|
|
856
|
+
const retries = Math.max(0, input.attemptNumber - 1);
|
|
857
|
+
const metrics = Object.freeze({
|
|
858
|
+
...(inputTokens === undefined ? {} : { inputTokens }),
|
|
859
|
+
...(outputTokens === undefined ? {} : { outputTokens }),
|
|
860
|
+
...(toolCalls === undefined ? {} : { toolCalls }),
|
|
861
|
+
...(latencyMs === undefined ? {} : { latencyMs }),
|
|
862
|
+
...(retries === 0 ? {} : { retries }),
|
|
863
|
+
});
|
|
864
|
+
const situation = automationSituation(input.automation.id);
|
|
865
|
+
const fixedHostObjective = input.executionMode === 'production'
|
|
866
|
+
&& isHostAutomationDefinition(input.automation.definition)
|
|
867
|
+
&& (input.outcome === 'succeeded' || input.outcome === 'failed' || input.outcome === 'timed_out');
|
|
868
|
+
const objectiveStatus = !fixedHostObjective
|
|
869
|
+
? 'unknown'
|
|
870
|
+
: input.outcome === 'succeeded' ? 'achieved' : 'not-achieved';
|
|
871
|
+
return Object.freeze({
|
|
872
|
+
executionMode: 'production',
|
|
873
|
+
scope: Object.freeze({
|
|
874
|
+
workspace: input.automation.definition.workspace,
|
|
875
|
+
preset: input.automation.definition.agentPreset,
|
|
876
|
+
}),
|
|
877
|
+
situation,
|
|
878
|
+
executionStatus: evaluationExecutionStatus(input.outcome),
|
|
879
|
+
objectiveStatus,
|
|
880
|
+
deliveryStatus: input.deliveryStatus,
|
|
881
|
+
source: Object.freeze({ kind: 'automation', id: 'assistant-automations' }),
|
|
882
|
+
trust: 'trusted',
|
|
883
|
+
evidence: Object.freeze([{ kind: 'automation-run', ref: input.runId }]),
|
|
884
|
+
metrics,
|
|
885
|
+
occurredAt: input.occurredAt,
|
|
886
|
+
idempotencyKey: `assistant-automations:terminal:${input.runId}:v1`,
|
|
887
|
+
evaluator: Object.freeze({
|
|
888
|
+
id: 'assistant-automations',
|
|
889
|
+
version: fixedHostObjective ? 'host-runbook-v1' : 'terminal-v1',
|
|
890
|
+
}),
|
|
891
|
+
});
|
|
892
|
+
}
|
|
263
893
|
export function stableOccurrenceId(automationId, triggerKind, triggerKey) {
|
|
264
894
|
return `occ-${createHash('sha256').update(`${automationId}\0${triggerKind}\0${triggerKey}`).digest('hex')}`;
|
|
265
895
|
}
|
|
@@ -391,6 +1021,610 @@ export class AutomationStore {
|
|
|
391
1021
|
return this.database.prepare('SELECT * FROM automation_definitions ORDER BY created_at, id').all()
|
|
392
1022
|
.map(record);
|
|
393
1023
|
}
|
|
1024
|
+
getDefinitionHash(automationId) {
|
|
1025
|
+
const row = this.database.prepare('SELECT definition_hash FROM automation_definitions WHERE id = ?')
|
|
1026
|
+
.get(automationId);
|
|
1027
|
+
return row?.definition_hash;
|
|
1028
|
+
}
|
|
1029
|
+
getCircuit(automationId, definitionHash) {
|
|
1030
|
+
if (!/^[a-f0-9]{64}$/u.test(definitionHash)) {
|
|
1031
|
+
throw new AutomationStoreError('invalid-definition', 'circuit definition hash is invalid');
|
|
1032
|
+
}
|
|
1033
|
+
const row = this.database.prepare(`
|
|
1034
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1035
|
+
`).get(automationId, definitionHash);
|
|
1036
|
+
return row === undefined ? undefined : circuit(row);
|
|
1037
|
+
}
|
|
1038
|
+
/** Read-only compatibility view; execution admission uses the transactional seam below. */
|
|
1039
|
+
getOpenCircuitForTask(taskId) {
|
|
1040
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
1041
|
+
if (taskRow === undefined || taskRow.attempt_count < 1)
|
|
1042
|
+
return undefined;
|
|
1043
|
+
const occurrenceRow = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
1044
|
+
.get(taskRow.occurrence_id);
|
|
1045
|
+
// Preview is an observation-only execution lane. It may exercise the
|
|
1046
|
+
// runner, but it can neither observe nor mutate the production circuit.
|
|
1047
|
+
if (occurrenceRow?.dry_run !== 0)
|
|
1048
|
+
return undefined;
|
|
1049
|
+
const attempt = this.database.prepare(`
|
|
1050
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
1051
|
+
`).get(taskId, taskRow.attempt_count);
|
|
1052
|
+
if (attempt === undefined)
|
|
1053
|
+
return undefined;
|
|
1054
|
+
const snapshot = automationSnapshot(attempt, taskRow.automation_id);
|
|
1055
|
+
if (snapshot === undefined)
|
|
1056
|
+
return undefined;
|
|
1057
|
+
const definitionHash = hash(snapshot.definition);
|
|
1058
|
+
const value = this.getCircuit(taskRow.automation_id, definitionHash);
|
|
1059
|
+
return value !== undefined && value.state !== 'closed' ? value : undefined;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* Host-only exact CAS. Arming never closes the circuit: it creates one
|
|
1063
|
+
* durable, expiring half-open capability that a task must atomically acquire.
|
|
1064
|
+
*/
|
|
1065
|
+
armCircuitProbe(input) {
|
|
1066
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1067
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1068
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1069
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1070
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1071
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1072
|
+
throw new AutomationStoreError('invalid-definition', 'circuit probe tuple is invalid');
|
|
1073
|
+
}
|
|
1074
|
+
const inputHash = hash({
|
|
1075
|
+
owner,
|
|
1076
|
+
automationId,
|
|
1077
|
+
definitionHash: input.definitionHash,
|
|
1078
|
+
expectedVersion: input.expectedVersion,
|
|
1079
|
+
leaseMs: input.leaseMs,
|
|
1080
|
+
});
|
|
1081
|
+
return this.transaction(() => {
|
|
1082
|
+
const prior = this.database.prepare(`
|
|
1083
|
+
SELECT input_hash, result_json FROM automation_circuit_operations WHERE operation_id = ?
|
|
1084
|
+
`).get(operationId);
|
|
1085
|
+
if (prior !== undefined) {
|
|
1086
|
+
if (prior.input_hash !== inputHash) {
|
|
1087
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit probe operation id was reused with different input');
|
|
1088
|
+
}
|
|
1089
|
+
return Object.freeze({
|
|
1090
|
+
operationId,
|
|
1091
|
+
circuit: parseCircuitResult(prior.result_json),
|
|
1092
|
+
replayed: true,
|
|
1093
|
+
});
|
|
1094
|
+
}
|
|
1095
|
+
const definition = this.database.prepare(`
|
|
1096
|
+
SELECT system_owner, definition_hash FROM automation_definitions WHERE id = ?
|
|
1097
|
+
`).get(automationId);
|
|
1098
|
+
if (definition === undefined || definition.system_owner !== owner
|
|
1099
|
+
|| definition.definition_hash !== input.definitionHash) {
|
|
1100
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1101
|
+
}
|
|
1102
|
+
const probeToken = `probe-${randomUUID()}`;
|
|
1103
|
+
const changed = this.database.prepare(`
|
|
1104
|
+
UPDATE automation_circuits
|
|
1105
|
+
SET state = 'half-open', probe_token = ?, probe_lease_until = ?, probe_task_id = NULL,
|
|
1106
|
+
updated_at = ?, version = version + 1
|
|
1107
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'open' AND version = ?
|
|
1108
|
+
`).run(probeToken, input.now + input.leaseMs, input.now, automationId, input.definitionHash, input.expectedVersion);
|
|
1109
|
+
const row = this.database.prepare(`
|
|
1110
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1111
|
+
`).get(automationId, input.definitionHash);
|
|
1112
|
+
if (row === undefined)
|
|
1113
|
+
throw new AutomationStoreError('not-found', 'automation circuit was not found');
|
|
1114
|
+
const current = circuit(row);
|
|
1115
|
+
if (changed.changes !== 1) {
|
|
1116
|
+
throw new AutomationStoreError('version-conflict', 'automation circuit changed before exact probe arm');
|
|
1117
|
+
}
|
|
1118
|
+
this.markIncidentRecoveringInTransaction(automationId, input.definitionHash, input.now);
|
|
1119
|
+
this.database.prepare(`
|
|
1120
|
+
INSERT INTO automation_circuit_operations(
|
|
1121
|
+
operation_id, system_owner, automation_id, definition_hash,
|
|
1122
|
+
expected_circuit_version, lease_ms, input_hash, result_json, created_at
|
|
1123
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1124
|
+
`).run(operationId, owner, automationId, input.definitionHash, input.expectedVersion, input.leaseMs, inputHash, JSON.stringify(current), input.now);
|
|
1125
|
+
return Object.freeze({ operationId, circuit: current, replayed: false });
|
|
1126
|
+
});
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* Atomically arms one exact production probe and schedules the only task that
|
|
1130
|
+
* may consume it. The operation ledger, circuit transition, occurrence and
|
|
1131
|
+
* task commit or roll back together.
|
|
1132
|
+
*/
|
|
1133
|
+
probeCircuitAndScheduleCanary(input) {
|
|
1134
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1135
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1136
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1137
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1138
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1139
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1140
|
+
throw new AutomationStoreError('invalid-definition', 'circuit canary tuple is invalid');
|
|
1141
|
+
}
|
|
1142
|
+
const inputHash = hash({
|
|
1143
|
+
lane: 'probe-and-schedule-canary-v1',
|
|
1144
|
+
owner,
|
|
1145
|
+
automationId,
|
|
1146
|
+
definitionHash: input.definitionHash,
|
|
1147
|
+
expectedVersion: input.expectedVersion,
|
|
1148
|
+
leaseMs: input.leaseMs,
|
|
1149
|
+
});
|
|
1150
|
+
return this.transaction(() => {
|
|
1151
|
+
const prior = this.database.prepare(`
|
|
1152
|
+
SELECT input_hash, result_json FROM automation_circuit_operations WHERE operation_id = ?
|
|
1153
|
+
`).get(operationId);
|
|
1154
|
+
if (prior !== undefined) {
|
|
1155
|
+
if (prior.input_hash !== inputHash) {
|
|
1156
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit canary operation id was reused with different input');
|
|
1157
|
+
}
|
|
1158
|
+
const replay = parseCircuitCanaryResult(prior.result_json);
|
|
1159
|
+
return Object.freeze({ operationId, ...replay, replayed: true });
|
|
1160
|
+
}
|
|
1161
|
+
const definition = this.database.prepare(`
|
|
1162
|
+
SELECT system_owner, definition_hash, status
|
|
1163
|
+
FROM automation_definitions WHERE id = ?
|
|
1164
|
+
`).get(automationId);
|
|
1165
|
+
if (definition === undefined || definition.system_owner !== owner
|
|
1166
|
+
|| definition.definition_hash !== input.definitionHash) {
|
|
1167
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1168
|
+
}
|
|
1169
|
+
if (definition.status !== 'active') {
|
|
1170
|
+
throw new AutomationStoreError('invalid-state', 'circuit canary automation is not active');
|
|
1171
|
+
}
|
|
1172
|
+
const probeToken = `probe-${randomUUID()}`;
|
|
1173
|
+
const changed = this.database.prepare(`
|
|
1174
|
+
UPDATE automation_circuits
|
|
1175
|
+
SET state = 'half-open', probe_token = ?, probe_lease_until = ?, probe_task_id = NULL,
|
|
1176
|
+
updated_at = ?, version = version + 1
|
|
1177
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'open' AND version = ?
|
|
1178
|
+
`).run(probeToken, input.now + input.leaseMs, input.now, automationId, input.definitionHash, input.expectedVersion);
|
|
1179
|
+
const row = this.database.prepare(`
|
|
1180
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1181
|
+
`).get(automationId, input.definitionHash);
|
|
1182
|
+
if (row === undefined)
|
|
1183
|
+
throw new AutomationStoreError('not-found', 'automation circuit was not found');
|
|
1184
|
+
const current = circuit(row);
|
|
1185
|
+
if (changed.changes !== 1) {
|
|
1186
|
+
throw new AutomationStoreError('version-conflict', 'automation circuit changed before exact canary arm');
|
|
1187
|
+
}
|
|
1188
|
+
this.markIncidentRecoveringInTransaction(automationId, input.definitionHash, input.now);
|
|
1189
|
+
const inserted = this.insertOccurrence({
|
|
1190
|
+
automationId,
|
|
1191
|
+
triggerKind: 'manual',
|
|
1192
|
+
triggerKey: `circuit-canary:${operationId}`,
|
|
1193
|
+
scheduledAt: input.now,
|
|
1194
|
+
status: 'pending',
|
|
1195
|
+
dryRun: false,
|
|
1196
|
+
});
|
|
1197
|
+
if (!inserted.created || inserted.value.dryRun || inserted.value.status !== 'pending') {
|
|
1198
|
+
throw new AutomationStoreError('idempotency-conflict', 'circuit canary occurrence already exists outside the atomic operation');
|
|
1199
|
+
}
|
|
1200
|
+
const occurrenceId = inserted.value.id;
|
|
1201
|
+
const taskId = `task-${occurrenceId}`;
|
|
1202
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?')
|
|
1203
|
+
.get(taskId);
|
|
1204
|
+
if (taskRow === undefined || taskRow.occurrence_id !== occurrenceId
|
|
1205
|
+
|| taskRow.automation_id !== automationId || taskRow.status !== 'scheduled'
|
|
1206
|
+
|| taskRow.attempt_count !== 0) {
|
|
1207
|
+
throw new AutomationStoreError('invalid-state', 'atomic circuit canary task was not created');
|
|
1208
|
+
}
|
|
1209
|
+
const stored = Object.freeze({
|
|
1210
|
+
circuit: current,
|
|
1211
|
+
occurrenceId,
|
|
1212
|
+
taskId,
|
|
1213
|
+
executionMode: 'production',
|
|
1214
|
+
});
|
|
1215
|
+
this.database.prepare(`
|
|
1216
|
+
INSERT INTO automation_circuit_operations(
|
|
1217
|
+
operation_id, system_owner, automation_id, definition_hash,
|
|
1218
|
+
expected_circuit_version, lease_ms, input_hash, result_json, created_at
|
|
1219
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1220
|
+
`).run(operationId, owner, automationId, input.definitionHash, input.expectedVersion, input.leaseMs, inputHash, JSON.stringify(stored), input.now);
|
|
1221
|
+
return Object.freeze({ operationId, ...stored, replayed: false });
|
|
1222
|
+
});
|
|
1223
|
+
}
|
|
1224
|
+
/**
|
|
1225
|
+
* Atomically admits an ordinary execution or consumes the one half-open
|
|
1226
|
+
* capability. A probing row is idempotent only for its exact task id.
|
|
1227
|
+
*/
|
|
1228
|
+
acquireCircuitExecutionForTask(input) {
|
|
1229
|
+
const taskId = text(input.taskId, 'taskId', 1_000);
|
|
1230
|
+
this.validateLease(input.now, input.leaseMs);
|
|
1231
|
+
return this.transaction(() => {
|
|
1232
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
1233
|
+
if (taskRow === undefined)
|
|
1234
|
+
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
1235
|
+
if (taskRow.status !== 'running' || taskRow.attempt_count < 1) {
|
|
1236
|
+
throw new AutomationStoreError('invalid-state', 'only a running task may acquire circuit execution');
|
|
1237
|
+
}
|
|
1238
|
+
const occurrenceRow = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
1239
|
+
.get(taskRow.occurrence_id);
|
|
1240
|
+
if (occurrenceRow === undefined || (occurrenceRow.dry_run !== 0 && occurrenceRow.dry_run !== 1)) {
|
|
1241
|
+
throw new AutomationStoreError('invalid-state', 'running task has no valid occurrence execution mode');
|
|
1242
|
+
}
|
|
1243
|
+
// A preview never consumes a production probe and is never blocked by a
|
|
1244
|
+
// production circuit. Its terminal receipt is still persisted normally.
|
|
1245
|
+
if (occurrenceRow.dry_run === 1)
|
|
1246
|
+
return { kind: 'normal' };
|
|
1247
|
+
const attempt = this.database.prepare(`
|
|
1248
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
1249
|
+
`).get(taskId, taskRow.attempt_count);
|
|
1250
|
+
if (attempt === undefined)
|
|
1251
|
+
throw new AutomationStoreError('invalid-state', 'running task has no immutable attempt');
|
|
1252
|
+
const snapshot = automationSnapshot(attempt, taskRow.automation_id);
|
|
1253
|
+
if (snapshot === undefined)
|
|
1254
|
+
throw new AutomationStoreError('invalid-state', 'running task has no immutable snapshot');
|
|
1255
|
+
const definitionHash = hash(snapshot.definition);
|
|
1256
|
+
const selected = this.database.prepare(`
|
|
1257
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1258
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1259
|
+
if (selected === undefined || selected.state === 'closed')
|
|
1260
|
+
return { kind: 'normal' };
|
|
1261
|
+
if ((selected.state === 'half-open' || selected.state === 'probing')
|
|
1262
|
+
&& selected.probe_lease_until !== null && selected.probe_lease_until <= input.now) {
|
|
1263
|
+
this.database.prepare(`
|
|
1264
|
+
UPDATE automation_circuits
|
|
1265
|
+
SET state = 'open', probe_token = NULL, probe_lease_until = NULL, probe_task_id = NULL,
|
|
1266
|
+
updated_at = ?, version = version + 1
|
|
1267
|
+
WHERE automation_id = ? AND definition_hash = ? AND version = ?
|
|
1268
|
+
`).run(input.now, taskRow.automation_id, definitionHash, selected.version);
|
|
1269
|
+
const expired = this.database.prepare(`
|
|
1270
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1271
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1272
|
+
return { kind: 'blocked', circuit: circuit(expired) };
|
|
1273
|
+
}
|
|
1274
|
+
if (selected.state === 'open')
|
|
1275
|
+
return { kind: 'blocked', circuit: circuit(selected) };
|
|
1276
|
+
if (selected.state === 'probing') {
|
|
1277
|
+
return selected.probe_task_id === taskId
|
|
1278
|
+
? { kind: 'probe', circuit: circuit(selected) }
|
|
1279
|
+
: { kind: 'blocked', circuit: circuit(selected) };
|
|
1280
|
+
}
|
|
1281
|
+
// The atomic canary lane binds a half-open transition to one durable
|
|
1282
|
+
// task. Legacy armCircuitProbe rows have no taskId and retain their
|
|
1283
|
+
// existing first-claim-wins behaviour.
|
|
1284
|
+
const designated = this.database.prepare(`
|
|
1285
|
+
SELECT json_extract(result_json, '$.taskId') AS task_id
|
|
1286
|
+
FROM automation_circuit_operations
|
|
1287
|
+
WHERE automation_id = ? AND definition_hash = ?
|
|
1288
|
+
AND expected_circuit_version = ?
|
|
1289
|
+
AND json_extract(result_json, '$.taskId') IS NOT NULL
|
|
1290
|
+
ORDER BY created_at DESC, operation_id DESC LIMIT 1
|
|
1291
|
+
`).get(taskRow.automation_id, definitionHash, selected.version - 1);
|
|
1292
|
+
if (designated !== undefined && designated.task_id !== taskId) {
|
|
1293
|
+
return { kind: 'blocked', circuit: circuit(selected) };
|
|
1294
|
+
}
|
|
1295
|
+
const probeLeaseUntil = Math.min(selected.probe_lease_until, input.now + input.leaseMs);
|
|
1296
|
+
const changed = this.database.prepare(`
|
|
1297
|
+
UPDATE automation_circuits
|
|
1298
|
+
SET state = 'probing', probe_task_id = ?, probe_lease_until = ?,
|
|
1299
|
+
updated_at = ?, version = version + 1
|
|
1300
|
+
WHERE automation_id = ? AND definition_hash = ?
|
|
1301
|
+
AND state = 'half-open' AND version = ? AND probe_token = ?
|
|
1302
|
+
`).run(taskId, probeLeaseUntil, input.now, taskRow.automation_id, definitionHash, selected.version, selected.probe_token);
|
|
1303
|
+
const acquired = this.database.prepare(`
|
|
1304
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
1305
|
+
`).get(taskRow.automation_id, definitionHash);
|
|
1306
|
+
if (changed.changes !== 1)
|
|
1307
|
+
return { kind: 'blocked', circuit: circuit(acquired) };
|
|
1308
|
+
return { kind: 'probe', circuit: circuit(acquired) };
|
|
1309
|
+
});
|
|
1310
|
+
}
|
|
1311
|
+
recoverExpiredCircuitProbes(input) {
|
|
1312
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
1313
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1314
|
+
}
|
|
1315
|
+
return this.transaction(() => {
|
|
1316
|
+
const expired = this.database.prepare(`
|
|
1317
|
+
SELECT automation_id, definition_hash FROM automation_circuits
|
|
1318
|
+
WHERE state IN ('half-open', 'probing') AND probe_lease_until <= ?
|
|
1319
|
+
`).all(input.now);
|
|
1320
|
+
const changed = Number(this.database.prepare(`
|
|
1321
|
+
UPDATE automation_circuits
|
|
1322
|
+
SET state = 'open', probe_token = NULL, probe_lease_until = NULL, probe_task_id = NULL,
|
|
1323
|
+
updated_at = ?, version = version + 1
|
|
1324
|
+
WHERE state IN ('half-open', 'probing') AND probe_lease_until <= ?
|
|
1325
|
+
`).run(input.now, input.now).changes);
|
|
1326
|
+
for (const row of expired) {
|
|
1327
|
+
this.markRecoveringIncidentOpenInTransaction(row.automation_id, row.definition_hash, input.now);
|
|
1328
|
+
}
|
|
1329
|
+
return changed;
|
|
1330
|
+
});
|
|
1331
|
+
}
|
|
1332
|
+
/** Content-free bounded inventory for one exact system owner. */
|
|
1333
|
+
listSystemOwned(input) {
|
|
1334
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1335
|
+
const limit = input.limit ?? 1_000;
|
|
1336
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1337
|
+
throw new AutomationStoreError('invalid-definition', 'system-owned inventory limit must be between 1 and 1000');
|
|
1338
|
+
}
|
|
1339
|
+
return this.transaction(() => Object.freeze(this.database.prepare(`
|
|
1340
|
+
SELECT id, system_owner, definition_hash, status, version
|
|
1341
|
+
FROM automation_definitions
|
|
1342
|
+
WHERE system_owner = ?
|
|
1343
|
+
ORDER BY id
|
|
1344
|
+
LIMIT ?
|
|
1345
|
+
`).all(owner, limit).map(row => Object.freeze({
|
|
1346
|
+
owner,
|
|
1347
|
+
automationId: row.id,
|
|
1348
|
+
automationStatus: row.status,
|
|
1349
|
+
definitionHash: row.definition_hash,
|
|
1350
|
+
definitionVersion: row.version,
|
|
1351
|
+
}))));
|
|
1352
|
+
}
|
|
1353
|
+
/**
|
|
1354
|
+
* Pauses one exact system-owned row without accepting or rewriting a
|
|
1355
|
+
* definition. The lifecycle CAS and durable receipt share one transaction.
|
|
1356
|
+
*/
|
|
1357
|
+
pauseSystemOwned(input) {
|
|
1358
|
+
const owner = text(input.owner, 'owner', 200);
|
|
1359
|
+
const operationId = text(input.operationId, 'operationId', 500);
|
|
1360
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1361
|
+
if (!/^[a-f0-9]{64}$/u.test(input.definitionHash)
|
|
1362
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1363
|
+
throw new AutomationStoreError('invalid-definition', 'system pause tuple is invalid');
|
|
1364
|
+
}
|
|
1365
|
+
const inputHash = hash({
|
|
1366
|
+
lane: 'pause-system-owned-v1',
|
|
1367
|
+
owner,
|
|
1368
|
+
automationId,
|
|
1369
|
+
definitionHash: input.definitionHash,
|
|
1370
|
+
expectedVersion: input.expectedVersion,
|
|
1371
|
+
});
|
|
1372
|
+
return this.transaction(() => {
|
|
1373
|
+
const prior = this.database.prepare(`
|
|
1374
|
+
SELECT input_hash, result_json
|
|
1375
|
+
FROM automation_system_reconciles WHERE idempotency_key = ?
|
|
1376
|
+
`).get(operationId);
|
|
1377
|
+
if (prior !== undefined) {
|
|
1378
|
+
if (prior.input_hash !== inputHash) {
|
|
1379
|
+
throw new AutomationStoreError('idempotency-conflict', 'system pause operation id was reused with different input');
|
|
1380
|
+
}
|
|
1381
|
+
const replay = parseSystemPauseResult(prior.result_json);
|
|
1382
|
+
if (replay.owner !== owner || replay.automationId !== automationId
|
|
1383
|
+
|| replay.definitionHash !== input.definitionHash
|
|
1384
|
+
|| replay.expectedVersion !== input.expectedVersion) {
|
|
1385
|
+
throw new AutomationStoreError('invalid-state', 'system pause receipt does not match its operation');
|
|
1386
|
+
}
|
|
1387
|
+
return Object.freeze({ operationId, ...replay, replayed: true });
|
|
1388
|
+
}
|
|
1389
|
+
const current = this.database.prepare(`
|
|
1390
|
+
SELECT * FROM automation_definitions WHERE id = ?
|
|
1391
|
+
`).get(automationId);
|
|
1392
|
+
if (current === undefined || current.system_owner !== owner
|
|
1393
|
+
|| current.definition_hash !== input.definitionHash) {
|
|
1394
|
+
throw new AutomationStoreError('not-found', 'exact system-owned automation definition was not found');
|
|
1395
|
+
}
|
|
1396
|
+
if (current.version !== input.expectedVersion) {
|
|
1397
|
+
throw new AutomationStoreError('version-conflict', 'system-owned automation version changed before pause');
|
|
1398
|
+
}
|
|
1399
|
+
if (current.status !== 'active') {
|
|
1400
|
+
throw new AutomationStoreError('invalid-state', 'only an active system-owned automation can be paused');
|
|
1401
|
+
}
|
|
1402
|
+
const now = this.now();
|
|
1403
|
+
const changed = this.database.prepare(`
|
|
1404
|
+
UPDATE automation_definitions
|
|
1405
|
+
SET status = 'paused', next_run_at = NULL, updated_at = ?, version = version + 1
|
|
1406
|
+
WHERE id = ? AND system_owner = ? AND definition_hash = ?
|
|
1407
|
+
AND version = ? AND status = 'active'
|
|
1408
|
+
`).run(now, automationId, owner, input.definitionHash, input.expectedVersion);
|
|
1409
|
+
if (changed.changes !== 1) {
|
|
1410
|
+
throw new AutomationStoreError('version-conflict', 'system-owned automation changed before pause commit');
|
|
1411
|
+
}
|
|
1412
|
+
const stored = Object.freeze({
|
|
1413
|
+
owner,
|
|
1414
|
+
automationId,
|
|
1415
|
+
definitionHash: input.definitionHash,
|
|
1416
|
+
expectedVersion: input.expectedVersion,
|
|
1417
|
+
definitionVersion: input.expectedVersion + 1,
|
|
1418
|
+
automationStatus: 'paused',
|
|
1419
|
+
});
|
|
1420
|
+
this.database.prepare(`
|
|
1421
|
+
INSERT INTO automation_system_reconciles(
|
|
1422
|
+
idempotency_key, system_owner, automation_id, input_hash, result_json, created_at
|
|
1423
|
+
) VALUES (?, ?, ?, ?, ?, ?)
|
|
1424
|
+
`).run(operationId, owner, automationId, inputHash, JSON.stringify(stored), now);
|
|
1425
|
+
return Object.freeze({ operationId, ...stored, replayed: false });
|
|
1426
|
+
});
|
|
1427
|
+
}
|
|
1428
|
+
inspectSystemOwned(input) {
|
|
1429
|
+
const owner = text(input.owner, 'owner', 500);
|
|
1430
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1431
|
+
return this.transaction(() => {
|
|
1432
|
+
const definitionRow = this.database.prepare('SELECT * FROM automation_definitions WHERE id = ?')
|
|
1433
|
+
.get(automationId);
|
|
1434
|
+
if (definitionRow === undefined || definitionRow.system_owner !== owner) {
|
|
1435
|
+
throw new AutomationStoreError('not-found', 'system-owned automation was not found for exact owner');
|
|
1436
|
+
}
|
|
1437
|
+
const automation = record(definitionRow);
|
|
1438
|
+
const projectRun = (executionMode) => {
|
|
1439
|
+
const latestRow = this.database.prepare(`
|
|
1440
|
+
SELECT * FROM automation_runs WHERE automation_id = ? AND execution_mode = ?
|
|
1441
|
+
ORDER BY created_at DESC, id DESC LIMIT 1
|
|
1442
|
+
`).get(automationId, executionMode);
|
|
1443
|
+
if (latestRow === undefined)
|
|
1444
|
+
return undefined;
|
|
1445
|
+
const latest = run(latestRow);
|
|
1446
|
+
let snapshot;
|
|
1447
|
+
try {
|
|
1448
|
+
snapshot = this.getRunExecutionSnapshot(latest.id);
|
|
1449
|
+
}
|
|
1450
|
+
catch {
|
|
1451
|
+
snapshot = undefined;
|
|
1452
|
+
}
|
|
1453
|
+
const immutableContext = snapshot !== undefined
|
|
1454
|
+
&& latest.definitionHash !== undefined
|
|
1455
|
+
&& hash(snapshot.definition) === latest.definitionHash
|
|
1456
|
+
? Object.freeze({
|
|
1457
|
+
state: 'verified',
|
|
1458
|
+
definitionHash: latest.definitionHash,
|
|
1459
|
+
definitionVersion: snapshot.version,
|
|
1460
|
+
scope: Object.freeze({
|
|
1461
|
+
workspace: snapshot.definition.workspace,
|
|
1462
|
+
agentPreset: snapshot.definition.agentPreset,
|
|
1463
|
+
}),
|
|
1464
|
+
})
|
|
1465
|
+
: Object.freeze({ state: 'unknown' });
|
|
1466
|
+
return Object.freeze({
|
|
1467
|
+
runId: latest.id,
|
|
1468
|
+
status: latest.status,
|
|
1469
|
+
executionMode,
|
|
1470
|
+
diagnostic: latest.diagnostic,
|
|
1471
|
+
createdAt: latest.createdAt,
|
|
1472
|
+
immutableContext,
|
|
1473
|
+
});
|
|
1474
|
+
};
|
|
1475
|
+
const latestProductionRun = projectRun('production');
|
|
1476
|
+
const latestPreviewRun = projectRun('preview');
|
|
1477
|
+
const latestTerminalRuns = Object.freeze({
|
|
1478
|
+
...(latestProductionRun === undefined ? {} : { production: latestProductionRun }),
|
|
1479
|
+
...(latestPreviewRun === undefined ? {} : { preview: latestPreviewRun }),
|
|
1480
|
+
});
|
|
1481
|
+
const current = this.getCircuit(automationId, definitionRow.definition_hash);
|
|
1482
|
+
const currentCircuit = current === undefined
|
|
1483
|
+
? undefined
|
|
1484
|
+
: Object.freeze({
|
|
1485
|
+
definitionHash: current.definitionHash,
|
|
1486
|
+
state: current.state,
|
|
1487
|
+
failureClass: current.failureClass,
|
|
1488
|
+
failurePhase: current.failurePhase,
|
|
1489
|
+
failureCode: current.failureCode,
|
|
1490
|
+
openedAt: current.openedAt,
|
|
1491
|
+
updatedAt: current.updatedAt,
|
|
1492
|
+
...(current.probeLeaseUntil === undefined ? {} : { probeLeaseUntil: current.probeLeaseUntil }),
|
|
1493
|
+
version: current.version,
|
|
1494
|
+
});
|
|
1495
|
+
const activeIncident = this.getCurrentIncident(automationId, definitionRow.definition_hash);
|
|
1496
|
+
const currentIncident = activeIncident === undefined
|
|
1497
|
+
? undefined
|
|
1498
|
+
: Object.freeze({
|
|
1499
|
+
definitionHash: activeIncident.definitionHash,
|
|
1500
|
+
stage: activeIncident.stage,
|
|
1501
|
+
failureClass: activeIncident.failureClass,
|
|
1502
|
+
failurePhase: activeIncident.failurePhase,
|
|
1503
|
+
failureCode: activeIncident.failureCode,
|
|
1504
|
+
state: activeIncident.state,
|
|
1505
|
+
lifecycleGeneration: activeIncident.lifecycleGeneration,
|
|
1506
|
+
presentationRevision: activeIncident.presentationRevision,
|
|
1507
|
+
alertStatus: activeIncident.alertStatus,
|
|
1508
|
+
openedAt: activeIncident.openedAt,
|
|
1509
|
+
updatedAt: activeIncident.updatedAt,
|
|
1510
|
+
version: activeIncident.version,
|
|
1511
|
+
});
|
|
1512
|
+
return Object.freeze({
|
|
1513
|
+
owner,
|
|
1514
|
+
automationId,
|
|
1515
|
+
automationStatus: automation.status,
|
|
1516
|
+
definitionHash: definitionRow.definition_hash,
|
|
1517
|
+
definitionVersion: automation.version,
|
|
1518
|
+
...(latestProductionRun === undefined ? {} : { latestTerminalRun: latestProductionRun }),
|
|
1519
|
+
latestTerminalRuns,
|
|
1520
|
+
...(currentCircuit === undefined ? {} : { currentCircuit }),
|
|
1521
|
+
...(currentIncident === undefined ? {} : { currentIncident }),
|
|
1522
|
+
});
|
|
1523
|
+
});
|
|
1524
|
+
}
|
|
1525
|
+
listIncidents(input) {
|
|
1526
|
+
if (!Number.isSafeInteger(input.limit) || input.limit < 1 || input.limit > 1_000) {
|
|
1527
|
+
throw new AutomationStoreError('invalid-definition', 'incident limit must be between 1 and 1000');
|
|
1528
|
+
}
|
|
1529
|
+
const rows = input.automationId === undefined
|
|
1530
|
+
? this.database.prepare(`
|
|
1531
|
+
SELECT * FROM automation_incidents ORDER BY opened_at DESC, id DESC LIMIT ?
|
|
1532
|
+
`).all(input.limit)
|
|
1533
|
+
: this.database.prepare(`
|
|
1534
|
+
SELECT * FROM automation_incidents WHERE automation_id = ?
|
|
1535
|
+
ORDER BY opened_at DESC, id DESC LIMIT ?
|
|
1536
|
+
`).all(input.automationId, input.limit);
|
|
1537
|
+
return rows.map(incident);
|
|
1538
|
+
}
|
|
1539
|
+
listPendingIncidentAlerts(limit) {
|
|
1540
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1541
|
+
throw new AutomationStoreError('invalid-definition', 'incident alert limit must be between 1 and 1000');
|
|
1542
|
+
}
|
|
1543
|
+
const output = [];
|
|
1544
|
+
const rows = this.database.prepare(`
|
|
1545
|
+
SELECT * FROM automation_incidents
|
|
1546
|
+
WHERE alert_status = 'pending'
|
|
1547
|
+
ORDER BY updated_at, id LIMIT 1000
|
|
1548
|
+
`).all();
|
|
1549
|
+
for (const row of rows) {
|
|
1550
|
+
if (output.length === limit)
|
|
1551
|
+
break;
|
|
1552
|
+
try {
|
|
1553
|
+
output.push(incident(row));
|
|
1554
|
+
}
|
|
1555
|
+
catch {
|
|
1556
|
+
// A malformed poison row is terminally suppressed so it cannot starve
|
|
1557
|
+
// later incident alerts or leak unvalidated content into Delivery.
|
|
1558
|
+
this.database.prepare(`
|
|
1559
|
+
UPDATE automation_incidents
|
|
1560
|
+
SET alert_status = 'suppressed',
|
|
1561
|
+
updated_at = ?, version = version + 1
|
|
1562
|
+
WHERE id = ? AND alert_status = 'pending'
|
|
1563
|
+
`).run(this.now(), row.id);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
return output;
|
|
1567
|
+
}
|
|
1568
|
+
completeIncidentAlert(input) {
|
|
1569
|
+
if (!Number.isSafeInteger(input.now)
|
|
1570
|
+
|| !Number.isSafeInteger(input.expectedLifecycleGeneration) || input.expectedLifecycleGeneration < 1
|
|
1571
|
+
|| !Number.isSafeInteger(input.expectedPresentationRevision) || input.expectedPresentationRevision < 1
|
|
1572
|
+
|| !Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1) {
|
|
1573
|
+
throw new AutomationStoreError('invalid-definition', 'incident alert time is invalid');
|
|
1574
|
+
}
|
|
1575
|
+
const alertRef = text(input.alertRef, 'alertRef', 500);
|
|
1576
|
+
const changed = this.database.prepare(`
|
|
1577
|
+
UPDATE automation_incidents
|
|
1578
|
+
SET alert_status = 'enqueued', alert_ref = ?, updated_at = ?, version = version + 1
|
|
1579
|
+
WHERE id = ? AND alert_status = ? AND lifecycle_generation = ?
|
|
1580
|
+
AND presentation_revision = ? AND version = ?
|
|
1581
|
+
`).run(alertRef, input.now, input.incidentId, input.expectedStatus, input.expectedLifecycleGeneration, input.expectedPresentationRevision, input.expectedVersion);
|
|
1582
|
+
const row = this.database.prepare('SELECT * FROM automation_incidents WHERE id = ?')
|
|
1583
|
+
.get(input.incidentId);
|
|
1584
|
+
if (row === undefined || (changed.changes !== 1
|
|
1585
|
+
&& !(row.alert_status === 'enqueued' && row.alert_ref === alertRef
|
|
1586
|
+
&& row.lifecycle_generation === input.expectedLifecycleGeneration
|
|
1587
|
+
&& row.presentation_revision === input.expectedPresentationRevision))) {
|
|
1588
|
+
throw new AutomationStoreError('version-conflict', 'incident alert state changed before completion');
|
|
1589
|
+
}
|
|
1590
|
+
return incident(row);
|
|
1591
|
+
}
|
|
1592
|
+
getCurrentIncident(automationId, definitionHash) {
|
|
1593
|
+
const row = this.database.prepare(`
|
|
1594
|
+
SELECT * FROM automation_incidents
|
|
1595
|
+
WHERE automation_id = ? AND definition_hash = ? AND state <> 'resolved'
|
|
1596
|
+
ORDER BY updated_at DESC, id DESC LIMIT 1
|
|
1597
|
+
`).get(automationId, definitionHash);
|
|
1598
|
+
return row === undefined ? undefined : incident(row);
|
|
1599
|
+
}
|
|
1600
|
+
/**
|
|
1601
|
+
* Re-prove the exact transport target before an incident leaves this store.
|
|
1602
|
+
* Terminal targets come from the winning claim snapshot rather than the
|
|
1603
|
+
* mutable current definition, so an edit/restart cannot redirect an alert.
|
|
1604
|
+
*/
|
|
1605
|
+
getIncidentNotificationTarget(incidentId) {
|
|
1606
|
+
const id = text(incidentId, 'incidentId', 1_000);
|
|
1607
|
+
const row = this.database.prepare('SELECT * FROM automation_incidents WHERE id = ?')
|
|
1608
|
+
.get(id);
|
|
1609
|
+
if (row === undefined)
|
|
1610
|
+
return undefined;
|
|
1611
|
+
const value = incident(row);
|
|
1612
|
+
if (value.stage !== 'terminal') {
|
|
1613
|
+
// Availability incidents can only be opened for a Host definition. The
|
|
1614
|
+
// route itself was copied into the incident in the opening transaction.
|
|
1615
|
+
return Object.freeze({ kind: 'owner-route', authorityId: value.notificationRouteId });
|
|
1616
|
+
}
|
|
1617
|
+
if (value.runId === undefined)
|
|
1618
|
+
return undefined;
|
|
1619
|
+
const proof = this.getProvenProductionRun(value.runId);
|
|
1620
|
+
if (proof === undefined || proof.run.automationId !== value.automationId
|
|
1621
|
+
|| proof.run.definitionHash !== value.definitionHash)
|
|
1622
|
+
return undefined;
|
|
1623
|
+
const route = incidentNotificationRoute(proof.automation.definition);
|
|
1624
|
+
if (route === undefined || route.id !== value.notificationRouteId)
|
|
1625
|
+
return undefined;
|
|
1626
|
+
return route.target;
|
|
1627
|
+
}
|
|
394
1628
|
health() {
|
|
395
1629
|
const scalar = (sql) => this.database.prepare(sql).get().count;
|
|
396
1630
|
return {
|
|
@@ -400,6 +1634,16 @@ export class AutomationStore {
|
|
|
400
1634
|
runningTasks: scalar("SELECT COUNT(*) AS count FROM automation_tasks WHERE status = 'running'"),
|
|
401
1635
|
failedRuns: scalar("SELECT COUNT(*) AS count FROM automation_runs WHERE status IN ('failed', 'timed_out', 'cancelled')"),
|
|
402
1636
|
unknownRuns: scalar("SELECT COUNT(*) AS count FROM automation_runs WHERE status = 'unknown'"),
|
|
1637
|
+
pendingEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'pending'"),
|
|
1638
|
+
retryingEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'pending' AND attempt_count > 0"),
|
|
1639
|
+
// This is an append-only lifetime counter. It supports rate/delta
|
|
1640
|
+
// monitoring but does not by itself describe current health.
|
|
1641
|
+
failedEvaluationAttempts: scalar('SELECT COALESCE(SUM(attempt_count), 0) AS count FROM automation_evaluation_outbox'),
|
|
1642
|
+
deadLetterEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'dead-letter'"),
|
|
1643
|
+
oldestPendingEvaluationAt: scalar("SELECT COALESCE(MIN(created_at), 0) AS count FROM automation_evaluation_outbox WHERE status = 'pending'"),
|
|
1644
|
+
openCircuits: scalar("SELECT COUNT(*) AS count FROM automation_circuits WHERE state <> 'closed'"),
|
|
1645
|
+
openIncidents: scalar("SELECT COUNT(*) AS count FROM automation_incidents WHERE state <> 'resolved'"),
|
|
1646
|
+
pendingIncidentAlerts: scalar("SELECT COUNT(*) AS count FROM automation_incidents WHERE alert_status = 'pending'"),
|
|
403
1647
|
};
|
|
404
1648
|
}
|
|
405
1649
|
changeApproved(input) {
|
|
@@ -445,6 +1689,58 @@ export class AutomationStore {
|
|
|
445
1689
|
return result;
|
|
446
1690
|
});
|
|
447
1691
|
}
|
|
1692
|
+
listDueHostExecutionRequirements(input) {
|
|
1693
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
1694
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement time is invalid');
|
|
1695
|
+
}
|
|
1696
|
+
const limit = input.limit ?? 1_000;
|
|
1697
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1698
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement limit is invalid');
|
|
1699
|
+
}
|
|
1700
|
+
const rows = this.database.prepare(`
|
|
1701
|
+
SELECT * FROM automation_definitions
|
|
1702
|
+
WHERE status = 'active' AND next_run_at IS NOT NULL AND next_run_at <= ?
|
|
1703
|
+
ORDER BY next_run_at, id LIMIT ?
|
|
1704
|
+
`).all(input.now, limit);
|
|
1705
|
+
return rows.flatMap(row => {
|
|
1706
|
+
const automation = record(row);
|
|
1707
|
+
return isHostAutomationDefinition(automation.definition)
|
|
1708
|
+
? [Object.freeze({
|
|
1709
|
+
automationId: automation.id,
|
|
1710
|
+
definitionHash: row.definition_hash,
|
|
1711
|
+
execution: automation.definition.execution,
|
|
1712
|
+
})]
|
|
1713
|
+
: [];
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
listClaimableHostExecutionRequirements(limit = 100) {
|
|
1717
|
+
if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) {
|
|
1718
|
+
throw new AutomationStoreError('invalid-definition', 'Host requirement limit is invalid');
|
|
1719
|
+
}
|
|
1720
|
+
const rows = this.database.prepare(`
|
|
1721
|
+
SELECT DISTINCT definition.* FROM automation_tasks AS task
|
|
1722
|
+
JOIN automation_definitions AS definition ON definition.id = task.automation_id
|
|
1723
|
+
WHERE task.status = 'scheduled' AND (
|
|
1724
|
+
definition.status = 'active'
|
|
1725
|
+
OR (
|
|
1726
|
+
definition.status = 'paused'
|
|
1727
|
+
AND definition.system_owner = ?
|
|
1728
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
1729
|
+
)
|
|
1730
|
+
)
|
|
1731
|
+
ORDER BY task.created_at, task.id LIMIT ?
|
|
1732
|
+
`).all(growthAutomationOwner, limit);
|
|
1733
|
+
return rows.flatMap(row => {
|
|
1734
|
+
const automation = record(row);
|
|
1735
|
+
return isHostAutomationDefinition(automation.definition)
|
|
1736
|
+
? [Object.freeze({
|
|
1737
|
+
automationId: automation.id,
|
|
1738
|
+
definitionHash: row.definition_hash,
|
|
1739
|
+
execution: automation.definition.execution,
|
|
1740
|
+
})]
|
|
1741
|
+
: [];
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
448
1744
|
materializeDue(input) {
|
|
449
1745
|
if (!Number.isSafeInteger(input.now) || !Number.isSafeInteger(input.misfireGraceMs)
|
|
450
1746
|
|| input.misfireGraceMs < 0 || !Number.isSafeInteger(input.maxCatchUp)
|
|
@@ -460,6 +1756,20 @@ export class AutomationStore {
|
|
|
460
1756
|
const created = [];
|
|
461
1757
|
for (const row of due) {
|
|
462
1758
|
const automation = record(row);
|
|
1759
|
+
if (isHostAutomationDefinition(automation.definition)) {
|
|
1760
|
+
const availability = this.exactHostAvailability(input.hostAvailability, automation.id, row.definition_hash, 'materialize');
|
|
1761
|
+
if (availability?.available !== true) {
|
|
1762
|
+
this.upsertAvailabilityIncidentInTransaction({
|
|
1763
|
+
automation,
|
|
1764
|
+
definitionHash: row.definition_hash,
|
|
1765
|
+
stage: 'materialize',
|
|
1766
|
+
failureCode: availability?.reasonCode ?? 'host-executor-proof-missing',
|
|
1767
|
+
now: input.now,
|
|
1768
|
+
});
|
|
1769
|
+
continue;
|
|
1770
|
+
}
|
|
1771
|
+
this.resolveIncidentInTransaction(automation.id, row.definition_hash, 'materialize', input.now);
|
|
1772
|
+
}
|
|
463
1773
|
const firstDue = automation.nextRunAt;
|
|
464
1774
|
const cutoff = input.now - input.misfireGraceMs;
|
|
465
1775
|
const current = dueOccurrences(automation.definition.schedule, Math.max(firstDue - 1, cutoff), input.now, input.maxCatchUp);
|
|
@@ -593,6 +1903,193 @@ export class AutomationStore {
|
|
|
593
1903
|
getTaskRecord(id) {
|
|
594
1904
|
return this.getTask(id);
|
|
595
1905
|
}
|
|
1906
|
+
/** Immutable definition captured by the winning claim, never the mutable current row. */
|
|
1907
|
+
getTaskExecutionSnapshot(taskId) {
|
|
1908
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
1909
|
+
if (taskRow === undefined || taskRow.attempt_count < 1)
|
|
1910
|
+
return undefined;
|
|
1911
|
+
const attempt = this.database.prepare(`
|
|
1912
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
1913
|
+
`).get(taskId, taskRow.attempt_count);
|
|
1914
|
+
return attempt === undefined ? undefined : automationSnapshot(attempt, taskRow.automation_id);
|
|
1915
|
+
}
|
|
1916
|
+
/** Immutable definition associated with a durable terminal run. */
|
|
1917
|
+
getRunExecutionSnapshot(runId) {
|
|
1918
|
+
const row = this.database.prepare(`
|
|
1919
|
+
SELECT attempt.*, run.automation_id AS expected_automation_id
|
|
1920
|
+
FROM automation_runs AS run
|
|
1921
|
+
JOIN automation_attempts AS attempt ON attempt.id = run.attempt_id
|
|
1922
|
+
WHERE run.id = ?
|
|
1923
|
+
`).get(runId);
|
|
1924
|
+
return row === undefined ? undefined : automationSnapshot(row, row.expected_automation_id);
|
|
1925
|
+
}
|
|
1926
|
+
/** Host-derived occurrence mode associated with a durable terminal run. */
|
|
1927
|
+
getRunExecutionMode(runId) {
|
|
1928
|
+
const row = this.database.prepare(`SELECT execution_mode FROM automation_runs WHERE id = ?`)
|
|
1929
|
+
.get(runId);
|
|
1930
|
+
if (row === undefined)
|
|
1931
|
+
return undefined;
|
|
1932
|
+
if (row.execution_mode === 'production' || row.execution_mode === 'preview' || row.execution_mode === 'unknown') {
|
|
1933
|
+
return row.execution_mode;
|
|
1934
|
+
}
|
|
1935
|
+
throw new AutomationStoreError('invalid-state', 'automation run has an invalid execution mode');
|
|
1936
|
+
}
|
|
1937
|
+
/**
|
|
1938
|
+
* Re-prove the exact production tuple immediately before an effect sink.
|
|
1939
|
+
* Legacy rows, mutable current definitions, and a mode flag alone are never
|
|
1940
|
+
* sufficient. Corrupt snapshots throw so dispatchers can quarantine the row.
|
|
1941
|
+
*/
|
|
1942
|
+
getProvenProductionRun(runId) {
|
|
1943
|
+
const row = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId);
|
|
1944
|
+
if (row === undefined || row.execution_mode !== 'production' || row.definition_hash === null)
|
|
1945
|
+
return undefined;
|
|
1946
|
+
// Parse all bounded run fields before treating the row as effect-capable.
|
|
1947
|
+
const value = run(row);
|
|
1948
|
+
const occurrenceRow = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
1949
|
+
.get(row.occurrence_id);
|
|
1950
|
+
if (occurrenceRow === undefined || occurrenceRow.dry_run !== 0)
|
|
1951
|
+
return undefined;
|
|
1952
|
+
const attempt = this.database.prepare('SELECT * FROM automation_attempts WHERE id = ?')
|
|
1953
|
+
.get(row.attempt_id);
|
|
1954
|
+
if (attempt === undefined)
|
|
1955
|
+
return undefined;
|
|
1956
|
+
const snapshot = automationSnapshot(attempt, row.automation_id);
|
|
1957
|
+
if (snapshot === undefined || hash(snapshot.definition) !== row.definition_hash)
|
|
1958
|
+
return undefined;
|
|
1959
|
+
const evidence = value.evidence;
|
|
1960
|
+
if (evidence !== undefined) {
|
|
1961
|
+
const expectedOutcome = row.status === 'succeeded' ? 'succeeded' : 'failed';
|
|
1962
|
+
if (evidence.situation !== automationSituation(snapshot.id)
|
|
1963
|
+
|| evidence.outcome !== expectedOutcome
|
|
1964
|
+
|| evidence.detail !== `automation "${snapshot.definition.name}": run ${row.status}`
|
|
1965
|
+
|| evidence.idempotencyKey !== `automation-run:${row.id}`
|
|
1966
|
+
|| evidence.occurredAt !== row.created_at
|
|
1967
|
+
|| evidence.workspace !== snapshot.definition.workspace
|
|
1968
|
+
|| evidence.agentPreset !== snapshot.definition.agentPreset
|
|
1969
|
+
|| evidence.automationId !== snapshot.id
|
|
1970
|
+
|| evidence.runId !== row.id) {
|
|
1971
|
+
throw new AutomationStoreError('invalid-state', 'automation evidence does not match its immutable run');
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
return Object.freeze({ run: value, automation: snapshot });
|
|
1975
|
+
}
|
|
1976
|
+
/**
|
|
1977
|
+
* Resolve an Evaluation automation-run reference into a content-minimal,
|
|
1978
|
+
* exactly re-provable production receipt. Callers must supply the Evaluation
|
|
1979
|
+
* tuple; this method never guesses scope or situation from mutable state.
|
|
1980
|
+
*/
|
|
1981
|
+
resolveQualityEvidence(input) {
|
|
1982
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
1983
|
+
const runId = text(input.runId, 'runId', 1_000);
|
|
1984
|
+
const workspace = text(input.expectedScope.workspace, 'expectedScope.workspace', 4_096);
|
|
1985
|
+
const preset = text(input.expectedScope.preset, 'expectedScope.preset', 200);
|
|
1986
|
+
const situation = text(input.expectedSituation, 'expectedSituation', 1_000);
|
|
1987
|
+
if (!Number.isSafeInteger(input.expectedOccurredAt)
|
|
1988
|
+
|| input.evidenceRef.kind !== 'automation-run' || input.evidenceRef.ref !== runId) {
|
|
1989
|
+
throw new AutomationStoreError('invalid-definition', 'quality evidence expectation is invalid');
|
|
1990
|
+
}
|
|
1991
|
+
const owner = input.owner === undefined ? undefined : text(input.owner, 'owner', 500);
|
|
1992
|
+
const proof = this.getProvenProductionRun(runId);
|
|
1993
|
+
if (proof === undefined || proof.run.automationId !== automationId
|
|
1994
|
+
|| (proof.run.status !== 'succeeded'
|
|
1995
|
+
&& proof.run.status !== 'failed' && proof.run.status !== 'timed_out')
|
|
1996
|
+
|| proof.run.definitionHash === undefined
|
|
1997
|
+
|| proof.run.evidence === undefined
|
|
1998
|
+
|| (proof.run.evidenceStatus !== 'pending' && proof.run.evidenceStatus !== 'recorded')
|
|
1999
|
+
|| proof.automation.definition.workspace !== workspace
|
|
2000
|
+
|| proof.automation.definition.agentPreset !== preset
|
|
2001
|
+
|| proof.run.evidence.situation !== situation
|
|
2002
|
+
|| proof.run.evidence.occurredAt !== input.expectedOccurredAt
|
|
2003
|
+
|| proof.run.evidence.runId !== runId
|
|
2004
|
+
|| proof.run.evidence.automationId !== automationId
|
|
2005
|
+
|| (owner !== undefined && proof.automation.owner !== owner))
|
|
2006
|
+
return undefined;
|
|
2007
|
+
const base = {
|
|
2008
|
+
schemaVersion: 1,
|
|
2009
|
+
source: 'assistant-automations',
|
|
2010
|
+
executionKind: isHostAutomationDefinition(proof.automation.definition) ? 'host' : 'agent',
|
|
2011
|
+
automationId,
|
|
2012
|
+
runId,
|
|
2013
|
+
definitionHash: proof.run.definitionHash,
|
|
2014
|
+
status: proof.run.status,
|
|
2015
|
+
scope: Object.freeze({ workspace, preset }),
|
|
2016
|
+
situation,
|
|
2017
|
+
occurredAt: input.expectedOccurredAt,
|
|
2018
|
+
evidenceRef: Object.freeze({ kind: 'automation-run', ref: runId }),
|
|
2019
|
+
...(proof.run.evidence.sessionId === undefined ? {} : { sessionId: proof.run.evidence.sessionId }),
|
|
2020
|
+
...(proof.run.evidence.ruleId === undefined ? {} : { ruleId: proof.run.evidence.ruleId }),
|
|
2021
|
+
...(proof.run.evidence.guidanceVersion === undefined
|
|
2022
|
+
? {}
|
|
2023
|
+
: { guidanceVersion: proof.run.evidence.guidanceVersion }),
|
|
2024
|
+
};
|
|
2025
|
+
return Object.freeze({ ...base, proofDigest: hash(base) });
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Prove that one exact owner-bound output is the immutable output of an
|
|
2029
|
+
* Agent production run. This is narrower than quality evidence and is used
|
|
2030
|
+
* only by Delivery's reserved learning-metadata lane.
|
|
2031
|
+
*/
|
|
2032
|
+
resolveDeliveryEvidence(input) {
|
|
2033
|
+
const automationId = text(input.automationId, 'automationId', 500);
|
|
2034
|
+
const runId = text(input.runId, 'runId', 1_000);
|
|
2035
|
+
const workspace = text(input.expectedWorkspace, 'expectedWorkspace', 4_096);
|
|
2036
|
+
const bindingId = text(input.expectedBindingId, 'expectedBindingId', 1_000);
|
|
2037
|
+
if (!/^[a-f0-9]{64}$/u.test(input.expectedOutputDigest)) {
|
|
2038
|
+
throw new AutomationStoreError('invalid-definition', 'delivery output digest is invalid');
|
|
2039
|
+
}
|
|
2040
|
+
const proof = this.getProvenProductionRun(runId);
|
|
2041
|
+
if (proof === undefined || proof.run.automationId !== automationId
|
|
2042
|
+
|| proof.run.status !== 'succeeded'
|
|
2043
|
+
|| (proof.run.deliveryStatus !== 'pending' && proof.run.deliveryStatus !== 'enqueued')
|
|
2044
|
+
|| proof.run.evidence === undefined
|
|
2045
|
+
|| (proof.run.evidenceStatus !== 'pending' && proof.run.evidenceStatus !== 'recorded')
|
|
2046
|
+
|| isHostAutomationDefinition(proof.automation.definition)
|
|
2047
|
+
|| proof.automation.definition.workspace !== workspace
|
|
2048
|
+
|| proof.automation.definition.deliveryBindingId !== bindingId
|
|
2049
|
+
|| proof.run.evidence.automationId !== automationId
|
|
2050
|
+
|| proof.run.evidence.runId !== runId
|
|
2051
|
+
|| proof.run.evidence.situation !== `automation:${automationId}`
|
|
2052
|
+
|| hashText(proof.run.outputPreview) !== input.expectedOutputDigest)
|
|
2053
|
+
return undefined;
|
|
2054
|
+
const base = {
|
|
2055
|
+
schemaVersion: 1,
|
|
2056
|
+
source: 'assistant-automations',
|
|
2057
|
+
executionKind: 'agent',
|
|
2058
|
+
automationId,
|
|
2059
|
+
runId,
|
|
2060
|
+
occurrenceId: proof.run.occurrenceId,
|
|
2061
|
+
workspace,
|
|
2062
|
+
agentPreset: proof.automation.definition.agentPreset,
|
|
2063
|
+
bindingId,
|
|
2064
|
+
situation: proof.run.evidence.situation,
|
|
2065
|
+
occurredAt: proof.run.evidence.occurredAt,
|
|
2066
|
+
executionStatus: 'succeeded',
|
|
2067
|
+
outputDigest: input.expectedOutputDigest,
|
|
2068
|
+
};
|
|
2069
|
+
return Object.freeze({ ...base, proofDigest: hash(base) });
|
|
2070
|
+
}
|
|
2071
|
+
validateQualityEvidence(receipt) {
|
|
2072
|
+
try {
|
|
2073
|
+
if (receipt.schemaVersion !== 1 || receipt.source !== 'assistant-automations')
|
|
2074
|
+
return false;
|
|
2075
|
+
const resolved = this.resolveQualityEvidence({
|
|
2076
|
+
automationId: receipt.automationId,
|
|
2077
|
+
runId: receipt.runId,
|
|
2078
|
+
expectedScope: receipt.scope,
|
|
2079
|
+
expectedSituation: receipt.situation,
|
|
2080
|
+
expectedOccurredAt: receipt.occurredAt,
|
|
2081
|
+
evidenceRef: receipt.evidenceRef,
|
|
2082
|
+
});
|
|
2083
|
+
return resolved !== undefined && JSON.stringify(resolved) === JSON.stringify(receipt);
|
|
2084
|
+
}
|
|
2085
|
+
catch {
|
|
2086
|
+
return false;
|
|
2087
|
+
}
|
|
2088
|
+
}
|
|
2089
|
+
getRun(runId) {
|
|
2090
|
+
const row = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId);
|
|
2091
|
+
return row === undefined ? undefined : run(row);
|
|
2092
|
+
}
|
|
596
2093
|
acquireDuty(input) {
|
|
597
2094
|
const ownerId = text(input.ownerId, 'ownerId', 500);
|
|
598
2095
|
this.validateLease(input.now, input.leaseMs);
|
|
@@ -628,6 +2125,34 @@ export class AutomationStore {
|
|
|
628
2125
|
return duty({ ...current, lease_until: leaseUntil }, true);
|
|
629
2126
|
});
|
|
630
2127
|
}
|
|
2128
|
+
/**
|
|
2129
|
+
* Relinquish a live coordinator lease without weakening its fencing token.
|
|
2130
|
+
*
|
|
2131
|
+
* A cleanly stopped process may expire only the exact lease it owns. The
|
|
2132
|
+
* row is deliberately retained so the next owner must advance the fencing
|
|
2133
|
+
* token; a late/stale owner can therefore never release its successor.
|
|
2134
|
+
*/
|
|
2135
|
+
releaseDuty(input) {
|
|
2136
|
+
const ownerId = text(input.ownerId, 'ownerId', 500);
|
|
2137
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
2138
|
+
throw new AutomationStoreError('invalid-definition', 'release time is invalid');
|
|
2139
|
+
}
|
|
2140
|
+
return this.transaction(() => {
|
|
2141
|
+
const current = this.database.prepare('SELECT * FROM duty_lease WHERE singleton = 1').get();
|
|
2142
|
+
if (current === undefined || current.owner_id !== ownerId || current.fencing_token !== input.fencingToken) {
|
|
2143
|
+
throw new AutomationStoreError('stale-fence', 'duty ownership is missing or fenced');
|
|
2144
|
+
}
|
|
2145
|
+
const leaseUntil = Math.min(current.lease_until, input.now);
|
|
2146
|
+
const changed = this.database.prepare(`
|
|
2147
|
+
UPDATE duty_lease SET lease_until = ?, updated_at = ?
|
|
2148
|
+
WHERE singleton = 1 AND owner_id = ? AND fencing_token = ?
|
|
2149
|
+
`).run(leaseUntil, input.now, ownerId, input.fencingToken);
|
|
2150
|
+
if (changed.changes !== 1) {
|
|
2151
|
+
throw new AutomationStoreError('stale-fence', 'duty ownership changed while releasing');
|
|
2152
|
+
}
|
|
2153
|
+
return duty({ ...current, lease_until: leaseUntil }, false);
|
|
2154
|
+
});
|
|
2155
|
+
}
|
|
631
2156
|
claimNextTask(input) {
|
|
632
2157
|
this.validateLease(input.now, input.leaseMs);
|
|
633
2158
|
return this.transaction(() => {
|
|
@@ -635,11 +2160,18 @@ export class AutomationStore {
|
|
|
635
2160
|
const candidates = this.database.prepare(`
|
|
636
2161
|
SELECT task.* FROM automation_tasks task
|
|
637
2162
|
JOIN automation_definitions definition ON definition.id = task.automation_id
|
|
638
|
-
WHERE task.status = 'scheduled' AND
|
|
2163
|
+
WHERE task.status = 'scheduled' AND (
|
|
2164
|
+
definition.status = 'active'
|
|
2165
|
+
OR (
|
|
2166
|
+
definition.status = 'paused'
|
|
2167
|
+
AND definition.system_owner = ?
|
|
2168
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
2169
|
+
)
|
|
2170
|
+
)
|
|
639
2171
|
ORDER BY task.created_at, task.id LIMIT 100
|
|
640
|
-
`).all();
|
|
2172
|
+
`).all(growthAutomationOwner);
|
|
641
2173
|
for (const candidate of candidates) {
|
|
642
|
-
const result = this.claimTaskInTransaction(candidate, input);
|
|
2174
|
+
const result = this.claimTaskInTransaction(candidate, input, input.hostAvailability);
|
|
643
2175
|
if (result !== undefined)
|
|
644
2176
|
return result;
|
|
645
2177
|
}
|
|
@@ -655,7 +2187,7 @@ export class AutomationStore {
|
|
|
655
2187
|
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
656
2188
|
if (row.status !== 'scheduled')
|
|
657
2189
|
throw new AutomationStoreError('invalid-state', 'only scheduled tasks can be claimed');
|
|
658
|
-
return this.claimTaskInTransaction(row, input);
|
|
2190
|
+
return this.claimTaskInTransaction(row, input, input.hostAvailability);
|
|
659
2191
|
});
|
|
660
2192
|
}
|
|
661
2193
|
startTask(input) {
|
|
@@ -671,6 +2203,28 @@ export class AutomationStore {
|
|
|
671
2203
|
UPDATE automation_attempts SET status = 'running', session_id = ?, started_at = ?, updated_at = ?
|
|
672
2204
|
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
673
2205
|
`).run(sessionId, input.now, input.now, row.id, row.attempt_count, input.fencingToken);
|
|
2206
|
+
const occurrence = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
2207
|
+
.get(row.occurrence_id);
|
|
2208
|
+
const attempt = this.database.prepare(`
|
|
2209
|
+
SELECT * FROM automation_attempts
|
|
2210
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2211
|
+
`).get(row.id, row.attempt_count, input.fencingToken);
|
|
2212
|
+
let snapshot;
|
|
2213
|
+
try {
|
|
2214
|
+
snapshot = attempt === undefined ? undefined : automationSnapshot(attempt, row.automation_id);
|
|
2215
|
+
}
|
|
2216
|
+
catch {
|
|
2217
|
+
// The coordinator's quarantine path owns malformed snapshots. Starting
|
|
2218
|
+
// the task must still commit so that path can write its terminal receipt.
|
|
2219
|
+
}
|
|
2220
|
+
// Host recovery is explicitly armed by its circuit probe. A routed Agent
|
|
2221
|
+
// has no separate Host probe, so a later production attempt itself is
|
|
2222
|
+
// the durable transition from open to recovering.
|
|
2223
|
+
if (occurrence?.dry_run === 0 && snapshot !== undefined
|
|
2224
|
+
&& !isHostAutomationDefinition(snapshot.definition)
|
|
2225
|
+
&& incidentNotificationRoute(snapshot.definition) !== undefined) {
|
|
2226
|
+
this.markIncidentRecoveringInTransaction(snapshot.id, hash(snapshot.definition), input.now);
|
|
2227
|
+
}
|
|
674
2228
|
return this.getTask(row.id);
|
|
675
2229
|
});
|
|
676
2230
|
}
|
|
@@ -683,6 +2237,62 @@ export class AutomationStore {
|
|
|
683
2237
|
return this.getTask(row.id);
|
|
684
2238
|
});
|
|
685
2239
|
}
|
|
2240
|
+
/**
|
|
2241
|
+
* Terminally quarantine a started task whose immutable execution snapshot
|
|
2242
|
+
* cannot be decoded. No run/evaluation is invented because historical scope
|
|
2243
|
+
* is unavailable; the occurrence and attempt still become durably unknown.
|
|
2244
|
+
*/
|
|
2245
|
+
quarantineInvalidExecutionSnapshot(input) {
|
|
2246
|
+
if (!Number.isSafeInteger(input.now)) {
|
|
2247
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2248
|
+
}
|
|
2249
|
+
return this.transaction(() => {
|
|
2250
|
+
const row = this.requireMutableTask(input, ['running']);
|
|
2251
|
+
const attempt = this.database.prepare(`
|
|
2252
|
+
SELECT * FROM automation_attempts
|
|
2253
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2254
|
+
`).get(row.id, row.attempt_count, input.fencingToken);
|
|
2255
|
+
const occurrence = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2256
|
+
.get(row.occurrence_id);
|
|
2257
|
+
if (attempt === undefined || occurrence === undefined) {
|
|
2258
|
+
throw new AutomationStoreError('invalid-state', 'invalid snapshot task has no exact attempt or occurrence');
|
|
2259
|
+
}
|
|
2260
|
+
const executionMode = occurrence.dry_run === 0
|
|
2261
|
+
? 'production'
|
|
2262
|
+
: occurrence.dry_run === 1 ? 'preview' : 'unknown';
|
|
2263
|
+
const diagnostic = Object.freeze({
|
|
2264
|
+
schemaVersion: 1,
|
|
2265
|
+
failureClass: 'infrastructure',
|
|
2266
|
+
failurePhase: 'recovery',
|
|
2267
|
+
failureCode: 'execution-snapshot-invalid',
|
|
2268
|
+
promptSubmissionState: 'not-submitted',
|
|
2269
|
+
sideEffectState: 'none',
|
|
2270
|
+
retryability: 'after-intervention',
|
|
2271
|
+
budgetSettlementState: 'unknown',
|
|
2272
|
+
});
|
|
2273
|
+
this.database.prepare(`
|
|
2274
|
+
INSERT INTO automation_runs(
|
|
2275
|
+
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
2276
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2277
|
+
delivery_status, delivery_ref, evidence_status, evidence_json, created_at, updated_at
|
|
2278
|
+
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', ?, NULL, ?, NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
2279
|
+
`).run(`run-${row.id}`, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'immutable execution snapshot is invalid', executionMode, JSON.stringify(diagnostic), input.now, input.now);
|
|
2280
|
+
this.database.prepare(`
|
|
2281
|
+
UPDATE automation_attempts
|
|
2282
|
+
SET status = 'unknown', failure_code = 'execution-snapshot-invalid',
|
|
2283
|
+
finished_at = ?, updated_at = ?
|
|
2284
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2285
|
+
`).run(input.now, input.now, row.id, row.attempt_count, input.fencingToken);
|
|
2286
|
+
this.database.prepare(`
|
|
2287
|
+
UPDATE automation_tasks SET status = 'unknown', lease_until = NULL, updated_at = ? WHERE id = ?
|
|
2288
|
+
`).run(input.now, row.id);
|
|
2289
|
+
this.database.prepare(`
|
|
2290
|
+
UPDATE automation_occurrences
|
|
2291
|
+
SET status = 'unknown', reason = 'execution-snapshot-invalid', updated_at = ? WHERE id = ?
|
|
2292
|
+
`).run(input.now, row.occurrence_id);
|
|
2293
|
+
return this.getTask(row.id);
|
|
2294
|
+
});
|
|
2295
|
+
}
|
|
686
2296
|
requestCancellation(input) {
|
|
687
2297
|
if (!Number.isSafeInteger(input.now))
|
|
688
2298
|
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
@@ -716,6 +2326,28 @@ export class AutomationStore {
|
|
|
716
2326
|
if (usageJson === undefined || Buffer.byteLength(usageJson, 'utf8') > 16_384) {
|
|
717
2327
|
throw new AutomationStoreError('invalid-definition', 'usage must be bounded JSON');
|
|
718
2328
|
}
|
|
2329
|
+
const diagnostic = executionDiagnostic(input.diagnostic ?? legacyAutomationExecutionDiagnostic, 'invalid-definition');
|
|
2330
|
+
if (input.diagnostic !== undefined && !isLegacyExecutionDiagnostic(diagnostic)) {
|
|
2331
|
+
assertDiagnosticOutcome(input.outcome, diagnostic, (() => {
|
|
2332
|
+
const row = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?')
|
|
2333
|
+
.get(input.taskId);
|
|
2334
|
+
if (row === undefined || row.attempt_count < 1) {
|
|
2335
|
+
throw new AutomationStoreError('not-found', 'automation task was not found');
|
|
2336
|
+
}
|
|
2337
|
+
const attempt = this.database.prepare(`
|
|
2338
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
2339
|
+
`).get(row.id, row.attempt_count);
|
|
2340
|
+
const snapshot = attempt === undefined ? undefined : automationSnapshot(attempt, row.automation_id);
|
|
2341
|
+
if (snapshot === undefined) {
|
|
2342
|
+
throw new AutomationStoreError('invalid-state', 'fresh diagnostic has no immutable definition');
|
|
2343
|
+
}
|
|
2344
|
+
return snapshot.definition;
|
|
2345
|
+
})());
|
|
2346
|
+
}
|
|
2347
|
+
const diagnosticJson = JSON.stringify(diagnostic);
|
|
2348
|
+
if (Buffer.byteLength(diagnosticJson, 'utf8') > 4_096) {
|
|
2349
|
+
throw new AutomationStoreError('invalid-definition', 'execution diagnostic exceeds 4096 bytes');
|
|
2350
|
+
}
|
|
719
2351
|
const evidenceAttribution = normalizeEvidenceAttribution(input.evidenceAttribution);
|
|
720
2352
|
return this.transaction(() => {
|
|
721
2353
|
const existing = this.database.prepare('SELECT * FROM automation_runs WHERE task_id = ?').get(input.taskId);
|
|
@@ -728,7 +2360,8 @@ export class AutomationStore {
|
|
|
728
2360
|
&& existing.session_id === (input.sessionId ?? null)
|
|
729
2361
|
&& existing.artifact_ref === (input.artifactRef ?? null)
|
|
730
2362
|
&& existing.output_preview === preview
|
|
731
|
-
&& existing.usage_json === usageJson
|
|
2363
|
+
&& existing.usage_json === usageJson
|
|
2364
|
+
&& existing.diagnostic_json === diagnosticJson;
|
|
732
2365
|
if (!exact)
|
|
733
2366
|
throw new AutomationStoreError('idempotency-conflict', 'task completion was replayed with different output');
|
|
734
2367
|
return run(existing);
|
|
@@ -740,37 +2373,146 @@ export class AutomationStore {
|
|
|
740
2373
|
if (attempt === undefined)
|
|
741
2374
|
throw new AutomationStoreError('stale-fence', 'winning attempt was not found');
|
|
742
2375
|
const runId = `run-${row.id}`;
|
|
743
|
-
const automation =
|
|
744
|
-
if (automation === undefined)
|
|
745
|
-
throw new AutomationStoreError('invalid-state', '
|
|
2376
|
+
const automation = automationSnapshot(attempt, row.automation_id);
|
|
2377
|
+
if (automation === undefined) {
|
|
2378
|
+
throw new AutomationStoreError('invalid-state', 'winning attempt has no immutable automation snapshot');
|
|
2379
|
+
}
|
|
2380
|
+
const occurrenceRow = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2381
|
+
.get(row.occurrence_id);
|
|
2382
|
+
if (occurrenceRow === undefined || (occurrenceRow.dry_run !== 0 && occurrenceRow.dry_run !== 1)) {
|
|
2383
|
+
throw new AutomationStoreError('invalid-state', 'winning task has no valid occurrence execution mode');
|
|
2384
|
+
}
|
|
2385
|
+
// `dryRun` remains the public compatibility field, while terminal sinks
|
|
2386
|
+
// reason about an explicit mode. Future evaluation/shadow modes must stay
|
|
2387
|
+
// non-production here unless they deliberately earn sink capabilities.
|
|
2388
|
+
const executionMode = occurrenceRow.dry_run === 1 ? 'preview' : 'production';
|
|
2389
|
+
const definitionHash = hash(automation.definition);
|
|
746
2390
|
const evidence = buildOutcomeEvidence({
|
|
747
2391
|
automation,
|
|
748
2392
|
runId,
|
|
749
2393
|
outcome: input.outcome,
|
|
2394
|
+
executionMode,
|
|
750
2395
|
occurredAt: input.now,
|
|
751
2396
|
attribution: evidenceAttribution,
|
|
752
2397
|
});
|
|
753
|
-
const
|
|
2398
|
+
const normalizedOutput = preview.normalize('NFC').trim();
|
|
2399
|
+
const runDeliveryStatus = input.outcome === 'succeeded'
|
|
2400
|
+
&& executionMode === 'production'
|
|
754
2401
|
&& automation.definition.deliveryBindingId !== undefined
|
|
755
|
-
? '
|
|
2402
|
+
? normalizedOutput === '' || automation.definition.deliverySuppressExact?.includes(normalizedOutput) === true
|
|
2403
|
+
? 'suppressed'
|
|
2404
|
+
: 'pending'
|
|
756
2405
|
: null;
|
|
2406
|
+
const evaluation = buildTerminalEvaluation({
|
|
2407
|
+
automation,
|
|
2408
|
+
runId,
|
|
2409
|
+
outcome: input.outcome,
|
|
2410
|
+
executionMode,
|
|
2411
|
+
occurredAt: input.now,
|
|
2412
|
+
usage: input.usage,
|
|
2413
|
+
startedAt: attempt.started_at,
|
|
2414
|
+
attemptNumber: attempt.attempt_number,
|
|
2415
|
+
deliveryStatus: runDeliveryStatus === 'pending' ? 'unknown' : 'not-required',
|
|
2416
|
+
});
|
|
2417
|
+
const evaluationJson = evaluation === undefined ? undefined : JSON.stringify(evaluation);
|
|
2418
|
+
if (evaluationJson !== undefined && Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
2419
|
+
throw new AutomationStoreError('invalid-definition', 'automation evaluation payload exceeds 32768 bytes');
|
|
2420
|
+
}
|
|
757
2421
|
this.database.prepare(`
|
|
758
2422
|
INSERT INTO automation_runs(
|
|
759
2423
|
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
760
|
-
output_preview, usage_json,
|
|
2424
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2425
|
+
delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
761
2426
|
created_at, updated_at
|
|
762
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
|
|
763
|
-
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson,
|
|
2427
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
|
|
2428
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson, executionMode, definitionHash, diagnosticJson, runDeliveryStatus, evidence.status, evidence.json, input.now, input.now);
|
|
2429
|
+
if (evaluationJson !== undefined) {
|
|
2430
|
+
this.database.prepare(`
|
|
2431
|
+
INSERT INTO automation_evaluation_outbox(
|
|
2432
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
2433
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
2434
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
2435
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
2436
|
+
}
|
|
764
2437
|
this.database.prepare(`
|
|
765
2438
|
UPDATE automation_attempts
|
|
766
2439
|
SET status = ?, failure_code = ?, finished_at = ?, updated_at = ? WHERE id = ?
|
|
767
|
-
`).run(input.outcome,
|
|
2440
|
+
`).run(input.outcome, diagnostic.failureClass === 'none' ? null : diagnostic.failureCode, input.now, input.now, attempt.id);
|
|
768
2441
|
this.database.prepare(`
|
|
769
2442
|
UPDATE automation_tasks SET status = ?, lease_until = NULL, updated_at = ? WHERE id = ?
|
|
770
2443
|
`).run(input.outcome, input.now, row.id);
|
|
771
2444
|
this.database.prepare(`
|
|
772
2445
|
UPDATE automation_occurrences SET status = ?, reason = ?, updated_at = ? WHERE id = ?
|
|
773
|
-
`).run(input.outcome,
|
|
2446
|
+
`).run(input.outcome, diagnostic.failureClass === 'none' ? null : diagnostic.failureCode, input.now, row.occurrence_id);
|
|
2447
|
+
const circuitRow = this.database.prepare(`
|
|
2448
|
+
SELECT * FROM automation_circuits WHERE automation_id = ? AND definition_hash = ?
|
|
2449
|
+
`).get(automation.id, definitionHash);
|
|
2450
|
+
const atomicCanary = occurrenceRow.trigger_kind === 'manual'
|
|
2451
|
+
&& occurrenceRow.trigger_key.startsWith('circuit-canary:');
|
|
2452
|
+
let terminalSuccessResolvesIncident = input.outcome === 'succeeded' && !atomicCanary;
|
|
2453
|
+
if (executionMode === 'production'
|
|
2454
|
+
&& circuitRow?.state === 'probing' && circuitRow.probe_task_id === row.id) {
|
|
2455
|
+
const probePassed = input.outcome === 'succeeded'
|
|
2456
|
+
&& diagnostic.failureClass === 'none'
|
|
2457
|
+
&& circuitRow.probe_lease_until !== null
|
|
2458
|
+
&& circuitRow.probe_lease_until > input.now;
|
|
2459
|
+
const classifiedProbeFailure = diagnostic.failureClass === 'configuration'
|
|
2460
|
+
|| diagnostic.failureClass === 'policy'
|
|
2461
|
+
|| diagnostic.failureClass === 'budget';
|
|
2462
|
+
this.database.prepare(`
|
|
2463
|
+
UPDATE automation_circuits
|
|
2464
|
+
SET state = ?, failure_class = ?, failure_phase = ?, failure_code = ?,
|
|
2465
|
+
opened_at = ?, updated_at = ?, probe_token = NULL,
|
|
2466
|
+
probe_lease_until = NULL, probe_task_id = NULL, version = version + 1
|
|
2467
|
+
WHERE automation_id = ? AND definition_hash = ? AND state = 'probing'
|
|
2468
|
+
AND probe_task_id = ? AND version = ?
|
|
2469
|
+
`).run(probePassed ? 'closed' : 'open', classifiedProbeFailure ? diagnostic.failureClass : circuitRow.failure_class, classifiedProbeFailure ? diagnostic.failurePhase : circuitRow.failure_phase, classifiedProbeFailure ? diagnostic.failureCode : circuitRow.failure_code, probePassed ? circuitRow.opened_at : input.now, input.now, automation.id, definitionHash, row.id, circuitRow.version);
|
|
2470
|
+
terminalSuccessResolvesIncident = probePassed;
|
|
2471
|
+
}
|
|
2472
|
+
if (executionMode === 'production'
|
|
2473
|
+
&& diagnostic.failureCode !== 'circuit-open'
|
|
2474
|
+
&& (diagnostic.failureClass === 'configuration'
|
|
2475
|
+
|| diagnostic.failureClass === 'policy'
|
|
2476
|
+
|| diagnostic.failureClass === 'budget')) {
|
|
2477
|
+
this.database.prepare(`
|
|
2478
|
+
INSERT INTO automation_circuits(
|
|
2479
|
+
automation_id, definition_hash, state, failure_class, failure_phase,
|
|
2480
|
+
failure_code, opened_at, updated_at, version
|
|
2481
|
+
) VALUES (?, ?, 'open', ?, ?, ?, ?, ?, 1)
|
|
2482
|
+
ON CONFLICT(automation_id, definition_hash) DO UPDATE SET
|
|
2483
|
+
state = 'open', failure_class = excluded.failure_class,
|
|
2484
|
+
failure_phase = excluded.failure_phase, failure_code = excluded.failure_code,
|
|
2485
|
+
opened_at = excluded.opened_at, updated_at = excluded.updated_at,
|
|
2486
|
+
version = automation_circuits.version + 1
|
|
2487
|
+
WHERE automation_circuits.state = 'closed'
|
|
2488
|
+
`).run(automation.id, definitionHash, diagnostic.failureClass, diagnostic.failurePhase, diagnostic.failureCode, input.now, input.now);
|
|
2489
|
+
}
|
|
2490
|
+
const notificationRoute = incidentNotificationRoute(automation.definition);
|
|
2491
|
+
if (executionMode === 'production' && notificationRoute !== undefined) {
|
|
2492
|
+
if (terminalSuccessResolvesIncident) {
|
|
2493
|
+
this.resolveIncidentInTransaction(automation.id, definitionHash, 'terminal', input.now);
|
|
2494
|
+
}
|
|
2495
|
+
else if (input.outcome === 'succeeded' && diagnostic.failureClass === 'none') {
|
|
2496
|
+
// A canary that completed after its exact probe lease is evidence of
|
|
2497
|
+
// execution, not evidence that the open circuit recovered.
|
|
2498
|
+
this.markRecoveringIncidentOpenInTransaction(automation.id, definitionHash, input.now);
|
|
2499
|
+
}
|
|
2500
|
+
else if (!isHostAutomationDefinition(automation.definition) && input.outcome === 'cancelled') {
|
|
2501
|
+
// Expected coordinator stop/cancel-previous is not an owner incident.
|
|
2502
|
+
// If it interrupted an Agent recovery attempt, retain the prior open
|
|
2503
|
+
// incident rather than leaving its lifecycle stuck at recovering.
|
|
2504
|
+
this.markRecoveringIncidentOpenInTransaction(automation.id, definitionHash, input.now);
|
|
2505
|
+
}
|
|
2506
|
+
else {
|
|
2507
|
+
this.upsertTerminalIncidentInTransaction({
|
|
2508
|
+
automation,
|
|
2509
|
+
definitionHash,
|
|
2510
|
+
runId,
|
|
2511
|
+
diagnostic,
|
|
2512
|
+
now: input.now,
|
|
2513
|
+
});
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
774
2516
|
return run(this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId));
|
|
775
2517
|
});
|
|
776
2518
|
}
|
|
@@ -778,9 +2520,69 @@ export class AutomationStore {
|
|
|
778
2520
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
779
2521
|
throw new AutomationStoreError('invalid-definition', 'evidence limit must be between 1 and 1000');
|
|
780
2522
|
}
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
2523
|
+
const output = [];
|
|
2524
|
+
const rows = this.database.prepare(`
|
|
2525
|
+
SELECT * FROM automation_runs
|
|
2526
|
+
WHERE evidence_status = 'pending' ORDER BY updated_at, id LIMIT 1000
|
|
2527
|
+
`).all();
|
|
2528
|
+
for (const row of rows) {
|
|
2529
|
+
if (output.length === limit)
|
|
2530
|
+
break;
|
|
2531
|
+
try {
|
|
2532
|
+
const proof = this.getProvenProductionRun(row.id);
|
|
2533
|
+
if (proof === undefined
|
|
2534
|
+
|| (proof.run.status !== 'succeeded' && proof.run.status !== 'failed' && proof.run.status !== 'timed_out')
|
|
2535
|
+
|| proof.run.evidenceStatus !== 'pending' || proof.run.evidence === undefined) {
|
|
2536
|
+
try {
|
|
2537
|
+
this.suppressRunEvidence({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2538
|
+
}
|
|
2539
|
+
catch {
|
|
2540
|
+
// Concurrent settlement already removed it from this lane.
|
|
2541
|
+
}
|
|
2542
|
+
continue;
|
|
2543
|
+
}
|
|
2544
|
+
output.push(proof.run);
|
|
2545
|
+
}
|
|
2546
|
+
catch {
|
|
2547
|
+
// A poison row is monotonically quarantined so it cannot occupy the
|
|
2548
|
+
// head of a bounded batch forever or block independent peers.
|
|
2549
|
+
try {
|
|
2550
|
+
this.suppressRunEvidence({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2551
|
+
}
|
|
2552
|
+
catch {
|
|
2553
|
+
// Concurrent settlement already removed it from this lane.
|
|
2554
|
+
}
|
|
2555
|
+
}
|
|
2556
|
+
}
|
|
2557
|
+
return output;
|
|
2558
|
+
}
|
|
2559
|
+
suppressRunEvidence(input) {
|
|
2560
|
+
if (!Number.isSafeInteger(input.now))
|
|
2561
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2562
|
+
const changed = this.database.prepare(`
|
|
2563
|
+
UPDATE automation_runs
|
|
2564
|
+
SET evidence_status = 'suppressed', evidence_json = NULL, updated_at = ?
|
|
2565
|
+
WHERE id = ? AND evidence_status = ?
|
|
2566
|
+
`).run(input.now, input.runId, input.expectedStatus);
|
|
2567
|
+
if (changed.changes !== 1) {
|
|
2568
|
+
const current = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId);
|
|
2569
|
+
if (current === undefined)
|
|
2570
|
+
return undefined;
|
|
2571
|
+
if (current.evidence_status !== 'suppressed') {
|
|
2572
|
+
throw new AutomationStoreError('version-conflict', 'run evidence state changed before suppression');
|
|
2573
|
+
}
|
|
2574
|
+
}
|
|
2575
|
+
const current = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId);
|
|
2576
|
+
if (current === undefined)
|
|
2577
|
+
return undefined;
|
|
2578
|
+
try {
|
|
2579
|
+
return run(current);
|
|
2580
|
+
}
|
|
2581
|
+
catch {
|
|
2582
|
+
// The effect intent is already terminal even if unrelated legacy fields
|
|
2583
|
+
// remain corrupt; callers do not need a public projection to stay safe.
|
|
2584
|
+
return undefined;
|
|
2585
|
+
}
|
|
784
2586
|
}
|
|
785
2587
|
deferRunEvidence(input) {
|
|
786
2588
|
if (!Number.isSafeInteger(input.now))
|
|
@@ -812,13 +2614,131 @@ export class AutomationStore {
|
|
|
812
2614
|
}
|
|
813
2615
|
return run(this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId));
|
|
814
2616
|
}
|
|
2617
|
+
listPendingEvaluations(limit, now = Number.MAX_SAFE_INTEGER) {
|
|
2618
|
+
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
2619
|
+
throw new AutomationStoreError('invalid-definition', 'evaluation limit must be between 1 and 1000');
|
|
2620
|
+
}
|
|
2621
|
+
if (!Number.isSafeInteger(now))
|
|
2622
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2623
|
+
const rows = this.database.prepare(`
|
|
2624
|
+
SELECT * FROM automation_evaluation_outbox
|
|
2625
|
+
WHERE status = 'pending' AND next_attempt_at <= ? ORDER BY next_attempt_at, id LIMIT ?
|
|
2626
|
+
`).all(now, limit);
|
|
2627
|
+
const output = [];
|
|
2628
|
+
for (const row of rows) {
|
|
2629
|
+
try {
|
|
2630
|
+
output.push(evaluationOutboxEntry(row));
|
|
2631
|
+
}
|
|
2632
|
+
catch {
|
|
2633
|
+
// A repaired or legacy poison row must not abort scheduler recovery or
|
|
2634
|
+
// starve valid observations behind it. Quarantine it without exposing
|
|
2635
|
+
// payload contents or exception text.
|
|
2636
|
+
this.database.prepare(`
|
|
2637
|
+
UPDATE automation_evaluation_outbox
|
|
2638
|
+
SET status = 'dead-letter', attempt_count = attempt_count + 1,
|
|
2639
|
+
last_failure_at = ?, last_error_code = 'invalid-outbox-payload', updated_at = ?
|
|
2640
|
+
WHERE id = ? AND status = 'pending'
|
|
2641
|
+
`).run(now, now, row.id);
|
|
2642
|
+
}
|
|
2643
|
+
}
|
|
2644
|
+
return output;
|
|
2645
|
+
}
|
|
2646
|
+
getPendingEvaluationForRun(runId, now = Number.MAX_SAFE_INTEGER) {
|
|
2647
|
+
if (!Number.isSafeInteger(now))
|
|
2648
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2649
|
+
const row = this.database.prepare(`
|
|
2650
|
+
SELECT * FROM automation_evaluation_outbox
|
|
2651
|
+
WHERE run_id = ? AND observation_kind = 'terminal' AND status = 'pending' AND next_attempt_at <= ?
|
|
2652
|
+
ORDER BY id LIMIT 1
|
|
2653
|
+
`).get(runId, now);
|
|
2654
|
+
if (row === undefined)
|
|
2655
|
+
return undefined;
|
|
2656
|
+
try {
|
|
2657
|
+
return evaluationOutboxEntry(row);
|
|
2658
|
+
}
|
|
2659
|
+
catch {
|
|
2660
|
+
this.database.prepare(`
|
|
2661
|
+
UPDATE automation_evaluation_outbox
|
|
2662
|
+
SET status = 'dead-letter', attempt_count = attempt_count + 1,
|
|
2663
|
+
last_failure_at = ?, last_error_code = 'invalid-outbox-payload', updated_at = ?
|
|
2664
|
+
WHERE id = ? AND status = 'pending'
|
|
2665
|
+
`).run(now, now, row.id);
|
|
2666
|
+
return undefined;
|
|
2667
|
+
}
|
|
2668
|
+
}
|
|
2669
|
+
deferEvaluation(input) {
|
|
2670
|
+
if (!Number.isSafeInteger(input.now))
|
|
2671
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2672
|
+
if (!Number.isSafeInteger(input.retryAt) || input.retryAt < input.now
|
|
2673
|
+
|| !Number.isSafeInteger(input.maxAttempts) || input.maxAttempts < 1 || input.maxAttempts > 100) {
|
|
2674
|
+
throw new AutomationStoreError('invalid-definition', 'evaluation retry bounds are invalid');
|
|
2675
|
+
}
|
|
2676
|
+
const errorCode = text(input.errorCode, 'evaluation errorCode', 64);
|
|
2677
|
+
const changed = this.database.prepare(`
|
|
2678
|
+
UPDATE automation_evaluation_outbox
|
|
2679
|
+
SET attempt_count = attempt_count + 1,
|
|
2680
|
+
status = CASE WHEN attempt_count + 1 >= ? THEN 'dead-letter' ELSE 'pending' END,
|
|
2681
|
+
next_attempt_at = CASE WHEN attempt_count + 1 >= ? THEN next_attempt_at ELSE ? END,
|
|
2682
|
+
last_failure_at = ?, last_error_code = ?, updated_at = ?
|
|
2683
|
+
WHERE id = ? AND status = ?
|
|
2684
|
+
`).run(input.maxAttempts, input.maxAttempts, input.retryAt, input.now, errorCode, input.now, input.id, input.expectedStatus);
|
|
2685
|
+
if (changed.changes !== 1) {
|
|
2686
|
+
const current = this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
2687
|
+
.get(input.id);
|
|
2688
|
+
if (current?.status === 'recorded')
|
|
2689
|
+
return evaluationOutboxEntry(current);
|
|
2690
|
+
throw new AutomationStoreError('version-conflict', 'evaluation outbox state changed before deferral');
|
|
2691
|
+
}
|
|
2692
|
+
return evaluationOutboxEntry(this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
2693
|
+
.get(input.id));
|
|
2694
|
+
}
|
|
2695
|
+
completeEvaluation(input) {
|
|
2696
|
+
if (!Number.isSafeInteger(input.now))
|
|
2697
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
2698
|
+
const changed = this.database.prepare(`
|
|
2699
|
+
UPDATE automation_evaluation_outbox SET status = 'recorded', updated_at = ?
|
|
2700
|
+
WHERE id = ? AND status = ?
|
|
2701
|
+
`).run(input.now, input.id, input.expectedStatus);
|
|
2702
|
+
if (changed.changes !== 1) {
|
|
2703
|
+
const current = this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
2704
|
+
.get(input.id);
|
|
2705
|
+
if (current?.status === 'recorded')
|
|
2706
|
+
return evaluationOutboxEntry(current);
|
|
2707
|
+
throw new AutomationStoreError('version-conflict', 'evaluation outbox state changed before completion');
|
|
2708
|
+
}
|
|
2709
|
+
return evaluationOutboxEntry(this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
2710
|
+
.get(input.id));
|
|
2711
|
+
}
|
|
815
2712
|
listPendingDeliveries(limit) {
|
|
816
2713
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
817
2714
|
throw new AutomationStoreError('invalid-definition', 'delivery limit must be between 1 and 1000');
|
|
818
2715
|
}
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
2716
|
+
const output = [];
|
|
2717
|
+
const rows = this.database.prepare(`
|
|
2718
|
+
SELECT * FROM automation_runs WHERE delivery_status = 'pending' ORDER BY created_at, id LIMIT 1000
|
|
2719
|
+
`).all();
|
|
2720
|
+
for (const row of rows) {
|
|
2721
|
+
if (output.length === limit)
|
|
2722
|
+
break;
|
|
2723
|
+
try {
|
|
2724
|
+
const proof = this.getProvenProductionRun(row.id);
|
|
2725
|
+
if (proof === undefined || proof.run.status !== 'succeeded'
|
|
2726
|
+
|| proof.automation.definition.deliveryBindingId === undefined) {
|
|
2727
|
+
this.suppressRunDelivery({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2728
|
+
continue;
|
|
2729
|
+
}
|
|
2730
|
+
output.push(proof.run);
|
|
2731
|
+
}
|
|
2732
|
+
catch {
|
|
2733
|
+
try {
|
|
2734
|
+
this.suppressRunDelivery({ runId: row.id, expectedStatus: 'pending', now: this.now() });
|
|
2735
|
+
}
|
|
2736
|
+
catch {
|
|
2737
|
+
// A concurrent terminal settlement owns this row.
|
|
2738
|
+
}
|
|
2739
|
+
}
|
|
2740
|
+
}
|
|
2741
|
+
return output;
|
|
822
2742
|
}
|
|
823
2743
|
completeRunDelivery(input) {
|
|
824
2744
|
if (!Number.isSafeInteger(input.now))
|
|
@@ -861,6 +2781,56 @@ export class AutomationStore {
|
|
|
861
2781
|
.all(input.automationId, input.limit);
|
|
862
2782
|
return rows.map(run);
|
|
863
2783
|
}
|
|
2784
|
+
/**
|
|
2785
|
+
* Newest-first terminal history whose immutable claim snapshot belongs to one
|
|
2786
|
+
* exact Agent scope. Legacy rows without a verifiable snapshot are omitted;
|
|
2787
|
+
* the current mutable definition is never used to guess historical scope.
|
|
2788
|
+
*/
|
|
2789
|
+
listRunsForExecutionScope(input) {
|
|
2790
|
+
if (!isAbsolute(input.workspace) || input.agentPreset.normalize('NFC').trim() === '') {
|
|
2791
|
+
throw new AutomationStoreError('invalid-definition', 'run history requires an absolute workspace and preset');
|
|
2792
|
+
}
|
|
2793
|
+
if (!Number.isSafeInteger(input.limit) || input.limit <= 0 || input.limit > 1_000) {
|
|
2794
|
+
throw new AutomationStoreError('invalid-definition', 'run limit must be between 1 and 1000');
|
|
2795
|
+
}
|
|
2796
|
+
const output = [];
|
|
2797
|
+
let cursor;
|
|
2798
|
+
while (output.length < input.limit) {
|
|
2799
|
+
const pageSize = Math.min(1_000, Math.max(100, (input.limit - output.length) * 4));
|
|
2800
|
+
const automationClause = input.automationId === undefined ? '' : 'AND run.automation_id = ?';
|
|
2801
|
+
const cursorClause = cursor === undefined
|
|
2802
|
+
? ''
|
|
2803
|
+
: 'AND (run.created_at < ? OR (run.created_at = ? AND run.id < ?))';
|
|
2804
|
+
const parameters = [];
|
|
2805
|
+
if (input.automationId !== undefined)
|
|
2806
|
+
parameters.push(input.automationId);
|
|
2807
|
+
if (cursor !== undefined)
|
|
2808
|
+
parameters.push(cursor.createdAt, cursor.createdAt, cursor.id);
|
|
2809
|
+
parameters.push(pageSize);
|
|
2810
|
+
const rows = this.database.prepare(`
|
|
2811
|
+
SELECT run.*,
|
|
2812
|
+
attempt.automation_snapshot_hash AS scope_snapshot_hash,
|
|
2813
|
+
attempt.automation_snapshot_json AS scope_snapshot_json
|
|
2814
|
+
FROM automation_runs AS run
|
|
2815
|
+
JOIN automation_attempts AS attempt ON attempt.id = run.attempt_id
|
|
2816
|
+
WHERE 1 = 1 ${automationClause} ${cursorClause}
|
|
2817
|
+
ORDER BY run.created_at DESC, run.id DESC
|
|
2818
|
+
LIMIT ?
|
|
2819
|
+
`).all(...parameters);
|
|
2820
|
+
for (const row of rows) {
|
|
2821
|
+
const snapshot = decodeAutomationSnapshot(row.scope_snapshot_hash, row.scope_snapshot_json, row.automation_id);
|
|
2822
|
+
if (snapshot !== undefined && executionScopeMatches(snapshot, input))
|
|
2823
|
+
output.push(run(row));
|
|
2824
|
+
if (output.length === input.limit)
|
|
2825
|
+
break;
|
|
2826
|
+
}
|
|
2827
|
+
const last = rows.at(-1);
|
|
2828
|
+
if (last === undefined || rows.length < pageSize)
|
|
2829
|
+
break;
|
|
2830
|
+
cursor = { createdAt: last.created_at, id: last.id };
|
|
2831
|
+
}
|
|
2832
|
+
return Object.freeze(output);
|
|
2833
|
+
}
|
|
864
2834
|
recoverExpiredTasks(input) {
|
|
865
2835
|
if (!Number.isSafeInteger(input.now))
|
|
866
2836
|
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
@@ -876,10 +2846,17 @@ export class AutomationStore {
|
|
|
876
2846
|
`).all(input.now, limit);
|
|
877
2847
|
const recovered = [];
|
|
878
2848
|
for (const row of rows) {
|
|
879
|
-
const
|
|
2849
|
+
const attempt = this.database.prepare(`
|
|
2850
|
+
SELECT * FROM automation_attempts
|
|
2851
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
2852
|
+
`).get(row.id, row.attempt_count, row.fencing_token);
|
|
2853
|
+
if (attempt === undefined) {
|
|
2854
|
+
throw new AutomationStoreError('invalid-state', 'expired task has no matching execution attempt');
|
|
2855
|
+
}
|
|
2856
|
+
const snapshot = automationSnapshot(attempt, row.automation_id);
|
|
880
2857
|
const retry = row.status === 'running'
|
|
881
|
-
&& definition.retrySafety === 'idempotent'
|
|
882
|
-
&& row.attempt_count <= definition.maxRetries;
|
|
2858
|
+
&& snapshot?.definition.retrySafety === 'idempotent'
|
|
2859
|
+
&& row.attempt_count <= snapshot.definition.maxRetries;
|
|
883
2860
|
const target = row.status === 'claimed' || retry ? 'scheduled' : 'unknown';
|
|
884
2861
|
const attemptStatus = row.status === 'claimed' ? 'lost' : 'unknown';
|
|
885
2862
|
this.database.prepare(`
|
|
@@ -893,6 +2870,64 @@ export class AutomationStore {
|
|
|
893
2870
|
WHERE id = ?
|
|
894
2871
|
`).run(target, input.now, row.id);
|
|
895
2872
|
if (target === 'unknown') {
|
|
2873
|
+
if (snapshot !== undefined) {
|
|
2874
|
+
const runId = `run-${row.id}`;
|
|
2875
|
+
const recoveredOccurrence = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?')
|
|
2876
|
+
.get(row.occurrence_id);
|
|
2877
|
+
const recoveredMode = recoveredOccurrence?.dry_run === 0
|
|
2878
|
+
? 'production'
|
|
2879
|
+
: recoveredOccurrence?.dry_run === 1 ? 'preview' : 'unknown';
|
|
2880
|
+
const recoveredDiagnostic = Object.freeze({
|
|
2881
|
+
schemaVersion: 1,
|
|
2882
|
+
failureClass: 'infrastructure',
|
|
2883
|
+
failurePhase: 'recovery',
|
|
2884
|
+
failureCode: 'runner-lease-expired',
|
|
2885
|
+
promptSubmissionState: 'unknown',
|
|
2886
|
+
sideEffectState: 'unknown',
|
|
2887
|
+
retryability: 'unsafe',
|
|
2888
|
+
budgetSettlementState: 'unknown',
|
|
2889
|
+
});
|
|
2890
|
+
const evaluation = buildTerminalEvaluation({
|
|
2891
|
+
automation: snapshot,
|
|
2892
|
+
runId,
|
|
2893
|
+
outcome: 'unknown',
|
|
2894
|
+
executionMode: recoveredMode,
|
|
2895
|
+
occurredAt: input.now,
|
|
2896
|
+
usage: {},
|
|
2897
|
+
startedAt: attempt.started_at,
|
|
2898
|
+
attemptNumber: attempt.attempt_number,
|
|
2899
|
+
deliveryStatus: 'not-required',
|
|
2900
|
+
});
|
|
2901
|
+
const evaluationJson = evaluation === undefined ? undefined : JSON.stringify(evaluation);
|
|
2902
|
+
if (evaluationJson !== undefined && Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
2903
|
+
throw new AutomationStoreError('invalid-state', 'recovered automation evaluation payload is oversized');
|
|
2904
|
+
}
|
|
2905
|
+
this.database.prepare(`
|
|
2906
|
+
INSERT INTO automation_runs(
|
|
2907
|
+
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
2908
|
+
output_preview, usage_json, execution_mode, definition_hash, diagnostic_json,
|
|
2909
|
+
delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
2910
|
+
created_at, updated_at
|
|
2911
|
+
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', ?, ?, ?, NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
2912
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'runner lease expired before terminal receipt', recoveredMode, hash(snapshot.definition), JSON.stringify(recoveredDiagnostic), input.now, input.now);
|
|
2913
|
+
if (evaluationJson !== undefined) {
|
|
2914
|
+
this.database.prepare(`
|
|
2915
|
+
INSERT INTO automation_evaluation_outbox(
|
|
2916
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
2917
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
2918
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
2919
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
2920
|
+
}
|
|
2921
|
+
if (recoveredMode === 'production') {
|
|
2922
|
+
this.upsertTerminalIncidentInTransaction({
|
|
2923
|
+
automation: snapshot,
|
|
2924
|
+
definitionHash: hash(snapshot.definition),
|
|
2925
|
+
runId,
|
|
2926
|
+
diagnostic: recoveredDiagnostic,
|
|
2927
|
+
now: input.now,
|
|
2928
|
+
});
|
|
2929
|
+
}
|
|
2930
|
+
}
|
|
896
2931
|
this.database.prepare(`
|
|
897
2932
|
UPDATE automation_occurrences SET status = 'unknown', reason = 'runner-lease-expired', updated_at = ? WHERE id = ?
|
|
898
2933
|
`).run(input.now, row.occurrence_id);
|
|
@@ -932,10 +2967,55 @@ export class AutomationStore {
|
|
|
932
2967
|
throw new AutomationStoreError('invalid-state', 'task state does not allow this transition');
|
|
933
2968
|
return row;
|
|
934
2969
|
}
|
|
935
|
-
claimTaskInTransaction(row, input) {
|
|
936
|
-
const
|
|
937
|
-
if (
|
|
2970
|
+
claimTaskInTransaction(row, input, hostAvailability) {
|
|
2971
|
+
const storedAutomation = this.get(row.automation_id);
|
|
2972
|
+
if (storedAutomation === undefined)
|
|
938
2973
|
return undefined;
|
|
2974
|
+
// claimTask() bypasses claimNextTask()'s candidate scan, so retain the
|
|
2975
|
+
// same exact paused-Growth proof here as an independent admission fence.
|
|
2976
|
+
const designatedGrowthTask = storedAutomation.status === 'paused'
|
|
2977
|
+
&& this.database.prepare(`
|
|
2978
|
+
SELECT 1 AS present
|
|
2979
|
+
FROM automation_tasks task
|
|
2980
|
+
JOIN automation_definitions definition ON definition.id = task.automation_id
|
|
2981
|
+
WHERE task.id = ?
|
|
2982
|
+
AND definition.status = 'paused'
|
|
2983
|
+
AND definition.system_owner = ?
|
|
2984
|
+
AND ${pausedGrowthTaskAdmission('task', 'definition')}
|
|
2985
|
+
`).get(row.id, growthAutomationOwner) !== undefined;
|
|
2986
|
+
if (storedAutomation.status !== 'active' && !designatedGrowthTask)
|
|
2987
|
+
return undefined;
|
|
2988
|
+
// A Growth rehearsal/exposure never activates the mutable definition. Its
|
|
2989
|
+
// one designated task receives an immutable active execution snapshot,
|
|
2990
|
+
// while every scheduler/materialization query still sees the artifact as
|
|
2991
|
+
// paused and therefore cannot create a second exposure.
|
|
2992
|
+
const automation = designatedGrowthTask
|
|
2993
|
+
? Object.freeze({ ...storedAutomation, status: 'active', nextRunAt: undefined })
|
|
2994
|
+
: storedAutomation;
|
|
2995
|
+
if (isHostAutomationDefinition(automation.definition)) {
|
|
2996
|
+
const occurrence = this.database.prepare('SELECT dry_run FROM automation_occurrences WHERE id = ?')
|
|
2997
|
+
.get(row.occurrence_id);
|
|
2998
|
+
if (occurrence === undefined || (occurrence.dry_run !== 0 && occurrence.dry_run !== 1)) {
|
|
2999
|
+
throw new AutomationStoreError('invalid-state', 'Host task has no exact execution mode');
|
|
3000
|
+
}
|
|
3001
|
+
const preview = occurrence.dry_run === 1;
|
|
3002
|
+
const definitionHash = hash(automation.definition);
|
|
3003
|
+
const availability = this.exactHostAvailability(hostAvailability, automation.id, definitionHash, 'claim');
|
|
3004
|
+
if (availability?.available !== true) {
|
|
3005
|
+
if (!preview) {
|
|
3006
|
+
this.upsertAvailabilityIncidentInTransaction({
|
|
3007
|
+
automation,
|
|
3008
|
+
definitionHash,
|
|
3009
|
+
stage: 'claim',
|
|
3010
|
+
failureCode: availability?.reasonCode ?? 'host-executor-proof-missing',
|
|
3011
|
+
now: input.now,
|
|
3012
|
+
});
|
|
3013
|
+
}
|
|
3014
|
+
return undefined;
|
|
3015
|
+
}
|
|
3016
|
+
if (!preview)
|
|
3017
|
+
this.resolveIncidentInTransaction(automation.id, definitionHash, 'claim', input.now);
|
|
3018
|
+
}
|
|
939
3019
|
const active = this.database.prepare(`
|
|
940
3020
|
SELECT * FROM automation_tasks
|
|
941
3021
|
WHERE automation_id = ? AND id != ? AND status IN ('claimed', 'running')
|
|
@@ -960,6 +3040,7 @@ export class AutomationStore {
|
|
|
960
3040
|
}
|
|
961
3041
|
const attemptNumber = row.attempt_count + 1;
|
|
962
3042
|
const attemptId = `attempt-${row.id}-${attemptNumber}`;
|
|
3043
|
+
const snapshot = encodeAutomationSnapshot(automation);
|
|
963
3044
|
const leaseUntil = input.now + input.leaseMs;
|
|
964
3045
|
this.database.prepare(`
|
|
965
3046
|
UPDATE automation_tasks
|
|
@@ -969,11 +3050,132 @@ export class AutomationStore {
|
|
|
969
3050
|
this.database.prepare(`
|
|
970
3051
|
INSERT INTO automation_attempts(
|
|
971
3052
|
id, task_id, attempt_number, owner_id, fencing_token, status, session_id,
|
|
972
|
-
failure_code, started_at, finished_at,
|
|
973
|
-
|
|
974
|
-
|
|
3053
|
+
failure_code, started_at, finished_at, automation_snapshot_hash,
|
|
3054
|
+
automation_snapshot_json, created_at, updated_at
|
|
3055
|
+
) VALUES (?, ?, ?, ?, ?, 'claimed', NULL, NULL, NULL, NULL, ?, ?, ?, ?)
|
|
3056
|
+
`).run(attemptId, row.id, attemptNumber, input.ownerId, input.fencingToken, snapshot.hash, snapshot.json, input.now, input.now);
|
|
975
3057
|
return this.getTask(row.id);
|
|
976
3058
|
}
|
|
3059
|
+
exactHostAvailability(decisions, automationId, definitionHash, stage) {
|
|
3060
|
+
const exact = decisions?.filter(decision => decision.automationId === automationId
|
|
3061
|
+
&& decision.definitionHash === definitionHash && decision.stage === stage) ?? [];
|
|
3062
|
+
if (exact.length !== 1)
|
|
3063
|
+
return undefined;
|
|
3064
|
+
const decision = exact[0];
|
|
3065
|
+
if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(decision.reasonCode))
|
|
3066
|
+
return undefined;
|
|
3067
|
+
return decision;
|
|
3068
|
+
}
|
|
3069
|
+
upsertAvailabilityIncidentInTransaction(input) {
|
|
3070
|
+
if (!isHostAutomationDefinition(input.automation.definition))
|
|
3071
|
+
return;
|
|
3072
|
+
const code = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,63}$/u.test(input.failureCode)
|
|
3073
|
+
? input.failureCode
|
|
3074
|
+
: 'host-executor-unavailable';
|
|
3075
|
+
const id = `incident-${hash([
|
|
3076
|
+
input.automation.id, input.definitionHash, input.stage,
|
|
3077
|
+
])}`;
|
|
3078
|
+
this.database.prepare(`
|
|
3079
|
+
INSERT INTO automation_incidents(
|
|
3080
|
+
id, automation_id, definition_hash, stage, state, failure_class,
|
|
3081
|
+
failure_phase, failure_code, side_effect_state, retryability,
|
|
3082
|
+
notification_route_id, lifecycle_generation, presentation_revision,
|
|
3083
|
+
alert_status, alert_ref, run_id,
|
|
3084
|
+
opened_at, updated_at, resolved_at, version
|
|
3085
|
+
) VALUES (?, ?, ?, ?, 'open', 'configuration', 'executor-availability', ?,
|
|
3086
|
+
'none', 'after-intervention', ?, 1, 1, 'pending', NULL, NULL, ?, ?, NULL, 1)
|
|
3087
|
+
ON CONFLICT(automation_id, definition_hash, stage) DO UPDATE SET
|
|
3088
|
+
state = 'open', failure_class = 'configuration',
|
|
3089
|
+
failure_phase = 'executor-availability', failure_code = excluded.failure_code,
|
|
3090
|
+
side_effect_state = 'none', retryability = 'after-intervention',
|
|
3091
|
+
notification_route_id = excluded.notification_route_id,
|
|
3092
|
+
lifecycle_generation = CASE
|
|
3093
|
+
WHEN automation_incidents.state = 'resolved'
|
|
3094
|
+
THEN automation_incidents.lifecycle_generation + 1
|
|
3095
|
+
ELSE automation_incidents.lifecycle_generation
|
|
3096
|
+
END,
|
|
3097
|
+
presentation_revision = automation_incidents.presentation_revision + 1,
|
|
3098
|
+
alert_status = 'pending',
|
|
3099
|
+
alert_ref = CASE
|
|
3100
|
+
WHEN automation_incidents.state = 'resolved' THEN NULL
|
|
3101
|
+
ELSE automation_incidents.alert_ref
|
|
3102
|
+
END,
|
|
3103
|
+
opened_at = CASE
|
|
3104
|
+
WHEN automation_incidents.state = 'resolved' THEN excluded.opened_at
|
|
3105
|
+
ELSE automation_incidents.opened_at
|
|
3106
|
+
END,
|
|
3107
|
+
updated_at = excluded.updated_at, resolved_at = NULL,
|
|
3108
|
+
version = automation_incidents.version + 1
|
|
3109
|
+
`).run(id, input.automation.id, input.definitionHash, input.stage, code, input.automation.definition.execution.ownerRouteId, input.now, input.now);
|
|
3110
|
+
}
|
|
3111
|
+
resolveIncidentInTransaction(automationId, definitionHash, stage, now) {
|
|
3112
|
+
this.database.prepare(`
|
|
3113
|
+
UPDATE automation_incidents
|
|
3114
|
+
SET state = 'resolved', resolved_at = ?, updated_at = ?,
|
|
3115
|
+
presentation_revision = presentation_revision + 1,
|
|
3116
|
+
alert_status = 'pending',
|
|
3117
|
+
version = version + 1
|
|
3118
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = ?
|
|
3119
|
+
AND state IN ('open', 'recovering')
|
|
3120
|
+
`).run(now, now, automationId, definitionHash, stage);
|
|
3121
|
+
}
|
|
3122
|
+
markIncidentRecoveringInTransaction(automationId, definitionHash, now) {
|
|
3123
|
+
this.database.prepare(`
|
|
3124
|
+
UPDATE automation_incidents
|
|
3125
|
+
SET state = 'recovering', resolved_at = NULL, updated_at = ?,
|
|
3126
|
+
presentation_revision = presentation_revision + 1,
|
|
3127
|
+
alert_status = 'pending', version = version + 1
|
|
3128
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = 'terminal' AND state = 'open'
|
|
3129
|
+
`).run(now, automationId, definitionHash);
|
|
3130
|
+
}
|
|
3131
|
+
markRecoveringIncidentOpenInTransaction(automationId, definitionHash, now) {
|
|
3132
|
+
this.database.prepare(`
|
|
3133
|
+
UPDATE automation_incidents
|
|
3134
|
+
SET state = 'open', updated_at = ?, presentation_revision = presentation_revision + 1,
|
|
3135
|
+
alert_status = 'pending', version = version + 1
|
|
3136
|
+
WHERE automation_id = ? AND definition_hash = ? AND stage = 'terminal' AND state = 'recovering'
|
|
3137
|
+
`).run(now, automationId, definitionHash);
|
|
3138
|
+
}
|
|
3139
|
+
upsertTerminalIncidentInTransaction(input) {
|
|
3140
|
+
const route = incidentNotificationRoute(input.automation.definition);
|
|
3141
|
+
if (route === undefined || input.diagnostic.failureClass === 'none')
|
|
3142
|
+
return;
|
|
3143
|
+
const id = `incident-${hash([input.automation.id, input.definitionHash, 'terminal'])}`;
|
|
3144
|
+
this.database.prepare(`
|
|
3145
|
+
INSERT INTO automation_incidents(
|
|
3146
|
+
id, automation_id, definition_hash, stage, state, failure_class,
|
|
3147
|
+
failure_phase, failure_code, side_effect_state, retryability,
|
|
3148
|
+
notification_route_id, lifecycle_generation, presentation_revision,
|
|
3149
|
+
alert_status, alert_ref, run_id,
|
|
3150
|
+
opened_at, updated_at, resolved_at, version
|
|
3151
|
+
) VALUES (?, ?, ?, 'terminal', 'open', ?, ?, ?, ?, ?, ?,
|
|
3152
|
+
1, 1, 'pending', NULL, ?, ?, ?, NULL, 1)
|
|
3153
|
+
ON CONFLICT(automation_id, definition_hash, stage) DO UPDATE SET
|
|
3154
|
+
state = 'open', failure_class = excluded.failure_class,
|
|
3155
|
+
failure_phase = excluded.failure_phase, failure_code = excluded.failure_code,
|
|
3156
|
+
side_effect_state = excluded.side_effect_state,
|
|
3157
|
+
retryability = excluded.retryability,
|
|
3158
|
+
notification_route_id = excluded.notification_route_id,
|
|
3159
|
+
run_id = excluded.run_id,
|
|
3160
|
+
lifecycle_generation = CASE
|
|
3161
|
+
WHEN automation_incidents.state = 'resolved'
|
|
3162
|
+
THEN automation_incidents.lifecycle_generation + 1
|
|
3163
|
+
ELSE automation_incidents.lifecycle_generation
|
|
3164
|
+
END,
|
|
3165
|
+
presentation_revision = automation_incidents.presentation_revision + 1,
|
|
3166
|
+
alert_status = 'pending',
|
|
3167
|
+
alert_ref = CASE
|
|
3168
|
+
WHEN automation_incidents.state = 'resolved' THEN NULL
|
|
3169
|
+
ELSE automation_incidents.alert_ref
|
|
3170
|
+
END,
|
|
3171
|
+
opened_at = CASE
|
|
3172
|
+
WHEN automation_incidents.state = 'resolved' THEN excluded.opened_at
|
|
3173
|
+
ELSE automation_incidents.opened_at
|
|
3174
|
+
END,
|
|
3175
|
+
updated_at = excluded.updated_at, resolved_at = NULL,
|
|
3176
|
+
version = automation_incidents.version + 1
|
|
3177
|
+
`).run(id, input.automation.id, input.definitionHash, input.diagnostic.failureClass, input.diagnostic.failurePhase, input.diagnostic.failureCode, input.diagnostic.sideEffectState, input.diagnostic.retryability, route.id, input.runId, input.now, input.now);
|
|
3178
|
+
}
|
|
977
3179
|
insertOccurrence(input) {
|
|
978
3180
|
const id = stableOccurrenceId(input.automationId, input.triggerKind, input.triggerKey);
|
|
979
3181
|
const existing = this.database.prepare('SELECT * FROM automation_occurrences WHERE id = ?').get(id);
|