@dsh-enhanced/assistant-automations 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/lib/coordinator.d.ts +13 -1
- package/lib/coordinator.d.ts.map +1 -1
- package/lib/coordinator.js +93 -5
- package/lib/coordinator.js.map +1 -1
- package/lib/runner.d.ts.map +1 -1
- package/lib/runner.js +2 -1
- package/lib/runner.js.map +1 -1
- package/lib/service.d.ts +2 -1
- package/lib/service.d.ts.map +1 -1
- package/lib/service.js +41 -7
- package/lib/service.js.map +1 -1
- package/lib/sqlite.d.ts +1 -1
- package/lib/sqlite.d.ts.map +1 -1
- package/lib/sqlite.js +122 -2
- package/lib/sqlite.js.map +1 -1
- package/lib/store.d.ts +37 -1
- package/lib/store.d.ts.map +1 -1
- package/lib/store.js +397 -13
- package/lib/store.js.map +1 -1
- package/lib/tools.d.ts.map +1 -1
- package/lib/tools.js +17 -5
- package/lib/tools.js.map +1 -1
- package/lib/types.d.ts +47 -0
- package/lib/types.d.ts.map +1 -1
- package/lib/version.d.ts +1 -1
- package/lib/version.js +1 -1
- package/package.json +13 -4
package/lib/store.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
-
import { isAbsolute } from 'node:path';
|
|
2
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
3
3
|
import { dueOccurrences, nextOccurrence, previousOccurrence, validateSchedule, } from './schedule.js';
|
|
4
4
|
import { AutomationDatabaseError, openAutomationDatabase } from './sqlite.js';
|
|
5
5
|
export class AutomationStoreError extends Error {
|
|
@@ -13,6 +13,9 @@ export class AutomationStoreError extends Error {
|
|
|
13
13
|
function hash(value) {
|
|
14
14
|
return createHash('sha256').update(JSON.stringify(value)).digest('hex');
|
|
15
15
|
}
|
|
16
|
+
function hashText(value) {
|
|
17
|
+
return createHash('sha256').update(value).digest('hex');
|
|
18
|
+
}
|
|
16
19
|
function text(value, field, maximum = 1_000) {
|
|
17
20
|
if (typeof value !== 'string' || value.trim() === '') {
|
|
18
21
|
throw new AutomationStoreError('invalid-definition', `${field} must be a non-empty string`);
|
|
@@ -125,6 +128,68 @@ export function normalizeAutomationDefinition(value, maxPromptBytes = 16_384, ma
|
|
|
125
128
|
...(deliverySuppressExact === undefined ? {} : { deliverySuppressExact }),
|
|
126
129
|
});
|
|
127
130
|
}
|
|
131
|
+
function freezeAutomationSnapshot(value, expectedId) {
|
|
132
|
+
if (typeof value !== 'object' || value === null || Array.isArray(value)) {
|
|
133
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot must be an object');
|
|
134
|
+
}
|
|
135
|
+
const input = value;
|
|
136
|
+
if (typeof input['id'] !== 'string' || (expectedId !== undefined && input['id'] !== expectedId)) {
|
|
137
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot id does not match its task');
|
|
138
|
+
}
|
|
139
|
+
if (input['status'] !== 'active' || !Number.isSafeInteger(input['version']) || input['version'] < 1
|
|
140
|
+
|| !Number.isSafeInteger(input['createdAt']) || !Number.isSafeInteger(input['updatedAt'])) {
|
|
141
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot metadata is invalid');
|
|
142
|
+
}
|
|
143
|
+
const nextRunAt = input['nextRunAt'];
|
|
144
|
+
if (nextRunAt !== undefined && !Number.isSafeInteger(nextRunAt)) {
|
|
145
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot nextRunAt is invalid');
|
|
146
|
+
}
|
|
147
|
+
const owner = input['owner'];
|
|
148
|
+
if (owner !== undefined && typeof owner !== 'string') {
|
|
149
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot owner is invalid');
|
|
150
|
+
}
|
|
151
|
+
const definition = normalizeAutomationDefinition(input['definition'], 16 * 1024 * 1024, 10_000);
|
|
152
|
+
return Object.freeze({
|
|
153
|
+
id: input['id'],
|
|
154
|
+
...(owner === undefined ? {} : { owner }),
|
|
155
|
+
definition,
|
|
156
|
+
status: 'active',
|
|
157
|
+
nextRunAt: nextRunAt,
|
|
158
|
+
createdAt: input['createdAt'],
|
|
159
|
+
updatedAt: input['updatedAt'],
|
|
160
|
+
version: input['version'],
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
function encodeAutomationSnapshot(value) {
|
|
164
|
+
const json = JSON.stringify(value);
|
|
165
|
+
if (Buffer.byteLength(json, 'utf8') > 16 * 1024 * 1024) {
|
|
166
|
+
throw new AutomationStoreError('invalid-definition', 'automation execution snapshot exceeds 16777216 bytes');
|
|
167
|
+
}
|
|
168
|
+
return { hash: hashText(json), json };
|
|
169
|
+
}
|
|
170
|
+
function decodeAutomationSnapshot(snapshotHash, snapshotJson, expectedId) {
|
|
171
|
+
if (snapshotHash === null || snapshotJson === null)
|
|
172
|
+
return undefined;
|
|
173
|
+
if (Buffer.byteLength(snapshotJson, 'utf8') > 16 * 1024 * 1024
|
|
174
|
+
|| hashText(snapshotJson) !== snapshotHash) {
|
|
175
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot integrity check failed');
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
return freezeAutomationSnapshot(JSON.parse(snapshotJson), expectedId);
|
|
179
|
+
}
|
|
180
|
+
catch (error) {
|
|
181
|
+
if (error instanceof AutomationStoreError)
|
|
182
|
+
throw error;
|
|
183
|
+
throw new AutomationStoreError('invalid-state', 'automation execution snapshot contains invalid JSON');
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
function automationSnapshot(row, expectedId) {
|
|
187
|
+
return decodeAutomationSnapshot(row.automation_snapshot_hash, row.automation_snapshot_json, expectedId);
|
|
188
|
+
}
|
|
189
|
+
function executionScopeMatches(snapshot, scope) {
|
|
190
|
+
return resolve(snapshot.definition.workspace.normalize('NFC')) === resolve(scope.workspace.normalize('NFC'))
|
|
191
|
+
&& snapshot.definition.agentPreset.normalize('NFC').trim() === scope.agentPreset.normalize('NFC').trim();
|
|
192
|
+
}
|
|
128
193
|
function record(row) {
|
|
129
194
|
return Object.freeze({
|
|
130
195
|
id: row.id,
|
|
@@ -214,6 +279,44 @@ function run(row) {
|
|
|
214
279
|
updatedAt: row.updated_at,
|
|
215
280
|
});
|
|
216
281
|
}
|
|
282
|
+
function deepFreezeJson(value) {
|
|
283
|
+
if (typeof value !== 'object' || value === null || Object.isFrozen(value))
|
|
284
|
+
return value;
|
|
285
|
+
for (const child of Object.values(value))
|
|
286
|
+
deepFreezeJson(child);
|
|
287
|
+
return Object.freeze(value);
|
|
288
|
+
}
|
|
289
|
+
function evaluationOutboxEntry(row) {
|
|
290
|
+
if (row.observation_kind !== 'terminal'
|
|
291
|
+
|| (row.status !== 'pending' && row.status !== 'recorded' && row.status !== 'dead-letter')
|
|
292
|
+
|| !Number.isSafeInteger(row.attempt_count) || row.attempt_count < 0
|
|
293
|
+
|| !Number.isSafeInteger(row.next_attempt_at)) {
|
|
294
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation outbox row has an invalid state');
|
|
295
|
+
}
|
|
296
|
+
if (Buffer.byteLength(row.payload_json, 'utf8') > 32_768) {
|
|
297
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation payload exceeds 32768 bytes');
|
|
298
|
+
}
|
|
299
|
+
let payload;
|
|
300
|
+
try {
|
|
301
|
+
payload = deepFreezeJson(JSON.parse(row.payload_json));
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
throw new AutomationStoreError('invalid-state', 'automation evaluation payload contains invalid JSON');
|
|
305
|
+
}
|
|
306
|
+
return Object.freeze({
|
|
307
|
+
id: row.id,
|
|
308
|
+
runId: row.run_id,
|
|
309
|
+
kind: row.observation_kind,
|
|
310
|
+
status: row.status,
|
|
311
|
+
payload,
|
|
312
|
+
attemptCount: row.attempt_count,
|
|
313
|
+
nextAttemptAt: row.next_attempt_at,
|
|
314
|
+
...(row.last_failure_at === null ? {} : { lastFailureAt: row.last_failure_at }),
|
|
315
|
+
...(row.last_error_code === null ? {} : { lastErrorCode: row.last_error_code }),
|
|
316
|
+
createdAt: row.created_at,
|
|
317
|
+
updatedAt: row.updated_at,
|
|
318
|
+
});
|
|
319
|
+
}
|
|
217
320
|
function duty(row, acquired) {
|
|
218
321
|
return Object.freeze({
|
|
219
322
|
acquired,
|
|
@@ -260,6 +363,48 @@ function buildOutcomeEvidence(input) {
|
|
|
260
363
|
}
|
|
261
364
|
return { status: 'pending', json };
|
|
262
365
|
}
|
|
366
|
+
function evaluationExecutionStatus(status) {
|
|
367
|
+
return status === 'timed_out' ? 'timed-out' : status;
|
|
368
|
+
}
|
|
369
|
+
function nonNegativeUsageMetric(usage, key) {
|
|
370
|
+
const value = usage[key];
|
|
371
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
372
|
+
}
|
|
373
|
+
function buildTerminalEvaluation(input) {
|
|
374
|
+
const inputTokens = nonNegativeUsageMetric(input.usage, 'inputTokens');
|
|
375
|
+
const outputTokens = nonNegativeUsageMetric(input.usage, 'outputTokens');
|
|
376
|
+
const toolCalls = nonNegativeUsageMetric(input.usage, 'toolCalls');
|
|
377
|
+
const latencyMs = input.startedAt === null ? undefined : Math.max(0, input.occurredAt - input.startedAt);
|
|
378
|
+
const retries = Math.max(0, input.attemptNumber - 1);
|
|
379
|
+
const metrics = Object.freeze({
|
|
380
|
+
...(inputTokens === undefined ? {} : { inputTokens }),
|
|
381
|
+
...(outputTokens === undefined ? {} : { outputTokens }),
|
|
382
|
+
...(toolCalls === undefined ? {} : { toolCalls }),
|
|
383
|
+
...(latencyMs === undefined ? {} : { latencyMs }),
|
|
384
|
+
...(retries === 0 ? {} : { retries }),
|
|
385
|
+
});
|
|
386
|
+
const rawSituation = `automation:${input.automation.id}`;
|
|
387
|
+
const situation = Buffer.byteLength(rawSituation, 'utf8') <= 200
|
|
388
|
+
? rawSituation
|
|
389
|
+
: `automation:${createHash('sha256').update(input.automation.id).digest('hex')}`;
|
|
390
|
+
return Object.freeze({
|
|
391
|
+
scope: Object.freeze({
|
|
392
|
+
workspace: input.automation.definition.workspace,
|
|
393
|
+
preset: input.automation.definition.agentPreset,
|
|
394
|
+
}),
|
|
395
|
+
situation,
|
|
396
|
+
executionStatus: evaluationExecutionStatus(input.outcome),
|
|
397
|
+
objectiveStatus: 'unknown',
|
|
398
|
+
deliveryStatus: input.deliveryStatus,
|
|
399
|
+
source: Object.freeze({ kind: 'automation', id: 'assistant-automations' }),
|
|
400
|
+
trust: 'trusted',
|
|
401
|
+
evidence: Object.freeze([{ kind: 'automation-run', ref: input.runId }]),
|
|
402
|
+
metrics,
|
|
403
|
+
occurredAt: input.occurredAt,
|
|
404
|
+
idempotencyKey: `assistant-automations:terminal:${input.runId}:v1`,
|
|
405
|
+
evaluator: Object.freeze({ id: 'assistant-automations', version: 'terminal-v1' }),
|
|
406
|
+
});
|
|
407
|
+
}
|
|
263
408
|
export function stableOccurrenceId(automationId, triggerKind, triggerKey) {
|
|
264
409
|
return `occ-${createHash('sha256').update(`${automationId}\0${triggerKind}\0${triggerKey}`).digest('hex')}`;
|
|
265
410
|
}
|
|
@@ -400,6 +545,13 @@ export class AutomationStore {
|
|
|
400
545
|
runningTasks: scalar("SELECT COUNT(*) AS count FROM automation_tasks WHERE status = 'running'"),
|
|
401
546
|
failedRuns: scalar("SELECT COUNT(*) AS count FROM automation_runs WHERE status IN ('failed', 'timed_out', 'cancelled')"),
|
|
402
547
|
unknownRuns: scalar("SELECT COUNT(*) AS count FROM automation_runs WHERE status = 'unknown'"),
|
|
548
|
+
pendingEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'pending'"),
|
|
549
|
+
retryingEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'pending' AND attempt_count > 0"),
|
|
550
|
+
// This is an append-only lifetime counter. It supports rate/delta
|
|
551
|
+
// monitoring but does not by itself describe current health.
|
|
552
|
+
failedEvaluationAttempts: scalar('SELECT COALESCE(SUM(attempt_count), 0) AS count FROM automation_evaluation_outbox'),
|
|
553
|
+
deadLetterEvaluations: scalar("SELECT COUNT(*) AS count FROM automation_evaluation_outbox WHERE status = 'dead-letter'"),
|
|
554
|
+
oldestPendingEvaluationAt: scalar("SELECT COALESCE(MIN(created_at), 0) AS count FROM automation_evaluation_outbox WHERE status = 'pending'"),
|
|
403
555
|
};
|
|
404
556
|
}
|
|
405
557
|
changeApproved(input) {
|
|
@@ -593,6 +745,30 @@ export class AutomationStore {
|
|
|
593
745
|
getTaskRecord(id) {
|
|
594
746
|
return this.getTask(id);
|
|
595
747
|
}
|
|
748
|
+
/** Immutable definition captured by the winning claim, never the mutable current row. */
|
|
749
|
+
getTaskExecutionSnapshot(taskId) {
|
|
750
|
+
const taskRow = this.database.prepare('SELECT * FROM automation_tasks WHERE id = ?').get(taskId);
|
|
751
|
+
if (taskRow === undefined || taskRow.attempt_count < 1)
|
|
752
|
+
return undefined;
|
|
753
|
+
const attempt = this.database.prepare(`
|
|
754
|
+
SELECT * FROM automation_attempts WHERE task_id = ? AND attempt_number = ?
|
|
755
|
+
`).get(taskId, taskRow.attempt_count);
|
|
756
|
+
return attempt === undefined ? undefined : automationSnapshot(attempt, taskRow.automation_id);
|
|
757
|
+
}
|
|
758
|
+
/** Immutable definition associated with a durable terminal run. */
|
|
759
|
+
getRunExecutionSnapshot(runId) {
|
|
760
|
+
const row = this.database.prepare(`
|
|
761
|
+
SELECT attempt.*, run.automation_id AS expected_automation_id
|
|
762
|
+
FROM automation_runs AS run
|
|
763
|
+
JOIN automation_attempts AS attempt ON attempt.id = run.attempt_id
|
|
764
|
+
WHERE run.id = ?
|
|
765
|
+
`).get(runId);
|
|
766
|
+
return row === undefined ? undefined : automationSnapshot(row, row.expected_automation_id);
|
|
767
|
+
}
|
|
768
|
+
getRun(runId) {
|
|
769
|
+
const row = this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(runId);
|
|
770
|
+
return row === undefined ? undefined : run(row);
|
|
771
|
+
}
|
|
596
772
|
acquireDuty(input) {
|
|
597
773
|
const ownerId = text(input.ownerId, 'ownerId', 500);
|
|
598
774
|
this.validateLease(input.now, input.leaseMs);
|
|
@@ -740,9 +916,10 @@ export class AutomationStore {
|
|
|
740
916
|
if (attempt === undefined)
|
|
741
917
|
throw new AutomationStoreError('stale-fence', 'winning attempt was not found');
|
|
742
918
|
const runId = `run-${row.id}`;
|
|
743
|
-
const automation =
|
|
744
|
-
if (automation === undefined)
|
|
745
|
-
throw new AutomationStoreError('invalid-state', '
|
|
919
|
+
const automation = automationSnapshot(attempt, row.automation_id);
|
|
920
|
+
if (automation === undefined) {
|
|
921
|
+
throw new AutomationStoreError('invalid-state', 'winning attempt has no immutable automation snapshot');
|
|
922
|
+
}
|
|
746
923
|
const evidence = buildOutcomeEvidence({
|
|
747
924
|
automation,
|
|
748
925
|
runId,
|
|
@@ -750,17 +927,40 @@ export class AutomationStore {
|
|
|
750
927
|
occurredAt: input.now,
|
|
751
928
|
attribution: evidenceAttribution,
|
|
752
929
|
});
|
|
753
|
-
const
|
|
930
|
+
const normalizedOutput = preview.normalize('NFC').trim();
|
|
931
|
+
const runDeliveryStatus = input.outcome === 'succeeded'
|
|
754
932
|
&& automation.definition.deliveryBindingId !== undefined
|
|
755
|
-
? '
|
|
933
|
+
? normalizedOutput === '' || automation.definition.deliverySuppressExact?.includes(normalizedOutput) === true
|
|
934
|
+
? 'suppressed'
|
|
935
|
+
: 'pending'
|
|
756
936
|
: null;
|
|
937
|
+
const evaluation = buildTerminalEvaluation({
|
|
938
|
+
automation,
|
|
939
|
+
runId,
|
|
940
|
+
outcome: input.outcome,
|
|
941
|
+
occurredAt: input.now,
|
|
942
|
+
usage: input.usage,
|
|
943
|
+
startedAt: attempt.started_at,
|
|
944
|
+
attemptNumber: attempt.attempt_number,
|
|
945
|
+
deliveryStatus: runDeliveryStatus === 'pending' ? 'unknown' : 'not-required',
|
|
946
|
+
});
|
|
947
|
+
const evaluationJson = JSON.stringify(evaluation);
|
|
948
|
+
if (Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
949
|
+
throw new AutomationStoreError('invalid-definition', 'automation evaluation payload exceeds 32768 bytes');
|
|
950
|
+
}
|
|
757
951
|
this.database.prepare(`
|
|
758
952
|
INSERT INTO automation_runs(
|
|
759
953
|
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
760
954
|
output_preview, usage_json, delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
761
955
|
created_at, updated_at
|
|
762
956
|
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?)
|
|
763
|
-
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson,
|
|
957
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, input.outcome, input.sessionId ?? null, input.artifactRef ?? null, preview, usageJson, runDeliveryStatus, evidence.status, evidence.json, input.now, input.now);
|
|
958
|
+
this.database.prepare(`
|
|
959
|
+
INSERT INTO automation_evaluation_outbox(
|
|
960
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
961
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
962
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
963
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
764
964
|
this.database.prepare(`
|
|
765
965
|
UPDATE automation_attempts
|
|
766
966
|
SET status = ?, failure_code = ?, finished_at = ?, updated_at = ? WHERE id = ?
|
|
@@ -812,6 +1012,101 @@ export class AutomationStore {
|
|
|
812
1012
|
}
|
|
813
1013
|
return run(this.database.prepare('SELECT * FROM automation_runs WHERE id = ?').get(input.runId));
|
|
814
1014
|
}
|
|
1015
|
+
listPendingEvaluations(limit, now = Number.MAX_SAFE_INTEGER) {
|
|
1016
|
+
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
1017
|
+
throw new AutomationStoreError('invalid-definition', 'evaluation limit must be between 1 and 1000');
|
|
1018
|
+
}
|
|
1019
|
+
if (!Number.isSafeInteger(now))
|
|
1020
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1021
|
+
const rows = this.database.prepare(`
|
|
1022
|
+
SELECT * FROM automation_evaluation_outbox
|
|
1023
|
+
WHERE status = 'pending' AND next_attempt_at <= ? ORDER BY next_attempt_at, id LIMIT ?
|
|
1024
|
+
`).all(now, limit);
|
|
1025
|
+
const output = [];
|
|
1026
|
+
for (const row of rows) {
|
|
1027
|
+
try {
|
|
1028
|
+
output.push(evaluationOutboxEntry(row));
|
|
1029
|
+
}
|
|
1030
|
+
catch {
|
|
1031
|
+
// A repaired or legacy poison row must not abort scheduler recovery or
|
|
1032
|
+
// starve valid observations behind it. Quarantine it without exposing
|
|
1033
|
+
// payload contents or exception text.
|
|
1034
|
+
this.database.prepare(`
|
|
1035
|
+
UPDATE automation_evaluation_outbox
|
|
1036
|
+
SET status = 'dead-letter', attempt_count = attempt_count + 1,
|
|
1037
|
+
last_failure_at = ?, last_error_code = 'invalid-outbox-payload', updated_at = ?
|
|
1038
|
+
WHERE id = ? AND status = 'pending'
|
|
1039
|
+
`).run(now, now, row.id);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
return output;
|
|
1043
|
+
}
|
|
1044
|
+
getPendingEvaluationForRun(runId, now = Number.MAX_SAFE_INTEGER) {
|
|
1045
|
+
if (!Number.isSafeInteger(now))
|
|
1046
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1047
|
+
const row = this.database.prepare(`
|
|
1048
|
+
SELECT * FROM automation_evaluation_outbox
|
|
1049
|
+
WHERE run_id = ? AND observation_kind = 'terminal' AND status = 'pending' AND next_attempt_at <= ?
|
|
1050
|
+
ORDER BY id LIMIT 1
|
|
1051
|
+
`).get(runId, now);
|
|
1052
|
+
if (row === undefined)
|
|
1053
|
+
return undefined;
|
|
1054
|
+
try {
|
|
1055
|
+
return evaluationOutboxEntry(row);
|
|
1056
|
+
}
|
|
1057
|
+
catch {
|
|
1058
|
+
this.database.prepare(`
|
|
1059
|
+
UPDATE automation_evaluation_outbox
|
|
1060
|
+
SET status = 'dead-letter', attempt_count = attempt_count + 1,
|
|
1061
|
+
last_failure_at = ?, last_error_code = 'invalid-outbox-payload', updated_at = ?
|
|
1062
|
+
WHERE id = ? AND status = 'pending'
|
|
1063
|
+
`).run(now, now, row.id);
|
|
1064
|
+
return undefined;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
deferEvaluation(input) {
|
|
1068
|
+
if (!Number.isSafeInteger(input.now))
|
|
1069
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1070
|
+
if (!Number.isSafeInteger(input.retryAt) || input.retryAt < input.now
|
|
1071
|
+
|| !Number.isSafeInteger(input.maxAttempts) || input.maxAttempts < 1 || input.maxAttempts > 100) {
|
|
1072
|
+
throw new AutomationStoreError('invalid-definition', 'evaluation retry bounds are invalid');
|
|
1073
|
+
}
|
|
1074
|
+
const errorCode = text(input.errorCode, 'evaluation errorCode', 64);
|
|
1075
|
+
const changed = this.database.prepare(`
|
|
1076
|
+
UPDATE automation_evaluation_outbox
|
|
1077
|
+
SET attempt_count = attempt_count + 1,
|
|
1078
|
+
status = CASE WHEN attempt_count + 1 >= ? THEN 'dead-letter' ELSE 'pending' END,
|
|
1079
|
+
next_attempt_at = CASE WHEN attempt_count + 1 >= ? THEN next_attempt_at ELSE ? END,
|
|
1080
|
+
last_failure_at = ?, last_error_code = ?, updated_at = ?
|
|
1081
|
+
WHERE id = ? AND status = ?
|
|
1082
|
+
`).run(input.maxAttempts, input.maxAttempts, input.retryAt, input.now, errorCode, input.now, input.id, input.expectedStatus);
|
|
1083
|
+
if (changed.changes !== 1) {
|
|
1084
|
+
const current = this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
1085
|
+
.get(input.id);
|
|
1086
|
+
if (current?.status === 'recorded')
|
|
1087
|
+
return evaluationOutboxEntry(current);
|
|
1088
|
+
throw new AutomationStoreError('version-conflict', 'evaluation outbox state changed before deferral');
|
|
1089
|
+
}
|
|
1090
|
+
return evaluationOutboxEntry(this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
1091
|
+
.get(input.id));
|
|
1092
|
+
}
|
|
1093
|
+
completeEvaluation(input) {
|
|
1094
|
+
if (!Number.isSafeInteger(input.now))
|
|
1095
|
+
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
1096
|
+
const changed = this.database.prepare(`
|
|
1097
|
+
UPDATE automation_evaluation_outbox SET status = 'recorded', updated_at = ?
|
|
1098
|
+
WHERE id = ? AND status = ?
|
|
1099
|
+
`).run(input.now, input.id, input.expectedStatus);
|
|
1100
|
+
if (changed.changes !== 1) {
|
|
1101
|
+
const current = this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
1102
|
+
.get(input.id);
|
|
1103
|
+
if (current?.status === 'recorded')
|
|
1104
|
+
return evaluationOutboxEntry(current);
|
|
1105
|
+
throw new AutomationStoreError('version-conflict', 'evaluation outbox state changed before completion');
|
|
1106
|
+
}
|
|
1107
|
+
return evaluationOutboxEntry(this.database.prepare('SELECT * FROM automation_evaluation_outbox WHERE id = ?')
|
|
1108
|
+
.get(input.id));
|
|
1109
|
+
}
|
|
815
1110
|
listPendingDeliveries(limit) {
|
|
816
1111
|
if (!Number.isSafeInteger(limit) || limit <= 0 || limit > 1_000) {
|
|
817
1112
|
throw new AutomationStoreError('invalid-definition', 'delivery limit must be between 1 and 1000');
|
|
@@ -861,6 +1156,56 @@ export class AutomationStore {
|
|
|
861
1156
|
.all(input.automationId, input.limit);
|
|
862
1157
|
return rows.map(run);
|
|
863
1158
|
}
|
|
1159
|
+
/**
|
|
1160
|
+
* Newest-first terminal history whose immutable claim snapshot belongs to one
|
|
1161
|
+
* exact Agent scope. Legacy rows without a verifiable snapshot are omitted;
|
|
1162
|
+
* the current mutable definition is never used to guess historical scope.
|
|
1163
|
+
*/
|
|
1164
|
+
listRunsForExecutionScope(input) {
|
|
1165
|
+
if (!isAbsolute(input.workspace) || input.agentPreset.normalize('NFC').trim() === '') {
|
|
1166
|
+
throw new AutomationStoreError('invalid-definition', 'run history requires an absolute workspace and preset');
|
|
1167
|
+
}
|
|
1168
|
+
if (!Number.isSafeInteger(input.limit) || input.limit <= 0 || input.limit > 1_000) {
|
|
1169
|
+
throw new AutomationStoreError('invalid-definition', 'run limit must be between 1 and 1000');
|
|
1170
|
+
}
|
|
1171
|
+
const output = [];
|
|
1172
|
+
let cursor;
|
|
1173
|
+
while (output.length < input.limit) {
|
|
1174
|
+
const pageSize = Math.min(1_000, Math.max(100, (input.limit - output.length) * 4));
|
|
1175
|
+
const automationClause = input.automationId === undefined ? '' : 'AND run.automation_id = ?';
|
|
1176
|
+
const cursorClause = cursor === undefined
|
|
1177
|
+
? ''
|
|
1178
|
+
: 'AND (run.created_at < ? OR (run.created_at = ? AND run.id < ?))';
|
|
1179
|
+
const parameters = [];
|
|
1180
|
+
if (input.automationId !== undefined)
|
|
1181
|
+
parameters.push(input.automationId);
|
|
1182
|
+
if (cursor !== undefined)
|
|
1183
|
+
parameters.push(cursor.createdAt, cursor.createdAt, cursor.id);
|
|
1184
|
+
parameters.push(pageSize);
|
|
1185
|
+
const rows = this.database.prepare(`
|
|
1186
|
+
SELECT run.*,
|
|
1187
|
+
attempt.automation_snapshot_hash AS scope_snapshot_hash,
|
|
1188
|
+
attempt.automation_snapshot_json AS scope_snapshot_json
|
|
1189
|
+
FROM automation_runs AS run
|
|
1190
|
+
JOIN automation_attempts AS attempt ON attempt.id = run.attempt_id
|
|
1191
|
+
WHERE 1 = 1 ${automationClause} ${cursorClause}
|
|
1192
|
+
ORDER BY run.created_at DESC, run.id DESC
|
|
1193
|
+
LIMIT ?
|
|
1194
|
+
`).all(...parameters);
|
|
1195
|
+
for (const row of rows) {
|
|
1196
|
+
const snapshot = decodeAutomationSnapshot(row.scope_snapshot_hash, row.scope_snapshot_json, row.automation_id);
|
|
1197
|
+
if (snapshot !== undefined && executionScopeMatches(snapshot, input))
|
|
1198
|
+
output.push(run(row));
|
|
1199
|
+
if (output.length === input.limit)
|
|
1200
|
+
break;
|
|
1201
|
+
}
|
|
1202
|
+
const last = rows.at(-1);
|
|
1203
|
+
if (last === undefined || rows.length < pageSize)
|
|
1204
|
+
break;
|
|
1205
|
+
cursor = { createdAt: last.created_at, id: last.id };
|
|
1206
|
+
}
|
|
1207
|
+
return Object.freeze(output);
|
|
1208
|
+
}
|
|
864
1209
|
recoverExpiredTasks(input) {
|
|
865
1210
|
if (!Number.isSafeInteger(input.now))
|
|
866
1211
|
throw new AutomationStoreError('invalid-definition', 'now must be a safe integer');
|
|
@@ -876,10 +1221,17 @@ export class AutomationStore {
|
|
|
876
1221
|
`).all(input.now, limit);
|
|
877
1222
|
const recovered = [];
|
|
878
1223
|
for (const row of rows) {
|
|
879
|
-
const
|
|
1224
|
+
const attempt = this.database.prepare(`
|
|
1225
|
+
SELECT * FROM automation_attempts
|
|
1226
|
+
WHERE task_id = ? AND attempt_number = ? AND fencing_token = ?
|
|
1227
|
+
`).get(row.id, row.attempt_count, row.fencing_token);
|
|
1228
|
+
if (attempt === undefined) {
|
|
1229
|
+
throw new AutomationStoreError('invalid-state', 'expired task has no matching execution attempt');
|
|
1230
|
+
}
|
|
1231
|
+
const snapshot = automationSnapshot(attempt, row.automation_id);
|
|
880
1232
|
const retry = row.status === 'running'
|
|
881
|
-
&& definition.retrySafety === 'idempotent'
|
|
882
|
-
&& row.attempt_count <= definition.maxRetries;
|
|
1233
|
+
&& snapshot?.definition.retrySafety === 'idempotent'
|
|
1234
|
+
&& row.attempt_count <= snapshot.definition.maxRetries;
|
|
883
1235
|
const target = row.status === 'claimed' || retry ? 'scheduled' : 'unknown';
|
|
884
1236
|
const attemptStatus = row.status === 'claimed' ? 'lost' : 'unknown';
|
|
885
1237
|
this.database.prepare(`
|
|
@@ -893,6 +1245,36 @@ export class AutomationStore {
|
|
|
893
1245
|
WHERE id = ?
|
|
894
1246
|
`).run(target, input.now, row.id);
|
|
895
1247
|
if (target === 'unknown') {
|
|
1248
|
+
if (snapshot !== undefined) {
|
|
1249
|
+
const runId = `run-${row.id}`;
|
|
1250
|
+
const evaluation = buildTerminalEvaluation({
|
|
1251
|
+
automation: snapshot,
|
|
1252
|
+
runId,
|
|
1253
|
+
outcome: 'unknown',
|
|
1254
|
+
occurredAt: input.now,
|
|
1255
|
+
usage: {},
|
|
1256
|
+
startedAt: attempt.started_at,
|
|
1257
|
+
attemptNumber: attempt.attempt_number,
|
|
1258
|
+
deliveryStatus: 'not-required',
|
|
1259
|
+
});
|
|
1260
|
+
const evaluationJson = JSON.stringify(evaluation);
|
|
1261
|
+
if (Buffer.byteLength(evaluationJson, 'utf8') > 32_768) {
|
|
1262
|
+
throw new AutomationStoreError('invalid-state', 'recovered automation evaluation payload is oversized');
|
|
1263
|
+
}
|
|
1264
|
+
this.database.prepare(`
|
|
1265
|
+
INSERT INTO automation_runs(
|
|
1266
|
+
id, occurrence_id, automation_id, task_id, attempt_id, status, session_id, artifact_ref,
|
|
1267
|
+
output_preview, usage_json, delivery_status, delivery_ref, evidence_status, evidence_json,
|
|
1268
|
+
created_at, updated_at
|
|
1269
|
+
) VALUES (?, ?, ?, ?, ?, 'unknown', ?, NULL, ?, '{}', NULL, NULL, 'suppressed', NULL, ?, ?)
|
|
1270
|
+
`).run(runId, row.occurrence_id, row.automation_id, row.id, attempt.id, attempt.session_id, 'runner lease expired before terminal receipt', input.now, input.now);
|
|
1271
|
+
this.database.prepare(`
|
|
1272
|
+
INSERT INTO automation_evaluation_outbox(
|
|
1273
|
+
id, run_id, observation_kind, status, payload_json, attempt_count,
|
|
1274
|
+
next_attempt_at, last_failure_at, last_error_code, created_at, updated_at
|
|
1275
|
+
) VALUES (?, ?, 'terminal', 'pending', ?, 0, ?, NULL, NULL, ?, ?)
|
|
1276
|
+
`).run(`evaluation-terminal:${runId}`, runId, evaluationJson, input.now, input.now, input.now);
|
|
1277
|
+
}
|
|
896
1278
|
this.database.prepare(`
|
|
897
1279
|
UPDATE automation_occurrences SET status = 'unknown', reason = 'runner-lease-expired', updated_at = ? WHERE id = ?
|
|
898
1280
|
`).run(input.now, row.occurrence_id);
|
|
@@ -960,6 +1342,7 @@ export class AutomationStore {
|
|
|
960
1342
|
}
|
|
961
1343
|
const attemptNumber = row.attempt_count + 1;
|
|
962
1344
|
const attemptId = `attempt-${row.id}-${attemptNumber}`;
|
|
1345
|
+
const snapshot = encodeAutomationSnapshot(automation);
|
|
963
1346
|
const leaseUntil = input.now + input.leaseMs;
|
|
964
1347
|
this.database.prepare(`
|
|
965
1348
|
UPDATE automation_tasks
|
|
@@ -969,9 +1352,10 @@ export class AutomationStore {
|
|
|
969
1352
|
this.database.prepare(`
|
|
970
1353
|
INSERT INTO automation_attempts(
|
|
971
1354
|
id, task_id, attempt_number, owner_id, fencing_token, status, session_id,
|
|
972
|
-
failure_code, started_at, finished_at,
|
|
973
|
-
|
|
974
|
-
|
|
1355
|
+
failure_code, started_at, finished_at, automation_snapshot_hash,
|
|
1356
|
+
automation_snapshot_json, created_at, updated_at
|
|
1357
|
+
) VALUES (?, ?, ?, ?, ?, 'claimed', NULL, NULL, NULL, NULL, ?, ?, ?, ?)
|
|
1358
|
+
`).run(attemptId, row.id, attemptNumber, input.ownerId, input.fencingToken, snapshot.hash, snapshot.json, input.now, input.now);
|
|
975
1359
|
return this.getTask(row.id);
|
|
976
1360
|
}
|
|
977
1361
|
insertOccurrence(input) {
|