@agent-native/core 0.213.1 → 0.216.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action.js +54 -1
- package/dist/agent/actions/disconnect-chatgpt-subscription.js +5 -0
- package/dist/agent/actions/get-agentkit-capabilities.js +3 -3
- package/dist/agent/actions/manage-builder-connection.js +5 -0
- package/dist/agent/actions/manage-provider-key-policy.js +5 -0
- package/dist/agent/actions/manage-provider-models.js +5 -0
- package/dist/agent/actions/manage-service-providers.js +5 -0
- package/dist/agent/actions/select-chatgpt-subscription-account.js +5 -0
- package/dist/agent/context-xray/actions/context-evict.js +5 -0
- package/dist/agent/context-xray/actions/context-pin.js +5 -0
- package/dist/agent/context-xray/actions/context-report.js +5 -0
- package/dist/agent/context-xray/actions/context-restore.js +5 -0
- package/dist/agent/execution-fence.d.ts +1 -0
- package/dist/agent/execution-fence.js +5 -0
- package/dist/agent/production-agent.d.ts +35 -2
- package/dist/agent/production-agent.js +629 -32
- package/dist/agent/run-manager.d.ts +2 -0
- package/dist/agent/run-manager.js +397 -115
- package/dist/agent/run-slot-lock.d.ts +2 -0
- package/dist/agent/run-slot-lock.js +6 -0
- package/dist/agent/run-store.d.ts +5 -0
- package/dist/agent/run-store.js +423 -156
- package/dist/agent/thread-data-builder.js +27 -5
- package/dist/agent/types.d.ts +6 -12
- package/dist/agentkit/capabilities.d.ts +1 -1
- package/dist/appearance/actions/get-theme-family.d.ts +8 -0
- package/dist/appearance/actions/get-theme-family.js +38 -0
- package/dist/appearance/actions/set-theme-family.d.ts +7 -0
- package/dist/appearance/actions/set-theme-family.js +33 -0
- package/dist/appearance/theme-family.d.ts +13 -0
- package/dist/appearance/theme-family.js +38 -0
- package/dist/application-state/store.d.ts +5 -0
- package/dist/application-state/store.js +26 -0
- package/dist/chat-threads/schema-migrations.js +17 -0
- package/dist/chat-threads/store.d.ts +21 -1
- package/dist/chat-threads/store.js +223 -41
- package/dist/cli/skills-content/turn-into-app-skill.d.ts +1 -1
- package/dist/cli/skills-content/turn-into-app-skill.js +20 -18
- package/dist/cli/templates-meta.js +14 -0
- package/dist/client/analytics.js +167 -17
- package/dist/client/chat/agentkit-agent-native.js +79 -3
- package/dist/client/chat/agentkit-protocol.js +80 -42
- package/dist/client/chat/runtime.d.ts +5 -1
- package/dist/client/chat/runtime.js +46 -3
- package/dist/client/consent.d.ts +36 -0
- package/dist/client/consent.js +283 -0
- package/dist/client/review/use-review.d.ts +2 -0
- package/dist/client/review/use-review.js +28 -1
- package/dist/client/sharing/useResourceAccessGate.d.ts +3 -2
- package/dist/client/sharing/useResourceAccessGate.js +9 -1
- package/dist/client/theme-family.d.ts +35 -0
- package/dist/client/theme-family.js +223 -0
- package/dist/client/theme.d.ts +1 -1
- package/dist/client/theme.js +5 -2
- package/dist/client/ui/index.d.ts +1 -0
- package/dist/client/ui/index.js +1 -0
- package/dist/client/use-chat-thread-list.d.ts +2 -0
- package/dist/client/use-chat-thread-list.js +12 -2
- package/dist/client/use-chat-threads.d.ts +1 -0
- package/dist/client/use-chat-threads.js +2 -0
- package/dist/connections/catalog.d.ts +5 -5
- package/dist/connections/catalog.js +22 -3
- package/dist/data-widgets/index.d.ts +374 -5
- package/dist/data-widgets/index.js +242 -11
- package/dist/db/migrations.js +4 -1
- package/dist/db/schema-snapshot.generated.js +8 -1
- package/dist/feature-flags/actions/set-feature-flag.js +5 -0
- package/dist/feature-flags/index.d.ts +1 -1
- package/dist/feature-flags/index.js +1 -1
- package/dist/feature-flags/store.d.ts +12 -0
- package/dist/feature-flags/store.js +79 -7
- package/dist/file-upload/actions/manage-file-storage.js +5 -0
- package/dist/file-upload/actions/upload-image.js +5 -0
- package/dist/framework-tools.js +2 -0
- package/dist/history/actions/create-resource-version.js +5 -0
- package/dist/history/actions/restore-resource-version.js +5 -0
- package/dist/ingestion/index.d.ts +1 -1
- package/dist/ingestion/index.js +1 -1
- package/dist/ingestion/pptx.d.ts +11 -0
- package/dist/ingestion/pptx.js +81 -1
- package/dist/integrations/actions/manage-messaging-channel.js +5 -0
- package/dist/jobs/scheduler.js +17 -1
- package/dist/labs/actions/set-lab.js +5 -0
- package/dist/localization/actions/set-localization-preference.js +5 -0
- package/dist/mcp/screen-memory-stdio.d.ts +7 -7
- package/dist/observability/actions/save-observability-instruction-update.js +5 -0
- package/dist/observability/actions/save-observability-review-summary.js +5 -0
- package/dist/provider-api/actions/custom-provider-registration.d.ts +19 -19
- package/dist/provider-api/actions/provider-api.d.ts +10 -10
- package/dist/provider-api/index.js +9 -2
- package/dist/review/actions/consume-review-feedback.js +5 -0
- package/dist/review/actions/create-review-comment.js +5 -0
- package/dist/review/actions/delete-review-comment.js +5 -0
- package/dist/review/actions/react-to-review-comment.js +5 -0
- package/dist/review/actions/reply-review-comment.js +5 -0
- package/dist/review/actions/resolve-review-thread.js +5 -0
- package/dist/review/actions/send-review-thread-to-agent.js +5 -0
- package/dist/review/actions/set-review-status.js +5 -0
- package/dist/review/actions/set-review-thread-muted.js +5 -0
- package/dist/review/actions/set-review-thread-unread.js +5 -0
- package/dist/review/actions/set-review-threads-unread.js +5 -0
- package/dist/review/actions/update-review-comment-anchor.js +5 -0
- package/dist/review/actions/update-review-comment.js +5 -0
- package/dist/review/suggestions/actions.js +60 -21
- package/dist/review/suggestions/store.d.ts +2 -0
- package/dist/review/suggestions/store.js +28 -0
- package/dist/secrets/actions/delete-api-key.js +5 -0
- package/dist/secrets/routes.d.ts +10 -10
- package/dist/server/action-discovery.js +8 -0
- package/dist/server/agent-chat/context-tools.js +105 -2
- package/dist/server/agent-chat-plugin.d.ts +2 -0
- package/dist/server/agent-chat-plugin.js +242 -54
- package/dist/server/agent-engine-default-model-route.d.ts +1 -1
- package/dist/server/agent-engine-ollama-models-route.d.ts +1 -1
- package/dist/server/application-state-maintenance.d.ts +2 -0
- package/dist/server/application-state-maintenance.js +2 -0
- package/dist/server/attribution.d.ts +14 -0
- package/dist/server/attribution.js +62 -2
- package/dist/server/auth.js +7 -0
- package/dist/server/consent-route.d.ts +41 -0
- package/dist/server/consent-route.js +102 -0
- package/dist/server/first-touch-route.d.ts +39 -0
- package/dist/server/first-touch-route.js +115 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/index.js +1 -0
- package/dist/server/queued-submission-inbox-drain.d.ts +9 -0
- package/dist/server/queued-submission-inbox-drain.js +12 -0
- package/dist/server/request-origin.d.ts +6 -0
- package/dist/server/request-origin.js +12 -0
- package/dist/settings/org-settings.d.ts +2 -1
- package/dist/settings/org-settings.js +4 -1
- package/dist/settings/store.d.ts +8 -0
- package/dist/settings/store.js +14 -5
- package/dist/shared/consent.d.ts +31 -0
- package/dist/shared/consent.js +145 -0
- package/dist/shared/first-touch-persistence.d.ts +31 -0
- package/dist/shared/first-touch-persistence.js +53 -0
- package/dist/sharing/actions/list-resource-shares.d.ts +1 -1
- package/dist/sharing/actions/set-resource-visibility.js +5 -0
- package/dist/sharing/actions/share-resource.js +5 -0
- package/dist/sharing/actions/unshare-resource.js +5 -0
- package/dist/templates/chat/.agents/skills/actions/references/action-fields.md +1 -1
- package/dist/templates/default/.agents/skills/actions/references/action-fields.md +1 -1
- package/dist/templates/factory/.agents/skills/review-latest-feedback/SKILL.md +36 -80
- package/dist/templates/factory/.agents/skills/review-latest-feedback/references/channel-inventory.md +54 -16
- package/dist/templates/factory/.agents/skills/review-latest-feedback/references/slack-reaction-replies.md +50 -73
- package/dist/templates/headless/.agents/skills/actions/references/action-fields.md +1 -1
- package/dist/templates/workspace-core/.agents/skills/actions/references/action-fields.md +1 -1
- package/dist/templates/workspace-core/.agents/skills/turn-into-app/SKILL.md +20 -18
- package/dist/usage/actions/get-usage-metrics.js +27 -4
- package/dist/usage/actions/manage-usage-alert.js +5 -0
- package/dist/vite/client.js +8 -6
- package/dist/workspace-connections/actions/bulk-update-workspace-user-groups.js +5 -0
- package/dist/workspace-connections/actions/delete-workspace-user-group.js +5 -0
- package/dist/workspace-connections/actions/upsert-workspace-user-group.js +5 -0
- package/docs/content/actions-access-control.mdx +8 -8
- package/docs/content/actions-defining.mdx +9 -8
- package/docs/content/template-design-developers.mdx +9 -9
- package/docs/content/template-design-features.mdx +1 -3
- package/docs/content/turn-into-app.mdx +4 -0
- package/package.json +4 -3
- package/src/templates/chat/.agents/skills/actions/references/action-fields.md +1 -1
- package/src/templates/default/.agents/skills/actions/references/action-fields.md +1 -1
- package/src/templates/factory/.agents/skills/review-latest-feedback/SKILL.md +36 -80
- package/src/templates/factory/.agents/skills/review-latest-feedback/references/channel-inventory.md +54 -16
- package/src/templates/factory/.agents/skills/review-latest-feedback/references/slack-reaction-replies.md +50 -73
- package/src/templates/headless/.agents/skills/actions/references/action-fields.md +1 -1
- package/src/templates/workspace-core/.agents/skills/actions/references/action-fields.md +1 -1
- package/src/templates/workspace-core/.agents/skills/turn-into-app/SKILL.md +20 -18
package/dist/agent/run-store.js
CHANGED
|
@@ -16,6 +16,7 @@ import { AGENT_CHAT_RECOVERY_OF_RUN_FIELD } from "./durable-background.js";
|
|
|
16
16
|
import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
|
|
17
17
|
import { currentExecutionFence, currentExecutionFenceForScope, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
|
|
18
18
|
import { AGENT_RUN_MIGRATIONS, AGENT_RUN_MIGRATIONS_TABLE, } from "./run-migrations.js";
|
|
19
|
+
import { lockThreadRunSlot } from "./run-slot-lock.js";
|
|
19
20
|
import { isConsistentToolCallInputFingerprint, isUnavailableToolCallInput, toolCallInputFingerprint, } from "./tool-call-journal.js";
|
|
20
21
|
import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
|
|
21
22
|
import { readWriteReceipt } from "./write-receipts.js";
|
|
@@ -30,6 +31,10 @@ export const STALE_RUN_ERROR_EVENT = {
|
|
|
30
31
|
details: "The run heartbeat stopped while the run was still marked running. Partial output and tool calls were preserved when available.",
|
|
31
32
|
};
|
|
32
33
|
export const STALE_RUN_TERMINAL_REASON = `error:${STALE_RUN_ERROR_EVENT.errorCode}`;
|
|
34
|
+
const CONTINUATION_TERMINAL_EVENT_REASONS = [
|
|
35
|
+
"auto_continue",
|
|
36
|
+
...CONTINUATION_REASONS,
|
|
37
|
+
];
|
|
33
38
|
export const UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT = {
|
|
34
39
|
type: "error",
|
|
35
40
|
error: "The agent run was handed off to a background worker that never started. It was recovered so you can try again.",
|
|
@@ -1329,10 +1334,7 @@ export async function tryClaimRunSlot(threadId, runId, maxStaleMs, options) {
|
|
|
1329
1334
|
if (denied)
|
|
1330
1335
|
return { refused: denied };
|
|
1331
1336
|
}
|
|
1332
|
-
await tx
|
|
1333
|
-
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
1334
|
-
args: [`agent-native:run-slot:${threadId}`],
|
|
1335
|
-
});
|
|
1337
|
+
await lockThreadRunSlot(tx, threadId);
|
|
1336
1338
|
await lockContinuationOrder(tx, threadId, turnId);
|
|
1337
1339
|
const abortMarker = await tx.execute({
|
|
1338
1340
|
sql: `SELECT id FROM agent_runs WHERE thread_id = ? AND turn_id = ? AND dispatch_mode = 'turn-abort' AND status = 'aborted' LIMIT 1`,
|
|
@@ -1440,6 +1442,15 @@ export async function tryClaimRunSlot(threadId, runId, maxStaleMs, options) {
|
|
|
1440
1442
|
};
|
|
1441
1443
|
}
|
|
1442
1444
|
}
|
|
1445
|
+
if (!continueOf && (await options?.hasPendingQueuedSubmission?.(tx))) {
|
|
1446
|
+
return {
|
|
1447
|
+
result: {
|
|
1448
|
+
claimed: false,
|
|
1449
|
+
activeRunId: null,
|
|
1450
|
+
queuedSubmissionPending: true,
|
|
1451
|
+
},
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1443
1454
|
const continuationOrder = normalizeContinuationOrder(options?.continuationOrder) ??
|
|
1444
1455
|
(await nextContinuationOrder(tx, threadId, turnId));
|
|
1445
1456
|
const inserted = await tx.execute({
|
|
@@ -2069,10 +2080,7 @@ async function reapSingleStaleRun(runId, source, maxStaleMs) {
|
|
|
2069
2080
|
return { reaped: false, projectionFailed: false };
|
|
2070
2081
|
}
|
|
2071
2082
|
const reapResult = await client.transaction(async (tx) => {
|
|
2072
|
-
await tx.
|
|
2073
|
-
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
2074
|
-
args: [`agent-native:run-slot:${runScope.thread_id}`],
|
|
2075
|
-
});
|
|
2083
|
+
await lockThreadRunSlot(tx, runScope.thread_id);
|
|
2076
2084
|
const lease = await lockTurnExecutionLease(tx, runScope.thread_id, runScope.turn_id);
|
|
2077
2085
|
const currentRun = await tx.execute({
|
|
2078
2086
|
sql: `SELECT status, worker_generation FROM agent_runs WHERE id = ? FOR UPDATE`,
|
|
@@ -2286,14 +2294,17 @@ export async function updateRunStatus(runId, status) {
|
|
|
2286
2294
|
}
|
|
2287
2295
|
export async function updateRunStatusIfRunning(runId, status, options = {}) {
|
|
2288
2296
|
await ensureRunTables();
|
|
2297
|
+
const terminalReason = options.terminalReason?.slice(0, 200);
|
|
2289
2298
|
const { rowsAffected } = await withFencedExecutionWrite((tx) => tx.execute({
|
|
2290
2299
|
sql: `UPDATE agent_runs SET status = ?, completed_at = ?, dispatch_payload = NULL,
|
|
2291
2300
|
thread_projection_pending = CASE WHEN ? THEN TRUE ELSE thread_projection_pending END
|
|
2301
|
+
${terminalReason === undefined ? "" : ", terminal_reason = ?"}
|
|
2292
2302
|
WHERE id = ? AND status = 'running'`,
|
|
2293
2303
|
args: [
|
|
2294
2304
|
status,
|
|
2295
2305
|
Date.now(),
|
|
2296
2306
|
options.threadProjectionPending === true,
|
|
2307
|
+
...(terminalReason === undefined ? [] : [terminalReason]),
|
|
2297
2308
|
runId,
|
|
2298
2309
|
],
|
|
2299
2310
|
}), { runId });
|
|
@@ -2341,10 +2352,7 @@ export async function finalizeClaimedBackgroundRunFailure(runId, failure) {
|
|
|
2341
2352
|
}
|
|
2342
2353
|
const expectedGeneration = scope.worker_generation == null ? null : String(scope.worker_generation);
|
|
2343
2354
|
return client.transaction(async (tx) => {
|
|
2344
|
-
await tx.
|
|
2345
|
-
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
2346
|
-
args: [`agent-native:run-slot:${scope.thread_id}`],
|
|
2347
|
-
});
|
|
2355
|
+
await lockThreadRunSlot(tx, scope.thread_id);
|
|
2348
2356
|
const lease = await lockTurnExecutionLease(tx, scope.thread_id, scope.turn_id);
|
|
2349
2357
|
const selected = await tx.execute({
|
|
2350
2358
|
sql: `SELECT status, dispatch_mode, worker_generation
|
|
@@ -2511,7 +2519,8 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
|
2511
2519
|
if (!client.transaction) {
|
|
2512
2520
|
throw new Error("Atomic run cancellation requires transaction support");
|
|
2513
2521
|
}
|
|
2514
|
-
const
|
|
2522
|
+
const abortEvent = terminalEventForAbortReason(reason);
|
|
2523
|
+
const commitAbort = async (tx) => {
|
|
2515
2524
|
await lockExecutionLeasesForRuns(tx, [runId]);
|
|
2516
2525
|
await lockAndAssertCurrentExecutionFences(tx);
|
|
2517
2526
|
const runResult = await tx.execute({
|
|
@@ -2558,6 +2567,7 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
|
2558
2567
|
const rowsAffected = result.rowsAffected ?? 0;
|
|
2559
2568
|
if (rowsAffected > 0) {
|
|
2560
2569
|
await revokeExecutionLeasesForRuns(tx, [runId]);
|
|
2570
|
+
await appendAbortTerminalRunEventInTransaction(tx, runId, abortEvent);
|
|
2561
2571
|
return { rowsAffected, appendTerminalEvent: true };
|
|
2562
2572
|
}
|
|
2563
2573
|
const terminalState = await tx.execute({
|
|
@@ -2565,21 +2575,16 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
|
2565
2575
|
args: [runId],
|
|
2566
2576
|
});
|
|
2567
2577
|
const terminalRun = terminalState.rows[0];
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
});
|
|
2575
|
-
if (abortResult.appendTerminalEvent) {
|
|
2576
|
-
try {
|
|
2577
|
-
await safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-aborted");
|
|
2578
|
-
}
|
|
2579
|
-
catch (error) {
|
|
2580
|
-
throw new AgentRunEventNotPersistedError(runId, null, true, error);
|
|
2578
|
+
const appendTerminalEvent = terminalRun?.status === "aborted" &&
|
|
2579
|
+
terminalRun.terminal_reason === `aborted:${reason ?? "user"}` &&
|
|
2580
|
+
(!requireSafeBoundary || terminalRun.in_flight_since == null);
|
|
2581
|
+
if (appendTerminalEvent) {
|
|
2582
|
+
await appendAbortTerminalRunEventInTransaction(tx, runId, abortEvent);
|
|
2583
|
+
return { rowsAffected: 0, appendTerminalEvent: true };
|
|
2581
2584
|
}
|
|
2582
|
-
|
|
2585
|
+
return { rowsAffected: 0, appendTerminalEvent: false };
|
|
2586
|
+
};
|
|
2587
|
+
const abortResult = await retryAbortTerminalEventTransaction("mark-aborted", () => client.transaction(commitAbort));
|
|
2583
2588
|
return abortResult.rowsAffected > 0 || abortResult.appendTerminalEvent;
|
|
2584
2589
|
}
|
|
2585
2590
|
async function lockExecutionLeasesForRuns(db, runIds) {
|
|
@@ -2654,12 +2659,9 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2654
2659
|
if (!client.transaction) {
|
|
2655
2660
|
throw new Error("Atomic turn cancellation requires transaction support");
|
|
2656
2661
|
}
|
|
2657
|
-
const
|
|
2658
|
-
|
|
2659
|
-
await tx
|
|
2660
|
-
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
2661
|
-
args: [`agent-native:run-slot:${threadId}`],
|
|
2662
|
-
});
|
|
2662
|
+
const commitAbort = async (tx) => {
|
|
2663
|
+
const runIds = [];
|
|
2664
|
+
await lockThreadRunSlot(tx, threadId);
|
|
2663
2665
|
await lockAndAssertCurrentExecutionFences(tx);
|
|
2664
2666
|
const lease = await lockTurnExecutionLease(tx, threadId, turnId);
|
|
2665
2667
|
if (fence &&
|
|
@@ -2727,6 +2729,40 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2727
2729
|
return "already_terminal";
|
|
2728
2730
|
}
|
|
2729
2731
|
}
|
|
2732
|
+
const continuationRun = await tx.execute({
|
|
2733
|
+
sql: `SELECT id FROM agent_runs
|
|
2734
|
+
WHERE thread_id = ? AND COALESCE(turn_id, id) = ?
|
|
2735
|
+
AND status = 'truncated'
|
|
2736
|
+
AND terminal_reason IN (${CONTINUATION_TERMINAL_EVENT_REASONS.map(() => "?").join(", ")})
|
|
2737
|
+
AND dispatch_mode IS DISTINCT FROM 'turn-abort'
|
|
2738
|
+
ORDER BY started_at DESC LIMIT 1 FOR UPDATE`,
|
|
2739
|
+
args: [threadId, turnId, ...CONTINUATION_TERMINAL_EVENT_REASONS],
|
|
2740
|
+
});
|
|
2741
|
+
const continuationRunId = String(continuationRun.rows[0]?.id ?? "");
|
|
2742
|
+
if (continuationRunId) {
|
|
2743
|
+
const abortedContinuation = await tx.execute({
|
|
2744
|
+
sql: `UPDATE agent_runs
|
|
2745
|
+
SET status = 'aborted', abort_reason = ?, completed_at = ?, terminal_reason = ?
|
|
2746
|
+
WHERE id = ? AND status = 'truncated'
|
|
2747
|
+
AND terminal_reason IN (${CONTINUATION_TERMINAL_EVENT_REASONS.map(() => "?").join(", ")})
|
|
2748
|
+
RETURNING id`,
|
|
2749
|
+
args: [
|
|
2750
|
+
reason,
|
|
2751
|
+
now,
|
|
2752
|
+
`aborted:${reason}`,
|
|
2753
|
+
continuationRunId,
|
|
2754
|
+
...CONTINUATION_TERMINAL_EVENT_REASONS,
|
|
2755
|
+
],
|
|
2756
|
+
});
|
|
2757
|
+
if (abortedContinuation.rows.length > 0) {
|
|
2758
|
+
runIds.push(continuationRunId);
|
|
2759
|
+
await revokeExecutionLeasesForRuns(tx, [continuationRunId]);
|
|
2760
|
+
}
|
|
2761
|
+
}
|
|
2762
|
+
const abortEvent = terminalEventForAbortReason(reason);
|
|
2763
|
+
for (const runId of runIds) {
|
|
2764
|
+
await appendAbortTerminalRunEventInTransaction(tx, runId, abortEvent);
|
|
2765
|
+
}
|
|
2730
2766
|
await tx.execute({
|
|
2731
2767
|
sql: `INSERT INTO agent_runs (id, thread_id, status, abort_reason, started_at, completed_at, heartbeat_at, last_progress_at, turn_id, terminal_reason, dispatch_mode) VALUES (?, ?, 'aborted', ?, ?, ?, ?, ?, ?, ?, 'turn-abort'), (?, ?, 'aborted', ?, ?, ?, ?, ?, ?, ?, 'turn-abort') ON CONFLICT (id) DO NOTHING`,
|
|
2732
2768
|
args: [
|
|
@@ -2751,8 +2787,8 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2751
2787
|
],
|
|
2752
2788
|
});
|
|
2753
2789
|
return "aborted";
|
|
2754
|
-
}
|
|
2755
|
-
|
|
2790
|
+
};
|
|
2791
|
+
const outcome = await retryAbortTerminalEventTransaction("mark-turn-aborted", () => client.transaction(commitAbort));
|
|
2756
2792
|
return outcome;
|
|
2757
2793
|
}
|
|
2758
2794
|
export async function getRunTurnRef(runId) {
|
|
@@ -2799,12 +2835,12 @@ export async function getRunAbortState(runId) {
|
|
|
2799
2835
|
...(row.abort_reason ? { reason: row.abort_reason } : {}),
|
|
2800
2836
|
};
|
|
2801
2837
|
}
|
|
2802
|
-
function persistedRunEventData(eventData, inputSource) {
|
|
2838
|
+
function persistedRunEventData(eventData, inputSource, parsedEvent) {
|
|
2803
2839
|
const stripped = stripInlineBytesFromJson(eventData, "placeholder");
|
|
2804
|
-
const original = JSON.parse(eventData);
|
|
2840
|
+
const original = parsedEvent ?? JSON.parse(eventData);
|
|
2805
2841
|
if (original.type === "approval_required") {
|
|
2806
2842
|
return JSON.stringify({
|
|
2807
|
-
...JSON.parse(stripped),
|
|
2843
|
+
...(stripped === eventData ? original : JSON.parse(stripped)),
|
|
2808
2844
|
approvalKey: approvalKeyForHistory(original.approvalKey),
|
|
2809
2845
|
});
|
|
2810
2846
|
}
|
|
@@ -2816,7 +2852,7 @@ function persistedRunEventData(eventData, inputSource) {
|
|
|
2816
2852
|
original.inputFingerprint !== undefined ||
|
|
2817
2853
|
original.inputStoredFingerprint !== undefined)
|
|
2818
2854
|
return stripped;
|
|
2819
|
-
const stored = JSON.parse(stripped);
|
|
2855
|
+
const stored = stripped === eventData ? original : JSON.parse(stripped);
|
|
2820
2856
|
// Only fresh execution input can establish identity; reserialization cannot recover lost arguments.
|
|
2821
2857
|
return JSON.stringify({
|
|
2822
2858
|
...stored,
|
|
@@ -2843,19 +2879,57 @@ export class AgentRunEventNotPersistedError extends Error {
|
|
|
2843
2879
|
}
|
|
2844
2880
|
export async function insertRunEvent(runId, seq, eventData, options) {
|
|
2845
2881
|
await ensureRunTables();
|
|
2846
|
-
const
|
|
2882
|
+
const event = parseRunEventData(eventData, `run ${runId} sequence ${seq}`);
|
|
2883
|
+
const continuationReason = options?.allowContinuationTerminal &&
|
|
2884
|
+
event.type === "auto_continue" &&
|
|
2885
|
+
isContinuationTerminalReason(event.reason)
|
|
2886
|
+
? event.reason
|
|
2887
|
+
: null;
|
|
2888
|
+
const storedEventData = persistedRunEventData(eventData, options?.toolInputSource, event);
|
|
2847
2889
|
const eventAt = options?.eventAt ?? Date.now();
|
|
2890
|
+
const resolvesCheckpoint = terminalStatusForEvent(event) !== null;
|
|
2848
2891
|
// The reaper must wait for the marker's commit before exposing a successor.
|
|
2849
2892
|
let result;
|
|
2850
2893
|
try {
|
|
2851
2894
|
result = await withFencedExecutionWrite(async (tx) => {
|
|
2852
|
-
const
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2895
|
+
const writeEvent = async (writeTx) => {
|
|
2896
|
+
if (options?.allowContinuationTerminal) {
|
|
2897
|
+
const runRef = await writeTx.execute({
|
|
2898
|
+
sql: "SELECT thread_id FROM agent_runs WHERE id = ?",
|
|
2899
|
+
args: [runId],
|
|
2900
|
+
});
|
|
2901
|
+
const threadId = String(runRef.rows[0]
|
|
2902
|
+
?.thread_id ?? "");
|
|
2903
|
+
if (threadId) {
|
|
2904
|
+
// Serialize eligibility with turn abort's marker transaction.
|
|
2905
|
+
await writeTx.execute({
|
|
2906
|
+
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
2907
|
+
args: [`agent-native:run-slot:${threadId}`],
|
|
2908
|
+
});
|
|
2909
|
+
}
|
|
2910
|
+
}
|
|
2911
|
+
const runAdmissionPredicate = continuationReason
|
|
2912
|
+
? `AND NOT EXISTS (
|
|
2913
|
+
SELECT 1 FROM agent_runs AS aborted_turn
|
|
2914
|
+
WHERE aborted_turn.thread_id = run.thread_id
|
|
2915
|
+
AND COALESCE(aborted_turn.turn_id, aborted_turn.id) = COALESCE(run.turn_id, run.id)
|
|
2916
|
+
AND aborted_turn.dispatch_mode = 'turn-abort'
|
|
2917
|
+
AND aborted_turn.status = 'aborted'
|
|
2918
|
+
)
|
|
2919
|
+
AND run.status = 'truncated'
|
|
2920
|
+
AND run.terminal_reason = ?`
|
|
2921
|
+
: "AND run.status = 'running'";
|
|
2922
|
+
const runAdmissionArgs = continuationReason
|
|
2923
|
+
? [continuationReason]
|
|
2924
|
+
: [];
|
|
2925
|
+
const insertSql = options?.requireInserted
|
|
2926
|
+
? `WITH eligible_run AS MATERIALIZED (
|
|
2927
|
+
SELECT run.id FROM agent_runs AS run
|
|
2928
|
+
WHERE run.id = ? ${runAdmissionPredicate}
|
|
2929
|
+
FOR UPDATE
|
|
2856
2930
|
)
|
|
2857
2931
|
INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
2858
|
-
SELECT id, ?, ?, ? FROM
|
|
2932
|
+
SELECT id, ?, ?, ? FROM eligible_run LIMIT 1
|
|
2859
2933
|
ON CONFLICT (run_id, seq) DO NOTHING`
|
|
2860
2934
|
: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
2861
2935
|
SELECT ?, ?, ?, ?
|
|
@@ -2863,40 +2937,83 @@ export async function insertRunEvent(runId, seq, eventData, options) {
|
|
|
2863
2937
|
SELECT 1 FROM agent_runs
|
|
2864
2938
|
WHERE id = ? AND status <> 'running'
|
|
2865
2939
|
)
|
|
2866
|
-
ON CONFLICT (run_id, seq) DO NOTHING
|
|
2867
|
-
|
|
2868
|
-
runId,
|
|
2869
|
-
seq,
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2940
|
+
ON CONFLICT (run_id, seq) DO NOTHING`;
|
|
2941
|
+
const insertArgs = options?.requireInserted
|
|
2942
|
+
? [runId, ...runAdmissionArgs, seq, eventAt, storedEventData]
|
|
2943
|
+
: [runId, seq, eventAt, storedEventData, runId];
|
|
2944
|
+
const { rowsAffected } = await writeTx.execute({
|
|
2945
|
+
sql: insertSql,
|
|
2946
|
+
args: insertArgs,
|
|
2947
|
+
});
|
|
2948
|
+
if (resolvesCheckpoint) {
|
|
2949
|
+
let terminalPersisted = rowsAffected === 1;
|
|
2950
|
+
if (!terminalPersisted) {
|
|
2951
|
+
const existing = await writeTx.execute({
|
|
2952
|
+
sql: "SELECT event_data FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
2953
|
+
args: [runId, seq],
|
|
2954
|
+
});
|
|
2955
|
+
terminalPersisted =
|
|
2956
|
+
String(existing.rows[0]
|
|
2957
|
+
?.event_data) === storedEventData;
|
|
2958
|
+
}
|
|
2959
|
+
if (terminalPersisted) {
|
|
2960
|
+
// The reserved checkpoint sequence sorts after ordinary events,
|
|
2961
|
+
// so an authoritative terminal event must retire it atomically.
|
|
2962
|
+
await writeTx.execute({
|
|
2963
|
+
sql: "DELETE FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
2964
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
2965
|
+
});
|
|
2966
|
+
}
|
|
2967
|
+
}
|
|
2968
|
+
if (!options?.requireInserted || rowsAffected === 1)
|
|
2969
|
+
return {
|
|
2970
|
+
rowsAffected,
|
|
2971
|
+
terminalRun: false,
|
|
2972
|
+
persistedIdentically: rowsAffected === 1,
|
|
2973
|
+
};
|
|
2974
|
+
const existing = await writeTx.execute({
|
|
2975
|
+
sql: `SELECT event_at, event_data FROM agent_run_events
|
|
2976
|
+
WHERE run_id = ? AND seq = ?`,
|
|
2977
|
+
args: [runId, seq],
|
|
2978
|
+
});
|
|
2979
|
+
const existingRow = existing.rows[0];
|
|
2980
|
+
const persistedIdentically = existingRow !== undefined &&
|
|
2981
|
+
Number(existingRow.event_at) === eventAt &&
|
|
2982
|
+
String(existingRow.event_data) === storedEventData;
|
|
2983
|
+
const run = await writeTx.execute({
|
|
2984
|
+
sql: `SELECT run.status, run.terminal_reason,
|
|
2985
|
+
EXISTS (
|
|
2986
|
+
SELECT 1 FROM agent_runs AS aborted_turn
|
|
2987
|
+
WHERE aborted_turn.thread_id = run.thread_id
|
|
2988
|
+
AND COALESCE(aborted_turn.turn_id, aborted_turn.id) = COALESCE(run.turn_id, run.id)
|
|
2989
|
+
AND aborted_turn.dispatch_mode = 'turn-abort'
|
|
2990
|
+
AND aborted_turn.status = 'aborted'
|
|
2991
|
+
) AS turn_aborted
|
|
2992
|
+
FROM agent_runs AS run WHERE run.id = ?`,
|
|
2993
|
+
args: [runId],
|
|
2994
|
+
});
|
|
2995
|
+
const runRow = run.rows[0];
|
|
2996
|
+
const status = runRow?.status;
|
|
2997
|
+
const continuationTerminalEligible = continuationReason !== null &&
|
|
2998
|
+
status === "truncated" &&
|
|
2999
|
+
runRow?.terminal_reason === continuationReason &&
|
|
3000
|
+
runRow?.turn_aborted !== true &&
|
|
3001
|
+
runRow?.turn_aborted !== "true";
|
|
2876
3002
|
return {
|
|
2877
3003
|
rowsAffected,
|
|
2878
|
-
terminalRun:
|
|
2879
|
-
|
|
3004
|
+
terminalRun: status !== undefined &&
|
|
3005
|
+
status !== "running" &&
|
|
3006
|
+
!continuationTerminalEligible,
|
|
3007
|
+
persistedIdentically,
|
|
2880
3008
|
};
|
|
2881
|
-
const existing = await tx.execute({
|
|
2882
|
-
sql: `SELECT event_at, event_data FROM agent_run_events
|
|
2883
|
-
WHERE run_id = ? AND seq = ?`,
|
|
2884
|
-
args: [runId, seq],
|
|
2885
|
-
});
|
|
2886
|
-
const existingRow = existing.rows[0];
|
|
2887
|
-
const persistedIdentically = existingRow !== undefined &&
|
|
2888
|
-
Number(existingRow.event_at) === eventAt &&
|
|
2889
|
-
String(existingRow.event_data) === storedEventData;
|
|
2890
|
-
const run = await tx.execute({
|
|
2891
|
-
sql: "SELECT status FROM agent_runs WHERE id = ?",
|
|
2892
|
-
args: [runId],
|
|
2893
|
-
});
|
|
2894
|
-
const status = run.rows[0]?.status;
|
|
2895
|
-
return {
|
|
2896
|
-
rowsAffected,
|
|
2897
|
-
terminalRun: status !== "running",
|
|
2898
|
-
persistedIdentically,
|
|
2899
3009
|
};
|
|
3010
|
+
if (!resolvesCheckpoint || currentExecutionFence()) {
|
|
3011
|
+
return writeEvent(tx);
|
|
3012
|
+
}
|
|
3013
|
+
if (!tx.transaction) {
|
|
3014
|
+
throw new Error("Checkpoint resolution requires run event transaction support");
|
|
3015
|
+
}
|
|
3016
|
+
return tx.transaction(writeEvent);
|
|
2900
3017
|
}, { runId });
|
|
2901
3018
|
}
|
|
2902
3019
|
catch (error) {
|
|
@@ -2987,8 +3104,62 @@ export async function insertRunEvents(runId, events) {
|
|
|
2987
3104
|
}
|
|
2988
3105
|
export const CHECKPOINT_TERMINAL_EVENT_SEQ = 1_000_000_000;
|
|
2989
3106
|
export async function persistRunCheckpointEvent(runId, event, terminalReason) {
|
|
2990
|
-
await
|
|
2991
|
-
|
|
3107
|
+
await ensureRunTables();
|
|
3108
|
+
const storedEventData = persistedRunEventData(JSON.stringify(event));
|
|
3109
|
+
const reason = terminalReason.slice(0, 200);
|
|
3110
|
+
const writeCheckpoint = async (tx) => {
|
|
3111
|
+
const { rows } = await tx.execute({
|
|
3112
|
+
sql: "SELECT status, terminal_reason FROM agent_runs WHERE id = ? FOR UPDATE",
|
|
3113
|
+
args: [runId],
|
|
3114
|
+
});
|
|
3115
|
+
const run = rows[0];
|
|
3116
|
+
if (!run || run.status !== "running") {
|
|
3117
|
+
throw new AgentRunEventNotPersistedError(runId, CHECKPOINT_TERMINAL_EVENT_SEQ, Boolean(run));
|
|
3118
|
+
}
|
|
3119
|
+
const existing = await tx.execute({
|
|
3120
|
+
sql: "SELECT event_data FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
3121
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
3122
|
+
});
|
|
3123
|
+
const existingEvent = existing.rows[0];
|
|
3124
|
+
if (existingEvent) {
|
|
3125
|
+
if (existingEvent.event_data !== storedEventData) {
|
|
3126
|
+
throw new AgentRunEventNotPersistedError(runId, CHECKPOINT_TERMINAL_EVENT_SEQ, true);
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
else {
|
|
3130
|
+
const inserted = await tx.execute({
|
|
3131
|
+
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
3132
|
+
VALUES (?, ?, ?, ?)
|
|
3133
|
+
ON CONFLICT (run_id, seq) DO NOTHING`,
|
|
3134
|
+
args: [
|
|
3135
|
+
runId,
|
|
3136
|
+
CHECKPOINT_TERMINAL_EVENT_SEQ,
|
|
3137
|
+
Date.now(),
|
|
3138
|
+
storedEventData,
|
|
3139
|
+
],
|
|
3140
|
+
});
|
|
3141
|
+
if (inserted.rowsAffected !== 1) {
|
|
3142
|
+
throw new AgentRunEventNotPersistedError(runId, CHECKPOINT_TERMINAL_EVENT_SEQ);
|
|
3143
|
+
}
|
|
3144
|
+
}
|
|
3145
|
+
const updated = await tx.execute({
|
|
3146
|
+
sql: `UPDATE agent_runs SET terminal_reason = ?
|
|
3147
|
+
WHERE id = ? AND status = 'running'
|
|
3148
|
+
AND (terminal_reason IS NULL OR terminal_reason = '' OR terminal_reason = ?)`,
|
|
3149
|
+
args: [reason, runId, reason],
|
|
3150
|
+
});
|
|
3151
|
+
if (updated.rowsAffected !== 1) {
|
|
3152
|
+
throw new AgentRunEventNotPersistedError(runId, CHECKPOINT_TERMINAL_EVENT_SEQ, true);
|
|
3153
|
+
}
|
|
3154
|
+
};
|
|
3155
|
+
await withFencedExecutionWrite((tx) => {
|
|
3156
|
+
if (currentExecutionFence())
|
|
3157
|
+
return writeCheckpoint(tx);
|
|
3158
|
+
if (!tx.transaction) {
|
|
3159
|
+
throw new Error("Run checkpoint writes require transaction support");
|
|
3160
|
+
}
|
|
3161
|
+
return tx.transaction(writeCheckpoint);
|
|
3162
|
+
}, { runId });
|
|
2992
3163
|
}
|
|
2993
3164
|
export async function getRunEventsSince(runId, fromSeq, opts) {
|
|
2994
3165
|
await ensureRunTables();
|
|
@@ -3082,8 +3253,10 @@ export async function getLastTerminalRunEvent(runId) {
|
|
|
3082
3253
|
await ensureRunTables();
|
|
3083
3254
|
const client = getDbExec();
|
|
3084
3255
|
const { rows } = await client.execute({
|
|
3085
|
-
sql: `SELECT seq, event_data FROM agent_run_events
|
|
3086
|
-
|
|
3256
|
+
sql: `SELECT seq, event_data FROM agent_run_events
|
|
3257
|
+
WHERE run_id = ? AND seq < ?
|
|
3258
|
+
ORDER BY seq DESC LIMIT 1`,
|
|
3259
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
3087
3260
|
});
|
|
3088
3261
|
const last = rows[0];
|
|
3089
3262
|
if (!last?.event_data)
|
|
@@ -3760,16 +3933,76 @@ async function finalizeStaleRun(runId, source, projectThread = true) {
|
|
|
3760
3933
|
return false;
|
|
3761
3934
|
}
|
|
3762
3935
|
}
|
|
3763
|
-
|
|
3936
|
+
class AbortTerminalEventWriteError extends Error {
|
|
3937
|
+
runId;
|
|
3938
|
+
constructor(runId, cause) {
|
|
3939
|
+
const detail = cause instanceof Error ? `: ${cause.message}` : "";
|
|
3940
|
+
super(`Abort terminal event was not persisted for run ${runId}${detail}`);
|
|
3941
|
+
this.name = "AbortTerminalEventWriteError";
|
|
3942
|
+
this.runId = runId;
|
|
3943
|
+
if (cause !== undefined) {
|
|
3944
|
+
Object.defineProperty(this, "cause", {
|
|
3945
|
+
value: cause,
|
|
3946
|
+
configurable: true,
|
|
3947
|
+
});
|
|
3948
|
+
}
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
async function retryAbortTerminalEventTransaction(source, transaction) {
|
|
3952
|
+
let firstError;
|
|
3953
|
+
try {
|
|
3954
|
+
return await transaction();
|
|
3955
|
+
}
|
|
3956
|
+
catch (error) {
|
|
3957
|
+
if (!(error instanceof AbortTerminalEventWriteError))
|
|
3958
|
+
throw error;
|
|
3959
|
+
firstError = error;
|
|
3960
|
+
}
|
|
3961
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
3962
|
+
try {
|
|
3963
|
+
return await transaction();
|
|
3964
|
+
}
|
|
3965
|
+
catch (error) {
|
|
3966
|
+
if (!(error instanceof AbortTerminalEventWriteError))
|
|
3967
|
+
throw error;
|
|
3968
|
+
const cause = error.cause ?? error;
|
|
3969
|
+
captureError(cause, {
|
|
3970
|
+
tags: {
|
|
3971
|
+
component: "agent-run-store",
|
|
3972
|
+
operation: "append-terminal-event",
|
|
3973
|
+
source,
|
|
3974
|
+
},
|
|
3975
|
+
extra: {
|
|
3976
|
+
runId: error.runId,
|
|
3977
|
+
eventType: "abort",
|
|
3978
|
+
firstError: firstError.cause instanceof Error
|
|
3979
|
+
? firstError.cause.message
|
|
3980
|
+
: String(firstError.cause ?? firstError),
|
|
3981
|
+
},
|
|
3982
|
+
});
|
|
3983
|
+
throw new AgentRunEventNotPersistedError(error.runId, null, false, cause);
|
|
3984
|
+
}
|
|
3985
|
+
}
|
|
3986
|
+
async function appendAbortTerminalRunEventInTransaction(tx, runId, event) {
|
|
3987
|
+
try {
|
|
3988
|
+
await appendTerminalRunEventInTransaction(tx, runId, event, true);
|
|
3989
|
+
}
|
|
3990
|
+
catch (error) {
|
|
3991
|
+
throw new AbortTerminalEventWriteError(runId, error);
|
|
3992
|
+
}
|
|
3993
|
+
}
|
|
3994
|
+
async function appendTerminalRunEventInTransaction(tx, runId, event, replaceExistingTerminal = false) {
|
|
3995
|
+
const eventData = stripInlineBytesFromJson(JSON.stringify(event), "placeholder");
|
|
3764
3996
|
const { rows } = await tx.execute({
|
|
3765
3997
|
sql: `SELECT seq, event_data
|
|
3766
3998
|
FROM agent_run_events
|
|
3767
3999
|
WHERE run_id = ?
|
|
3768
4000
|
ORDER BY seq DESC
|
|
3769
|
-
LIMIT 1`,
|
|
4001
|
+
LIMIT 1 FOR UPDATE`,
|
|
3770
4002
|
args: [runId],
|
|
3771
4003
|
});
|
|
3772
4004
|
const last = rows[0];
|
|
4005
|
+
let nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
|
|
3773
4006
|
if (last?.event_data) {
|
|
3774
4007
|
let parsed;
|
|
3775
4008
|
try {
|
|
@@ -3780,31 +4013,70 @@ async function appendTerminalRunEventInTransaction(tx, runId, event) {
|
|
|
3780
4013
|
cause: error,
|
|
3781
4014
|
});
|
|
3782
4015
|
}
|
|
3783
|
-
if (parsed
|
|
3784
|
-
|
|
3785
|
-
|
|
3786
|
-
|
|
4016
|
+
if (parsed) {
|
|
4017
|
+
const lastEventIsTerminal = Boolean(terminalStatusForEvent(parsed));
|
|
4018
|
+
if (lastEventIsTerminal && last.event_data === eventData)
|
|
4019
|
+
return;
|
|
4020
|
+
if (lastEventIsTerminal && !replaceExistingTerminal)
|
|
4021
|
+
return;
|
|
4022
|
+
if (lastEventIsTerminal && replaceExistingTerminal) {
|
|
4023
|
+
await tx.execute({
|
|
4024
|
+
sql: "DELETE FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
4025
|
+
args: [runId, Number(last.seq)],
|
|
4026
|
+
});
|
|
4027
|
+
if (Number(last.seq) === CHECKPOINT_TERMINAL_EVENT_SEQ) {
|
|
4028
|
+
const regularEvents = await tx.execute({
|
|
4029
|
+
sql: `SELECT COALESCE(MAX(seq), -1) AS seq FROM agent_run_events
|
|
4030
|
+
WHERE run_id = ? AND seq < ?`,
|
|
4031
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
4032
|
+
});
|
|
4033
|
+
nextSeq =
|
|
4034
|
+
Number(regularEvents.rows[0]
|
|
4035
|
+
?.seq ?? -1) + 1;
|
|
4036
|
+
}
|
|
4037
|
+
else {
|
|
4038
|
+
nextSeq = Number(last.seq);
|
|
4039
|
+
}
|
|
4040
|
+
}
|
|
3787
4041
|
}
|
|
3788
4042
|
}
|
|
3789
|
-
const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
|
|
3790
4043
|
if (!Number.isSafeInteger(nextSeq)) {
|
|
3791
4044
|
throw new Error(`Run ${runId} has no safe sequence for its terminal event`);
|
|
3792
4045
|
}
|
|
3793
|
-
|
|
3794
|
-
|
|
3795
|
-
|
|
3796
|
-
|
|
3797
|
-
|
|
3798
|
-
|
|
3799
|
-
runId,
|
|
3800
|
-
|
|
3801
|
-
|
|
3802
|
-
|
|
3803
|
-
|
|
3804
|
-
|
|
3805
|
-
|
|
3806
|
-
|
|
4046
|
+
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
4047
|
+
const inserted = await tx.execute({
|
|
4048
|
+
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
4049
|
+
VALUES (?, ?, ?, ?)
|
|
4050
|
+
ON CONFLICT (run_id, seq) DO NOTHING
|
|
4051
|
+
RETURNING seq`,
|
|
4052
|
+
args: [runId, nextSeq, Date.now(), eventData],
|
|
4053
|
+
});
|
|
4054
|
+
if ((inserted.rowsAffected ?? inserted.rows.length) === 1)
|
|
4055
|
+
return;
|
|
4056
|
+
const conflict = await tx.execute({
|
|
4057
|
+
sql: "SELECT event_data FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
4058
|
+
args: [runId, nextSeq],
|
|
4059
|
+
});
|
|
4060
|
+
const conflictingEventData = conflict.rows[0]?.event_data;
|
|
4061
|
+
if (typeof conflictingEventData !== "string") {
|
|
4062
|
+
throw new Error(`Run ${runId} terminal event sequence ${nextSeq} was not persisted`);
|
|
4063
|
+
}
|
|
4064
|
+
const conflictingEvent = parseRunEventData(conflictingEventData, `conflicting event at sequence ${nextSeq} for run ${runId}`);
|
|
4065
|
+
if (terminalStatusForEvent(conflictingEvent))
|
|
4066
|
+
return;
|
|
4067
|
+
const regularEvents = await tx.execute({
|
|
4068
|
+
sql: `SELECT COALESCE(MAX(seq), -1) AS seq FROM agent_run_events
|
|
4069
|
+
WHERE run_id = ? AND seq < ?`,
|
|
4070
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
4071
|
+
});
|
|
4072
|
+
nextSeq =
|
|
4073
|
+
Number(regularEvents.rows[0]?.seq ??
|
|
4074
|
+
-1) + 1;
|
|
4075
|
+
if (!Number.isSafeInteger(nextSeq)) {
|
|
4076
|
+
throw new Error(`Run ${runId} has no safe sequence for its terminal event`);
|
|
4077
|
+
}
|
|
3807
4078
|
}
|
|
4079
|
+
throw new Error(`Run ${runId} terminal event conflicted repeatedly at sequence ${nextSeq}`);
|
|
3808
4080
|
}
|
|
3809
4081
|
async function retryPendingStaleRunThreadProjections(client) {
|
|
3810
4082
|
const { rows } = await client.execute({
|
|
@@ -3826,69 +4098,64 @@ async function retryPendingStaleRunThreadProjections(client) {
|
|
|
3826
4098
|
}
|
|
3827
4099
|
return { failed, truncated };
|
|
3828
4100
|
}
|
|
3829
|
-
async function safeAppendTerminalRunEvent(runId, event, source) {
|
|
3830
|
-
let firstError;
|
|
3831
|
-
try {
|
|
3832
|
-
await appendTerminalRunEvent(runId, event);
|
|
3833
|
-
return;
|
|
3834
|
-
}
|
|
3835
|
-
catch (err) {
|
|
3836
|
-
firstError = err;
|
|
3837
|
-
}
|
|
3838
|
-
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
3839
|
-
try {
|
|
3840
|
-
await appendTerminalRunEvent(runId, event);
|
|
3841
|
-
}
|
|
3842
|
-
catch (retryErr) {
|
|
3843
|
-
captureError(retryErr, {
|
|
3844
|
-
tags: {
|
|
3845
|
-
component: "agent-run-store",
|
|
3846
|
-
operation: "append-terminal-event",
|
|
3847
|
-
source,
|
|
3848
|
-
},
|
|
3849
|
-
extra: {
|
|
3850
|
-
runId,
|
|
3851
|
-
eventType: typeof event.type === "string" ? event.type : "(unknown)",
|
|
3852
|
-
firstError: firstError instanceof Error ? firstError.message : String(firstError),
|
|
3853
|
-
},
|
|
3854
|
-
});
|
|
3855
|
-
throw retryErr;
|
|
3856
|
-
}
|
|
3857
|
-
}
|
|
3858
4101
|
async function appendTerminalRunEvent(runId, event) {
|
|
3859
4102
|
await ensureRunTables();
|
|
3860
4103
|
const client = getDbExec();
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
4104
|
+
if (!client.transaction) {
|
|
4105
|
+
throw new Error("Terminal run event writes require transaction support");
|
|
4106
|
+
}
|
|
4107
|
+
const storedEventData = stripInlineBytesFromJson(JSON.stringify(event), "placeholder");
|
|
4108
|
+
await client.transaction(async (tx) => {
|
|
4109
|
+
const { rows } = await tx.execute({
|
|
4110
|
+
sql: `SELECT seq, event_data FROM agent_run_events
|
|
4111
|
+
WHERE run_id = ? ORDER BY seq DESC LIMIT 1 FOR UPDATE`,
|
|
4112
|
+
args: [runId],
|
|
4113
|
+
});
|
|
4114
|
+
const last = rows[0];
|
|
4115
|
+
let nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
|
|
4116
|
+
if (last?.event_data) {
|
|
4117
|
+
const parsed = parseRunEventData(last.event_data, `last persisted event for run ${runId}`);
|
|
4118
|
+
const lastEventIsTerminal = parsed.type === "done" ||
|
|
4119
|
+
parsed.type === "error" ||
|
|
4120
|
+
parsed.type === "missing_api_key" ||
|
|
4121
|
+
parsed.type === "loop_limit" ||
|
|
4122
|
+
parsed.type === "auto_continue";
|
|
4123
|
+
const replacesCheckpoint = Number(last.seq) === CHECKPOINT_TERMINAL_EVENT_SEQ &&
|
|
4124
|
+
parsed.type === "auto_continue" &&
|
|
4125
|
+
(event.type !== "auto_continue" || parsed.reason !== event.reason);
|
|
4126
|
+
if (lastEventIsTerminal && !replacesCheckpoint)
|
|
3874
4127
|
return;
|
|
4128
|
+
if (replacesCheckpoint) {
|
|
4129
|
+
await tx.execute({
|
|
4130
|
+
sql: "DELETE FROM agent_run_events WHERE run_id = ? AND seq = ?",
|
|
4131
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
4132
|
+
});
|
|
4133
|
+
const regularEvents = await tx.execute({
|
|
4134
|
+
sql: `SELECT COALESCE(MAX(seq), -1) AS seq FROM agent_run_events
|
|
4135
|
+
WHERE run_id = ? AND seq < ?`,
|
|
4136
|
+
args: [runId, CHECKPOINT_TERMINAL_EVENT_SEQ],
|
|
4137
|
+
});
|
|
4138
|
+
nextSeq =
|
|
4139
|
+
Number(regularEvents.rows[0]
|
|
4140
|
+
?.seq ?? -1) + 1;
|
|
3875
4141
|
}
|
|
3876
4142
|
}
|
|
3877
|
-
|
|
3878
|
-
|
|
4143
|
+
const inserted = await tx.execute({
|
|
4144
|
+
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING RETURNING seq`,
|
|
4145
|
+
args: [runId, nextSeq, Date.now(), storedEventData],
|
|
4146
|
+
});
|
|
4147
|
+
if (inserted.rows.length === 0) {
|
|
4148
|
+
throw new Error(`Run ${runId} terminal event conflicted with sequence ${nextSeq}`);
|
|
3879
4149
|
}
|
|
3880
|
-
}
|
|
3881
|
-
const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
|
|
3882
|
-
const inserted = await client.execute({
|
|
3883
|
-
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING RETURNING seq`,
|
|
3884
|
-
args: [
|
|
3885
|
-
runId,
|
|
3886
|
-
nextSeq,
|
|
3887
|
-
Date.now(),
|
|
3888
|
-
stripInlineBytesFromJson(JSON.stringify(event), "placeholder"),
|
|
3889
|
-
],
|
|
3890
4150
|
});
|
|
3891
|
-
|
|
3892
|
-
|
|
4151
|
+
}
|
|
4152
|
+
function parseRunEventData(eventData, context) {
|
|
4153
|
+
const parsed = JSON.parse(eventData);
|
|
4154
|
+
if (parsed === null ||
|
|
4155
|
+
typeof parsed !== "object" ||
|
|
4156
|
+
Array.isArray(parsed) ||
|
|
4157
|
+
typeof parsed.type !== "string") {
|
|
4158
|
+
throw new Error(`Run event data is malformed for ${context}`);
|
|
3893
4159
|
}
|
|
4160
|
+
return parsed;
|
|
3894
4161
|
}
|