@agent-native/core 0.210.1-nightly-20261020124228 → 0.210.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/a2a/server.js +2 -1
- package/dist/agent/contain-principal.js +2 -3
- package/dist/agent/engine/ai-sdk-engine.js +29 -20
- package/dist/agent/engine/anthropic-engine.js +10 -1
- package/dist/agent/engine/builder-engine.js +28 -14
- package/dist/agent/engine/types.d.ts +11 -0
- package/dist/agent/engine/types.js +8 -0
- package/dist/agent/execution-fence.d.ts +5 -0
- package/dist/agent/execution-fence.js +15 -0
- package/dist/agent/harness/background.js +28 -7
- package/dist/agent/production-agent.d.ts +4 -3
- package/dist/agent/production-agent.js +20 -3
- package/dist/agent/run-manager.d.ts +8 -1
- package/dist/agent/run-manager.js +273 -34
- package/dist/agent/run-store.d.ts +3 -2
- package/dist/agent/run-store.fencing.worker.js +19 -1
- package/dist/agent/run-store.js +216 -67
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +2 -2
- package/dist/observability/routes.d.ts +5 -5
- package/dist/org/index.d.ts +1 -1
- package/dist/secrets/routes.d.ts +4 -4
- package/dist/server/agent-chat-plugin.js +27 -11
- package/dist/server/agent-teams.d.ts +3 -0
- package/dist/server/agent-teams.js +192 -30
- package/package.json +4 -4
package/dist/agent/run-store.js
CHANGED
|
@@ -14,7 +14,7 @@ import { approvalKeyForHistory } from "./approval-key.js";
|
|
|
14
14
|
import { admitAutoContinue, admitManualContinue, } from "./auto-continue.js";
|
|
15
15
|
import { AGENT_CHAT_RECOVERY_OF_RUN_FIELD } from "./durable-background.js";
|
|
16
16
|
import { LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
|
|
17
|
-
import { currentExecutionFence, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
|
|
17
|
+
import { currentExecutionFence, currentExecutionFenceForScope, ExecutionFenceError, MissingExecutionFenceError, lockAndAssertCurrentExecutionFences, StaleExecutionFenceError, withFencedExecutionWrite, } from "./execution-fence.js";
|
|
18
18
|
import { AGENT_RUN_MIGRATIONS, AGENT_RUN_MIGRATIONS_TABLE, } from "./run-migrations.js";
|
|
19
19
|
import { isConsistentToolCallInputFingerprint, isUnavailableToolCallInput, toolCallInputFingerprint, } from "./tool-call-journal.js";
|
|
20
20
|
import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
|
|
@@ -2228,32 +2228,51 @@ export async function reapIfStale(runId, maxStaleMs) {
|
|
|
2228
2228
|
export async function reapUnclaimedBackgroundRun(runId) {
|
|
2229
2229
|
await ensureRunTables();
|
|
2230
2230
|
const client = getDbExec();
|
|
2231
|
+
if (!client.transaction) {
|
|
2232
|
+
throw new Error("Atomic unclaimed-background reaping requires transaction support");
|
|
2233
|
+
}
|
|
2231
2234
|
const completedAt = Date.now();
|
|
2232
2235
|
const cutoff = completedAt - UNCLAIMED_BACKGROUND_RUN_GRACE_MS;
|
|
2233
|
-
const
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2236
|
+
const reaped = await client.transaction(async (tx) => {
|
|
2237
|
+
const { rowsAffected } = await tx.execute({
|
|
2238
|
+
sql: `UPDATE agent_runs
|
|
2239
|
+
SET status = 'errored',
|
|
2240
|
+
completed_at = ?,
|
|
2241
|
+
error_code = ?,
|
|
2242
|
+
error_detail = ?,
|
|
2243
|
+
terminal_reason = ?
|
|
2244
|
+
WHERE id = ?
|
|
2245
|
+
AND status = 'running'
|
|
2246
|
+
AND dispatch_mode = 'background'
|
|
2247
|
+
AND COALESCE(heartbeat_at, started_at) < ?`,
|
|
2248
|
+
args: [
|
|
2249
|
+
completedAt,
|
|
2250
|
+
UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
|
|
2251
|
+
UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.details,
|
|
2252
|
+
UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT.errorCode,
|
|
2253
|
+
runId,
|
|
2254
|
+
cutoff,
|
|
2255
|
+
],
|
|
2256
|
+
});
|
|
2257
|
+
const updated = (rowsAffected ?? 0) > 0;
|
|
2258
|
+
if (updated) {
|
|
2259
|
+
await appendTerminalRunEventInTransaction(tx, runId, UNCLAIMED_BACKGROUND_RUN_ERROR_EVENT);
|
|
2260
|
+
}
|
|
2261
|
+
return updated;
|
|
2252
2262
|
});
|
|
2253
|
-
const reaped = (rowsAffected ?? 0) > 0;
|
|
2254
2263
|
if (reaped) {
|
|
2255
|
-
|
|
2256
|
-
|
|
2264
|
+
try {
|
|
2265
|
+
await recordRunDiagnostic(runId, RUN_DIAG_STAGE.workerThrew, "unclaimed background dispatch reaped (worker never claimed the run)");
|
|
2266
|
+
}
|
|
2267
|
+
catch (error) {
|
|
2268
|
+
captureError(error, {
|
|
2269
|
+
tags: {
|
|
2270
|
+
component: "agent-run-store",
|
|
2271
|
+
operation: "record-unclaimed-background-reap-diagnostic",
|
|
2272
|
+
},
|
|
2273
|
+
extra: { runId },
|
|
2274
|
+
});
|
|
2275
|
+
}
|
|
2257
2276
|
}
|
|
2258
2277
|
return reaped;
|
|
2259
2278
|
}
|
|
@@ -2478,7 +2497,7 @@ export function terminalEventForAbortReason(reason) {
|
|
|
2478
2497
|
};
|
|
2479
2498
|
}
|
|
2480
2499
|
export async function markRunAborted(runId, reason) {
|
|
2481
|
-
|
|
2500
|
+
return updateRunAborted(runId, reason, false);
|
|
2482
2501
|
}
|
|
2483
2502
|
/** Abort only when no model or tool work has crossed its dispatch boundary. */
|
|
2484
2503
|
export async function markRunAbortedAtSafeBoundary(runId, reason) {
|
|
@@ -2486,22 +2505,32 @@ export async function markRunAbortedAtSafeBoundary(runId, reason) {
|
|
|
2486
2505
|
}
|
|
2487
2506
|
async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
2488
2507
|
await ensureRunTables();
|
|
2508
|
+
const fence = currentExecutionFenceForScope({ runId });
|
|
2489
2509
|
const client = getDbExec();
|
|
2490
2510
|
if (!client.transaction) {
|
|
2491
2511
|
throw new Error("Atomic run cancellation requires transaction support");
|
|
2492
2512
|
}
|
|
2493
|
-
const
|
|
2513
|
+
const abortResult = await client.transaction(async (tx) => {
|
|
2494
2514
|
await lockExecutionLeasesForRuns(tx, [runId]);
|
|
2515
|
+
await lockAndAssertCurrentExecutionFences(tx);
|
|
2516
|
+
const runResult = await tx.execute({
|
|
2517
|
+
sql: `SELECT thread_id, COALESCE(turn_id, id) AS turn_id,
|
|
2518
|
+
worker_generation
|
|
2519
|
+
FROM agent_runs WHERE id = ? FOR UPDATE`,
|
|
2520
|
+
args: [runId],
|
|
2521
|
+
});
|
|
2522
|
+
const lockedRun = runResult.rows[0];
|
|
2523
|
+
if (fence &&
|
|
2524
|
+
(lockedRun?.thread_id !== fence.threadId ||
|
|
2525
|
+
lockedRun.turn_id !== fence.turnId ||
|
|
2526
|
+
lockedRun.worker_generation == null ||
|
|
2527
|
+
String(lockedRun.worker_generation) !== fence.generation)) {
|
|
2528
|
+
throw new StaleExecutionFenceError(fence);
|
|
2529
|
+
}
|
|
2495
2530
|
if (requireSafeBoundary) {
|
|
2496
|
-
|
|
2497
|
-
|
|
2498
|
-
|
|
2499
|
-
args: [runId],
|
|
2500
|
-
});
|
|
2501
|
-
const runRef = run.rows[0];
|
|
2502
|
-
if (runRef) {
|
|
2503
|
-
if (typeof runRef.thread_id !== "string" ||
|
|
2504
|
-
typeof runRef.turn_id !== "string") {
|
|
2531
|
+
if (lockedRun) {
|
|
2532
|
+
if (typeof lockedRun.thread_id !== "string" ||
|
|
2533
|
+
typeof lockedRun.turn_id !== "string") {
|
|
2505
2534
|
throw new Error(`Run ${runId} has an invalid thread or turn ID`);
|
|
2506
2535
|
}
|
|
2507
2536
|
const dispatchedStep = await tx.execute({
|
|
@@ -2509,10 +2538,11 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
|
2509
2538
|
WHERE thread_id = ? AND turn_id = ? AND status = 'dispatched'
|
|
2510
2539
|
AND run_id = ?
|
|
2511
2540
|
LIMIT 1`,
|
|
2512
|
-
args: [
|
|
2541
|
+
args: [lockedRun.thread_id, lockedRun.turn_id, runId],
|
|
2513
2542
|
});
|
|
2514
|
-
if (dispatchedStep.rows.length > 0)
|
|
2515
|
-
return 0;
|
|
2543
|
+
if (dispatchedStep.rows.length > 0) {
|
|
2544
|
+
return { rowsAffected: 0, appendTerminalEvent: false };
|
|
2545
|
+
}
|
|
2516
2546
|
}
|
|
2517
2547
|
}
|
|
2518
2548
|
const result = await tx.execute({
|
|
@@ -2524,15 +2554,32 @@ async function updateRunAborted(runId, reason, requireSafeBoundary) {
|
|
|
2524
2554
|
runId,
|
|
2525
2555
|
],
|
|
2526
2556
|
});
|
|
2527
|
-
|
|
2557
|
+
const rowsAffected = result.rowsAffected ?? 0;
|
|
2558
|
+
if (rowsAffected > 0) {
|
|
2528
2559
|
await revokeExecutionLeasesForRuns(tx, [runId]);
|
|
2560
|
+
return { rowsAffected, appendTerminalEvent: true };
|
|
2529
2561
|
}
|
|
2530
|
-
|
|
2562
|
+
const terminalState = await tx.execute({
|
|
2563
|
+
sql: `SELECT status, terminal_reason, in_flight_since FROM agent_runs WHERE id = ?`,
|
|
2564
|
+
args: [runId],
|
|
2565
|
+
});
|
|
2566
|
+
const terminalRun = terminalState.rows[0];
|
|
2567
|
+
return {
|
|
2568
|
+
rowsAffected,
|
|
2569
|
+
appendTerminalEvent: terminalRun?.status === "aborted" &&
|
|
2570
|
+
terminalRun.terminal_reason === `aborted:${reason ?? "user"}` &&
|
|
2571
|
+
(!requireSafeBoundary || terminalRun.in_flight_since == null),
|
|
2572
|
+
};
|
|
2531
2573
|
});
|
|
2532
|
-
if (
|
|
2533
|
-
|
|
2574
|
+
if (abortResult.appendTerminalEvent) {
|
|
2575
|
+
try {
|
|
2576
|
+
await safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-aborted");
|
|
2577
|
+
}
|
|
2578
|
+
catch (error) {
|
|
2579
|
+
throw new AgentRunEventNotPersistedError(runId, null, true, error);
|
|
2580
|
+
}
|
|
2534
2581
|
}
|
|
2535
|
-
return
|
|
2582
|
+
return abortResult.rowsAffected > 0 || abortResult.appendTerminalEvent;
|
|
2536
2583
|
}
|
|
2537
2584
|
async function lockExecutionLeasesForRuns(db, runIds) {
|
|
2538
2585
|
const ids = [...new Set(runIds)].sort();
|
|
@@ -2612,6 +2659,7 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2612
2659
|
sql: "SELECT pg_advisory_xact_lock(hashtextextended(?, 0::bigint))",
|
|
2613
2660
|
args: [`agent-native:run-slot:${threadId}`],
|
|
2614
2661
|
});
|
|
2662
|
+
await lockAndAssertCurrentExecutionFences(tx);
|
|
2615
2663
|
const lease = await lockTurnExecutionLease(tx, threadId, turnId);
|
|
2616
2664
|
if (fence &&
|
|
2617
2665
|
(!lease ||
|
|
@@ -2640,7 +2688,7 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2640
2688
|
}
|
|
2641
2689
|
if (runIds.length === 0) {
|
|
2642
2690
|
const existing = await tx.execute({
|
|
2643
|
-
sql: `SELECT id, terminal_reason,
|
|
2691
|
+
sql: `SELECT id, status, terminal_reason,
|
|
2644
2692
|
EXISTS (
|
|
2645
2693
|
SELECT 1 FROM agent_run_events
|
|
2646
2694
|
WHERE agent_run_events.run_id = agent_runs.id
|
|
@@ -2661,6 +2709,17 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2661
2709
|
],
|
|
2662
2710
|
});
|
|
2663
2711
|
const latest = existing.rows[0];
|
|
2712
|
+
const aborted = await tx.execute({
|
|
2713
|
+
sql: `SELECT id FROM agent_runs
|
|
2714
|
+
WHERE thread_id = ? AND COALESCE(turn_id, id) = ?
|
|
2715
|
+
AND status = 'aborted'
|
|
2716
|
+
AND dispatch_mode IS DISTINCT FROM 'turn-abort'
|
|
2717
|
+
AND terminal_reason = ?`,
|
|
2718
|
+
args: [threadId, turnId, `aborted:${reason}`],
|
|
2719
|
+
});
|
|
2720
|
+
runIds.push(...aborted.rows
|
|
2721
|
+
.map((row) => String(row.id ?? ""))
|
|
2722
|
+
.filter(Boolean));
|
|
2664
2723
|
if (latest &&
|
|
2665
2724
|
latest.continuation_pending !== true &&
|
|
2666
2725
|
!isContinuationTerminalReason(latest.terminal_reason ?? "")) {
|
|
@@ -2692,8 +2751,6 @@ export async function markTurnAborted(threadId, turnId, reason = "user") {
|
|
|
2692
2751
|
});
|
|
2693
2752
|
return "aborted";
|
|
2694
2753
|
});
|
|
2695
|
-
if (outcome === "already_terminal")
|
|
2696
|
-
return outcome;
|
|
2697
2754
|
await Promise.all(runIds.map((runId) => safeAppendTerminalRunEvent(runId, terminalEventForAbortReason(reason), "mark-turn-aborted")));
|
|
2698
2755
|
return outcome;
|
|
2699
2756
|
}
|
|
@@ -2767,40 +2824,100 @@ function persistedRunEventData(eventData, inputSource) {
|
|
|
2767
2824
|
});
|
|
2768
2825
|
}
|
|
2769
2826
|
export class AgentRunEventNotPersistedError extends Error {
|
|
2770
|
-
|
|
2771
|
-
|
|
2827
|
+
terminalRun;
|
|
2828
|
+
constructor(runId, seq, terminalRun = false, cause) {
|
|
2829
|
+
const detail = cause instanceof Error ? `: ${cause.message}` : "";
|
|
2830
|
+
super(seq === null
|
|
2831
|
+
? `Terminal run event was not persisted: ${runId}${detail}`
|
|
2832
|
+
: `Run event was not persisted: ${runId} sequence ${seq}${detail}`);
|
|
2772
2833
|
this.name = "AgentRunEventNotPersistedError";
|
|
2834
|
+
this.terminalRun = terminalRun;
|
|
2835
|
+
if (cause !== undefined) {
|
|
2836
|
+
Object.defineProperty(this, "cause", {
|
|
2837
|
+
value: cause,
|
|
2838
|
+
configurable: true,
|
|
2839
|
+
});
|
|
2840
|
+
}
|
|
2773
2841
|
}
|
|
2774
2842
|
}
|
|
2775
2843
|
export async function insertRunEvent(runId, seq, eventData, options) {
|
|
2776
2844
|
await ensureRunTables();
|
|
2777
2845
|
const storedEventData = persistedRunEventData(eventData, options?.toolInputSource);
|
|
2846
|
+
const eventAt = options?.eventAt ?? Date.now();
|
|
2778
2847
|
// The reaper must wait for the marker's commit before exposing a successor.
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2848
|
+
let result;
|
|
2849
|
+
try {
|
|
2850
|
+
result = await withFencedExecutionWrite(async (tx) => {
|
|
2851
|
+
const { rowsAffected } = await tx.execute({
|
|
2852
|
+
sql: options?.requireInserted
|
|
2853
|
+
? `WITH running_run AS MATERIALIZED (
|
|
2782
2854
|
SELECT id FROM agent_runs WHERE id = ? AND status = 'running' FOR UPDATE
|
|
2783
2855
|
)
|
|
2784
2856
|
INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
2785
2857
|
SELECT id, ?, ?, ? FROM running_run LIMIT 1
|
|
2786
2858
|
ON CONFLICT (run_id, seq) DO NOTHING`
|
|
2787
|
-
|
|
2859
|
+
: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
2788
2860
|
SELECT ?, ?, ?, ?
|
|
2789
2861
|
WHERE NOT EXISTS (
|
|
2790
2862
|
SELECT 1 FROM agent_runs
|
|
2791
2863
|
WHERE id = ? AND status <> 'running'
|
|
2792
2864
|
)
|
|
2793
2865
|
ON CONFLICT (run_id, seq) DO NOTHING`,
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2866
|
+
args: [
|
|
2867
|
+
runId,
|
|
2868
|
+
seq,
|
|
2869
|
+
eventAt,
|
|
2870
|
+
storedEventData,
|
|
2871
|
+
...(options?.requireInserted ? [] : [runId]),
|
|
2872
|
+
],
|
|
2873
|
+
});
|
|
2874
|
+
if (!options?.requireInserted || rowsAffected === 1)
|
|
2875
|
+
return {
|
|
2876
|
+
rowsAffected,
|
|
2877
|
+
terminalRun: false,
|
|
2878
|
+
persistedIdentically: rowsAffected === 1,
|
|
2879
|
+
};
|
|
2880
|
+
const existing = await tx.execute({
|
|
2881
|
+
sql: `SELECT event_at, event_data FROM agent_run_events
|
|
2882
|
+
WHERE run_id = ? AND seq = ?`,
|
|
2883
|
+
args: [runId, seq],
|
|
2884
|
+
});
|
|
2885
|
+
const existingRow = existing.rows[0];
|
|
2886
|
+
const persistedIdentically = existingRow !== undefined &&
|
|
2887
|
+
Number(existingRow.event_at) === eventAt &&
|
|
2888
|
+
String(existingRow.event_data) === storedEventData;
|
|
2889
|
+
const run = await tx.execute({
|
|
2890
|
+
sql: "SELECT status FROM agent_runs WHERE id = ?",
|
|
2891
|
+
args: [runId],
|
|
2892
|
+
});
|
|
2893
|
+
const status = run.rows[0]?.status;
|
|
2894
|
+
return {
|
|
2895
|
+
rowsAffected,
|
|
2896
|
+
terminalRun: status !== "running",
|
|
2897
|
+
persistedIdentically,
|
|
2898
|
+
};
|
|
2899
|
+
}, { runId });
|
|
2900
|
+
}
|
|
2901
|
+
catch (error) {
|
|
2902
|
+
if (error instanceof StaleExecutionFenceError) {
|
|
2903
|
+
let status;
|
|
2904
|
+
try {
|
|
2905
|
+
status = await getRunStatus(runId);
|
|
2906
|
+
}
|
|
2907
|
+
catch (statusError) {
|
|
2908
|
+
throw new AggregateError([error, statusError], `Could not confirm run status after a stale execution fence: ${runId}`);
|
|
2909
|
+
}
|
|
2910
|
+
if (status !== null && status !== "running") {
|
|
2911
|
+
throw new AgentRunEventNotPersistedError(runId, seq, true, error);
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
throw error;
|
|
2915
|
+
}
|
|
2916
|
+
const { rowsAffected, terminalRun, persistedIdentically } = result;
|
|
2917
|
+
if (options?.requireInserted &&
|
|
2918
|
+
rowsAffected !== 1 &&
|
|
2919
|
+
(!persistedIdentically || terminalRun))
|
|
2920
|
+
throw new AgentRunEventNotPersistedError(runId, seq, terminalRun);
|
|
2804
2921
|
}
|
|
2805
2922
|
// Stays well under PostgreSQL's 65,535 bind-parameter limit at 4 per row.
|
|
2806
2923
|
const RUN_EVENT_BATCH_ROWS = 500;
|
|
@@ -2812,9 +2929,13 @@ export async function insertRunEvents(runId, events) {
|
|
|
2812
2929
|
if (events.length === 0)
|
|
2813
2930
|
return;
|
|
2814
2931
|
await ensureRunTables();
|
|
2815
|
-
|
|
2932
|
+
const writeBatch = async (tx) => {
|
|
2816
2933
|
for (let start = 0; start < events.length; start += RUN_EVENT_BATCH_ROWS) {
|
|
2817
2934
|
const chunk = events.slice(start, start + RUN_EVENT_BATCH_ROWS);
|
|
2935
|
+
const storedChunk = chunk.map((event) => ({
|
|
2936
|
+
...event,
|
|
2937
|
+
eventData: persistedRunEventData(event.eventData),
|
|
2938
|
+
}));
|
|
2818
2939
|
await tx.execute({
|
|
2819
2940
|
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data)
|
|
2820
2941
|
SELECT v.run_id, v.seq, v.event_at, v.event_data
|
|
@@ -2827,17 +2948,41 @@ export async function insertRunEvents(runId, events) {
|
|
|
2827
2948
|
)
|
|
2828
2949
|
ON CONFLICT (run_id, seq) DO NOTHING`,
|
|
2829
2950
|
args: [
|
|
2830
|
-
...
|
|
2951
|
+
...storedChunk.flatMap(({ seq, eventAt, eventData }) => [
|
|
2831
2952
|
runId,
|
|
2832
2953
|
seq,
|
|
2833
2954
|
eventAt,
|
|
2834
|
-
|
|
2955
|
+
eventData,
|
|
2835
2956
|
]),
|
|
2836
2957
|
runId,
|
|
2837
2958
|
],
|
|
2838
2959
|
});
|
|
2960
|
+
const persisted = await tx.execute({
|
|
2961
|
+
sql: `SELECT seq, event_at, event_data
|
|
2962
|
+
FROM agent_run_events
|
|
2963
|
+
WHERE run_id = ? AND seq IN (${chunk.map(() => "?").join(", ")})`,
|
|
2964
|
+
args: [runId, ...chunk.map(({ seq }) => seq)],
|
|
2965
|
+
});
|
|
2966
|
+
const persistedBySeq = new Map(persisted.rows.map((row) => [Number(row.seq), row]));
|
|
2967
|
+
for (const event of storedChunk) {
|
|
2968
|
+
const row = persistedBySeq.get(event.seq);
|
|
2969
|
+
if (!row ||
|
|
2970
|
+
Number(row.event_at) !== event.eventAt ||
|
|
2971
|
+
String(row.event_data) !== event.eventData) {
|
|
2972
|
+
const run = await tx.execute({
|
|
2973
|
+
sql: "SELECT status FROM agent_runs WHERE id = ?",
|
|
2974
|
+
args: [runId],
|
|
2975
|
+
});
|
|
2976
|
+
const status = run.rows[0]
|
|
2977
|
+
?.status;
|
|
2978
|
+
throw new AgentRunEventNotPersistedError(runId, event.seq, status !== "running");
|
|
2979
|
+
}
|
|
2980
|
+
}
|
|
2839
2981
|
}
|
|
2840
|
-
}
|
|
2982
|
+
};
|
|
2983
|
+
await withFencedExecutionWrite((tx) => currentExecutionFence()
|
|
2984
|
+
? writeBatch(tx)
|
|
2985
|
+
: (tx.transaction?.(writeBatch) ?? writeBatch(tx)), { runId });
|
|
2841
2986
|
}
|
|
2842
2987
|
export const CHECKPOINT_TERMINAL_EVENT_SEQ = 1_000_000_000;
|
|
2843
2988
|
export async function persistRunCheckpointEvent(runId, event, terminalReason) {
|
|
@@ -3681,6 +3826,7 @@ async function safeAppendTerminalRunEvent(runId, event, source) {
|
|
|
3681
3826
|
firstError: firstError instanceof Error ? firstError.message : String(firstError),
|
|
3682
3827
|
},
|
|
3683
3828
|
});
|
|
3829
|
+
throw retryErr;
|
|
3684
3830
|
}
|
|
3685
3831
|
}
|
|
3686
3832
|
async function appendTerminalRunEvent(runId, event) {
|
|
@@ -3707,8 +3853,8 @@ async function appendTerminalRunEvent(runId, event) {
|
|
|
3707
3853
|
}
|
|
3708
3854
|
}
|
|
3709
3855
|
const nextSeq = last ? Number(last.seq ?? -1) + 1 : 0;
|
|
3710
|
-
await client.execute({
|
|
3711
|
-
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING`,
|
|
3856
|
+
const inserted = await client.execute({
|
|
3857
|
+
sql: `INSERT INTO agent_run_events (run_id, seq, event_at, event_data) VALUES (?, ?, ?, ?) ON CONFLICT (run_id, seq) DO NOTHING RETURNING seq`,
|
|
3712
3858
|
args: [
|
|
3713
3859
|
runId,
|
|
3714
3860
|
nextSeq,
|
|
@@ -3716,4 +3862,7 @@ async function appendTerminalRunEvent(runId, event) {
|
|
|
3716
3862
|
stripInlineBytesFromJson(JSON.stringify(event), "placeholder"),
|
|
3717
3863
|
],
|
|
3718
3864
|
});
|
|
3865
|
+
if (inserted.rows.length === 0) {
|
|
3866
|
+
throw new Error(`Run ${runId} terminal event conflicted with sequence ${nextSeq}`);
|
|
3867
|
+
}
|
|
3719
3868
|
}
|
|
@@ -38,19 +38,19 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
|
|
|
38
38
|
error: string;
|
|
39
39
|
states?: undefined;
|
|
40
40
|
} | {
|
|
41
|
+
error?: undefined;
|
|
41
42
|
states: {
|
|
42
43
|
clientId: number;
|
|
43
44
|
state: string;
|
|
44
45
|
}[];
|
|
45
|
-
error?: undefined;
|
|
46
46
|
}>>;
|
|
47
47
|
export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
48
48
|
error: string;
|
|
49
49
|
users?: undefined;
|
|
50
50
|
} | {
|
|
51
|
+
error?: undefined;
|
|
51
52
|
users: {
|
|
52
53
|
clientId: number;
|
|
53
54
|
lastSeen: number;
|
|
54
55
|
}[];
|
|
55
|
-
error?: undefined;
|
|
56
56
|
}>>;
|
|
@@ -33,9 +33,9 @@ declare const _default: import("../../action.js").ActionDefinition<{
|
|
|
33
33
|
idempotencyKey: string;
|
|
34
34
|
deleted: boolean;
|
|
35
35
|
} | {
|
|
36
|
+
id?: undefined;
|
|
36
37
|
error: string;
|
|
37
38
|
url?: undefined;
|
|
38
|
-
id?: undefined;
|
|
39
39
|
provider?: undefined;
|
|
40
40
|
configured?: undefined;
|
|
41
41
|
} | {
|
|
@@ -45,8 +45,8 @@ declare const _default: import("../../action.js").ActionDefinition<{
|
|
|
45
45
|
configured?: undefined;
|
|
46
46
|
error?: undefined;
|
|
47
47
|
} | {
|
|
48
|
-
url?: undefined;
|
|
49
48
|
id?: undefined;
|
|
49
|
+
url?: undefined;
|
|
50
50
|
provider?: undefined;
|
|
51
51
|
error: string;
|
|
52
52
|
configured: boolean;
|
|
@@ -27,39 +27,39 @@ export declare function createObservabilityHandler(): import("h3").EventHandlerW
|
|
|
27
27
|
thumbsUpRate: number;
|
|
28
28
|
avgEvalScore: number;
|
|
29
29
|
} | {
|
|
30
|
-
ok?: undefined;
|
|
31
30
|
error: string;
|
|
32
31
|
message: string;
|
|
33
32
|
summary?: undefined;
|
|
34
33
|
spans?: undefined;
|
|
34
|
+
ok?: undefined;
|
|
35
35
|
} | {
|
|
36
36
|
error?: undefined;
|
|
37
|
-
ok?: undefined;
|
|
38
37
|
message?: undefined;
|
|
39
38
|
summary: import("./types.js").TraceSummary;
|
|
40
39
|
spans: import("./types.js").TraceSpan[];
|
|
40
|
+
ok?: undefined;
|
|
41
41
|
} | {
|
|
42
42
|
error?: undefined;
|
|
43
|
-
ok?: undefined;
|
|
44
43
|
message?: undefined;
|
|
45
44
|
summary?: undefined;
|
|
46
45
|
spans?: undefined;
|
|
47
46
|
traceMissing: true;
|
|
48
47
|
id: string;
|
|
48
|
+
ok?: undefined;
|
|
49
49
|
} | {
|
|
50
50
|
error?: undefined;
|
|
51
|
-
ok?: undefined;
|
|
52
51
|
message?: undefined;
|
|
53
52
|
summary?: undefined;
|
|
54
53
|
spans?: undefined;
|
|
55
54
|
traceMissing?: undefined;
|
|
56
55
|
id: string;
|
|
57
|
-
} | {
|
|
58
56
|
ok?: undefined;
|
|
57
|
+
} | {
|
|
59
58
|
message?: undefined;
|
|
60
59
|
summary?: undefined;
|
|
61
60
|
spans?: undefined;
|
|
62
61
|
error: any;
|
|
62
|
+
ok?: undefined;
|
|
63
63
|
} | {
|
|
64
64
|
error?: undefined;
|
|
65
65
|
message?: undefined;
|
package/dist/org/index.d.ts
CHANGED
|
@@ -164,8 +164,8 @@ export declare const getMyOrgHandler: import("h3").EventHandlerWithFetch<import(
|
|
|
164
164
|
export declare const createOrgHandler: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
165
165
|
name?: undefined;
|
|
166
166
|
success: boolean;
|
|
167
|
-
role?: undefined;
|
|
168
167
|
id?: undefined;
|
|
168
|
+
role?: undefined;
|
|
169
169
|
} | {
|
|
170
170
|
success?: undefined;
|
|
171
171
|
id: string;
|
package/dist/secrets/routes.d.ts
CHANGED
|
@@ -57,31 +57,31 @@ export declare function createWriteSecretHandler(): import("h3").EventHandlerWit
|
|
|
57
57
|
} | {
|
|
58
58
|
error: string;
|
|
59
59
|
errorCode?: undefined;
|
|
60
|
-
ok?: undefined;
|
|
61
60
|
status?: undefined;
|
|
61
|
+
ok?: undefined;
|
|
62
62
|
} | {
|
|
63
63
|
error: string;
|
|
64
64
|
errorCode: string;
|
|
65
|
-
ok?: undefined;
|
|
66
65
|
status?: undefined;
|
|
66
|
+
ok?: undefined;
|
|
67
67
|
} | {
|
|
68
68
|
error?: undefined;
|
|
69
69
|
errorCode?: undefined;
|
|
70
70
|
ok: boolean;
|
|
71
71
|
status: string;
|
|
72
72
|
} | {
|
|
73
|
-
ok?: undefined;
|
|
74
73
|
error: string;
|
|
75
74
|
removed?: undefined;
|
|
75
|
+
ok?: undefined;
|
|
76
76
|
} | {
|
|
77
77
|
error?: undefined;
|
|
78
78
|
ok: boolean;
|
|
79
79
|
removed: boolean;
|
|
80
80
|
}>>;
|
|
81
81
|
export declare function createTestSecretHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
|
|
82
|
-
ok?: undefined;
|
|
83
82
|
error: string;
|
|
84
83
|
note?: undefined;
|
|
84
|
+
ok?: undefined;
|
|
85
85
|
} | {
|
|
86
86
|
error?: undefined;
|
|
87
87
|
ok: boolean;
|
|
@@ -20,7 +20,7 @@ import { SYSTEM_PROMPT_CACHE_SPLIT } from "../agent/engine/prompt-cache.js";
|
|
|
20
20
|
import { PROVIDER_TO_ENV } from "../agent/engine/provider-env-vars.js";
|
|
21
21
|
import { currentExecutionFence, withExecutionFence, } from "../agent/execution-fence.js";
|
|
22
22
|
import { hostedHarnessSystemPrompt } from "../agent/harness/hosted.js";
|
|
23
|
-
import { createProductionAgentHandler, endsAtContinuationBoundary, actionsToEngineTools, executeAgentToolCall, filterActionsByAllowedNames, normalizeAgentActionSurfaceResolution, getActiveRunForThreadAsync, abortRunDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, subscribeToRun, } from "../agent/production-agent.js";
|
|
23
|
+
import { createProductionAgentHandler, endsAtContinuationBoundary, actionsToEngineTools, executeAgentToolCall, filterActionsByAllowedNames, normalizeAgentActionSurfaceResolution, getActiveRunForThreadAsync, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, subscribeToRun, } from "../agent/production-agent.js";
|
|
24
24
|
import { applyProviderModelSelection, providerForEngineName, resolveProviderModelSelectionAtScope, } from "../agent/provider-model-selection.js";
|
|
25
25
|
import { startRun } from "../agent/run-manager.js";
|
|
26
26
|
import { callerHasRunAccess, callerHasThreadAccess, } from "../agent/run-ownership.js";
|
|
@@ -420,7 +420,7 @@ export async function runA2AAgentLoopWithRunManager(runOptions, pluginOptions, t
|
|
|
420
420
|
model: runOptions.model,
|
|
421
421
|
engineName: runOptions.engine.name,
|
|
422
422
|
});
|
|
423
|
-
let cancellation = Promise.resolve(false);
|
|
423
|
+
let cancellation = Promise.resolve({ status: "fulfilled", aborted: false });
|
|
424
424
|
const abortFromTask = () => {
|
|
425
425
|
const reason = taskSignal?.reason;
|
|
426
426
|
const reasonCode = reason && typeof reason === "object" && "code" in reason
|
|
@@ -431,7 +431,7 @@ export async function runA2AAgentLoopWithRunManager(runOptions, pluginOptions, t
|
|
|
431
431
|
run.abort.abort("execution_lease_lost");
|
|
432
432
|
return;
|
|
433
433
|
}
|
|
434
|
-
cancellation = abortRunDurably(runId, clientAbortReason(reason));
|
|
434
|
+
cancellation = abortRunDurably(runId, clientAbortReason(reason)).then((aborted) => ({ status: "fulfilled", aborted }), (error) => ({ status: "rejected", error }));
|
|
435
435
|
};
|
|
436
436
|
if (taskSignal?.aborted) {
|
|
437
437
|
abortFromTask();
|
|
@@ -440,8 +440,17 @@ export async function runA2AAgentLoopWithRunManager(runOptions, pluginOptions, t
|
|
|
440
440
|
taskSignal?.addEventListener("abort", abortFromTask, { once: true });
|
|
441
441
|
}
|
|
442
442
|
try {
|
|
443
|
-
await run.finalized;
|
|
444
|
-
await cancellation;
|
|
443
|
+
const finalization = await run.finalized.then(() => ({ status: "fulfilled" }), (error) => ({ status: "rejected", error }));
|
|
444
|
+
const cancellationResult = await cancellation;
|
|
445
|
+
if (finalization.status === "rejected" &&
|
|
446
|
+
cancellationResult.status === "rejected") {
|
|
447
|
+
throw new AggregateError([finalization.error, cancellationResult.error], "A2A run finalization and durable cancellation both failed");
|
|
448
|
+
}
|
|
449
|
+
if (finalization.status === "rejected")
|
|
450
|
+
throw finalization.error;
|
|
451
|
+
if (cancellationResult.status === "rejected") {
|
|
452
|
+
throw cancellationResult.error;
|
|
453
|
+
}
|
|
445
454
|
}
|
|
446
455
|
finally {
|
|
447
456
|
taskSignal?.removeEventListener("abort", abortFromTask);
|
|
@@ -5262,10 +5271,15 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
5262
5271
|
// Empty/invalid body — keep the default user abort reason.
|
|
5263
5272
|
}
|
|
5264
5273
|
if (safeBoundary) {
|
|
5265
|
-
const
|
|
5266
|
-
|
|
5267
|
-
|
|
5268
|
-
|
|
5274
|
+
const isUserStop = USER_STOP_ABORT_REASONS.has(reason);
|
|
5275
|
+
const aborted = isUserStop
|
|
5276
|
+
? await abortRunAndTurnDurably(runId, reason, {
|
|
5277
|
+
safeBoundary: true,
|
|
5278
|
+
})
|
|
5279
|
+
: await abortRunAtSafeBoundaryDurably(runId, reason);
|
|
5280
|
+
if (isUserStop && !aborted) {
|
|
5281
|
+
const latest = await getRunById(runId);
|
|
5282
|
+
if (latest?.status === "aborted") {
|
|
5269
5283
|
await abortTurnDurably(runId, reason);
|
|
5270
5284
|
}
|
|
5271
5285
|
}
|
|
@@ -5274,9 +5288,11 @@ Non-code requests are still fine on this surface: read data, navigate the UI, su
|
|
|
5274
5288
|
// Recovery starts as soon as this response resolves, so wait for
|
|
5275
5289
|
// the cross-isolate abort + terminal event to be durable. Returning
|
|
5276
5290
|
// early lets Retry collide with the still-running row.
|
|
5277
|
-
await abortRunDurably(runId, reason);
|
|
5278
5291
|
if (USER_STOP_ABORT_REASONS.has(reason)) {
|
|
5279
|
-
await
|
|
5292
|
+
await abortRunAndTurnDurably(runId, reason);
|
|
5293
|
+
}
|
|
5294
|
+
else {
|
|
5295
|
+
await abortRunDurably(runId, reason);
|
|
5280
5296
|
}
|
|
5281
5297
|
return { ok: true };
|
|
5282
5298
|
}
|
|
@@ -44,6 +44,9 @@ export interface AgentTask {
|
|
|
44
44
|
terminalEffectsVersion?: 1;
|
|
45
45
|
terminalEffectsReconciled?: boolean;
|
|
46
46
|
terminalProgressStatus?: TerminalProgressStatus;
|
|
47
|
+
stopRequestedAt?: number;
|
|
48
|
+
stopRequestedRunId?: string;
|
|
49
|
+
stopRequestedReason?: string;
|
|
47
50
|
}
|
|
48
51
|
export interface AgentTeamOwnerScope {
|
|
49
52
|
ownerEmail: string | null;
|