@agent-native/core 0.220.0-nightly-20261024033422 → 0.220.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/a2a/handlers.js +7 -6
- package/dist/a2a/types.d.ts +1 -0
- package/dist/agent/engine/tool-call-journal-seed.js +24 -20
- package/dist/agent/execution-fence.d.ts +1 -1
- package/dist/agent/execution-fence.js +8 -1
- package/dist/agent/production-agent.js +31 -6
- package/dist/agent/reaped-run-thread.js +38 -3
- package/dist/agent/run-migrations.js +22 -0
- package/dist/agent/run-store.d.ts +19 -2
- package/dist/agent/run-store.fencing.worker.js +21 -1
- package/dist/agent/run-store.js +359 -35
- package/dist/agent/tool-approval-store.d.ts +13 -0
- package/dist/agent/tool-approval-store.js +64 -0
- package/dist/automation/index.d.ts +1 -1
- package/dist/client/background-agent-session.d.ts +4 -0
- package/dist/client/background-agent-session.js +17 -1
- package/dist/client/chat/runtime.js +38 -6
- package/dist/collab/routes.d.ts +2 -2
- package/dist/db/schema-snapshot.generated.js +14 -1
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/jobs/automation-recovery.d.ts +4 -0
- package/dist/jobs/automation-recovery.js +7 -0
- package/dist/jobs/background-automation-runner.js +19 -2
- package/dist/jobs/scheduler.js +3 -1
- package/dist/mcp/org-directory.d.ts +8 -7
- package/dist/mcp/org-directory.js +82 -1
- package/dist/notifications/routes.d.ts +1 -1
- package/dist/observability/routes.d.ts +7 -7
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +2 -2
- package/dist/secrets/routes.d.ts +10 -10
- package/dist/server/agent-chat-plugin.d.ts +10 -1
- package/dist/server/agent-chat-plugin.js +178 -11
- package/dist/server/agent-discovery.js +11 -49
- package/dist/server/agent-engine-default-model-route.d.ts +1 -1
- package/dist/server/agent-engine-ollama-models-route.d.ts +2 -2
- package/dist/usage/store.js +36 -33
- package/package.json +4 -4
package/dist/a2a/handlers.js
CHANGED
|
@@ -264,7 +264,7 @@ export async function processA2ATaskFromQueue(taskId, config, event) {
|
|
|
264
264
|
}, A2A_PROCESSING_HEARTBEAT_MS);
|
|
265
265
|
heartbeat.unref?.();
|
|
266
266
|
try {
|
|
267
|
-
await processClaimedA2ATask(taskId, claimed, config, workerAbort.signal, event);
|
|
267
|
+
await processClaimedA2ATask(taskId, claimed, config, workerAbort.signal, event, fence.generation);
|
|
268
268
|
}
|
|
269
269
|
finally {
|
|
270
270
|
clearInterval(heartbeat);
|
|
@@ -285,7 +285,7 @@ export async function processA2ATaskFromQueue(taskId, config, event) {
|
|
|
285
285
|
}
|
|
286
286
|
}
|
|
287
287
|
}
|
|
288
|
-
async function processClaimedA2ATask(taskId, claimed, config, signal, event) {
|
|
288
|
+
async function processClaimedA2ATask(taskId, claimed, config, signal, event, processingAttemptId) {
|
|
289
289
|
const message = claimed.history?.[0];
|
|
290
290
|
if (!message) {
|
|
291
291
|
await settleProcessingA2ATask(taskId, {
|
|
@@ -378,7 +378,7 @@ async function processClaimedA2ATask(taskId, claimed, config, signal, event) {
|
|
|
378
378
|
? { orgScope: "personal" }
|
|
379
379
|
: {}),
|
|
380
380
|
...(requestOrigin ? { requestOrigin } : {}),
|
|
381
|
-
}, () => runHandlerAndPersist(taskId, message, config, contextId, callerMetadata, event, sourceContext, verifiedEmail, signal));
|
|
381
|
+
}, () => runHandlerAndPersist(taskId, message, config, contextId, callerMetadata, event, sourceContext, verifiedEmail, signal, processingAttemptId));
|
|
382
382
|
}
|
|
383
383
|
catch (err) {
|
|
384
384
|
try {
|
|
@@ -471,11 +471,12 @@ function jsonRpcError(id, code, message) {
|
|
|
471
471
|
function jsonRpcResult(id, result) {
|
|
472
472
|
return { jsonrpc: "2.0", id, result };
|
|
473
473
|
}
|
|
474
|
-
function makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal) {
|
|
474
|
+
function makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId) {
|
|
475
475
|
const artifacts = [];
|
|
476
476
|
const pendingFileArtifacts = [];
|
|
477
477
|
const context = {
|
|
478
478
|
taskId,
|
|
479
|
+
processingAttemptId,
|
|
479
480
|
contextId,
|
|
480
481
|
metadata,
|
|
481
482
|
event,
|
|
@@ -573,8 +574,8 @@ async function withA2ARequestContext(metadata, event, fn) {
|
|
|
573
574
|
...(requestOrigin ? { requestOrigin } : {}),
|
|
574
575
|
}, fn);
|
|
575
576
|
}
|
|
576
|
-
async function runHandlerAndPersist(taskId, message, config, contextId, metadata, event, sourceContext, ownerEmail, signal) {
|
|
577
|
-
const { context, artifacts, persistFileArtifacts } = makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal);
|
|
577
|
+
async function runHandlerAndPersist(taskId, message, config, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId) {
|
|
578
|
+
const { context, artifacts, persistFileArtifacts } = makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId);
|
|
578
579
|
try {
|
|
579
580
|
signal?.throwIfAborted();
|
|
580
581
|
const result = getHandler(config)(message, context);
|
package/dist/a2a/types.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { isArtifactReceipt, } from "../../artifacts/detect.js";
|
|
2
|
-
import { getCompletedAgentExecutionSteps, getCurrentTurnEventsForThread, } from "../run-store.js";
|
|
2
|
+
import { AgentRunJournalUnreadableError, getCompletedAgentExecutionSteps, getCurrentTurnEventsForThread, } from "../run-store.js";
|
|
3
3
|
import { classifyToolCallJournal, isConsistentToolCallInputFingerprint, toolCallInputFingerprint, } from "../tool-call-journal.js";
|
|
4
4
|
const LEDGER_READ_RETRY_MS = 250;
|
|
5
5
|
function isRecord(value) {
|
|
@@ -36,15 +36,15 @@ function durableStepFileMutation(value) {
|
|
|
36
36
|
}
|
|
37
37
|
return { path: value.path, contentSha256: value.contentSha256 };
|
|
38
38
|
}
|
|
39
|
-
function durableStepToolDone(event, row) {
|
|
39
|
+
function durableStepToolDone(event, row, unreadable) {
|
|
40
40
|
const executionStepId = event.executionStepId ?? event.id;
|
|
41
41
|
if (!executionStepId ||
|
|
42
42
|
executionStepId !== row.stepId ||
|
|
43
43
|
row.operationId !== row.stepId) {
|
|
44
|
-
|
|
44
|
+
return unreadable(`Completed execution step ${row.stepId} does not match its ledger identity`);
|
|
45
45
|
}
|
|
46
46
|
if (event.tool !== row.toolName) {
|
|
47
|
-
|
|
47
|
+
return unreadable(`Completed execution step ${row.stepId} does not match ledger tool ${event.tool}`);
|
|
48
48
|
}
|
|
49
49
|
let input;
|
|
50
50
|
let outcome;
|
|
@@ -53,7 +53,7 @@ function durableStepToolDone(event, row) {
|
|
|
53
53
|
outcome = JSON.parse(row.resultJson);
|
|
54
54
|
}
|
|
55
55
|
catch (error) {
|
|
56
|
-
|
|
56
|
+
return unreadable(`Completed execution step ${row.stepId} has unreadable stored intent or result`, error);
|
|
57
57
|
}
|
|
58
58
|
if (!isRecord(input) ||
|
|
59
59
|
typeof input.argumentsHash !== "string" ||
|
|
@@ -63,11 +63,11 @@ function durableStepToolDone(event, row) {
|
|
|
63
63
|
typeof outcome.result !== "string" ||
|
|
64
64
|
typeof outcome.isError !== "boolean" ||
|
|
65
65
|
typeof outcome.completedSideEffect !== "boolean") {
|
|
66
|
-
|
|
66
|
+
return unreadable(`Completed execution step ${row.stepId} is missing replay identity or outcome fields`);
|
|
67
67
|
}
|
|
68
68
|
if (outcome.errorCode !== undefined &&
|
|
69
69
|
typeof outcome.errorCode !== "string") {
|
|
70
|
-
|
|
70
|
+
return unreadable(`Completed execution step ${row.stepId} has an invalid error code`);
|
|
71
71
|
}
|
|
72
72
|
const artifacts = Array.isArray(outcome.artifacts)
|
|
73
73
|
? outcome.artifacts.filter(isArtifactReceipt)
|
|
@@ -129,56 +129,60 @@ export async function overlayCompletedExecutionStepOutcomes(threadId, turnId, ev
|
|
|
129
129
|
const replacedIndexes = new Set();
|
|
130
130
|
const replacementsAt = new Map();
|
|
131
131
|
const replacementsAfter = new Map();
|
|
132
|
-
for (const row of rows) {
|
|
132
|
+
for (const [rowIndex, row] of rows.entries()) {
|
|
133
|
+
const unreadable = (message, cause) => {
|
|
134
|
+
const detail = new Error(message, cause === undefined ? undefined : { cause });
|
|
135
|
+
throw new AgentRunJournalUnreadableError(threadId, turnId, rowIndex, "invalid_execution_step", { cause: detail });
|
|
136
|
+
};
|
|
133
137
|
if (!candidateIds.includes(row.stepId)) {
|
|
134
|
-
|
|
138
|
+
return unreadable(`Completed execution step ${row.stepId} was not requested by the ledger`);
|
|
135
139
|
}
|
|
136
140
|
if (rowsById.has(row.stepId)) {
|
|
137
|
-
|
|
141
|
+
return unreadable(`Completed execution step ${row.stepId} has ambiguous stored rows`);
|
|
138
142
|
}
|
|
139
143
|
const starts = startsById.get(row.stepId);
|
|
140
144
|
if (!starts || starts.length === 0) {
|
|
141
|
-
|
|
145
|
+
return unreadable(`Completed execution step ${row.stepId} has no matching ledger call`);
|
|
142
146
|
}
|
|
143
147
|
const canonicalStart = starts[0].event;
|
|
144
|
-
const canonicalDone = durableStepToolDone(canonicalStart, row);
|
|
148
|
+
const canonicalDone = durableStepToolDone(canonicalStart, row, unreadable);
|
|
145
149
|
if (!isConsistentToolCallInputFingerprint(canonicalStart.input, canonicalStart.inputFingerprint, canonicalStart.inputStoredFingerprint)) {
|
|
146
|
-
|
|
150
|
+
return unreadable(`Completed execution step ${row.stepId} has unreadable ledger call identity`);
|
|
147
151
|
}
|
|
148
152
|
const canonicalFingerprint = getToolCallEventFingerprint(canonicalStart);
|
|
149
153
|
for (const { event } of starts.slice(1)) {
|
|
150
154
|
if (event.tool !== canonicalStart.tool ||
|
|
151
155
|
!isConsistentToolCallInputFingerprint(event.input, event.inputFingerprint, event.inputStoredFingerprint) ||
|
|
152
156
|
getToolCallEventFingerprint(event) !== canonicalFingerprint) {
|
|
153
|
-
|
|
157
|
+
return unreadable(`Completed execution step ${row.stepId} has conflicting ledger call identities`);
|
|
154
158
|
}
|
|
155
159
|
}
|
|
156
160
|
const dones = donesById.get(row.stepId) ?? [];
|
|
157
161
|
for (const { event } of dones) {
|
|
158
162
|
if (event.tool !== row.toolName ||
|
|
159
163
|
!isConsistentToolCallInputFingerprint(event.input, event.inputFingerprint, event.inputStoredFingerprint)) {
|
|
160
|
-
|
|
164
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done identity`);
|
|
161
165
|
}
|
|
162
166
|
const doneFingerprint = getToolCallEventFingerprint(event);
|
|
163
167
|
if (canonicalFingerprint !== undefined &&
|
|
164
168
|
doneFingerprint !== undefined &&
|
|
165
169
|
doneFingerprint !== canonicalFingerprint) {
|
|
166
|
-
|
|
170
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done input`);
|
|
167
171
|
}
|
|
168
172
|
if (event.argumentsHash !== undefined &&
|
|
169
173
|
event.argumentsHash !== canonicalDone.argumentsHash) {
|
|
170
|
-
|
|
174
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done arguments`);
|
|
171
175
|
}
|
|
172
176
|
if (event.completedSideEffect !== undefined &&
|
|
173
177
|
event.completedSideEffect !== canonicalDone.completedSideEffect) {
|
|
174
|
-
|
|
178
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done side-effect outcome`);
|
|
175
179
|
}
|
|
176
180
|
if (event.isError === true && canonicalDone.isError !== true) {
|
|
177
|
-
|
|
181
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done error outcome`);
|
|
178
182
|
}
|
|
179
183
|
if (event.errorCode !== undefined &&
|
|
180
184
|
event.errorCode !== canonicalDone.errorCode) {
|
|
181
|
-
|
|
185
|
+
return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done error code`);
|
|
182
186
|
}
|
|
183
187
|
}
|
|
184
188
|
rowsById.set(row.stepId, row);
|
|
@@ -24,7 +24,7 @@ export declare function currentExecutionFenceForScope(scope: {
|
|
|
24
24
|
export declare function withExecutionLeaseRequested<T>(work: () => T): T;
|
|
25
25
|
export declare function withExecutionFence<T>(fence: ExecutionFence, work: () => T): T;
|
|
26
26
|
export declare function runWithoutExecutionFence<T>(work: () => T): T;
|
|
27
|
-
export declare function lockAndAssertCurrentExecutionFences(tx: DbExec): Promise<void>;
|
|
27
|
+
export declare function lockAndAssertCurrentExecutionFences(tx: DbExec, ignoredFence?: ExecutionFence): Promise<void>;
|
|
28
28
|
export declare function withFencedExecutionWrite<T>(write: (tx: DbExec) => Promise<T>, scope?: {
|
|
29
29
|
runId?: string;
|
|
30
30
|
threadId?: string;
|
|
@@ -66,9 +66,16 @@ export function runWithoutExecutionFence(work) {
|
|
|
66
66
|
// now-expired lease through AsyncLocalStorage.
|
|
67
67
|
return executionFenceStorage.run({ fences: [], executionLeaseRequested: false }, work);
|
|
68
68
|
}
|
|
69
|
-
export async function lockAndAssertCurrentExecutionFences(tx) {
|
|
69
|
+
export async function lockAndAssertCurrentExecutionFences(tx, ignoredFence) {
|
|
70
70
|
const fences = executionFenceStorage.getStore()?.fences ?? [];
|
|
71
71
|
for (const activeFence of fences) {
|
|
72
|
+
if (ignoredFence &&
|
|
73
|
+
activeFence.threadId === ignoredFence.threadId &&
|
|
74
|
+
activeFence.turnId === ignoredFence.turnId &&
|
|
75
|
+
activeFence.runId === ignoredFence.runId &&
|
|
76
|
+
activeFence.generation === ignoredFence.generation) {
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
72
79
|
const { rows } = await tx.execute({
|
|
73
80
|
sql: `SELECT generation
|
|
74
81
|
FROM agent_execution_leases
|
|
@@ -89,13 +89,13 @@ import { maybeCompactThread, buildObservationalContext, hasObservationalMemory,
|
|
|
89
89
|
import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js";
|
|
90
90
|
import { applyUncheckedDefaultModelReplacement } from "./provider-model-selection.js";
|
|
91
91
|
import { startRun, subscribeToRun, replayCompletedTurn, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, getSlotHoldingRunId, tryClaimRunSlot, isHostedRuntime, resolveRunSoftTimeoutMs, resolveRunToolTimeoutCeilingMs, endsAfterToolResultWithoutAssistantFinal, endsDuringActionPreparation, } from "./run-manager.js";
|
|
92
|
-
import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
|
|
92
|
+
import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, resetAgentExecutionStepBeforeInvocation, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
|
|
93
93
|
import { setRunTurnIdHeader } from "./run-turn-id-header.js";
|
|
94
94
|
import { buildCurrentTimeUserContext } from "./runtime-context.js";
|
|
95
95
|
import { claimSetupResume, setupResumeRefusedRunId, } from "./setup-resume-claim.js";
|
|
96
96
|
import { getStrictToolInputValidator } from "./strict-tool-input.js";
|
|
97
97
|
import { normalizeSubmissionDeadlineMetadata } from "./submission-deadline.js";
|
|
98
|
-
import { consumeAgentToolApproval, createAgentToolApproval, isAgentToolAlwaysAllowed, resolveAgentToolApprovalTurnId, } from "./tool-approval-store.js";
|
|
98
|
+
import { consumeAgentToolApproval, createAgentToolApproval, ensureAgentToolApprovalTable, hasPendingAgentToolApproval, isAgentToolAlwaysAllowed, resolveAgentToolApprovalTurnId, } from "./tool-approval-store.js";
|
|
99
99
|
import { toolCallInputSignature } from "./tool-call-input-signature.js";
|
|
100
100
|
import { ambiguousLegacyCallEffectKey, findAmbiguousLegacyJournalEntryForCall, findInterruptedJournalEntryForCall, findInterruptedLegacyJournalEntryForCall, findReplayableJournalEntry, isUnavailableToolCallInput, journalEntriesAmbiguousInHistory, journalEntriesRepresentedInHistory, journalCallEffects, journalCallIdentities, matchToolCallResultsInHistory, toolCallIdentityKey, toolCallInputFingerprint, unmatchedAmbiguousLegacyToolCallResultsInHistory, } from "./tool-call-journal.js";
|
|
101
101
|
import { redactSensitiveFields, sanitizeToolErrorText, sanitizeToolErrorValue, } from "./tool-error-redaction.js";
|
|
@@ -6150,6 +6150,8 @@ export async function runAgentLoop(opts) {
|
|
|
6150
6150
|
let toolDoneEmitted = false;
|
|
6151
6151
|
let toolDonePersistenceFailed = false;
|
|
6152
6152
|
let executionStepFinalizationAttempted = false;
|
|
6153
|
+
let executionStepDispatched = false;
|
|
6154
|
+
let executionStepDispatchWrite;
|
|
6153
6155
|
let executionStepOutcomeRecorded = false;
|
|
6154
6156
|
let actionInvoked = false;
|
|
6155
6157
|
// Live once invoked, so an abandoned call still reads what ran in it.
|
|
@@ -7038,6 +7040,7 @@ export async function runAgentLoop(opts) {
|
|
|
7038
7040
|
if (cacheKey && cachedResult) {
|
|
7039
7041
|
if (executionStepIntent) {
|
|
7040
7042
|
await markAgentExecutionStepDispatched(executionStepIntent);
|
|
7043
|
+
executionStepDispatched = true;
|
|
7041
7044
|
}
|
|
7042
7045
|
const previousResult = cachedResult.content;
|
|
7043
7046
|
const visible = isCachedToolResultVisibleInContext(contextMessages, toolCall, previousResult);
|
|
@@ -7165,7 +7168,10 @@ export async function runAgentLoop(opts) {
|
|
|
7165
7168
|
throw new Error("Run aborted");
|
|
7166
7169
|
}
|
|
7167
7170
|
if (executionStepIntent) {
|
|
7168
|
-
|
|
7171
|
+
executionStepDispatchWrite = markAgentExecutionStepDispatched(executionStepIntent).then(() => {
|
|
7172
|
+
executionStepDispatched = true;
|
|
7173
|
+
});
|
|
7174
|
+
await executionStepDispatchWrite;
|
|
7169
7175
|
}
|
|
7170
7176
|
if (timeoutSignal.aborted) {
|
|
7171
7177
|
throw new Error(timeoutMessage());
|
|
@@ -7432,7 +7438,7 @@ export async function runAgentLoop(opts) {
|
|
|
7432
7438
|
thrown = err;
|
|
7433
7439
|
toolReportedError = false;
|
|
7434
7440
|
toolResultLost =
|
|
7435
|
-
signal.aborted ||
|
|
7441
|
+
(signal.aborted && actionInvoked) ||
|
|
7436
7442
|
isToolCallTimeoutResult(String(err?.message ?? err));
|
|
7437
7443
|
toolErrorCode = isActionContractError(err)
|
|
7438
7444
|
? err.errorCode
|
|
@@ -7511,9 +7517,16 @@ export async function runAgentLoop(opts) {
|
|
|
7511
7517
|
console.warn("Could not notify the action-change poller after a tool call", error);
|
|
7512
7518
|
}
|
|
7513
7519
|
}
|
|
7514
|
-
if (executionStepIntent
|
|
7520
|
+
if (executionStepIntent &&
|
|
7521
|
+
(executionStepDispatched || executionStepDispatchWrite)) {
|
|
7515
7522
|
executionStepFinalizationAttempted = true;
|
|
7516
|
-
if (
|
|
7523
|
+
if (!actionInvoked) {
|
|
7524
|
+
// The dispatch marker can persist while cancellation wins; it does
|
|
7525
|
+
// not prove the action handler started.
|
|
7526
|
+
await executionStepDispatchWrite?.catch(() => { });
|
|
7527
|
+
await resetAgentExecutionStepBeforeInvocation(executionStepIntent);
|
|
7528
|
+
}
|
|
7529
|
+
else if (toolResultLost || callEffect === "unknown") {
|
|
7517
7530
|
await markAgentExecutionStepUnknown(executionStepIntent);
|
|
7518
7531
|
}
|
|
7519
7532
|
else {
|
|
@@ -11047,6 +11060,9 @@ export function createProductionAgentHandler(options) {
|
|
|
11047
11060
|
const queuedSubmissionStore = shouldPreserveQueuedOrder
|
|
11048
11061
|
? await import("../chat-threads/store.js")
|
|
11049
11062
|
: undefined;
|
|
11063
|
+
if (ownerEmail && !isAuthorizedApprovalContinuation) {
|
|
11064
|
+
await ensureAgentToolApprovalTable();
|
|
11065
|
+
}
|
|
11050
11066
|
// The inbox read runs under the slot lock, but its schema must be ready first.
|
|
11051
11067
|
await queuedSubmissionStore?.ensureChatThreadTables();
|
|
11052
11068
|
slot = await tryClaimRunSlot(threadId, runId, undefined, {
|
|
@@ -11069,6 +11085,15 @@ export function createProductionAgentHandler(options) {
|
|
|
11069
11085
|
hasPendingQueuedSubmission: (tx) => queuedSubmissionStore.hasPendingQueuedSubmissionReceipt(tx, runThreadId),
|
|
11070
11086
|
}
|
|
11071
11087
|
: {}),
|
|
11088
|
+
...(ownerEmail && !isAuthorizedApprovalContinuation
|
|
11089
|
+
? {
|
|
11090
|
+
hasPendingApproval: (tx) => hasPendingAgentToolApproval(tx, {
|
|
11091
|
+
ownerEmail,
|
|
11092
|
+
orgId: getRequestOrgId() ?? null,
|
|
11093
|
+
threadId: runThreadId,
|
|
11094
|
+
}),
|
|
11095
|
+
}
|
|
11096
|
+
: {}),
|
|
11072
11097
|
});
|
|
11073
11098
|
}
|
|
11074
11099
|
catch (error) {
|
|
@@ -1,7 +1,42 @@
|
|
|
1
1
|
import { getThread, updateThreadData, withThreadDataLock, } from "../chat-threads/store.js";
|
|
2
2
|
import { RUN_NOT_STARTED_METADATA_KEY } from "../shared/agent-chat-run-not-started.js";
|
|
3
|
-
import { readStoppedRunForThreadFold } from "./run-store.js";
|
|
3
|
+
import { AgentRunJournalUnreadableError, readStoppedRunForThreadFold, } from "./run-store.js";
|
|
4
4
|
import { buildAssistantMessage, extractThreadMeta, foldAssistantTurn, isRunPrompt, normalizeThreadRepository, } from "./thread-data-builder.js";
|
|
5
|
+
function readThreadRepository(threadId, turnId, threadData) {
|
|
6
|
+
let repository;
|
|
7
|
+
try {
|
|
8
|
+
repository = JSON.parse(threadData || "{}");
|
|
9
|
+
}
|
|
10
|
+
catch (cause) {
|
|
11
|
+
throw new AgentRunJournalUnreadableError(threadId, turnId, 0, "invalid_thread_data", { cause });
|
|
12
|
+
}
|
|
13
|
+
const unreadable = (message) => new AgentRunJournalUnreadableError(threadId, turnId, 0, "invalid_thread_data", { cause: new Error(message) });
|
|
14
|
+
if (typeof repository !== "object" ||
|
|
15
|
+
repository === null ||
|
|
16
|
+
Array.isArray(repository)) {
|
|
17
|
+
throw unreadable("Stored thread data is not a repository object");
|
|
18
|
+
}
|
|
19
|
+
const messages = repository.messages;
|
|
20
|
+
if (messages !== undefined &&
|
|
21
|
+
(!Array.isArray(messages) ||
|
|
22
|
+
!messages.every((entry) => {
|
|
23
|
+
if (typeof entry !== "object" ||
|
|
24
|
+
entry === null ||
|
|
25
|
+
Array.isArray(entry)) {
|
|
26
|
+
return false;
|
|
27
|
+
}
|
|
28
|
+
const storedMessage = entry.message;
|
|
29
|
+
const message = storedMessage ?? entry;
|
|
30
|
+
return (typeof message === "object" &&
|
|
31
|
+
message !== null &&
|
|
32
|
+
!Array.isArray(message) &&
|
|
33
|
+
typeof message.id === "string" &&
|
|
34
|
+
message.id.length > 0);
|
|
35
|
+
}))) {
|
|
36
|
+
throw unreadable("Stored thread data has malformed messages");
|
|
37
|
+
}
|
|
38
|
+
return normalizeThreadRepository(repository);
|
|
39
|
+
}
|
|
5
40
|
/**
|
|
6
41
|
* Saves the reply of a run the server ended because its worker stopped
|
|
7
42
|
* (a crash, a killed function) into the thread, the way a run that finishes
|
|
@@ -51,7 +86,7 @@ export async function foldReapedRunIntoThread(runId, terminalError) {
|
|
|
51
86
|
const thread = await getThread(run.threadId);
|
|
52
87
|
if (!thread)
|
|
53
88
|
return "superseded";
|
|
54
|
-
const repo =
|
|
89
|
+
const repo = readThreadRepository(run.threadId, run.turnId, thread.threadData);
|
|
55
90
|
const folded = foldAssistantTurn(repo, assistant, {
|
|
56
91
|
runId,
|
|
57
92
|
turnId: run.turnId,
|
|
@@ -88,7 +123,7 @@ export async function foldReapedRunIntoThread(runId, terminalError) {
|
|
|
88
123
|
const currentThreadData = rows[0]?.thread_data;
|
|
89
124
|
if (typeof currentThreadData !== "string")
|
|
90
125
|
return false;
|
|
91
|
-
const currentRepo =
|
|
126
|
+
const currentRepo = readThreadRepository(run.threadId, run.turnId, currentThreadData);
|
|
92
127
|
const last = currentRepo.messages.at(-1)?.message ?? currentRepo.messages.at(-1);
|
|
93
128
|
const ownsPrompt = last?.role === "user"
|
|
94
129
|
? isRunPrompt(last, { runId, ...run })
|
|
@@ -139,4 +139,26 @@ export const AGENT_RUN_MIGRATIONS = [
|
|
|
139
139
|
WHERE thread_projection_pending IS TRUE
|
|
140
140
|
`,
|
|
141
141
|
},
|
|
142
|
+
{
|
|
143
|
+
version: 10,
|
|
144
|
+
name: "agent-run-thread-projection-retry-fairness",
|
|
145
|
+
sql: `
|
|
146
|
+
CREATE SEQUENCE IF NOT EXISTS agent_run_projection_retry_order_seq;
|
|
147
|
+
ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS thread_projection_retry_order BIGINT;
|
|
148
|
+
UPDATE agent_runs
|
|
149
|
+
SET thread_projection_retry_order = nextval('agent_run_projection_retry_order_seq')
|
|
150
|
+
WHERE thread_projection_pending IS TRUE
|
|
151
|
+
AND thread_projection_retry_order IS NULL;
|
|
152
|
+
CREATE INDEX IF NOT EXISTS agent_runs_thread_projection_retry_order_idx
|
|
153
|
+
ON agent_runs (thread_projection_retry_order ASC NULLS FIRST, completed_at ASC, id ASC)
|
|
154
|
+
WHERE thread_projection_pending IS TRUE
|
|
155
|
+
`,
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
version: 11,
|
|
159
|
+
name: "agent-run-thread-projection-quarantine",
|
|
160
|
+
sql: `
|
|
161
|
+
ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS thread_projection_error_code TEXT
|
|
162
|
+
`,
|
|
163
|
+
},
|
|
142
164
|
];
|
|
@@ -149,6 +149,7 @@ export type AgentExecutionStepAdmission = {
|
|
|
149
149
|
/** Persist intent before dispatch and decide whether an interrupted call may run again. */
|
|
150
150
|
export declare function beginAgentExecutionStep(intent: AgentExecutionStepIntent): Promise<AgentExecutionStepAdmission>;
|
|
151
151
|
export declare function markAgentExecutionStepDispatched(intent: AgentExecutionStepIntent): Promise<void>;
|
|
152
|
+
export declare function resetAgentExecutionStepBeforeInvocation(intent: AgentExecutionStepIntent): Promise<void>;
|
|
152
153
|
export declare function completeAgentExecutionStep(intent: AgentExecutionStepIntent, resultJson: string, errorJson?: string): Promise<void>;
|
|
153
154
|
export declare function markAgentExecutionStepUnknown(intent: AgentExecutionStepIntent): Promise<void>;
|
|
154
155
|
/** A detached action may attach late evidence only while its lease is current. */
|
|
@@ -265,11 +266,14 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
|
|
|
265
266
|
};
|
|
266
267
|
/** Defer a fresh turn while an earlier accepted inbox submission is queued. */
|
|
267
268
|
hasPendingQueuedSubmission?: (tx: DbExec) => Promise<boolean>;
|
|
269
|
+
/** Keep ordinary turns behind a pending tool approval on this thread. */
|
|
270
|
+
hasPendingApproval?: (tx: DbExec) => Promise<boolean>;
|
|
268
271
|
afterInsert?: (tx: DbExec) => Promise<void>;
|
|
269
272
|
}): Promise<{
|
|
270
273
|
claimed: boolean;
|
|
271
274
|
activeRunId: string | null;
|
|
272
275
|
queuedSubmissionPending?: boolean;
|
|
276
|
+
approvalPending?: boolean;
|
|
273
277
|
completedRunId?: string;
|
|
274
278
|
turnAborted?: boolean;
|
|
275
279
|
continueRefused?: ContinueRefusalCode;
|
|
@@ -411,6 +415,7 @@ export declare function getRunById(runId: string): Promise<{
|
|
|
411
415
|
errorCode: string | null;
|
|
412
416
|
errorDetail: string | null;
|
|
413
417
|
terminalReason: string | null;
|
|
418
|
+
threadProjectionErrorCode: string | null;
|
|
414
419
|
} | null>;
|
|
415
420
|
/**
|
|
416
421
|
* What saving a run the server ended on its behalf needs: the run's ledger,
|
|
@@ -469,6 +474,7 @@ export interface AgentRunSummary {
|
|
|
469
474
|
dispatchMode: string | null;
|
|
470
475
|
terminalReason: string | null;
|
|
471
476
|
diagStage: string | null;
|
|
477
|
+
threadProjectionErrorCode: string | null;
|
|
472
478
|
inFlightSince: number | null;
|
|
473
479
|
}
|
|
474
480
|
export declare function listRunsForThread(threadId: string, options?: {
|
|
@@ -484,9 +490,9 @@ export declare class AgentRunJournalUnreadableError extends Error {
|
|
|
484
490
|
readonly threadId: string;
|
|
485
491
|
readonly turnId: string | undefined;
|
|
486
492
|
readonly rowIndex: number;
|
|
487
|
-
readonly reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event";
|
|
493
|
+
readonly reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event" | "invalid_thread_data" | "invalid_execution_step";
|
|
488
494
|
readonly errorCode = "tool_call_journal_unreadable";
|
|
489
|
-
constructor(threadId: string, turnId: string | undefined, rowIndex: number, reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event", options?: ErrorOptions);
|
|
495
|
+
constructor(threadId: string, turnId: string | undefined, rowIndex: number, reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event" | "invalid_thread_data" | "invalid_execution_step", options?: ErrorOptions);
|
|
490
496
|
}
|
|
491
497
|
export declare function getCurrentTurnRunEventsForThread(threadId: string, knownTurnId?: string): Promise<CurrentTurnRunEvent[]>;
|
|
492
498
|
export declare function getCurrentTurnRunIdsForThread(threadId: string, turnId: string): Promise<string[]>;
|
|
@@ -522,3 +528,14 @@ export declare function listErroredRuns(options?: {
|
|
|
522
528
|
durationMs: number | null;
|
|
523
529
|
}>>;
|
|
524
530
|
export declare function ensureTerminalRunEvent(runId: string, event: Record<string, unknown>): Promise<void>;
|
|
531
|
+
export type RunThreadProjectionRetry = {
|
|
532
|
+
state: "untracked";
|
|
533
|
+
} | {
|
|
534
|
+
state: "already_projected";
|
|
535
|
+
} | {
|
|
536
|
+
state: "projected";
|
|
537
|
+
} | {
|
|
538
|
+
state: "deferred";
|
|
539
|
+
};
|
|
540
|
+
/** Retry after repairing persisted source data; until success, the marker keeps later same-turn runs gated. */
|
|
541
|
+
export declare function retryRunThreadProjection(runId: string): Promise<RunThreadProjectionRetry>;
|
|
@@ -6,7 +6,7 @@ import { toPostgresParams, withDbExec } from "../db/client.js";
|
|
|
6
6
|
import { withMigrationRuntime } from "../db/migration-runtime.js";
|
|
7
7
|
import { withExecutionFence } from "./execution-fence.js";
|
|
8
8
|
import { abortRunAtSafeBoundaryDurably } from "./run-manager.js";
|
|
9
|
-
import { beginAgentExecutionStep, claimTurnExecutionLease, completeAgentExecutionStep, ensureRunTables, insertRun, insertRunEvent, markAgentExecutionStepDispatched, } from "./run-store.js";
|
|
9
|
+
import { beginAgentExecutionStep, claimTurnExecutionLease, completeAgentExecutionStep, ensureRunTables, insertRun, insertRunEvent, markAgentExecutionStepDispatched, persistRunCheckpointEvent, reapIfStale, } from "./run-store.js";
|
|
10
10
|
import { applySubmittedUserMessage, buildUserMessage, extractThreadMeta, } from "./thread-data-builder.js";
|
|
11
11
|
function executor(sql) {
|
|
12
12
|
return {
|
|
@@ -115,6 +115,26 @@ async function main() {
|
|
|
115
115
|
}));
|
|
116
116
|
return;
|
|
117
117
|
}
|
|
118
|
+
if (mode === "checkpoint-crash") {
|
|
119
|
+
const payload = JSON.parse(extra ?? "null");
|
|
120
|
+
await insertRun(runId, threadId, turnId, {
|
|
121
|
+
dispatchMode: "background",
|
|
122
|
+
dispatchPayload: JSON.stringify(payload),
|
|
123
|
+
});
|
|
124
|
+
const fence = await claimTurnExecutionLease(threadId, turnId, runId);
|
|
125
|
+
await withExecutionFence(fence, async () => {
|
|
126
|
+
await insertRunEvent(runId, 0, JSON.stringify({ type: "text", text: "Completed before timeout" }));
|
|
127
|
+
await persistRunCheckpointEvent(runId, { type: "auto_continue", reason: "run_timeout" }, "run_timeout");
|
|
128
|
+
});
|
|
129
|
+
writeSync(1, JSON.stringify({ status: "checkpointed", fence }));
|
|
130
|
+
process.exit(83);
|
|
131
|
+
}
|
|
132
|
+
if (mode === "stale-reap") {
|
|
133
|
+
await withMigrationRuntime(() => ensureChatThreadTables());
|
|
134
|
+
const reaped = await reapIfStale(runId);
|
|
135
|
+
writeSync(1, JSON.stringify({ reaped }));
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
118
138
|
if (mode === "durable-abort") {
|
|
119
139
|
const [{ pid }] = await sql `SELECT pg_backend_pid() AS pid`;
|
|
120
140
|
databaseBackendPid = Number(pid);
|