@agent-native/core 0.218.0 → 0.220.0-nightly-20261023183804
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/a2a/server.d.ts +1 -1
- package/dist/a2a/server.js +8 -6
- package/dist/agent/execution-fence.js +5 -1
- package/dist/agent/production-agent.d.ts +1 -1
- package/dist/agent/production-agent.js +42 -9
- package/dist/agent/run-manager.js +110 -47
- package/dist/agent/run-store.d.ts +1 -0
- package/dist/agent/run-store.js +260 -31
- package/dist/agent/run-terminal-tracking.d.ts +26 -0
- package/dist/agent/run-terminal-tracking.js +54 -0
- package/dist/agent/run-turn-id-header.d.ts +9 -0
- package/dist/agent/run-turn-id-header.js +13 -0
- package/dist/automation/index.d.ts +1 -1
- package/dist/chat-threads/store.d.ts +12 -6
- package/dist/chat-threads/store.js +35 -28
- package/dist/cli/amplify-stream.js +1 -1
- package/dist/client/agent-chat/index.d.ts +1 -0
- package/dist/client/agent-chat/index.js +1 -0
- package/dist/client/agent-chat.d.ts +2 -0
- package/dist/client/agent-chat.js +31 -5
- package/dist/client/chat/agentkit-agent-native.js +8 -1
- package/dist/client/chat/agentkit-protocol.js +7 -1
- package/dist/client/chat/run-outcome-telemetry.js +1 -0
- package/dist/client/chat/run-outcome.d.ts +5 -0
- package/dist/client/chat/runtime.d.ts +34 -1
- package/dist/client/chat/runtime.js +96 -12
- package/dist/client/chat/turn-submitted-telemetry.d.ts +4 -0
- package/dist/client/chat/turn-submitted-telemetry.js +39 -0
- package/dist/client/chat/turn-submitted.d.ts +39 -0
- package/dist/client/chat/turn-submitted.js +16 -0
- package/dist/client/route-chunk-recovery.d.ts +3 -8
- package/dist/client/route-chunk-recovery.js +11 -9
- package/dist/client/route-warmup.js +101 -3
- package/dist/collab/routes.d.ts +1 -1
- package/dist/db/client.d.ts +1 -0
- package/dist/db/client.js +28 -18
- package/dist/jobs/actions/manage-recurring-job.d.ts +1 -1
- package/dist/mcp/embed-app.js +2 -1
- package/dist/mcp/screen-memory-stdio.d.ts +7 -7
- package/dist/notifications/routes.d.ts +4 -4
- package/dist/observability/routes.d.ts +7 -7
- package/dist/observability/traces.d.ts +2 -0
- package/dist/observability/traces.js +4 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +16 -16
- package/dist/provider-api/actions/provider-api.d.ts +13 -13
- package/dist/provider-api/corpus-jobs.d.ts +2 -2
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/review/index.d.ts +1 -1
- package/dist/review/index.js +1 -1
- package/dist/review/suggestions/actions.js +25 -17
- package/dist/review/suggestions/store.d.ts +14 -1
- package/dist/review/suggestions/store.js +16 -3
- package/dist/review/suggestions/types.d.ts +15 -1
- package/dist/secrets/routes.d.ts +10 -10
- package/dist/server/agent-chat/browser-team-tools.js +39 -6
- package/dist/server/agent-chat/script-entries.js +10 -4
- package/dist/server/agent-chat-plugin.d.ts +2 -0
- package/dist/server/agent-chat-plugin.js +38 -13
- package/dist/server/agent-engine-ollama-models-route.d.ts +1 -1
- package/dist/server/first-touch-route.d.ts +2 -2
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/shared/route-chunk-recovery-bootstrap.d.ts +9 -0
- package/dist/shared/route-chunk-recovery-bootstrap.js +25 -0
- package/dist/sharing/actions/list-resource-shares.d.ts +1 -1
- package/dist/templates/workspace-core/.agents/skills/external-agents/references/mcp-apps-embedding.md +12 -5
- package/dist/tracking/providers.js +3 -0
- package/dist/triggers/actions/manage-automation.d.ts +4 -4
- package/docs/content/external-agents.mdx +1 -1
- package/docs/content/mcp-apps.mdx +3 -3
- package/docs/content/mcp-protocol.mdx +1 -1
- package/docs/content/template-assets-integrations.mdx +5 -3
- package/package.json +4 -4
- package/src/templates/workspace-core/.agents/skills/external-agents/references/mcp-apps-embedding.md +12 -5
package/dist/a2a/server.d.ts
CHANGED
|
@@ -50,7 +50,7 @@ export declare function verifyA2AToken(token: string, event?: any, audienceOptio
|
|
|
50
50
|
allowAnthropicManagedContinuation?: boolean;
|
|
51
51
|
requireAudience?: boolean;
|
|
52
52
|
}): Promise<A2ATokenPayload>;
|
|
53
|
-
export type A2ATokenFailureCause = "malformed" | "secret-missing" | "audience-unresolvable" | "audience" | "issuer" | "expired" | "signature" | "invalid-claims" | "mcp-credential" | "missing-subject" | "org-binding" | "org-lookup-failed" | "undetermined";
|
|
53
|
+
export type A2ATokenFailureCause = "malformed" | "secret-missing" | "audience-unresolvable" | "audience" | "issuer" | "expired" | "signature" | "algorithm" | "invalid-claims" | "mcp-credential" | "missing-subject" | "org-binding" | "org-lookup-failed" | "undetermined";
|
|
54
54
|
/**
|
|
55
55
|
* Names why a token that `verifyA2AToken` resolved no identity for was
|
|
56
56
|
* refused. `verifyA2AToken` collapses a wrong audience, a wrong secret and an
|
package/dist/a2a/server.js
CHANGED
|
@@ -21,6 +21,7 @@ import { hasConfiguredA2ASecret, isA2AProductionRuntime, } from "./auth-policy.j
|
|
|
21
21
|
import { handleJsonRpcH3, processA2ATaskFromQueue } from "./handlers.js";
|
|
22
22
|
import { A2A_ORG_MEMBER_DELEGATION_TYPE, organizationPrincipalClaims, verifyA2AOrganizationIdentity, } from "./organization-identity.js";
|
|
23
23
|
import { claimA2AApproval, a2aTaskLeaseThreadId, getA2AApprovalForOwner, settleA2AApproval, } from "./task-store.js";
|
|
24
|
+
const A2A_TOKEN_ALGORITHMS = ["HS256"];
|
|
24
25
|
let _warnedUnauthA2A = false;
|
|
25
26
|
function warnA2AUnauthOnce() {
|
|
26
27
|
if (_warnedUnauthA2A)
|
|
@@ -196,6 +197,8 @@ async function checkAgainstSecret(token, secret, options) {
|
|
|
196
197
|
}
|
|
197
198
|
catch (error) {
|
|
198
199
|
const { code, claim } = error;
|
|
200
|
+
if (code === "ERR_JOSE_ALG_NOT_ALLOWED")
|
|
201
|
+
return { cause: "algorithm" };
|
|
199
202
|
if (code === "ERR_JWT_EXPIRED")
|
|
200
203
|
return { cause: "expired" };
|
|
201
204
|
if (code === "ERR_JWS_SIGNATURE_VERIFICATION_FAILED") {
|
|
@@ -243,7 +246,7 @@ export async function diagnoseA2ATokenFailure(token, event) {
|
|
|
243
246
|
return "malformed";
|
|
244
247
|
}
|
|
245
248
|
const secret = readDeployCredentialEnv("A2A_SECRET")?.trim();
|
|
246
|
-
const options = {};
|
|
249
|
+
const options = { algorithms: A2A_TOKEN_ALGORITHMS };
|
|
247
250
|
if (typeof claims.aud !== "undefined") {
|
|
248
251
|
const audience = expectedJwtAudience(event);
|
|
249
252
|
if (!audience)
|
|
@@ -331,7 +334,9 @@ async function verifyA2ATokenInternal(token, event, audienceOptions) {
|
|
|
331
334
|
if (audienceOptions?.requireAudience && !hasAudience) {
|
|
332
335
|
return { payload: { email: null, orgDomain: null } };
|
|
333
336
|
}
|
|
334
|
-
const verifyOptions = {
|
|
337
|
+
const verifyOptions = {
|
|
338
|
+
algorithms: A2A_TOKEN_ALGORITHMS,
|
|
339
|
+
};
|
|
335
340
|
if (hasAudience) {
|
|
336
341
|
// Fail closed: the token was minted for a specific audience, but this
|
|
337
342
|
// receiver can't derive its own expected audience (no APP_URL/URL and no
|
|
@@ -424,10 +429,7 @@ async function verifyA2ATokenInternal(token, event, audienceOptions) {
|
|
|
424
429
|
}
|
|
425
430
|
if (!organization)
|
|
426
431
|
return { payload: { email: null, orgDomain: null } };
|
|
427
|
-
const
|
|
428
|
-
? { ...verifyOptions, algorithms: ["HS256"] }
|
|
429
|
-
: verifyOptions;
|
|
430
|
-
const payload = await verifyJwtWithSecret(token, organization.secret, orgVerifyOptions);
|
|
432
|
+
const payload = await verifyJwtWithSecret(token, organization.secret, verifyOptions);
|
|
431
433
|
if (!payload)
|
|
432
434
|
return { payload: { email: null, orgDomain: null } };
|
|
433
435
|
const claims = organizationPrincipalClaims(payload, organization);
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
2
|
-
import { getDbExec } from "../db/client.js";
|
|
2
|
+
import { getDbExec, getScopedTransactionDbExec, } from "../db/client.js";
|
|
3
3
|
export class ExecutionFenceError extends Error {
|
|
4
4
|
code;
|
|
5
5
|
constructor(code, message) {
|
|
@@ -106,6 +106,10 @@ export async function withFencedExecutionWrite(write, scope) {
|
|
|
106
106
|
if (!scopedFence) {
|
|
107
107
|
throw new StaleExecutionFenceError(fence);
|
|
108
108
|
}
|
|
109
|
+
if (getScopedTransactionDbExec() === client) {
|
|
110
|
+
await lockAndAssertCurrentExecutionFences(client);
|
|
111
|
+
return write(client);
|
|
112
|
+
}
|
|
109
113
|
if (!client.transaction) {
|
|
110
114
|
throw new Error("Fenced execution writes require transaction support");
|
|
111
115
|
}
|
|
@@ -384,7 +384,7 @@ export interface ProductionAgentOptions {
|
|
|
384
384
|
runSoftTimeoutMs?: number;
|
|
385
385
|
runNoProgressTimeoutMs?: number;
|
|
386
386
|
durableBackgroundRuns?: boolean;
|
|
387
|
-
onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string) => void | Promise<void>;
|
|
387
|
+
onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string, turnId: string) => void | Promise<void>;
|
|
388
388
|
/**
|
|
389
389
|
* Called when the run's engine, model, and effort resolve, and again before
|
|
390
390
|
* each model call, because a call can run on a rate-limit fallback model or
|
|
@@ -90,6 +90,7 @@ import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js
|
|
|
90
90
|
import { applyUncheckedDefaultModelReplacement } from "./provider-model-selection.js";
|
|
91
91
|
import { startRun, subscribeToRun, replayCompletedTurn, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, getSlotHoldingRunId, tryClaimRunSlot, isHostedRuntime, resolveRunSoftTimeoutMs, resolveRunToolTimeoutCeilingMs, endsAfterToolResultWithoutAssistantFinal, endsDuringActionPreparation, } from "./run-manager.js";
|
|
92
92
|
import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
|
|
93
|
+
import { setRunTurnIdHeader } from "./run-turn-id-header.js";
|
|
93
94
|
import { buildCurrentTimeUserContext } from "./runtime-context.js";
|
|
94
95
|
import { claimSetupResume, setupResumeRefusedRunId, } from "./setup-resume-claim.js";
|
|
95
96
|
import { getStrictToolInputValidator } from "./strict-tool-input.js";
|
|
@@ -4448,6 +4449,20 @@ async function prepareToolCall(entry, input, signal, timeoutMs) {
|
|
|
4448
4449
|
function intentIsReadOnly(intent) {
|
|
4449
4450
|
return intent?.valid === true && intent.readOnly === true;
|
|
4450
4451
|
}
|
|
4452
|
+
function isBoundedTaskWait(toolName, input) {
|
|
4453
|
+
if (toolName !== "agent-teams" || !input || typeof input !== "object") {
|
|
4454
|
+
return false;
|
|
4455
|
+
}
|
|
4456
|
+
const args = input;
|
|
4457
|
+
// Only these operations honor the task reader's bounded wait contract.
|
|
4458
|
+
return ((args.action === "status" || args.action === "read-result") &&
|
|
4459
|
+
typeof args.taskId === "string" &&
|
|
4460
|
+
args.taskId.trim().length > 0 &&
|
|
4461
|
+
typeof args.waitMs === "number" &&
|
|
4462
|
+
Number.isInteger(args.waitMs) &&
|
|
4463
|
+
args.waitMs > 0 &&
|
|
4464
|
+
args.waitMs <= 20_000);
|
|
4465
|
+
}
|
|
4451
4466
|
function normalizeOptionalToolPlaceholders(schema, input) {
|
|
4452
4467
|
if (!schema?.properties || !input || typeof input !== "object") {
|
|
4453
4468
|
return { input, changed: false };
|
|
@@ -5977,7 +5992,12 @@ export async function runAgentLoop(opts) {
|
|
|
5977
5992
|
const actionRetrySafe = actionEntry && intent?.valid
|
|
5978
5993
|
? actionCallIsRetrySafe(actionEntry, preparedCallArguments(intent))
|
|
5979
5994
|
: false;
|
|
5980
|
-
const repeatGuardStopFromThisCall =
|
|
5995
|
+
const repeatGuardStopFromThisCall = actionIsReadOnly &&
|
|
5996
|
+
actionEntry?.dedupe === false &&
|
|
5997
|
+
intent?.valid === true &&
|
|
5998
|
+
isBoundedTaskWait(toolCall.name, preparedCallArguments(intent))
|
|
5999
|
+
? null
|
|
6000
|
+
: noteRepeatedToolCall(toolCall.name, toolCall.input, toolCall.id);
|
|
5981
6001
|
const rollbackRepeatedToolCall = () => {
|
|
5982
6002
|
const repeatKey = toolCallCacheKey(toolCall.name, toolCall.input);
|
|
5983
6003
|
const repeatCall = inFlightRepeatToolCalls.get(toolCall.id);
|
|
@@ -9398,6 +9418,7 @@ export function createProductionAgentHandler(options) {
|
|
|
9398
9418
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
9399
9419
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
9400
9420
|
setResponseHeader(event, "X-Run-Id", result.submittedRunId);
|
|
9421
|
+
setRunTurnIdHeader(event, getRun(result.submittedRunId)?.turnId);
|
|
9401
9422
|
setResponseHeader(event, "X-Dispatch-Mode", "replay");
|
|
9402
9423
|
return stream;
|
|
9403
9424
|
}
|
|
@@ -9486,6 +9507,7 @@ export function createProductionAgentHandler(options) {
|
|
|
9486
9507
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
9487
9508
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
9488
9509
|
setResponseHeader(event, "X-Run-Id", submittedRunId);
|
|
9510
|
+
setRunTurnIdHeader(event, getRun(submittedRunId)?.turnId);
|
|
9489
9511
|
setResponseHeader(event, "X-Dispatch-Mode", "replay");
|
|
9490
9512
|
return stream;
|
|
9491
9513
|
}
|
|
@@ -10780,6 +10802,11 @@ export function createProductionAgentHandler(options) {
|
|
|
10780
10802
|
!queuedMessageId &&
|
|
10781
10803
|
!queuedMessageClaimId &&
|
|
10782
10804
|
!isBackgroundWorker);
|
|
10805
|
+
const queuedSubmissionStore = shouldPreserveQueuedOrder
|
|
10806
|
+
? await import("../chat-threads/store.js")
|
|
10807
|
+
: undefined;
|
|
10808
|
+
// The inbox read runs under the slot lock, but its schema must be ready first.
|
|
10809
|
+
await queuedSubmissionStore?.ensureChatThreadTables();
|
|
10783
10810
|
slot = await tryClaimRunSlot(threadId, runId, undefined, {
|
|
10784
10811
|
turnId: effectiveTurnId,
|
|
10785
10812
|
...(turnInitiator ? { turnInitiator } : {}),
|
|
@@ -10795,12 +10822,9 @@ export function createProductionAgentHandler(options) {
|
|
|
10795
10822
|
? { dispatchPayload: durableDispatchPayload }
|
|
10796
10823
|
: {}),
|
|
10797
10824
|
...(continueOf ? { continueOf } : {}),
|
|
10798
|
-
...(
|
|
10825
|
+
...(queuedSubmissionStore
|
|
10799
10826
|
? {
|
|
10800
|
-
hasPendingQueuedSubmission:
|
|
10801
|
-
const { hasPendingQueuedSubmissionReceipt } = await import("../chat-threads/store.js");
|
|
10802
|
-
return hasPendingQueuedSubmissionReceipt(tx, runThreadId);
|
|
10803
|
-
},
|
|
10827
|
+
hasPendingQueuedSubmission: (tx) => queuedSubmissionStore.hasPendingQueuedSubmissionReceipt(tx, runThreadId),
|
|
10804
10828
|
}
|
|
10805
10829
|
: {}),
|
|
10806
10830
|
});
|
|
@@ -10833,6 +10857,9 @@ export function createProductionAgentHandler(options) {
|
|
|
10833
10857
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
10834
10858
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
10835
10859
|
setResponseHeader(event, "X-Run-Id", slot.completedRunId);
|
|
10860
|
+
// The client leaves a replay's turn out unless named here, and this
|
|
10861
|
+
// replay was found by its turn, so the stored turn is known.
|
|
10862
|
+
setRunTurnIdHeader(event, effectiveTurnId);
|
|
10836
10863
|
setResponseHeader(event, "X-Dispatch-Mode", "replay");
|
|
10837
10864
|
return stream;
|
|
10838
10865
|
}
|
|
@@ -11074,6 +11101,7 @@ export function createProductionAgentHandler(options) {
|
|
|
11074
11101
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
11075
11102
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
11076
11103
|
setResponseHeader(event, "X-Run-Id", duplicateRunId);
|
|
11104
|
+
setRunTurnIdHeader(event, getRun(duplicateRunId)?.turnId);
|
|
11077
11105
|
setResponseHeader(event, "X-Dispatch-Mode", "replay");
|
|
11078
11106
|
return stream;
|
|
11079
11107
|
}
|
|
@@ -11174,6 +11202,7 @@ export function createProductionAgentHandler(options) {
|
|
|
11174
11202
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
11175
11203
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
11176
11204
|
setResponseHeader(event, "X-Run-Id", runId);
|
|
11205
|
+
setRunTurnIdHeader(event, effectiveTurnId);
|
|
11177
11206
|
setResponseHeader(event, "X-Dispatch-Mode", "background");
|
|
11178
11207
|
return stream;
|
|
11179
11208
|
}
|
|
@@ -11503,7 +11532,7 @@ export function createProductionAgentHandler(options) {
|
|
|
11503
11532
|
void retainRequestDbPoolScope(recordRunDiagnostic(runId, RUN_DIAG_STAGE.setupTimings, setupDetail).catch(() => { })).catch(() => { });
|
|
11504
11533
|
}
|
|
11505
11534
|
if (options.onRunStart) {
|
|
11506
|
-
await options.onRunStart(send, threadId ?? runId, runId);
|
|
11535
|
+
await options.onRunStart(send, threadId ?? runId, runId, effectiveTurnId);
|
|
11507
11536
|
}
|
|
11508
11537
|
const unknownOutcomes = noUnknownOutcomes();
|
|
11509
11538
|
if (customAgentRefs.length > 0) {
|
|
@@ -11938,6 +11967,7 @@ export function createProductionAgentHandler(options) {
|
|
|
11938
11967
|
loopOpts: agentLoopOpts,
|
|
11939
11968
|
runId,
|
|
11940
11969
|
threadId: threadId ?? null,
|
|
11970
|
+
turnId: effectiveTurnId,
|
|
11941
11971
|
userId: ownerEmail,
|
|
11942
11972
|
config: obsConfig,
|
|
11943
11973
|
spanName: turnUsageLabel && turnUsageLabel !== "chat"
|
|
@@ -12047,10 +12077,12 @@ export function createProductionAgentHandler(options) {
|
|
|
12047
12077
|
: "foreground",
|
|
12048
12078
|
runRowAlreadyInserted: foregroundRunRowInserted,
|
|
12049
12079
|
executionLease: true,
|
|
12080
|
+
// An anonymous run is not a user; `run_started` does not name one either.
|
|
12081
|
+
userId: turnInitiator?.anonymous
|
|
12082
|
+
? undefined
|
|
12083
|
+
: (ownerEmail ?? undefined),
|
|
12050
12084
|
// Resolved AFTER stored-model/experiment overrides — the same value
|
|
12051
12085
|
// actually sent to the engine, not the raw client-requested model.
|
|
12052
|
-
// No userId here: `ownerEmail` is the only identity known at this
|
|
12053
|
-
// scope and is PII (email), which the terminal event must not carry.
|
|
12054
12086
|
model: effectiveModel,
|
|
12055
12087
|
engineName: engine.name,
|
|
12056
12088
|
attemptCount: backgroundContinuationCount,
|
|
@@ -12068,6 +12100,7 @@ export function createProductionAgentHandler(options) {
|
|
|
12068
12100
|
setResponseHeader(event, "Cache-Control", "no-cache");
|
|
12069
12101
|
setResponseHeader(event, "Connection", "keep-alive");
|
|
12070
12102
|
setResponseHeader(event, "X-Run-Id", runId);
|
|
12103
|
+
setRunTurnIdHeader(event, effectiveTurnId);
|
|
12071
12104
|
setResponseHeader(event, "X-Dispatch-Mode", foregroundSelfChainEligible ? "foreground-self-chain" : "foreground");
|
|
12072
12105
|
return stream;
|
|
12073
12106
|
});
|
|
@@ -7,11 +7,11 @@ import { ServicePrincipalRefusedError } from "../org/service-principal-guard.js"
|
|
|
7
7
|
import { captureError } from "../server/capture-error.js";
|
|
8
8
|
import { isLlmCredentialError, LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
|
|
9
9
|
import { classifyTerminalErrorCode, describeErrorWithCauses, isProviderConnectionError, } from "./engine/error-detail.js";
|
|
10
|
-
import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
|
|
11
10
|
import { getAgentEngineEntry } from "./engine/registry.js";
|
|
12
11
|
import { EngineError } from "./engine/types.js";
|
|
13
12
|
import { currentExecutionFence, MissingExecutionFenceError, StaleExecutionFenceError, withExecutionFence, withExecutionLeaseRequested, } from "./execution-fence.js";
|
|
14
|
-
import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
|
|
13
|
+
import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, setRunTerminalReasonIfStatus, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
|
|
14
|
+
import { trackRunTerminal } from "./run-terminal-tracking.js";
|
|
15
15
|
import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
|
|
16
16
|
const activeRuns = new Map();
|
|
17
17
|
const threadToRun = new Map();
|
|
@@ -435,39 +435,24 @@ function emitRunTerminalTrackingEvent(args) {
|
|
|
435
435
|
catch {
|
|
436
436
|
// Metrics must never affect the agent run or its persisted status.
|
|
437
437
|
}
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
438
|
+
// An abort is reported by the SQL write that ended the row, which may be
|
|
439
|
+
// another process's stop; reporting it here too would double-count it.
|
|
440
|
+
if (args.status === "aborted")
|
|
441
|
+
return;
|
|
442
|
+
void trackRunTerminal({
|
|
443
|
+
runId: args.runId,
|
|
444
|
+
threadId: args.threadId,
|
|
445
|
+
turnId: args.turnId,
|
|
443
446
|
status: args.status,
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
duration_ms: args.durationMs,
|
|
447
|
+
terminalReason: args.terminalReason,
|
|
448
|
+
errorCode: args.errorCode,
|
|
449
|
+
dispatchMode: args.dispatchMode,
|
|
450
|
+
abortReason: args.abortReason,
|
|
451
|
+
durationMs: args.durationMs,
|
|
450
452
|
model: args.model,
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
};
|
|
454
|
-
for (const key of Object.keys(properties)) {
|
|
455
|
-
if (properties[key] === undefined)
|
|
456
|
-
delete properties[key];
|
|
457
|
-
}
|
|
458
|
-
try {
|
|
459
|
-
void Promise.all([
|
|
460
|
-
import("../tracking/registry.js"),
|
|
461
|
-
import("../observability/tracking-identity.js"),
|
|
462
|
-
])
|
|
463
|
-
.then(([{ track }, { trackingIdentityProperties }]) => {
|
|
464
|
-
track("agent_run_terminal", { ...properties, ...trackingIdentityProperties() }, { userId: args.userId });
|
|
465
|
-
})
|
|
466
|
-
.catch(() => { });
|
|
467
|
-
}
|
|
468
|
-
catch {
|
|
469
|
-
// Tracking must never affect the agent run or its persisted status.
|
|
470
|
-
}
|
|
453
|
+
engineName: args.engineName,
|
|
454
|
+
attemptCount: args.attemptCount,
|
|
455
|
+
}, { userId: args.userId });
|
|
471
456
|
}
|
|
472
457
|
function abortInMemoryRun(run, reason = "user") {
|
|
473
458
|
run.abortReason = reason;
|
|
@@ -1205,6 +1190,10 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1205
1190
|
};
|
|
1206
1191
|
let lastAbortCheck = Date.now() - 3000;
|
|
1207
1192
|
let consecutiveAbortCheckFailures = 0;
|
|
1193
|
+
let completionInProgress = false;
|
|
1194
|
+
let ownTerminalStatusEstablished = false;
|
|
1195
|
+
let terminalStatusWrite = null;
|
|
1196
|
+
let terminalStatusWriteOutcomeUnknown = false;
|
|
1208
1197
|
const checkSqlAbort = () => {
|
|
1209
1198
|
const now = Date.now();
|
|
1210
1199
|
if (now - lastAbortCheck < 3000)
|
|
@@ -1216,9 +1205,26 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1216
1205
|
abortInMemoryRun(run, state.reason ?? "user");
|
|
1217
1206
|
return;
|
|
1218
1207
|
}
|
|
1219
|
-
if (!abort.signal.aborted) {
|
|
1208
|
+
if (!abort.signal.aborted && !ownTerminalStatusEstablished) {
|
|
1220
1209
|
const status = await getRunStatus(runId);
|
|
1221
1210
|
if (status !== null && status !== "running") {
|
|
1211
|
+
const pendingStatusWrite = terminalStatusWrite;
|
|
1212
|
+
if (pendingStatusWrite) {
|
|
1213
|
+
try {
|
|
1214
|
+
await pendingStatusWrite;
|
|
1215
|
+
}
|
|
1216
|
+
catch (error) {
|
|
1217
|
+
if (!terminalStatusWriteOutcomeUnknown)
|
|
1218
|
+
throw error;
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
if (terminalStatusWriteOutcomeUnknown)
|
|
1223
|
+
return;
|
|
1224
|
+
}
|
|
1225
|
+
if (status !== null &&
|
|
1226
|
+
status !== "running" &&
|
|
1227
|
+
!ownTerminalStatusEstablished) {
|
|
1222
1228
|
abortInMemoryRun(run, "displaced");
|
|
1223
1229
|
}
|
|
1224
1230
|
}
|
|
@@ -1249,7 +1255,6 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1249
1255
|
};
|
|
1250
1256
|
let consecutiveHeartbeatFailures = 0;
|
|
1251
1257
|
let heartbeatInFlight = false;
|
|
1252
|
-
let completionInProgress = false;
|
|
1253
1258
|
const heartbeatTimer = setInterval(() => {
|
|
1254
1259
|
if (!heartbeatInFlight) {
|
|
1255
1260
|
heartbeatInFlight = true;
|
|
@@ -1748,11 +1753,69 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1748
1753
|
const persistTerminalStatus = () => Object.keys(terminalStatusOptions).length === 0
|
|
1749
1754
|
? updateRunStatusIfRunning(runId, persistedStatus)
|
|
1750
1755
|
: updateRunStatusIfRunning(runId, persistedStatus, terminalStatusOptions);
|
|
1751
|
-
const
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
+
const persistOwnTerminalState = async (persist) => {
|
|
1757
|
+
let writeStarted = false;
|
|
1758
|
+
const write = withWorkerFence(() => {
|
|
1759
|
+
writeStarted = true;
|
|
1760
|
+
return persist();
|
|
1761
|
+
}).catch((error) => {
|
|
1762
|
+
if (writeStarted)
|
|
1763
|
+
terminalStatusWriteOutcomeUnknown = true;
|
|
1764
|
+
throw error;
|
|
1765
|
+
});
|
|
1766
|
+
terminalStatusWrite = write;
|
|
1767
|
+
try {
|
|
1768
|
+
const established = await write;
|
|
1769
|
+
if (established && !terminalStatusWriteOutcomeUnknown) {
|
|
1770
|
+
ownTerminalStatusEstablished = true;
|
|
1771
|
+
}
|
|
1772
|
+
return established;
|
|
1773
|
+
}
|
|
1774
|
+
finally {
|
|
1775
|
+
if (terminalStatusWrite === write)
|
|
1776
|
+
terminalStatusWrite = null;
|
|
1777
|
+
}
|
|
1778
|
+
};
|
|
1779
|
+
const persistOwnTerminalStatus = () => persistOwnTerminalState(persistTerminalStatus);
|
|
1780
|
+
const confirmAmbiguousTerminalStatus = async () => {
|
|
1781
|
+
const persistedRun = await getRunById(runId);
|
|
1782
|
+
if (persistedRun?.status !== persistedStatus)
|
|
1783
|
+
return false;
|
|
1784
|
+
const expectedTerminalReason = terminalReason.slice(0, 200);
|
|
1785
|
+
if (persistedRun.terminalReason === expectedTerminalReason) {
|
|
1786
|
+
return true;
|
|
1787
|
+
}
|
|
1788
|
+
if (persistedRun.terminalReason)
|
|
1789
|
+
return false;
|
|
1790
|
+
return withWorkerFence(() => setRunTerminalReasonIfStatus(runId, persistedStatus, expectedTerminalReason));
|
|
1791
|
+
};
|
|
1792
|
+
const reconcileTerminalRun = async () => {
|
|
1793
|
+
let reconciled = false;
|
|
1794
|
+
try {
|
|
1795
|
+
reconciled = await persistOwnTerminalState(() => threadProjectionPending
|
|
1796
|
+
? reconcileTerminalRunFromEvents(runId, {
|
|
1797
|
+
threadProjectionPending: true,
|
|
1798
|
+
})
|
|
1799
|
+
: reconcileTerminalRunFromEvents(runId));
|
|
1800
|
+
}
|
|
1801
|
+
catch (error) {
|
|
1802
|
+
if (!terminalStatusWriteOutcomeUnknown)
|
|
1803
|
+
throw error;
|
|
1804
|
+
}
|
|
1805
|
+
if (reconciled) {
|
|
1806
|
+
terminalStatusWriteOutcomeUnknown = false;
|
|
1807
|
+
ownTerminalStatusEstablished = true;
|
|
1808
|
+
return true;
|
|
1809
|
+
}
|
|
1810
|
+
if (!terminalStatusWriteOutcomeUnknown) {
|
|
1811
|
+
return reconciled;
|
|
1812
|
+
}
|
|
1813
|
+
const confirmed = await confirmAmbiguousTerminalStatus();
|
|
1814
|
+
terminalStatusWriteOutcomeUnknown = false;
|
|
1815
|
+
if (confirmed)
|
|
1816
|
+
ownTerminalStatusEstablished = true;
|
|
1817
|
+
return confirmed;
|
|
1818
|
+
};
|
|
1756
1819
|
const reportTerminalStatusPersistenceError = (error) => {
|
|
1757
1820
|
captureError(error, {
|
|
1758
1821
|
route: "/_agent-native/agent-chat",
|
|
@@ -1795,9 +1858,10 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1795
1858
|
else {
|
|
1796
1859
|
try {
|
|
1797
1860
|
await insertRunPromise;
|
|
1798
|
-
continuationStatusPersistedBeforeEvent =
|
|
1861
|
+
continuationStatusPersistedBeforeEvent =
|
|
1862
|
+
await persistOwnTerminalStatus();
|
|
1799
1863
|
if (!continuationStatusPersistedBeforeEvent) {
|
|
1800
|
-
await
|
|
1864
|
+
await reconcileTerminalRun();
|
|
1801
1865
|
const persistedRun = await getRunById(runId);
|
|
1802
1866
|
continuationStatusPersistedBeforeEvent =
|
|
1803
1867
|
persistedRun?.status === "truncated" &&
|
|
@@ -1875,7 +1939,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1875
1939
|
: await withWorkerFence(persistAtomicAbort);
|
|
1876
1940
|
terminalPersistenceEstablished = aborted
|
|
1877
1941
|
? true
|
|
1878
|
-
: await
|
|
1942
|
+
: await reconcileTerminalRun();
|
|
1879
1943
|
}
|
|
1880
1944
|
catch (error) {
|
|
1881
1945
|
terminalPersistenceEstablished = false;
|
|
@@ -1893,7 +1957,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1893
1957
|
(!terminalPersistenceError || eventPersistenceError)) {
|
|
1894
1958
|
let statusUpdated = false;
|
|
1895
1959
|
try {
|
|
1896
|
-
statusUpdated = await
|
|
1960
|
+
statusUpdated = await persistOwnTerminalStatus();
|
|
1897
1961
|
}
|
|
1898
1962
|
catch (error) {
|
|
1899
1963
|
reportTerminalStatusPersistenceError(error);
|
|
@@ -1905,8 +1969,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1905
1969
|
terminalPersistenceEstablished = !terminalPersistenceError;
|
|
1906
1970
|
}
|
|
1907
1971
|
else {
|
|
1908
|
-
terminalPersistenceEstablished =
|
|
1909
|
-
await withWorkerFence(reconcileTerminalRun);
|
|
1972
|
+
terminalPersistenceEstablished = await reconcileTerminalRun();
|
|
1910
1973
|
}
|
|
1911
1974
|
}
|
|
1912
1975
|
}
|
|
@@ -1957,7 +2020,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
|
|
|
1957
2020
|
if (terminalPersistenceError) {
|
|
1958
2021
|
const reconciled = eventPersistenceError
|
|
1959
2022
|
? false
|
|
1960
|
-
: await
|
|
2023
|
+
: await reconcileTerminalRun();
|
|
1961
2024
|
if (!reconciled)
|
|
1962
2025
|
throw terminalPersistenceError;
|
|
1963
2026
|
terminalPersistenceEstablished = true;
|
|
@@ -276,6 +276,7 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
|
|
|
276
276
|
}>;
|
|
277
277
|
export declare function setRunError(runId: string, errorCode: string | undefined, errorDetail: string | undefined): Promise<void>;
|
|
278
278
|
export declare function setRunTerminalReason(runId: string, terminalReason: string | undefined): Promise<void>;
|
|
279
|
+
export declare function setRunTerminalReasonIfStatus(runId: string, expectedStatus: "completed" | "truncated" | "errored" | "aborted", terminalReason: string): Promise<boolean>;
|
|
279
280
|
/**
|
|
280
281
|
* Repair a run whose terminal event was durably appended but whose final
|
|
281
282
|
* `agent_runs.status` write lost a race with reconnect/reaper code.
|