@agent-native/core 0.218.0 → 0.220.0-nightly-20261023183804

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/dist/a2a/server.d.ts +1 -1
  2. package/dist/a2a/server.js +8 -6
  3. package/dist/agent/execution-fence.js +5 -1
  4. package/dist/agent/production-agent.d.ts +1 -1
  5. package/dist/agent/production-agent.js +42 -9
  6. package/dist/agent/run-manager.js +110 -47
  7. package/dist/agent/run-store.d.ts +1 -0
  8. package/dist/agent/run-store.js +260 -31
  9. package/dist/agent/run-terminal-tracking.d.ts +26 -0
  10. package/dist/agent/run-terminal-tracking.js +54 -0
  11. package/dist/agent/run-turn-id-header.d.ts +9 -0
  12. package/dist/agent/run-turn-id-header.js +13 -0
  13. package/dist/automation/index.d.ts +1 -1
  14. package/dist/chat-threads/store.d.ts +12 -6
  15. package/dist/chat-threads/store.js +35 -28
  16. package/dist/cli/amplify-stream.js +1 -1
  17. package/dist/client/agent-chat/index.d.ts +1 -0
  18. package/dist/client/agent-chat/index.js +1 -0
  19. package/dist/client/agent-chat.d.ts +2 -0
  20. package/dist/client/agent-chat.js +31 -5
  21. package/dist/client/chat/agentkit-agent-native.js +8 -1
  22. package/dist/client/chat/agentkit-protocol.js +7 -1
  23. package/dist/client/chat/run-outcome-telemetry.js +1 -0
  24. package/dist/client/chat/run-outcome.d.ts +5 -0
  25. package/dist/client/chat/runtime.d.ts +34 -1
  26. package/dist/client/chat/runtime.js +96 -12
  27. package/dist/client/chat/turn-submitted-telemetry.d.ts +4 -0
  28. package/dist/client/chat/turn-submitted-telemetry.js +39 -0
  29. package/dist/client/chat/turn-submitted.d.ts +39 -0
  30. package/dist/client/chat/turn-submitted.js +16 -0
  31. package/dist/client/route-chunk-recovery.d.ts +3 -8
  32. package/dist/client/route-chunk-recovery.js +11 -9
  33. package/dist/client/route-warmup.js +101 -3
  34. package/dist/collab/routes.d.ts +1 -1
  35. package/dist/db/client.d.ts +1 -0
  36. package/dist/db/client.js +28 -18
  37. package/dist/jobs/actions/manage-recurring-job.d.ts +1 -1
  38. package/dist/mcp/embed-app.js +2 -1
  39. package/dist/mcp/screen-memory-stdio.d.ts +7 -7
  40. package/dist/notifications/routes.d.ts +4 -4
  41. package/dist/observability/routes.d.ts +7 -7
  42. package/dist/observability/traces.d.ts +2 -0
  43. package/dist/observability/traces.js +4 -1
  44. package/dist/provider-api/actions/custom-provider-registration.d.ts +16 -16
  45. package/dist/provider-api/actions/provider-api.d.ts +13 -13
  46. package/dist/provider-api/corpus-jobs.d.ts +2 -2
  47. package/dist/resources/handlers.d.ts +1 -1
  48. package/dist/review/index.d.ts +1 -1
  49. package/dist/review/index.js +1 -1
  50. package/dist/review/suggestions/actions.js +25 -17
  51. package/dist/review/suggestions/store.d.ts +14 -1
  52. package/dist/review/suggestions/store.js +16 -3
  53. package/dist/review/suggestions/types.d.ts +15 -1
  54. package/dist/secrets/routes.d.ts +10 -10
  55. package/dist/server/agent-chat/browser-team-tools.js +39 -6
  56. package/dist/server/agent-chat/script-entries.js +10 -4
  57. package/dist/server/agent-chat-plugin.d.ts +2 -0
  58. package/dist/server/agent-chat-plugin.js +38 -13
  59. package/dist/server/agent-engine-ollama-models-route.d.ts +1 -1
  60. package/dist/server/first-touch-route.d.ts +2 -2
  61. package/dist/server/realtime-token.d.ts +1 -1
  62. package/dist/shared/route-chunk-recovery-bootstrap.d.ts +9 -0
  63. package/dist/shared/route-chunk-recovery-bootstrap.js +25 -0
  64. package/dist/sharing/actions/list-resource-shares.d.ts +1 -1
  65. package/dist/templates/workspace-core/.agents/skills/external-agents/references/mcp-apps-embedding.md +12 -5
  66. package/dist/tracking/providers.js +3 -0
  67. package/dist/triggers/actions/manage-automation.d.ts +4 -4
  68. package/docs/content/external-agents.mdx +1 -1
  69. package/docs/content/mcp-apps.mdx +3 -3
  70. package/docs/content/mcp-protocol.mdx +1 -1
  71. package/docs/content/template-assets-integrations.mdx +5 -3
  72. package/package.json +4 -4
  73. package/src/templates/workspace-core/.agents/skills/external-agents/references/mcp-apps-embedding.md +12 -5
@@ -50,7 +50,7 @@ export declare function verifyA2AToken(token: string, event?: any, audienceOptio
50
50
  allowAnthropicManagedContinuation?: boolean;
51
51
  requireAudience?: boolean;
52
52
  }): Promise<A2ATokenPayload>;
53
- export type A2ATokenFailureCause = "malformed" | "secret-missing" | "audience-unresolvable" | "audience" | "issuer" | "expired" | "signature" | "invalid-claims" | "mcp-credential" | "missing-subject" | "org-binding" | "org-lookup-failed" | "undetermined";
53
+ export type A2ATokenFailureCause = "malformed" | "secret-missing" | "audience-unresolvable" | "audience" | "issuer" | "expired" | "signature" | "algorithm" | "invalid-claims" | "mcp-credential" | "missing-subject" | "org-binding" | "org-lookup-failed" | "undetermined";
54
54
  /**
55
55
  * Names why a token that `verifyA2AToken` resolved no identity for was
56
56
  * refused. `verifyA2AToken` collapses a wrong audience, a wrong secret and an
@@ -21,6 +21,7 @@ import { hasConfiguredA2ASecret, isA2AProductionRuntime, } from "./auth-policy.j
21
21
  import { handleJsonRpcH3, processA2ATaskFromQueue } from "./handlers.js";
22
22
  import { A2A_ORG_MEMBER_DELEGATION_TYPE, organizationPrincipalClaims, verifyA2AOrganizationIdentity, } from "./organization-identity.js";
23
23
  import { claimA2AApproval, a2aTaskLeaseThreadId, getA2AApprovalForOwner, settleA2AApproval, } from "./task-store.js";
24
+ const A2A_TOKEN_ALGORITHMS = ["HS256"];
24
25
  let _warnedUnauthA2A = false;
25
26
  function warnA2AUnauthOnce() {
26
27
  if (_warnedUnauthA2A)
@@ -196,6 +197,8 @@ async function checkAgainstSecret(token, secret, options) {
196
197
  }
197
198
  catch (error) {
198
199
  const { code, claim } = error;
200
+ if (code === "ERR_JOSE_ALG_NOT_ALLOWED")
201
+ return { cause: "algorithm" };
199
202
  if (code === "ERR_JWT_EXPIRED")
200
203
  return { cause: "expired" };
201
204
  if (code === "ERR_JWS_SIGNATURE_VERIFICATION_FAILED") {
@@ -243,7 +246,7 @@ export async function diagnoseA2ATokenFailure(token, event) {
243
246
  return "malformed";
244
247
  }
245
248
  const secret = readDeployCredentialEnv("A2A_SECRET")?.trim();
246
- const options = {};
249
+ const options = { algorithms: A2A_TOKEN_ALGORITHMS };
247
250
  if (typeof claims.aud !== "undefined") {
248
251
  const audience = expectedJwtAudience(event);
249
252
  if (!audience)
@@ -331,7 +334,9 @@ async function verifyA2ATokenInternal(token, event, audienceOptions) {
331
334
  if (audienceOptions?.requireAudience && !hasAudience) {
332
335
  return { payload: { email: null, orgDomain: null } };
333
336
  }
334
- const verifyOptions = {};
337
+ const verifyOptions = {
338
+ algorithms: A2A_TOKEN_ALGORITHMS,
339
+ };
335
340
  if (hasAudience) {
336
341
  // Fail closed: the token was minted for a specific audience, but this
337
342
  // receiver can't derive its own expected audience (no APP_URL/URL and no
@@ -424,10 +429,7 @@ async function verifyA2ATokenInternal(token, event, audienceOptions) {
424
429
  }
425
430
  if (!organization)
426
431
  return { payload: { email: null, orgDomain: null } };
427
- const orgVerifyOptions = unverifiedPayload?.typ === A2A_ORG_MEMBER_DELEGATION_TYPE
428
- ? { ...verifyOptions, algorithms: ["HS256"] }
429
- : verifyOptions;
430
- const payload = await verifyJwtWithSecret(token, organization.secret, orgVerifyOptions);
432
+ const payload = await verifyJwtWithSecret(token, organization.secret, verifyOptions);
431
433
  if (!payload)
432
434
  return { payload: { email: null, orgDomain: null } };
433
435
  const claims = organizationPrincipalClaims(payload, organization);
@@ -1,5 +1,5 @@
1
1
  import { AsyncLocalStorage } from "node:async_hooks";
2
- import { getDbExec } from "../db/client.js";
2
+ import { getDbExec, getScopedTransactionDbExec, } from "../db/client.js";
3
3
  export class ExecutionFenceError extends Error {
4
4
  code;
5
5
  constructor(code, message) {
@@ -106,6 +106,10 @@ export async function withFencedExecutionWrite(write, scope) {
106
106
  if (!scopedFence) {
107
107
  throw new StaleExecutionFenceError(fence);
108
108
  }
109
+ if (getScopedTransactionDbExec() === client) {
110
+ await lockAndAssertCurrentExecutionFences(client);
111
+ return write(client);
112
+ }
109
113
  if (!client.transaction) {
110
114
  throw new Error("Fenced execution writes require transaction support");
111
115
  }
@@ -384,7 +384,7 @@ export interface ProductionAgentOptions {
384
384
  runSoftTimeoutMs?: number;
385
385
  runNoProgressTimeoutMs?: number;
386
386
  durableBackgroundRuns?: boolean;
387
- onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string) => void | Promise<void>;
387
+ onRunStart?: (send: (event: AgentChatEvent) => void, threadId: string, runId: string, turnId: string) => void | Promise<void>;
388
388
  /**
389
389
  * Called when the run's engine, model, and effort resolve, and again before
390
390
  * each model call, because a call can run on a rate-limit fallback model or
@@ -90,6 +90,7 @@ import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js
90
90
  import { applyUncheckedDefaultModelReplacement } from "./provider-model-selection.js";
91
91
  import { startRun, subscribeToRun, replayCompletedTurn, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, getSlotHoldingRunId, tryClaimRunSlot, isHostedRuntime, resolveRunSoftTimeoutMs, resolveRunToolTimeoutCeilingMs, endsAfterToolResultWithoutAssistantFinal, endsDuringActionPreparation, } from "./run-manager.js";
92
92
  import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
93
+ import { setRunTurnIdHeader } from "./run-turn-id-header.js";
93
94
  import { buildCurrentTimeUserContext } from "./runtime-context.js";
94
95
  import { claimSetupResume, setupResumeRefusedRunId, } from "./setup-resume-claim.js";
95
96
  import { getStrictToolInputValidator } from "./strict-tool-input.js";
@@ -4448,6 +4449,20 @@ async function prepareToolCall(entry, input, signal, timeoutMs) {
4448
4449
  function intentIsReadOnly(intent) {
4449
4450
  return intent?.valid === true && intent.readOnly === true;
4450
4451
  }
4452
+ function isBoundedTaskWait(toolName, input) {
4453
+ if (toolName !== "agent-teams" || !input || typeof input !== "object") {
4454
+ return false;
4455
+ }
4456
+ const args = input;
4457
+ // Only these operations honor the task reader's bounded wait contract.
4458
+ return ((args.action === "status" || args.action === "read-result") &&
4459
+ typeof args.taskId === "string" &&
4460
+ args.taskId.trim().length > 0 &&
4461
+ typeof args.waitMs === "number" &&
4462
+ Number.isInteger(args.waitMs) &&
4463
+ args.waitMs > 0 &&
4464
+ args.waitMs <= 20_000);
4465
+ }
4451
4466
  function normalizeOptionalToolPlaceholders(schema, input) {
4452
4467
  if (!schema?.properties || !input || typeof input !== "object") {
4453
4468
  return { input, changed: false };
@@ -5977,7 +5992,12 @@ export async function runAgentLoop(opts) {
5977
5992
  const actionRetrySafe = actionEntry && intent?.valid
5978
5993
  ? actionCallIsRetrySafe(actionEntry, preparedCallArguments(intent))
5979
5994
  : false;
5980
- const repeatGuardStopFromThisCall = noteRepeatedToolCall(toolCall.name, toolCall.input, toolCall.id);
5995
+ const repeatGuardStopFromThisCall = actionIsReadOnly &&
5996
+ actionEntry?.dedupe === false &&
5997
+ intent?.valid === true &&
5998
+ isBoundedTaskWait(toolCall.name, preparedCallArguments(intent))
5999
+ ? null
6000
+ : noteRepeatedToolCall(toolCall.name, toolCall.input, toolCall.id);
5981
6001
  const rollbackRepeatedToolCall = () => {
5982
6002
  const repeatKey = toolCallCacheKey(toolCall.name, toolCall.input);
5983
6003
  const repeatCall = inFlightRepeatToolCalls.get(toolCall.id);
@@ -9398,6 +9418,7 @@ export function createProductionAgentHandler(options) {
9398
9418
  setResponseHeader(event, "Cache-Control", "no-cache");
9399
9419
  setResponseHeader(event, "Connection", "keep-alive");
9400
9420
  setResponseHeader(event, "X-Run-Id", result.submittedRunId);
9421
+ setRunTurnIdHeader(event, getRun(result.submittedRunId)?.turnId);
9401
9422
  setResponseHeader(event, "X-Dispatch-Mode", "replay");
9402
9423
  return stream;
9403
9424
  }
@@ -9486,6 +9507,7 @@ export function createProductionAgentHandler(options) {
9486
9507
  setResponseHeader(event, "Cache-Control", "no-cache");
9487
9508
  setResponseHeader(event, "Connection", "keep-alive");
9488
9509
  setResponseHeader(event, "X-Run-Id", submittedRunId);
9510
+ setRunTurnIdHeader(event, getRun(submittedRunId)?.turnId);
9489
9511
  setResponseHeader(event, "X-Dispatch-Mode", "replay");
9490
9512
  return stream;
9491
9513
  }
@@ -10780,6 +10802,11 @@ export function createProductionAgentHandler(options) {
10780
10802
  !queuedMessageId &&
10781
10803
  !queuedMessageClaimId &&
10782
10804
  !isBackgroundWorker);
10805
+ const queuedSubmissionStore = shouldPreserveQueuedOrder
10806
+ ? await import("../chat-threads/store.js")
10807
+ : undefined;
10808
+ // The inbox read runs under the slot lock, but its schema must be ready first.
10809
+ await queuedSubmissionStore?.ensureChatThreadTables();
10783
10810
  slot = await tryClaimRunSlot(threadId, runId, undefined, {
10784
10811
  turnId: effectiveTurnId,
10785
10812
  ...(turnInitiator ? { turnInitiator } : {}),
@@ -10795,12 +10822,9 @@ export function createProductionAgentHandler(options) {
10795
10822
  ? { dispatchPayload: durableDispatchPayload }
10796
10823
  : {}),
10797
10824
  ...(continueOf ? { continueOf } : {}),
10798
- ...(shouldPreserveQueuedOrder
10825
+ ...(queuedSubmissionStore
10799
10826
  ? {
10800
- hasPendingQueuedSubmission: async (tx) => {
10801
- const { hasPendingQueuedSubmissionReceipt } = await import("../chat-threads/store.js");
10802
- return hasPendingQueuedSubmissionReceipt(tx, runThreadId);
10803
- },
10827
+ hasPendingQueuedSubmission: (tx) => queuedSubmissionStore.hasPendingQueuedSubmissionReceipt(tx, runThreadId),
10804
10828
  }
10805
10829
  : {}),
10806
10830
  });
@@ -10833,6 +10857,9 @@ export function createProductionAgentHandler(options) {
10833
10857
  setResponseHeader(event, "Cache-Control", "no-cache");
10834
10858
  setResponseHeader(event, "Connection", "keep-alive");
10835
10859
  setResponseHeader(event, "X-Run-Id", slot.completedRunId);
10860
+ // The client leaves a replay's turn out unless named here, and this
10861
+ // replay was found by its turn, so the stored turn is known.
10862
+ setRunTurnIdHeader(event, effectiveTurnId);
10836
10863
  setResponseHeader(event, "X-Dispatch-Mode", "replay");
10837
10864
  return stream;
10838
10865
  }
@@ -11074,6 +11101,7 @@ export function createProductionAgentHandler(options) {
11074
11101
  setResponseHeader(event, "Cache-Control", "no-cache");
11075
11102
  setResponseHeader(event, "Connection", "keep-alive");
11076
11103
  setResponseHeader(event, "X-Run-Id", duplicateRunId);
11104
+ setRunTurnIdHeader(event, getRun(duplicateRunId)?.turnId);
11077
11105
  setResponseHeader(event, "X-Dispatch-Mode", "replay");
11078
11106
  return stream;
11079
11107
  }
@@ -11174,6 +11202,7 @@ export function createProductionAgentHandler(options) {
11174
11202
  setResponseHeader(event, "Cache-Control", "no-cache");
11175
11203
  setResponseHeader(event, "Connection", "keep-alive");
11176
11204
  setResponseHeader(event, "X-Run-Id", runId);
11205
+ setRunTurnIdHeader(event, effectiveTurnId);
11177
11206
  setResponseHeader(event, "X-Dispatch-Mode", "background");
11178
11207
  return stream;
11179
11208
  }
@@ -11503,7 +11532,7 @@ export function createProductionAgentHandler(options) {
11503
11532
  void retainRequestDbPoolScope(recordRunDiagnostic(runId, RUN_DIAG_STAGE.setupTimings, setupDetail).catch(() => { })).catch(() => { });
11504
11533
  }
11505
11534
  if (options.onRunStart) {
11506
- await options.onRunStart(send, threadId ?? runId, runId);
11535
+ await options.onRunStart(send, threadId ?? runId, runId, effectiveTurnId);
11507
11536
  }
11508
11537
  const unknownOutcomes = noUnknownOutcomes();
11509
11538
  if (customAgentRefs.length > 0) {
@@ -11938,6 +11967,7 @@ export function createProductionAgentHandler(options) {
11938
11967
  loopOpts: agentLoopOpts,
11939
11968
  runId,
11940
11969
  threadId: threadId ?? null,
11970
+ turnId: effectiveTurnId,
11941
11971
  userId: ownerEmail,
11942
11972
  config: obsConfig,
11943
11973
  spanName: turnUsageLabel && turnUsageLabel !== "chat"
@@ -12047,10 +12077,12 @@ export function createProductionAgentHandler(options) {
12047
12077
  : "foreground",
12048
12078
  runRowAlreadyInserted: foregroundRunRowInserted,
12049
12079
  executionLease: true,
12080
+ // An anonymous run is not a user; `run_started` does not name one either.
12081
+ userId: turnInitiator?.anonymous
12082
+ ? undefined
12083
+ : (ownerEmail ?? undefined),
12050
12084
  // Resolved AFTER stored-model/experiment overrides — the same value
12051
12085
  // actually sent to the engine, not the raw client-requested model.
12052
- // No userId here: `ownerEmail` is the only identity known at this
12053
- // scope and is PII (email), which the terminal event must not carry.
12054
12086
  model: effectiveModel,
12055
12087
  engineName: engine.name,
12056
12088
  attemptCount: backgroundContinuationCount,
@@ -12068,6 +12100,7 @@ export function createProductionAgentHandler(options) {
12068
12100
  setResponseHeader(event, "Cache-Control", "no-cache");
12069
12101
  setResponseHeader(event, "Connection", "keep-alive");
12070
12102
  setResponseHeader(event, "X-Run-Id", runId);
12103
+ setRunTurnIdHeader(event, effectiveTurnId);
12071
12104
  setResponseHeader(event, "X-Dispatch-Mode", foregroundSelfChainEligible ? "foreground-self-chain" : "foreground");
12072
12105
  return stream;
12073
12106
  });
@@ -7,11 +7,11 @@ import { ServicePrincipalRefusedError } from "../org/service-principal-guard.js"
7
7
  import { captureError } from "../server/capture-error.js";
8
8
  import { isLlmCredentialError, LLM_MISSING_CREDENTIALS_ERROR_CODE, LLM_MISSING_CREDENTIALS_MESSAGE, } from "./engine/credential-errors.js";
9
9
  import { classifyTerminalErrorCode, describeErrorWithCauses, isProviderConnectionError, } from "./engine/error-detail.js";
10
- import { runErrorTelemetryProperties } from "./engine/error-telemetry.js";
11
10
  import { getAgentEngineEntry } from "./engine/registry.js";
12
11
  import { EngineError } from "./engine/types.js";
13
12
  import { currentExecutionFence, MissingExecutionFenceError, StaleExecutionFenceError, withExecutionFence, withExecutionLeaseRequested, } from "./execution-fence.js";
14
- import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
13
+ import { AgentRunEventNotPersistedError, insertRun, insertRunEvent, insertRunEvents, updateRunStatusIfRunning, markRunAborted, markRunAbortedAtSafeBoundary, getRunAbortState, getRunStatus, getRunEventsSince, getCurrentTurnRunEventsForThread, getRunById, getRunByThread, getRunTurnRef, listRunsForThread, markTurnAborted, cleanupOldRuns, updateRunHeartbeat, bumpRunProgress, setRunInFlightMarker, reapIfStale, reapUnclaimedBackgroundRun, shouldRedispatchUnclaimedBackgroundRun, reconcileTerminalRunFromEvents, ensureTerminalRunEvent, getLastTerminalRunEvent, resolveErroredRunTerminalEvent, setRunError, setRunTerminalReason, setRunTerminalReasonIfStatus, persistRunCheckpointEvent, recordRunDiagnostic, RUN_DIAG_STAGE, claimTurnExecutionLease, renewTurnExecutionLease, releaseTurnExecutionLease, TurnExecutionLeaseBusyError, terminalEventForAbortReason, RUN_RECORD_MISSING_ERROR_EVENT, RUN_RECORD_MISSING_GRACE_MS, RUN_TERMINAL_LOOKUP_FAILED_ERROR_EVENT, UNKNOWN_RUN_STATUS_ERROR_EVENT, } from "./run-store.js";
14
+ import { trackRunTerminal } from "./run-terminal-tracking.js";
15
15
  import { CONTINUATION_REASONS, isContinuationTerminalReason } from "./types.js";
16
16
  const activeRuns = new Map();
17
17
  const threadToRun = new Map();
@@ -435,39 +435,24 @@ function emitRunTerminalTrackingEvent(args) {
435
435
  catch {
436
436
  // Metrics must never affect the agent run or its persisted status.
437
437
  }
438
- const properties = {
439
- source: "agent_run_manager",
440
- run_id: args.runId,
441
- thread_id: args.threadId,
442
- turn_id: args.turnId,
438
+ // An abort is reported by the SQL write that ended the row, which may be
439
+ // another process's stop; reporting it here too would double-count it.
440
+ if (args.status === "aborted")
441
+ return;
442
+ void trackRunTerminal({
443
+ runId: args.runId,
444
+ threadId: args.threadId,
445
+ turnId: args.turnId,
443
446
  status: args.status,
444
- terminal_reason: args.terminalReason,
445
- error_code: args.errorCode,
446
- ...(args.errorCode ? runErrorTelemetryProperties(args.errorCode) : {}),
447
- dispatch_mode: args.dispatchMode,
448
- abort_reason: args.abortReason,
449
- duration_ms: args.durationMs,
447
+ terminalReason: args.terminalReason,
448
+ errorCode: args.errorCode,
449
+ dispatchMode: args.dispatchMode,
450
+ abortReason: args.abortReason,
451
+ durationMs: args.durationMs,
450
452
  model: args.model,
451
- engine: args.engineName,
452
- attempt_count: args.attemptCount,
453
- };
454
- for (const key of Object.keys(properties)) {
455
- if (properties[key] === undefined)
456
- delete properties[key];
457
- }
458
- try {
459
- void Promise.all([
460
- import("../tracking/registry.js"),
461
- import("../observability/tracking-identity.js"),
462
- ])
463
- .then(([{ track }, { trackingIdentityProperties }]) => {
464
- track("agent_run_terminal", { ...properties, ...trackingIdentityProperties() }, { userId: args.userId });
465
- })
466
- .catch(() => { });
467
- }
468
- catch {
469
- // Tracking must never affect the agent run or its persisted status.
470
- }
453
+ engineName: args.engineName,
454
+ attemptCount: args.attemptCount,
455
+ }, { userId: args.userId });
471
456
  }
472
457
  function abortInMemoryRun(run, reason = "user") {
473
458
  run.abortReason = reason;
@@ -1205,6 +1190,10 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1205
1190
  };
1206
1191
  let lastAbortCheck = Date.now() - 3000;
1207
1192
  let consecutiveAbortCheckFailures = 0;
1193
+ let completionInProgress = false;
1194
+ let ownTerminalStatusEstablished = false;
1195
+ let terminalStatusWrite = null;
1196
+ let terminalStatusWriteOutcomeUnknown = false;
1208
1197
  const checkSqlAbort = () => {
1209
1198
  const now = Date.now();
1210
1199
  if (now - lastAbortCheck < 3000)
@@ -1216,9 +1205,26 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1216
1205
  abortInMemoryRun(run, state.reason ?? "user");
1217
1206
  return;
1218
1207
  }
1219
- if (!abort.signal.aborted) {
1208
+ if (!abort.signal.aborted && !ownTerminalStatusEstablished) {
1220
1209
  const status = await getRunStatus(runId);
1221
1210
  if (status !== null && status !== "running") {
1211
+ const pendingStatusWrite = terminalStatusWrite;
1212
+ if (pendingStatusWrite) {
1213
+ try {
1214
+ await pendingStatusWrite;
1215
+ }
1216
+ catch (error) {
1217
+ if (!terminalStatusWriteOutcomeUnknown)
1218
+ throw error;
1219
+ return;
1220
+ }
1221
+ }
1222
+ if (terminalStatusWriteOutcomeUnknown)
1223
+ return;
1224
+ }
1225
+ if (status !== null &&
1226
+ status !== "running" &&
1227
+ !ownTerminalStatusEstablished) {
1222
1228
  abortInMemoryRun(run, "displaced");
1223
1229
  }
1224
1230
  }
@@ -1249,7 +1255,6 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1249
1255
  };
1250
1256
  let consecutiveHeartbeatFailures = 0;
1251
1257
  let heartbeatInFlight = false;
1252
- let completionInProgress = false;
1253
1258
  const heartbeatTimer = setInterval(() => {
1254
1259
  if (!heartbeatInFlight) {
1255
1260
  heartbeatInFlight = true;
@@ -1748,11 +1753,69 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1748
1753
  const persistTerminalStatus = () => Object.keys(terminalStatusOptions).length === 0
1749
1754
  ? updateRunStatusIfRunning(runId, persistedStatus)
1750
1755
  : updateRunStatusIfRunning(runId, persistedStatus, terminalStatusOptions);
1751
- const reconcileTerminalRun = () => threadProjectionPending
1752
- ? reconcileTerminalRunFromEvents(runId, {
1753
- threadProjectionPending: true,
1754
- })
1755
- : reconcileTerminalRunFromEvents(runId);
1756
+ const persistOwnTerminalState = async (persist) => {
1757
+ let writeStarted = false;
1758
+ const write = withWorkerFence(() => {
1759
+ writeStarted = true;
1760
+ return persist();
1761
+ }).catch((error) => {
1762
+ if (writeStarted)
1763
+ terminalStatusWriteOutcomeUnknown = true;
1764
+ throw error;
1765
+ });
1766
+ terminalStatusWrite = write;
1767
+ try {
1768
+ const established = await write;
1769
+ if (established && !terminalStatusWriteOutcomeUnknown) {
1770
+ ownTerminalStatusEstablished = true;
1771
+ }
1772
+ return established;
1773
+ }
1774
+ finally {
1775
+ if (terminalStatusWrite === write)
1776
+ terminalStatusWrite = null;
1777
+ }
1778
+ };
1779
+ const persistOwnTerminalStatus = () => persistOwnTerminalState(persistTerminalStatus);
1780
+ const confirmAmbiguousTerminalStatus = async () => {
1781
+ const persistedRun = await getRunById(runId);
1782
+ if (persistedRun?.status !== persistedStatus)
1783
+ return false;
1784
+ const expectedTerminalReason = terminalReason.slice(0, 200);
1785
+ if (persistedRun.terminalReason === expectedTerminalReason) {
1786
+ return true;
1787
+ }
1788
+ if (persistedRun.terminalReason)
1789
+ return false;
1790
+ return withWorkerFence(() => setRunTerminalReasonIfStatus(runId, persistedStatus, expectedTerminalReason));
1791
+ };
1792
+ const reconcileTerminalRun = async () => {
1793
+ let reconciled = false;
1794
+ try {
1795
+ reconciled = await persistOwnTerminalState(() => threadProjectionPending
1796
+ ? reconcileTerminalRunFromEvents(runId, {
1797
+ threadProjectionPending: true,
1798
+ })
1799
+ : reconcileTerminalRunFromEvents(runId));
1800
+ }
1801
+ catch (error) {
1802
+ if (!terminalStatusWriteOutcomeUnknown)
1803
+ throw error;
1804
+ }
1805
+ if (reconciled) {
1806
+ terminalStatusWriteOutcomeUnknown = false;
1807
+ ownTerminalStatusEstablished = true;
1808
+ return true;
1809
+ }
1810
+ if (!terminalStatusWriteOutcomeUnknown) {
1811
+ return reconciled;
1812
+ }
1813
+ const confirmed = await confirmAmbiguousTerminalStatus();
1814
+ terminalStatusWriteOutcomeUnknown = false;
1815
+ if (confirmed)
1816
+ ownTerminalStatusEstablished = true;
1817
+ return confirmed;
1818
+ };
1756
1819
  const reportTerminalStatusPersistenceError = (error) => {
1757
1820
  captureError(error, {
1758
1821
  route: "/_agent-native/agent-chat",
@@ -1795,9 +1858,10 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1795
1858
  else {
1796
1859
  try {
1797
1860
  await insertRunPromise;
1798
- continuationStatusPersistedBeforeEvent = await withWorkerFence(persistTerminalStatus);
1861
+ continuationStatusPersistedBeforeEvent =
1862
+ await persistOwnTerminalStatus();
1799
1863
  if (!continuationStatusPersistedBeforeEvent) {
1800
- await withWorkerFence(reconcileTerminalRun);
1864
+ await reconcileTerminalRun();
1801
1865
  const persistedRun = await getRunById(runId);
1802
1866
  continuationStatusPersistedBeforeEvent =
1803
1867
  persistedRun?.status === "truncated" &&
@@ -1875,7 +1939,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1875
1939
  : await withWorkerFence(persistAtomicAbort);
1876
1940
  terminalPersistenceEstablished = aborted
1877
1941
  ? true
1878
- : await withWorkerFence(reconcileTerminalRun);
1942
+ : await reconcileTerminalRun();
1879
1943
  }
1880
1944
  catch (error) {
1881
1945
  terminalPersistenceEstablished = false;
@@ -1893,7 +1957,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1893
1957
  (!terminalPersistenceError || eventPersistenceError)) {
1894
1958
  let statusUpdated = false;
1895
1959
  try {
1896
- statusUpdated = await withWorkerFence(persistTerminalStatus);
1960
+ statusUpdated = await persistOwnTerminalStatus();
1897
1961
  }
1898
1962
  catch (error) {
1899
1963
  reportTerminalStatusPersistenceError(error);
@@ -1905,8 +1969,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1905
1969
  terminalPersistenceEstablished = !terminalPersistenceError;
1906
1970
  }
1907
1971
  else {
1908
- terminalPersistenceEstablished =
1909
- await withWorkerFence(reconcileTerminalRun);
1972
+ terminalPersistenceEstablished = await reconcileTerminalRun();
1910
1973
  }
1911
1974
  }
1912
1975
  }
@@ -1957,7 +2020,7 @@ export function startRun(runId, threadId, runFn, onComplete, options) {
1957
2020
  if (terminalPersistenceError) {
1958
2021
  const reconciled = eventPersistenceError
1959
2022
  ? false
1960
- : await withWorkerFence(reconcileTerminalRun);
2023
+ : await reconcileTerminalRun();
1961
2024
  if (!reconciled)
1962
2025
  throw terminalPersistenceError;
1963
2026
  terminalPersistenceEstablished = true;
@@ -276,6 +276,7 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
276
276
  }>;
277
277
  export declare function setRunError(runId: string, errorCode: string | undefined, errorDetail: string | undefined): Promise<void>;
278
278
  export declare function setRunTerminalReason(runId: string, terminalReason: string | undefined): Promise<void>;
279
+ export declare function setRunTerminalReasonIfStatus(runId: string, expectedStatus: "completed" | "truncated" | "errored" | "aborted", terminalReason: string): Promise<boolean>;
279
280
  /**
280
281
  * Repair a run whose terminal event was durably appended but whose final
281
282
  * `agent_runs.status` write lost a race with reconnect/reaper code.