@agent-native/core 0.220.0-nightly-20261024033422 → 0.220.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/dist/a2a/handlers.js +7 -6
  2. package/dist/a2a/types.d.ts +1 -0
  3. package/dist/agent/engine/tool-call-journal-seed.js +24 -20
  4. package/dist/agent/execution-fence.d.ts +1 -1
  5. package/dist/agent/execution-fence.js +8 -1
  6. package/dist/agent/production-agent.js +31 -6
  7. package/dist/agent/reaped-run-thread.js +38 -3
  8. package/dist/agent/run-migrations.js +22 -0
  9. package/dist/agent/run-store.d.ts +19 -2
  10. package/dist/agent/run-store.fencing.worker.js +21 -1
  11. package/dist/agent/run-store.js +359 -35
  12. package/dist/agent/tool-approval-store.d.ts +13 -0
  13. package/dist/agent/tool-approval-store.js +64 -0
  14. package/dist/automation/index.d.ts +1 -1
  15. package/dist/client/background-agent-session.d.ts +4 -0
  16. package/dist/client/background-agent-session.js +17 -1
  17. package/dist/client/chat/runtime.js +38 -6
  18. package/dist/collab/routes.d.ts +2 -2
  19. package/dist/db/schema-snapshot.generated.js +14 -1
  20. package/dist/file-upload/actions/upload-image.d.ts +1 -1
  21. package/dist/jobs/automation-recovery.d.ts +4 -0
  22. package/dist/jobs/automation-recovery.js +7 -0
  23. package/dist/jobs/background-automation-runner.js +19 -2
  24. package/dist/jobs/scheduler.js +3 -1
  25. package/dist/mcp/org-directory.d.ts +8 -7
  26. package/dist/mcp/org-directory.js +82 -1
  27. package/dist/notifications/routes.d.ts +1 -1
  28. package/dist/observability/routes.d.ts +7 -7
  29. package/dist/progress/routes.d.ts +1 -1
  30. package/dist/resources/handlers.d.ts +2 -2
  31. package/dist/secrets/routes.d.ts +10 -10
  32. package/dist/server/agent-chat-plugin.d.ts +10 -1
  33. package/dist/server/agent-chat-plugin.js +178 -11
  34. package/dist/server/agent-discovery.js +11 -49
  35. package/dist/server/agent-engine-default-model-route.d.ts +1 -1
  36. package/dist/server/agent-engine-ollama-models-route.d.ts +2 -2
  37. package/dist/usage/store.js +36 -33
  38. package/package.json +4 -4
@@ -264,7 +264,7 @@ export async function processA2ATaskFromQueue(taskId, config, event) {
264
264
  }, A2A_PROCESSING_HEARTBEAT_MS);
265
265
  heartbeat.unref?.();
266
266
  try {
267
- await processClaimedA2ATask(taskId, claimed, config, workerAbort.signal, event);
267
+ await processClaimedA2ATask(taskId, claimed, config, workerAbort.signal, event, fence.generation);
268
268
  }
269
269
  finally {
270
270
  clearInterval(heartbeat);
@@ -285,7 +285,7 @@ export async function processA2ATaskFromQueue(taskId, config, event) {
285
285
  }
286
286
  }
287
287
  }
288
- async function processClaimedA2ATask(taskId, claimed, config, signal, event) {
288
+ async function processClaimedA2ATask(taskId, claimed, config, signal, event, processingAttemptId) {
289
289
  const message = claimed.history?.[0];
290
290
  if (!message) {
291
291
  await settleProcessingA2ATask(taskId, {
@@ -378,7 +378,7 @@ async function processClaimedA2ATask(taskId, claimed, config, signal, event) {
378
378
  ? { orgScope: "personal" }
379
379
  : {}),
380
380
  ...(requestOrigin ? { requestOrigin } : {}),
381
- }, () => runHandlerAndPersist(taskId, message, config, contextId, callerMetadata, event, sourceContext, verifiedEmail, signal));
381
+ }, () => runHandlerAndPersist(taskId, message, config, contextId, callerMetadata, event, sourceContext, verifiedEmail, signal, processingAttemptId));
382
382
  }
383
383
  catch (err) {
384
384
  try {
@@ -471,11 +471,12 @@ function jsonRpcError(id, code, message) {
471
471
  function jsonRpcResult(id, result) {
472
472
  return { jsonrpc: "2.0", id, result };
473
473
  }
474
- function makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal) {
474
+ function makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId) {
475
475
  const artifacts = [];
476
476
  const pendingFileArtifacts = [];
477
477
  const context = {
478
478
  taskId,
479
+ processingAttemptId,
479
480
  contextId,
480
481
  metadata,
481
482
  event,
@@ -573,8 +574,8 @@ async function withA2ARequestContext(metadata, event, fn) {
573
574
  ...(requestOrigin ? { requestOrigin } : {}),
574
575
  }, fn);
575
576
  }
576
- async function runHandlerAndPersist(taskId, message, config, contextId, metadata, event, sourceContext, ownerEmail, signal) {
577
- const { context, artifacts, persistFileArtifacts } = makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal);
577
+ async function runHandlerAndPersist(taskId, message, config, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId) {
578
+ const { context, artifacts, persistFileArtifacts } = makeHandlerContext(taskId, contextId, metadata, event, sourceContext, ownerEmail, signal, processingAttemptId);
578
579
  try {
579
580
  signal?.throwIfAborted();
580
581
  const result = getHandler(config)(message, context);
@@ -149,6 +149,7 @@ export interface A2ACorrelationMetadata {
149
149
  }
150
150
  export interface A2AHandlerContext {
151
151
  taskId: string;
152
+ processingAttemptId?: string;
152
153
  contextId?: string;
153
154
  metadata?: Record<string, unknown>;
154
155
  event?: unknown;
@@ -1,5 +1,5 @@
1
1
  import { isArtifactReceipt, } from "../../artifacts/detect.js";
2
- import { getCompletedAgentExecutionSteps, getCurrentTurnEventsForThread, } from "../run-store.js";
2
+ import { AgentRunJournalUnreadableError, getCompletedAgentExecutionSteps, getCurrentTurnEventsForThread, } from "../run-store.js";
3
3
  import { classifyToolCallJournal, isConsistentToolCallInputFingerprint, toolCallInputFingerprint, } from "../tool-call-journal.js";
4
4
  const LEDGER_READ_RETRY_MS = 250;
5
5
  function isRecord(value) {
@@ -36,15 +36,15 @@ function durableStepFileMutation(value) {
36
36
  }
37
37
  return { path: value.path, contentSha256: value.contentSha256 };
38
38
  }
39
- function durableStepToolDone(event, row) {
39
+ function durableStepToolDone(event, row, unreadable) {
40
40
  const executionStepId = event.executionStepId ?? event.id;
41
41
  if (!executionStepId ||
42
42
  executionStepId !== row.stepId ||
43
43
  row.operationId !== row.stepId) {
44
- throw new Error(`Completed execution step ${row.stepId} does not match its ledger identity`);
44
+ return unreadable(`Completed execution step ${row.stepId} does not match its ledger identity`);
45
45
  }
46
46
  if (event.tool !== row.toolName) {
47
- throw new Error(`Completed execution step ${row.stepId} does not match ledger tool ${event.tool}`);
47
+ return unreadable(`Completed execution step ${row.stepId} does not match ledger tool ${event.tool}`);
48
48
  }
49
49
  let input;
50
50
  let outcome;
@@ -53,7 +53,7 @@ function durableStepToolDone(event, row) {
53
53
  outcome = JSON.parse(row.resultJson);
54
54
  }
55
55
  catch (error) {
56
- throw new Error(`Completed execution step ${row.stepId} has unreadable stored intent or result`, { cause: error });
56
+ return unreadable(`Completed execution step ${row.stepId} has unreadable stored intent or result`, error);
57
57
  }
58
58
  if (!isRecord(input) ||
59
59
  typeof input.argumentsHash !== "string" ||
@@ -63,11 +63,11 @@ function durableStepToolDone(event, row) {
63
63
  typeof outcome.result !== "string" ||
64
64
  typeof outcome.isError !== "boolean" ||
65
65
  typeof outcome.completedSideEffect !== "boolean") {
66
- throw new Error(`Completed execution step ${row.stepId} is missing replay identity or outcome fields`);
66
+ return unreadable(`Completed execution step ${row.stepId} is missing replay identity or outcome fields`);
67
67
  }
68
68
  if (outcome.errorCode !== undefined &&
69
69
  typeof outcome.errorCode !== "string") {
70
- throw new Error(`Completed execution step ${row.stepId} has an invalid error code`);
70
+ return unreadable(`Completed execution step ${row.stepId} has an invalid error code`);
71
71
  }
72
72
  const artifacts = Array.isArray(outcome.artifacts)
73
73
  ? outcome.artifacts.filter(isArtifactReceipt)
@@ -129,56 +129,60 @@ export async function overlayCompletedExecutionStepOutcomes(threadId, turnId, ev
129
129
  const replacedIndexes = new Set();
130
130
  const replacementsAt = new Map();
131
131
  const replacementsAfter = new Map();
132
- for (const row of rows) {
132
+ for (const [rowIndex, row] of rows.entries()) {
133
+ const unreadable = (message, cause) => {
134
+ const detail = new Error(message, cause === undefined ? undefined : { cause });
135
+ throw new AgentRunJournalUnreadableError(threadId, turnId, rowIndex, "invalid_execution_step", { cause: detail });
136
+ };
133
137
  if (!candidateIds.includes(row.stepId)) {
134
- throw new Error(`Completed execution step ${row.stepId} was not requested by the ledger`);
138
+ return unreadable(`Completed execution step ${row.stepId} was not requested by the ledger`);
135
139
  }
136
140
  if (rowsById.has(row.stepId)) {
137
- throw new Error(`Completed execution step ${row.stepId} has ambiguous stored rows`);
141
+ return unreadable(`Completed execution step ${row.stepId} has ambiguous stored rows`);
138
142
  }
139
143
  const starts = startsById.get(row.stepId);
140
144
  if (!starts || starts.length === 0) {
141
- throw new Error(`Completed execution step ${row.stepId} has no matching ledger call`);
145
+ return unreadable(`Completed execution step ${row.stepId} has no matching ledger call`);
142
146
  }
143
147
  const canonicalStart = starts[0].event;
144
- const canonicalDone = durableStepToolDone(canonicalStart, row);
148
+ const canonicalDone = durableStepToolDone(canonicalStart, row, unreadable);
145
149
  if (!isConsistentToolCallInputFingerprint(canonicalStart.input, canonicalStart.inputFingerprint, canonicalStart.inputStoredFingerprint)) {
146
- throw new Error(`Completed execution step ${row.stepId} has unreadable ledger call identity`);
150
+ return unreadable(`Completed execution step ${row.stepId} has unreadable ledger call identity`);
147
151
  }
148
152
  const canonicalFingerprint = getToolCallEventFingerprint(canonicalStart);
149
153
  for (const { event } of starts.slice(1)) {
150
154
  if (event.tool !== canonicalStart.tool ||
151
155
  !isConsistentToolCallInputFingerprint(event.input, event.inputFingerprint, event.inputStoredFingerprint) ||
152
156
  getToolCallEventFingerprint(event) !== canonicalFingerprint) {
153
- throw new Error(`Completed execution step ${row.stepId} has conflicting ledger call identities`);
157
+ return unreadable(`Completed execution step ${row.stepId} has conflicting ledger call identities`);
154
158
  }
155
159
  }
156
160
  const dones = donesById.get(row.stepId) ?? [];
157
161
  for (const { event } of dones) {
158
162
  if (event.tool !== row.toolName ||
159
163
  !isConsistentToolCallInputFingerprint(event.input, event.inputFingerprint, event.inputStoredFingerprint)) {
160
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done identity`);
164
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done identity`);
161
165
  }
162
166
  const doneFingerprint = getToolCallEventFingerprint(event);
163
167
  if (canonicalFingerprint !== undefined &&
164
168
  doneFingerprint !== undefined &&
165
169
  doneFingerprint !== canonicalFingerprint) {
166
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done input`);
170
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done input`);
167
171
  }
168
172
  if (event.argumentsHash !== undefined &&
169
173
  event.argumentsHash !== canonicalDone.argumentsHash) {
170
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done arguments`);
174
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done arguments`);
171
175
  }
172
176
  if (event.completedSideEffect !== undefined &&
173
177
  event.completedSideEffect !== canonicalDone.completedSideEffect) {
174
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done side-effect outcome`);
178
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done side-effect outcome`);
175
179
  }
176
180
  if (event.isError === true && canonicalDone.isError !== true) {
177
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done error outcome`);
181
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done error outcome`);
178
182
  }
179
183
  if (event.errorCode !== undefined &&
180
184
  event.errorCode !== canonicalDone.errorCode) {
181
- throw new Error(`Completed execution step ${row.stepId} conflicts with its tool_done error code`);
185
+ return unreadable(`Completed execution step ${row.stepId} conflicts with its tool_done error code`);
182
186
  }
183
187
  }
184
188
  rowsById.set(row.stepId, row);
@@ -24,7 +24,7 @@ export declare function currentExecutionFenceForScope(scope: {
24
24
  export declare function withExecutionLeaseRequested<T>(work: () => T): T;
25
25
  export declare function withExecutionFence<T>(fence: ExecutionFence, work: () => T): T;
26
26
  export declare function runWithoutExecutionFence<T>(work: () => T): T;
27
- export declare function lockAndAssertCurrentExecutionFences(tx: DbExec): Promise<void>;
27
+ export declare function lockAndAssertCurrentExecutionFences(tx: DbExec, ignoredFence?: ExecutionFence): Promise<void>;
28
28
  export declare function withFencedExecutionWrite<T>(write: (tx: DbExec) => Promise<T>, scope?: {
29
29
  runId?: string;
30
30
  threadId?: string;
@@ -66,9 +66,16 @@ export function runWithoutExecutionFence(work) {
66
66
  // now-expired lease through AsyncLocalStorage.
67
67
  return executionFenceStorage.run({ fences: [], executionLeaseRequested: false }, work);
68
68
  }
69
- export async function lockAndAssertCurrentExecutionFences(tx) {
69
+ export async function lockAndAssertCurrentExecutionFences(tx, ignoredFence) {
70
70
  const fences = executionFenceStorage.getStore()?.fences ?? [];
71
71
  for (const activeFence of fences) {
72
+ if (ignoredFence &&
73
+ activeFence.threadId === ignoredFence.threadId &&
74
+ activeFence.turnId === ignoredFence.turnId &&
75
+ activeFence.runId === ignoredFence.runId &&
76
+ activeFence.generation === ignoredFence.generation) {
77
+ continue;
78
+ }
72
79
  const { rows } = await tx.execute({
73
80
  sql: `SELECT generation
74
81
  FROM agent_execution_leases
@@ -89,13 +89,13 @@ import { maybeCompactThread, buildObservationalContext, hasObservationalMemory,
89
89
  import { ProcessorChain, TripWire, toolCallsFromContent, } from "./processors.js";
90
90
  import { applyUncheckedDefaultModelReplacement } from "./provider-model-selection.js";
91
91
  import { startRun, subscribeToRun, replayCompletedTurn, getActiveRunForThread, getActiveRunForThreadAsync, getRun, abortRun, abortRunDurably, abortRunAndTurnDurably, abortRunAtSafeBoundaryDurably, abortTurnByRefDurably, abortTurnDurably, getSlotHoldingRunId, tryClaimRunSlot, isHostedRuntime, resolveRunSoftTimeoutMs, resolveRunToolTimeoutCeilingMs, endsAfterToolResultWithoutAssistantFinal, endsDuringActionPreparation, } from "./run-manager.js";
92
- import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
92
+ import { writeLedgerEntry, readLedgerEntry, beginAgentExecutionStep, completeAgentExecutionStep, markAgentExecutionStepDispatched, resetAgentExecutionStepBeforeInvocation, markAgentExecutionStepUnknown, recordLateAgentExecutionStepOutcome, clearLedgerForThread, AgentTurnInitiatorMismatchError, AgentTurnInitiatorUnavailableError, insertRun, insertRunEvent, isRunAborted, isTurnAborted, markRunAborted, updateRunHeartbeat, updateRunStatusIfRunning, updateRunTerminalStateIfRunning, setRunError, setRunTerminalReason, claimBackgroundRun, readBackgroundRunClaim, recordRunDiagnostic, countRunsForTurn, RUN_DIAG_STAGE, UNCLAIMED_BACKGROUND_RUN_GRACE_MS, turnRunLedgerExhausted, } from "./run-store.js";
93
93
  import { setRunTurnIdHeader } from "./run-turn-id-header.js";
94
94
  import { buildCurrentTimeUserContext } from "./runtime-context.js";
95
95
  import { claimSetupResume, setupResumeRefusedRunId, } from "./setup-resume-claim.js";
96
96
  import { getStrictToolInputValidator } from "./strict-tool-input.js";
97
97
  import { normalizeSubmissionDeadlineMetadata } from "./submission-deadline.js";
98
- import { consumeAgentToolApproval, createAgentToolApproval, isAgentToolAlwaysAllowed, resolveAgentToolApprovalTurnId, } from "./tool-approval-store.js";
98
+ import { consumeAgentToolApproval, createAgentToolApproval, ensureAgentToolApprovalTable, hasPendingAgentToolApproval, isAgentToolAlwaysAllowed, resolveAgentToolApprovalTurnId, } from "./tool-approval-store.js";
99
99
  import { toolCallInputSignature } from "./tool-call-input-signature.js";
100
100
  import { ambiguousLegacyCallEffectKey, findAmbiguousLegacyJournalEntryForCall, findInterruptedJournalEntryForCall, findInterruptedLegacyJournalEntryForCall, findReplayableJournalEntry, isUnavailableToolCallInput, journalEntriesAmbiguousInHistory, journalEntriesRepresentedInHistory, journalCallEffects, journalCallIdentities, matchToolCallResultsInHistory, toolCallIdentityKey, toolCallInputFingerprint, unmatchedAmbiguousLegacyToolCallResultsInHistory, } from "./tool-call-journal.js";
101
101
  import { redactSensitiveFields, sanitizeToolErrorText, sanitizeToolErrorValue, } from "./tool-error-redaction.js";
@@ -6150,6 +6150,8 @@ export async function runAgentLoop(opts) {
6150
6150
  let toolDoneEmitted = false;
6151
6151
  let toolDonePersistenceFailed = false;
6152
6152
  let executionStepFinalizationAttempted = false;
6153
+ let executionStepDispatched = false;
6154
+ let executionStepDispatchWrite;
6153
6155
  let executionStepOutcomeRecorded = false;
6154
6156
  let actionInvoked = false;
6155
6157
  // Live once invoked, so an abandoned call still reads what ran in it.
@@ -7038,6 +7040,7 @@ export async function runAgentLoop(opts) {
7038
7040
  if (cacheKey && cachedResult) {
7039
7041
  if (executionStepIntent) {
7040
7042
  await markAgentExecutionStepDispatched(executionStepIntent);
7043
+ executionStepDispatched = true;
7041
7044
  }
7042
7045
  const previousResult = cachedResult.content;
7043
7046
  const visible = isCachedToolResultVisibleInContext(contextMessages, toolCall, previousResult);
@@ -7165,7 +7168,10 @@ export async function runAgentLoop(opts) {
7165
7168
  throw new Error("Run aborted");
7166
7169
  }
7167
7170
  if (executionStepIntent) {
7168
- await markAgentExecutionStepDispatched(executionStepIntent);
7171
+ executionStepDispatchWrite = markAgentExecutionStepDispatched(executionStepIntent).then(() => {
7172
+ executionStepDispatched = true;
7173
+ });
7174
+ await executionStepDispatchWrite;
7169
7175
  }
7170
7176
  if (timeoutSignal.aborted) {
7171
7177
  throw new Error(timeoutMessage());
@@ -7432,7 +7438,7 @@ export async function runAgentLoop(opts) {
7432
7438
  thrown = err;
7433
7439
  toolReportedError = false;
7434
7440
  toolResultLost =
7435
- signal.aborted ||
7441
+ (signal.aborted && actionInvoked) ||
7436
7442
  isToolCallTimeoutResult(String(err?.message ?? err));
7437
7443
  toolErrorCode = isActionContractError(err)
7438
7444
  ? err.errorCode
@@ -7511,9 +7517,16 @@ export async function runAgentLoop(opts) {
7511
7517
  console.warn("Could not notify the action-change poller after a tool call", error);
7512
7518
  }
7513
7519
  }
7514
- if (executionStepIntent) {
7520
+ if (executionStepIntent &&
7521
+ (executionStepDispatched || executionStepDispatchWrite)) {
7515
7522
  executionStepFinalizationAttempted = true;
7516
- if (toolResultLost || callEffect === "unknown") {
7523
+ if (!actionInvoked) {
7524
+ // The dispatch marker can persist while cancellation wins; it does
7525
+ // not prove the action handler started.
7526
+ await executionStepDispatchWrite?.catch(() => { });
7527
+ await resetAgentExecutionStepBeforeInvocation(executionStepIntent);
7528
+ }
7529
+ else if (toolResultLost || callEffect === "unknown") {
7517
7530
  await markAgentExecutionStepUnknown(executionStepIntent);
7518
7531
  }
7519
7532
  else {
@@ -11047,6 +11060,9 @@ export function createProductionAgentHandler(options) {
11047
11060
  const queuedSubmissionStore = shouldPreserveQueuedOrder
11048
11061
  ? await import("../chat-threads/store.js")
11049
11062
  : undefined;
11063
+ if (ownerEmail && !isAuthorizedApprovalContinuation) {
11064
+ await ensureAgentToolApprovalTable();
11065
+ }
11050
11066
  // The inbox read runs under the slot lock, but its schema must be ready first.
11051
11067
  await queuedSubmissionStore?.ensureChatThreadTables();
11052
11068
  slot = await tryClaimRunSlot(threadId, runId, undefined, {
@@ -11069,6 +11085,15 @@ export function createProductionAgentHandler(options) {
11069
11085
  hasPendingQueuedSubmission: (tx) => queuedSubmissionStore.hasPendingQueuedSubmissionReceipt(tx, runThreadId),
11070
11086
  }
11071
11087
  : {}),
11088
+ ...(ownerEmail && !isAuthorizedApprovalContinuation
11089
+ ? {
11090
+ hasPendingApproval: (tx) => hasPendingAgentToolApproval(tx, {
11091
+ ownerEmail,
11092
+ orgId: getRequestOrgId() ?? null,
11093
+ threadId: runThreadId,
11094
+ }),
11095
+ }
11096
+ : {}),
11072
11097
  });
11073
11098
  }
11074
11099
  catch (error) {
@@ -1,7 +1,42 @@
1
1
  import { getThread, updateThreadData, withThreadDataLock, } from "../chat-threads/store.js";
2
2
  import { RUN_NOT_STARTED_METADATA_KEY } from "../shared/agent-chat-run-not-started.js";
3
- import { readStoppedRunForThreadFold } from "./run-store.js";
3
+ import { AgentRunJournalUnreadableError, readStoppedRunForThreadFold, } from "./run-store.js";
4
4
  import { buildAssistantMessage, extractThreadMeta, foldAssistantTurn, isRunPrompt, normalizeThreadRepository, } from "./thread-data-builder.js";
5
+ function readThreadRepository(threadId, turnId, threadData) {
6
+ let repository;
7
+ try {
8
+ repository = JSON.parse(threadData || "{}");
9
+ }
10
+ catch (cause) {
11
+ throw new AgentRunJournalUnreadableError(threadId, turnId, 0, "invalid_thread_data", { cause });
12
+ }
13
+ const unreadable = (message) => new AgentRunJournalUnreadableError(threadId, turnId, 0, "invalid_thread_data", { cause: new Error(message) });
14
+ if (typeof repository !== "object" ||
15
+ repository === null ||
16
+ Array.isArray(repository)) {
17
+ throw unreadable("Stored thread data is not a repository object");
18
+ }
19
+ const messages = repository.messages;
20
+ if (messages !== undefined &&
21
+ (!Array.isArray(messages) ||
22
+ !messages.every((entry) => {
23
+ if (typeof entry !== "object" ||
24
+ entry === null ||
25
+ Array.isArray(entry)) {
26
+ return false;
27
+ }
28
+ const storedMessage = entry.message;
29
+ const message = storedMessage ?? entry;
30
+ return (typeof message === "object" &&
31
+ message !== null &&
32
+ !Array.isArray(message) &&
33
+ typeof message.id === "string" &&
34
+ message.id.length > 0);
35
+ }))) {
36
+ throw unreadable("Stored thread data has malformed messages");
37
+ }
38
+ return normalizeThreadRepository(repository);
39
+ }
5
40
  /**
6
41
  * Saves the reply of a run the server ended because its worker stopped
7
42
  * (a crash, a killed function) into the thread, the way a run that finishes
@@ -51,7 +86,7 @@ export async function foldReapedRunIntoThread(runId, terminalError) {
51
86
  const thread = await getThread(run.threadId);
52
87
  if (!thread)
53
88
  return "superseded";
54
- const repo = normalizeThreadRepository(JSON.parse(thread.threadData || "{}"));
89
+ const repo = readThreadRepository(run.threadId, run.turnId, thread.threadData);
55
90
  const folded = foldAssistantTurn(repo, assistant, {
56
91
  runId,
57
92
  turnId: run.turnId,
@@ -88,7 +123,7 @@ export async function foldReapedRunIntoThread(runId, terminalError) {
88
123
  const currentThreadData = rows[0]?.thread_data;
89
124
  if (typeof currentThreadData !== "string")
90
125
  return false;
91
- const currentRepo = normalizeThreadRepository(JSON.parse(currentThreadData || "{}"));
126
+ const currentRepo = readThreadRepository(run.threadId, run.turnId, currentThreadData);
92
127
  const last = currentRepo.messages.at(-1)?.message ?? currentRepo.messages.at(-1);
93
128
  const ownsPrompt = last?.role === "user"
94
129
  ? isRunPrompt(last, { runId, ...run })
@@ -139,4 +139,26 @@ export const AGENT_RUN_MIGRATIONS = [
139
139
  WHERE thread_projection_pending IS TRUE
140
140
  `,
141
141
  },
142
+ {
143
+ version: 10,
144
+ name: "agent-run-thread-projection-retry-fairness",
145
+ sql: `
146
+ CREATE SEQUENCE IF NOT EXISTS agent_run_projection_retry_order_seq;
147
+ ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS thread_projection_retry_order BIGINT;
148
+ UPDATE agent_runs
149
+ SET thread_projection_retry_order = nextval('agent_run_projection_retry_order_seq')
150
+ WHERE thread_projection_pending IS TRUE
151
+ AND thread_projection_retry_order IS NULL;
152
+ CREATE INDEX IF NOT EXISTS agent_runs_thread_projection_retry_order_idx
153
+ ON agent_runs (thread_projection_retry_order ASC NULLS FIRST, completed_at ASC, id ASC)
154
+ WHERE thread_projection_pending IS TRUE
155
+ `,
156
+ },
157
+ {
158
+ version: 11,
159
+ name: "agent-run-thread-projection-quarantine",
160
+ sql: `
161
+ ALTER TABLE agent_runs ADD COLUMN IF NOT EXISTS thread_projection_error_code TEXT
162
+ `,
163
+ },
142
164
  ];
@@ -149,6 +149,7 @@ export type AgentExecutionStepAdmission = {
149
149
  /** Persist intent before dispatch and decide whether an interrupted call may run again. */
150
150
  export declare function beginAgentExecutionStep(intent: AgentExecutionStepIntent): Promise<AgentExecutionStepAdmission>;
151
151
  export declare function markAgentExecutionStepDispatched(intent: AgentExecutionStepIntent): Promise<void>;
152
+ export declare function resetAgentExecutionStepBeforeInvocation(intent: AgentExecutionStepIntent): Promise<void>;
152
153
  export declare function completeAgentExecutionStep(intent: AgentExecutionStepIntent, resultJson: string, errorJson?: string): Promise<void>;
153
154
  export declare function markAgentExecutionStepUnknown(intent: AgentExecutionStepIntent): Promise<void>;
154
155
  /** A detached action may attach late evidence only while its lease is current. */
@@ -265,11 +266,14 @@ export declare function tryClaimRunSlot(threadId: string, runId: string, maxStal
265
266
  };
266
267
  /** Defer a fresh turn while an earlier accepted inbox submission is queued. */
267
268
  hasPendingQueuedSubmission?: (tx: DbExec) => Promise<boolean>;
269
+ /** Keep ordinary turns behind a pending tool approval on this thread. */
270
+ hasPendingApproval?: (tx: DbExec) => Promise<boolean>;
268
271
  afterInsert?: (tx: DbExec) => Promise<void>;
269
272
  }): Promise<{
270
273
  claimed: boolean;
271
274
  activeRunId: string | null;
272
275
  queuedSubmissionPending?: boolean;
276
+ approvalPending?: boolean;
273
277
  completedRunId?: string;
274
278
  turnAborted?: boolean;
275
279
  continueRefused?: ContinueRefusalCode;
@@ -411,6 +415,7 @@ export declare function getRunById(runId: string): Promise<{
411
415
  errorCode: string | null;
412
416
  errorDetail: string | null;
413
417
  terminalReason: string | null;
418
+ threadProjectionErrorCode: string | null;
414
419
  } | null>;
415
420
  /**
416
421
  * What saving a run the server ended on its behalf needs: the run's ledger,
@@ -469,6 +474,7 @@ export interface AgentRunSummary {
469
474
  dispatchMode: string | null;
470
475
  terminalReason: string | null;
471
476
  diagStage: string | null;
477
+ threadProjectionErrorCode: string | null;
472
478
  inFlightSince: number | null;
473
479
  }
474
480
  export declare function listRunsForThread(threadId: string, options?: {
@@ -484,9 +490,9 @@ export declare class AgentRunJournalUnreadableError extends Error {
484
490
  readonly threadId: string;
485
491
  readonly turnId: string | undefined;
486
492
  readonly rowIndex: number;
487
- readonly reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event";
493
+ readonly reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event" | "invalid_thread_data" | "invalid_execution_step";
488
494
  readonly errorCode = "tool_call_journal_unreadable";
489
- constructor(threadId: string, turnId: string | undefined, rowIndex: number, reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event", options?: ErrorOptions);
495
+ constructor(threadId: string, turnId: string | undefined, rowIndex: number, reason: "invalid_turn" | "invalid_row" | "invalid_event_json" | "invalid_event" | "invalid_thread_data" | "invalid_execution_step", options?: ErrorOptions);
490
496
  }
491
497
  export declare function getCurrentTurnRunEventsForThread(threadId: string, knownTurnId?: string): Promise<CurrentTurnRunEvent[]>;
492
498
  export declare function getCurrentTurnRunIdsForThread(threadId: string, turnId: string): Promise<string[]>;
@@ -522,3 +528,14 @@ export declare function listErroredRuns(options?: {
522
528
  durationMs: number | null;
523
529
  }>>;
524
530
  export declare function ensureTerminalRunEvent(runId: string, event: Record<string, unknown>): Promise<void>;
531
+ export type RunThreadProjectionRetry = {
532
+ state: "untracked";
533
+ } | {
534
+ state: "already_projected";
535
+ } | {
536
+ state: "projected";
537
+ } | {
538
+ state: "deferred";
539
+ };
540
+ /** Retry after repairing persisted source data; until success, the marker keeps later same-turn runs gated. */
541
+ export declare function retryRunThreadProjection(runId: string): Promise<RunThreadProjectionRetry>;
@@ -6,7 +6,7 @@ import { toPostgresParams, withDbExec } from "../db/client.js";
6
6
  import { withMigrationRuntime } from "../db/migration-runtime.js";
7
7
  import { withExecutionFence } from "./execution-fence.js";
8
8
  import { abortRunAtSafeBoundaryDurably } from "./run-manager.js";
9
- import { beginAgentExecutionStep, claimTurnExecutionLease, completeAgentExecutionStep, ensureRunTables, insertRun, insertRunEvent, markAgentExecutionStepDispatched, } from "./run-store.js";
9
+ import { beginAgentExecutionStep, claimTurnExecutionLease, completeAgentExecutionStep, ensureRunTables, insertRun, insertRunEvent, markAgentExecutionStepDispatched, persistRunCheckpointEvent, reapIfStale, } from "./run-store.js";
10
10
  import { applySubmittedUserMessage, buildUserMessage, extractThreadMeta, } from "./thread-data-builder.js";
11
11
  function executor(sql) {
12
12
  return {
@@ -115,6 +115,26 @@ async function main() {
115
115
  }));
116
116
  return;
117
117
  }
118
+ if (mode === "checkpoint-crash") {
119
+ const payload = JSON.parse(extra ?? "null");
120
+ await insertRun(runId, threadId, turnId, {
121
+ dispatchMode: "background",
122
+ dispatchPayload: JSON.stringify(payload),
123
+ });
124
+ const fence = await claimTurnExecutionLease(threadId, turnId, runId);
125
+ await withExecutionFence(fence, async () => {
126
+ await insertRunEvent(runId, 0, JSON.stringify({ type: "text", text: "Completed before timeout" }));
127
+ await persistRunCheckpointEvent(runId, { type: "auto_continue", reason: "run_timeout" }, "run_timeout");
128
+ });
129
+ writeSync(1, JSON.stringify({ status: "checkpointed", fence }));
130
+ process.exit(83);
131
+ }
132
+ if (mode === "stale-reap") {
133
+ await withMigrationRuntime(() => ensureChatThreadTables());
134
+ const reaped = await reapIfStale(runId);
135
+ writeSync(1, JSON.stringify({ reaped }));
136
+ return;
137
+ }
118
138
  if (mode === "durable-abort") {
119
139
  const [{ pid }] = await sql `SELECT pg_backend_pid() AS pid`;
120
140
  databaseBackendPid = Number(pid);