@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/dist/activities/agent-turn/errors.d.ts +5 -4
  2. package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
  3. package/dist/activities/agent-turn/history-sink.d.ts +6 -2
  4. package/dist/activities/agent-turn/quiescence.d.ts +4 -2
  5. package/dist/activities/sandbox-lease.d.ts +7 -2
  6. package/dist/activities/xai-quota.d.ts +3 -0
  7. package/dist/{activities-control-CTEMLTHG.js → activities-control-3SKLDTXQ.js} +205 -86
  8. package/dist/activities-control-3SKLDTXQ.js.map +1 -0
  9. package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
  10. package/dist/activities-turn-GHSFZB64.js.map +1 -0
  11. package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
  12. package/dist/chunk-O2LQWJIK.js.map +1 -0
  13. package/dist/index.js +1 -1
  14. package/dist/sandbox-routing.d.ts +6 -0
  15. package/dist/workflow-bundle.js +24 -2
  16. package/package.json +17 -17
  17. package/src/activities/agent-turn/agent-build.ts +1 -0
  18. package/src/activities/agent-turn/errors.ts +44 -11
  19. package/src/activities/agent-turn/failure-settlement.ts +21 -8
  20. package/src/activities/agent-turn/history-prefix.ts +51 -0
  21. package/src/activities/agent-turn/history-sink.ts +30 -6
  22. package/src/activities/agent-turn/quiescence.ts +8 -2
  23. package/src/activities/agent-turn/run.ts +4 -0
  24. package/src/activities/agent-turn/stream-attempt.ts +20 -7
  25. package/src/activities/agent-turn/tool-environment.ts +12 -0
  26. package/src/activities/agent-turn/xai-capacity.ts +11 -0
  27. package/src/activities/codex-capacity.ts +18 -0
  28. package/src/activities/run-input.ts +6 -7
  29. package/src/activities/sandbox-lease.ts +229 -87
  30. package/src/activities/session-state.ts +8 -7
  31. package/src/activities/xai-quota.ts +35 -0
  32. package/src/sandbox-routing.ts +115 -0
  33. package/src/workflows/session.ts +29 -1
  34. package/dist/activities-control-CTEMLTHG.js.map +0 -1
  35. package/dist/activities-turn-S5IU347A.js.map +0 -1
  36. package/dist/chunk-TAUE3GYT.js.map +0 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.26.0-canary.0",
3
+ "version": "0.26.0-canary.2",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -54,22 +54,22 @@
54
54
  },
55
55
  "dependencies": {
56
56
  "@llamaindex/liteparse": "2.14.2",
57
- "@opengeni/agent-proto": "^0.5.1-canary.9",
58
- "@opengeni/capabilities": "^0.3.2-canary.0",
59
- "@opengeni/codemode": "^0.4.27-canary.0",
60
- "@opengeni/codex": "^0.2.21-canary.0",
61
- "@opengeni/config": "^1.0.0-canary.0",
62
- "@opengeni/contracts": "^2.13.0-canary.0",
63
- "@opengeni/core": "^2.7.5-canary.0",
64
- "@opengeni/db": "^4.0.0-canary.0",
65
- "@opengeni/documents": "^0.8.19-canary.0",
66
- "@opengeni/events": "^0.4.17-canary.0",
67
- "@opengeni/github": "^0.7.3-canary.0",
68
- "@opengeni/network": "^0.3.0-canary.0",
69
- "@opengeni/observability": "^0.8.19-canary.0",
70
- "@opengeni/runtime": "^2.3.0-canary.0",
71
- "@opengeni/storage": "^0.2.120-canary.0",
72
- "@opengeni/xai-subscription": "^0.1.3-canary.0",
57
+ "@opengeni/agent-proto": "^0.5.1-canary.11",
58
+ "@opengeni/capabilities": "^0.3.2-canary.2",
59
+ "@opengeni/codemode": "^0.4.27-canary.2",
60
+ "@opengeni/codex": "^0.2.21-canary.2",
61
+ "@opengeni/config": "^1.0.0-canary.2",
62
+ "@opengeni/contracts": "^2.13.0-canary.2",
63
+ "@opengeni/core": "^2.7.5-canary.2",
64
+ "@opengeni/db": "^4.0.0-canary.2",
65
+ "@opengeni/documents": "^0.8.19-canary.2",
66
+ "@opengeni/events": "^0.4.17-canary.2",
67
+ "@opengeni/github": "^0.7.3-canary.2",
68
+ "@opengeni/network": "^0.3.0-canary.2",
69
+ "@opengeni/observability": "^0.8.19-canary.2",
70
+ "@opengeni/runtime": "^2.3.0-canary.2",
71
+ "@opengeni/storage": "^0.2.120-canary.2",
72
+ "@opengeni/xai-subscription": "^0.1.3-canary.2",
73
73
  "@temporalio/activity": "^1.17.0",
74
74
  "@temporalio/client": "^1.17.0",
75
75
  "@temporalio/worker": "^1.17.0",
@@ -599,6 +599,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
599
599
  let agentConstructionOutcome: "completed" | "failed" = "completed";
600
600
  try {
601
601
  return runtime.buildAgent(eventing.modelRunSettings, runtimeResources, {
602
+ ...(preparedTools.inputWaitYield ? { inputWaitYield: preparedTools.inputWaitYield } : {}),
602
603
  reasoningEffort: turn.reasoningEffort,
603
604
  latencyMode: turnExecutionPolicy.latencyMode,
604
605
  ...(serviceTier ? { serviceTier } : {}),
@@ -1,6 +1,8 @@
1
1
  import {
2
2
  ActiveSessionHistoryLimitExceededError,
3
3
  ApprovalRunStateLimitExceededError,
4
+ nestedPostgresSqlState,
5
+ safeDatabaseErrorFacts,
4
6
  isRetryableDatabaseTransportFailure,
5
7
  isSessionEventPersistenceError,
6
8
  SandboxLeaseTransitionError,
@@ -266,22 +268,21 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
266
268
  * Convert the atomic claim transaction's failure into a small, stable
267
269
  * Temporal wire contract. The original error remains in activity diagnostics,
268
270
  * but SQL text, parameters, and arbitrary invariant messages never enter
269
- * workflow history. Contention and operational database unavailability are
270
- * safe to re-read after backoff because the claim transaction contains no
271
- * model/tool effects; permanent database and state failures require terminal
272
- * settlement.
271
+ * workflow history. Every database failure is safe to re-read after backoff
272
+ * because the claim transaction contains no model/tool effects. Database
273
+ * constraints and authorization guards can also be repaired by a deployment or
274
+ * authority refresh, so they must preserve pending work. Only a non-database
275
+ * claim invariant is terminal.
273
276
  */
274
277
  export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
275
278
  const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
276
279
  const retryableCode = retryableDatabaseFailureCode(error);
277
- // The database transaction itself retries only the two contention failures
278
- // proven safe for immediate replay. Once the activity has failed, the
279
- // workflow may also retry operational outages after a durable re-read and
280
- // bounded delay. Unknown driver/database failures stay recoverable because
281
- // they commonly represent a lost connection; known constraint, auth, and
282
- // application SQLSTATEs are permanent and must not create an infinite loop.
280
+ // The workflow retries after a durable re-read and bounded delay. A
281
+ // SessionEventPersistenceError proves the failure happened inside the atomic
282
+ // claim boundary, before provider or tool work; retaining it is safer than
283
+ // destroying accepted machine input on an application SQLSTATE.
283
284
  const detail: PreClaimFailureDetail = {
284
- disposition: retryableCode ? "retryable" : "permanent",
285
+ disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
285
286
  code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
286
287
  };
287
288
  return ApplicationFailure.create({
@@ -826,6 +827,26 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
826
827
  return null;
827
828
  }
828
829
 
830
+ // The generic turn-failure boundary also receives application/provider errors.
831
+ // A five-character code or generic severity alone does not establish a driver error.
832
+ function findPostgresDriverError(error: unknown): Record<string, unknown> | null {
833
+ const queue: unknown[] = [error];
834
+ const seen = new Set<unknown>();
835
+ for (let index = 0; index < queue.length && index < 64; index += 1) {
836
+ const current = queue[index];
837
+ if (!current || typeof current !== "object" || seen.has(current)) continue;
838
+ seen.add(current);
839
+ const record = current as Record<string, unknown>;
840
+ if (record.name === "PostgresError") return record;
841
+ for (const key of ["cause", "original", "driverError", "error", "errors"]) {
842
+ const nested = record[key];
843
+ if (Array.isArray(nested)) queue.push(...nested.slice(0, 64));
844
+ else if (nested !== undefined) queue.push(nested);
845
+ }
846
+ }
847
+ return null;
848
+ }
849
+
829
850
  export function agentRunFailurePayload(
830
851
  error: unknown,
831
852
  options: { isCodexTurn?: boolean } = {},
@@ -1066,6 +1087,18 @@ export function agentRunFailurePayload(
1066
1087
  }
1067
1088
  return { error: message, code: "provider_unavailable", retryable: true };
1068
1089
  }
1090
+ const postgresDriverError = findPostgresDriverError(error);
1091
+ if (postgresDriverError) {
1092
+ const database = safeDatabaseErrorFacts(postgresDriverError);
1093
+ const sqlState = nestedPostgresSqlState(postgresDriverError);
1094
+ if (sqlState !== null || Object.keys(database).length > 0) {
1095
+ return {
1096
+ error: message,
1097
+ sqlState,
1098
+ ...(Object.keys(database).length > 0 ? { database } : {}),
1099
+ };
1100
+ }
1101
+ }
1069
1102
  return { error: message };
1070
1103
  }
1071
1104
 
@@ -263,9 +263,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
263
263
  // finished yet.
264
264
  const recoveryTurnId = attempt.turnId;
265
265
  // A true epoch supersession and a provider lifecycle transition are both
266
- // recoverable control-plane states, never session failures. The latter is
267
- // paced briefly; the next acquire waits on the durable claim and resumes
268
- // immediately when it clears.
266
+ // recoverable control-plane states, never session failures. A rotation
267
+ // persists an exact group/epoch wait marker so the workflow parks before
268
+ // another turn-worker dispatch; shorter non-rotation transitions retain
269
+ // their existing paced retry.
269
270
  const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
270
271
  const leaseControlError =
271
272
  error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
@@ -282,6 +283,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
282
283
  const transitionPending = lifecycleTransition !== null || rotationPending;
283
284
  const deadlineRotationPending =
284
285
  rotationPending && fencedLease?.rotationReason === "provider_deadline";
286
+ const sandboxLifecycleWait = rotationPending
287
+ ? {
288
+ version: 1 as const,
289
+ sandboxGroupId: leaseControlError.sandboxGroupId,
290
+ leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
291
+ reason: "rotation_in_progress" as const,
292
+ }
293
+ : undefined;
285
294
  const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
286
295
  sessionId: input.sessionId,
287
296
  turnId: recoveryTurnId,
@@ -306,6 +315,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
306
315
  },
307
316
  }
308
317
  : {}),
318
+ ...(sandboxLifecycleWait ? { sandboxLifecycleWait } : {}),
309
319
  });
310
320
  if (recovery.action === "stale") {
311
321
  acknowledgeLostAttemptOwnership();
@@ -319,7 +329,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
319
329
  control.turnMetricOutcome = "recovering";
320
330
  return claimedResult({
321
331
  status: "recovering",
322
- ...(transitionPending
332
+ ...(transitionPending && !sandboxLifecycleWait
323
333
  ? {
324
334
  continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
325
335
  }
@@ -393,6 +403,12 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
393
403
  sandboxGroupId: rotation.sandboxGroupId,
394
404
  leaseEpoch: rotation.leaseEpoch,
395
405
  },
406
+ sandboxLifecycleWait: {
407
+ version: 1,
408
+ sandboxGroupId: rotation.sandboxGroupId,
409
+ leaseEpoch: rotation.leaseEpoch,
410
+ reason: "rotation_in_progress",
411
+ },
396
412
  });
397
413
  if (recovery.action === "stale") {
398
414
  acknowledgeLostAttemptOwnership();
@@ -404,10 +420,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
404
420
  await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
405
421
  control.activityStatus = "recovering";
406
422
  control.turnMetricOutcome = "recovering";
407
- return claimedResult({
408
- status: "recovering",
409
- continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
410
- });
423
+ return claimedResult({ status: "recovering" });
411
424
  } catch (recoveryError) {
412
425
  console.error(
413
426
  "sandbox deadline rotation recovery failed",
@@ -0,0 +1,51 @@
1
+ import { createHash } from "node:crypto";
2
+ import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
3
+
4
+ /** A position is an append cursor only while the items before it stay identical. */
5
+ export class HistoryPrefixGuard {
6
+ private persisted: string[] = [];
7
+
8
+ seed(items: Array<Record<string, unknown>>, count: number, truncationTokens?: number) {
9
+ const keys = this.keys(items, truncationTokens);
10
+ if (keys.length < count)
11
+ throw new Error("Conversation history seed is shorter than its durable prefix");
12
+ this.persisted = keys.slice(0, count);
13
+ }
14
+
15
+ verify(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
16
+ const keys = this.keys(items, truncationTokens);
17
+ for (let i = 0; i < this.persisted.length; i++) {
18
+ if (keys[i] !== this.persisted[i]) {
19
+ // No content in diagnostics: this may include private user/tool data.
20
+ throw new Error(
21
+ `Conversation history durable prefix changed at item ${i}; refusing to skip unsaved work`,
22
+ );
23
+ }
24
+ }
25
+ return keys;
26
+ }
27
+
28
+ acknowledge(keys: string[], count: number) {
29
+ this.persisted = keys.slice(0, count);
30
+ }
31
+
32
+ private keys(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
33
+ return sanitizeHistoryItemsForModel(items, truncationTokens).map((item) =>
34
+ createHash("sha256")
35
+ .update(JSON.stringify(sorted(normalizeProtocolJsonValue(item))))
36
+ .digest("hex"),
37
+ );
38
+ }
39
+ }
40
+
41
+ function sorted(value: unknown): unknown {
42
+ if (Array.isArray(value)) return value.map(sorted);
43
+ if (value !== null && typeof value === "object") {
44
+ return Object.fromEntries(
45
+ Object.entries(value)
46
+ .sort(([a], [b]) => a.localeCompare(b))
47
+ .map(([key, entry]) => [key, sorted(entry)]),
48
+ );
49
+ }
50
+ return value;
51
+ }
@@ -10,6 +10,7 @@ import { safeErrorDiagnostic } from "./errors";
10
10
  import { historyRowsToAppend, isModelOrToolProgressHistoryItem } from "./history";
11
11
  import { runMandatoryHistoryPersistenceStep } from "./quiescence";
12
12
  import type { TurnMediaArtifacts } from "./media-artifacts";
13
+ import { HistoryPrefixGuard } from "./history-prefix";
13
14
 
14
15
  export type TurnHistorySinkDeps = {
15
16
  db: SharedActivityServices["db"];
@@ -28,8 +29,8 @@ export type TurnHistorySinkDeps = {
28
29
  * Dual-write of conversation truth (issue #35): completed items are reconciled
29
30
  * into session_history_items after every model response and at every turn-end
30
31
  * path (idempotent on position), and the sandbox recovery envelope is upserted
31
- * alongside. Best-effort by design: persistence problems must never fail the
32
- * run.
32
+ * alongside. Provider dispatch and settlement require durable persistence;
33
+ * the prefix guard rejects SDK history removal/reordering before slicing.
33
34
  *
34
35
  * Orphaned-tool-output guard: `stream.state.history` is NOT a plain
35
36
  * append-only array — it is a computed getter
@@ -48,6 +49,23 @@ export type TurnHistorySinkDeps = {
48
49
  * other reconcile.
49
50
  */
50
51
  export class TurnHistorySink {
52
+ private readonly prefix = new HistoryPrefixGuard();
53
+
54
+ seedHistory(input: string | readonly unknown[] | { readonly history: unknown[] }, count: number) {
55
+ const items =
56
+ typeof input === "string" && count === 0
57
+ ? []
58
+ : Array.isArray(input)
59
+ ? input
60
+ : (input as { history?: unknown[] })?.history;
61
+ if (!Array.isArray(items)) throw new Error("Conversation history seed is unavailable");
62
+ this.prefix.seed(
63
+ items as Array<Record<string, unknown>>,
64
+ count,
65
+ this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
66
+ );
67
+ this.persistedHistoryCount = count;
68
+ }
51
69
  persistedHistoryCount = 0;
52
70
  nextHistoryPosition = 0;
53
71
  providerArtifactCandidates: Awaited<ReturnType<typeof turnInput>>["providerArtifactCandidates"] =
@@ -72,6 +90,10 @@ export class TurnHistorySink {
72
90
  const rawHistory = (stream.state as { history?: unknown[] }).history;
73
91
  if (Array.isArray(rawHistory)) {
74
92
  const typedHistory = rawHistory as Array<Record<string, unknown>>;
93
+ const verifiedKeys = this.prefix.verify(
94
+ typedHistory,
95
+ this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
96
+ );
75
97
  await media.retainNativeGeneratedImagesFromHistory(typedHistory);
76
98
  const durableHistory = compactGeneratedImageHistory(
77
99
  compactRetainedScreenshotHistory(typedHistory, media.retainedScreenshotReceiptsByCallId),
@@ -109,9 +131,14 @@ export class TurnHistorySink {
109
131
  });
110
132
  }
111
133
  if (shouldAppendRows || !options.skipInputOnlyRows) {
134
+ this.prefix.acknowledge(verifiedKeys, nextWatermark);
112
135
  this.persistedHistoryCount = nextWatermark;
113
136
  this.nextHistoryPosition = nextPosition;
114
137
  }
138
+ } else if (options.requireDurable) {
139
+ throw new Error(
140
+ "SDK conversation history is unavailable at a mandatory persistence boundary",
141
+ );
115
142
  }
116
143
  const envelope = sandboxStateEntryFromRunState(stream.state);
117
144
  if (envelope) {
@@ -125,10 +152,7 @@ export class TurnHistorySink {
125
152
  );
126
153
  }
127
154
  } catch (persistError) {
128
- console.error(
129
- "session history dual-write failed (run unaffected)",
130
- safeErrorDiagnostic(persistError),
131
- );
155
+ console.error("session history persistence failed", safeErrorDiagnostic(persistError));
132
156
  if (options.requireDurable) throw persistError;
133
157
  }
134
158
  };
@@ -346,8 +346,14 @@ export function assertAgentStreamNotCancelled(cancelled: boolean): void {
346
346
  }
347
347
 
348
348
  /** Interruption streams do not produce a normal final output. Call this only
349
- * after the interruption branch has settled any required action. */
350
- export function requireAgentStreamFinalOutput(finalOutput: unknown | undefined): unknown {
349
+ * after the interruption branch has settled any required action. A successful
350
+ * runtime-owned input wait is a normal terminal result without assistant text;
351
+ * its authority comes from prepared tools, never the provider's output. */
352
+ export function requireAgentStreamFinalOutput(
353
+ finalOutput: unknown | undefined,
354
+ inputWaitYielded = false,
355
+ ): unknown {
356
+ if (inputWaitYielded) return "";
351
357
  if (finalOutput === undefined) {
352
358
  throw new IncompleteAgentStreamError();
353
359
  }
@@ -1571,6 +1571,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
1571
1571
  videoGenerationAcceptancesByCallId,
1572
1572
  });
1573
1573
  } catch (error) {
1574
+ // Stop new external wait mutations before asynchronous failure
1575
+ // persistence. Do not drain here: the original failure/cancellation and
1576
+ // durable attempt fence retain authority over already-running calls.
1577
+ eventing.preparedTools?.inputWaitYield?.closeAdmission();
1574
1578
  return await settleTurnFailure({
1575
1579
  error,
1576
1580
  input,
@@ -471,7 +471,7 @@ export async function runTurnStreamAttempt(
471
471
  // prepareInput already sanitized the exact durable prefix represented
472
472
  // by state.history. Carry its count forward instead of loading and
473
473
  // retaining the full active transcript a second time beside runInput.
474
- historySink.persistedHistoryCount = prepared.persistedHistoryCount;
474
+ historySink.seedHistory(prepared.input.input, prepared.persistedHistoryCount);
475
475
  preparedHistoryCount = prepared.persistedHistoryCount;
476
476
  const historyPositionStartedAt = performance.now();
477
477
  let historyPositionOutcome: "completed" | "failed" = "completed";
@@ -961,6 +961,7 @@ export async function runTurnStreamAttempt(
961
961
  ]);
962
962
  };
963
963
  const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
964
+ const closeStreamWaitAdmission = eventing.preparedTools?.inputWaitYield?.captureStreamClose();
964
965
  let streamDone = false;
965
966
  try {
966
967
  while (true) {
@@ -1330,6 +1331,10 @@ export async function runTurnStreamAttempt(
1330
1331
  }
1331
1332
  }
1332
1333
  } catch (error) {
1334
+ // Event processing can fail while SDK completion is still pending.
1335
+ // Close this stream before any failure publication; a legitimate
1336
+ // compaction retry may start a new, independently fenced generation.
1337
+ closeStreamWaitAdmission?.();
1333
1338
  if (fallbackProviderRequestLifecycleStartedAt !== null) {
1334
1339
  const durationMs = Math.max(
1335
1340
  0,
@@ -1399,8 +1404,13 @@ export async function runTurnStreamAttempt(
1399
1404
  temporalCancellationSignal: cancellationSignal,
1400
1405
  runtimeCancellationSignal,
1401
1406
  });
1407
+ // External Codemode stays reachable until finalization. Close wait
1408
+ // admission before any terminal output/history decision, and drain an
1409
+ // already-admitted wait before consulting the actual runner-yield latch.
1410
+ await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
1402
1411
  if (
1403
1412
  options.requireTerminalModelResponse &&
1413
+ !eventing.preparedTools?.inputWaitYield?.yielded &&
1404
1414
  eventing.stream.interruptions.length === 0 &&
1405
1415
  modelResponseState.responseCount === responseCountBeforeStream
1406
1416
  ) {
@@ -1409,7 +1419,8 @@ export async function runTurnStreamAttempt(
1409
1419
  // is not a completed logical turn: accepting finalOutput's undefined ->
1410
1420
  // empty-string fallback would release the queue and start newer user
1411
1421
  // work. Cancellation retains priority; otherwise checkpoint and recover
1412
- // this exact turn from the durable compacted history.
1422
+ // this exact turn from the durable compacted history. An actual runtime
1423
+ // wait yield deliberately needs no subsequent provider response.
1413
1424
  throwIfWorkerShuttingDown();
1414
1425
  throwIfTurnCancelled();
1415
1426
  throw new PostCompactionContinuationEmptyError();
@@ -1652,7 +1663,10 @@ export async function runTurnStreamAttempt(
1652
1663
  return claimedResult({ status: "requires_action" });
1653
1664
  }
1654
1665
 
1655
- const finalOutput = String(requireAgentStreamFinalOutput(eventing.stream.finalOutput));
1666
+ const inputWaitYielded = eventing.preparedTools?.inputWaitYield?.yielded === true;
1667
+ const finalOutput = String(
1668
+ requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded),
1669
+ );
1656
1670
  await historySink.reconcileConversationTruth({ requireDurable: true });
1657
1671
  // Op-stream durability fence: the tool outputs are now durably in the
1658
1672
  // history store (a redispatch would NOT re-execute them), so this
@@ -1665,10 +1679,9 @@ export async function runTurnStreamAttempt(
1665
1679
  if (
1666
1680
  !(await eventing.settle!({
1667
1681
  events: [
1668
- {
1669
- type: "agent.message.completed",
1670
- payload: { text: finalOutput },
1671
- },
1682
+ ...(inputWaitYielded
1683
+ ? []
1684
+ : [{ type: "agent.message.completed" as const, payload: { text: finalOutput } }]),
1672
1685
  { type: "turn.completed", payload: { output: finalOutput } },
1673
1686
  { type: "session.status.changed", payload: { status: "idle" } },
1674
1687
  ],
@@ -757,6 +757,18 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
757
757
  resolveCredential,
758
758
  onAuthNeeded: publishToolAuthNeeded,
759
759
  materializeConnectorAttachments,
760
+ refreshOwnedCommand: async (commandId) => {
761
+ throwIfWorkerShuttingDown();
762
+ throwIfTurnCancelled();
763
+ // Read only the existing attempt-owned object. A retained read must
764
+ // not provision a sandbox or follow an active-pointer change.
765
+ const owned = (sandboxState.lazyOwnedSandbox?.session ??
766
+ sandboxState.resolvedSandbox?.established.session ??
767
+ media.sdkOwnedSandboxSession) as {
768
+ refreshOwnedCommand?: (id: string) => Promise<boolean>;
769
+ } | null;
770
+ return (await owned?.refreshOwnedCommand?.(commandId)) ?? false;
771
+ },
760
772
  spillOversizedModelToolResult: async ({ operationId, result }) =>
761
773
  await toolResultSpill.spill({ operationId, result }),
762
774
  localMcpServers,
@@ -10,6 +10,7 @@ import {
10
10
  import { publishDurableSessionEvents } from "@opengeni/events";
11
11
 
12
12
  import type { CapacityPhaseDeps, CapacityPhaseOutcome } from "./codex-capacity";
13
+ import { refreshExhaustedXaiQuota } from "../xai-quota";
13
14
 
14
15
  export async function selectXaiTurnCapacity(
15
16
  deps: CapacityPhaseDeps,
@@ -43,6 +44,16 @@ export async function selectXaiTurnCapacity(
43
44
  authoritySnapshot,
44
45
  });
45
46
  const leaseStartedAtMs = performance.now();
47
+ await refreshExhaustedXaiQuota({
48
+ db,
49
+ settings: deps.settings,
50
+ accountId: input.accountId,
51
+ workspaceId: input.workspaceId,
52
+ subjectId,
53
+ sessionId: input.sessionId,
54
+ turnId: turn.id,
55
+ authoritySnapshot,
56
+ });
46
57
  const leased = await acquireXaiCredentialLease(db, {
47
58
  accountId: input.accountId,
48
59
  workspaceId: input.workspaceId,
@@ -3,6 +3,7 @@ import {
3
3
  fetchCodexUsageForAccount,
4
4
  getCodexCapacityWaitForSession,
5
5
  getXaiCapacityWaitForSession,
6
+ resolveXaiWaiterSubject,
6
7
  listCodexAccountStatuses,
7
8
  listPendingCodexCapacityWakeTargets,
8
9
  reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
@@ -11,6 +12,7 @@ import {
11
12
  type CodexCapacitySelectionContext,
12
13
  } from "@opengeni/db";
13
14
  import { publishDurableSessionEvents } from "@opengeni/events";
15
+ import { refreshExhaustedXaiQuota } from "./xai-quota";
14
16
  import {
15
17
  authoritativeCodexCapacityResetAt,
16
18
  codexAccountNeedsLiveCapacityRefresh,
@@ -278,6 +280,22 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
278
280
  if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
279
281
  return { action: "stale" };
280
282
  }
283
+ const authority = await resolveXaiWaiterSubject(
284
+ resolved.db,
285
+ input.workspaceId,
286
+ input.sessionId,
287
+ );
288
+ if (authority)
289
+ await refreshExhaustedXaiQuota({
290
+ db: resolved.db,
291
+ settings: resolved.settings,
292
+ accountId: input.accountId,
293
+ workspaceId: input.workspaceId,
294
+ sessionId: input.sessionId,
295
+ turnId: authority.turnId,
296
+ subjectId: authority.subjectId,
297
+ authoritySnapshot: authority.snapshot,
298
+ });
281
299
  const result = await reconcileXaiCapacityWaitDb(resolved.db, {
282
300
  accountId: input.accountId,
283
301
  workspaceId: input.workspaceId,
@@ -416,13 +416,12 @@ export async function turnInput(
416
416
  options.turnId,
417
417
  ),
418
418
  );
419
- if (updates.length > 0) {
420
- const historyItemIds = new Set(
421
- updates.map((update) => update.deliveredHistoryItemId).filter(Boolean),
422
- );
423
- if (historyItemIds.size !== 1 || updates.some((update) => !update.deliveredHistoryItemId)) {
424
- throw new Error("Delivered internal updates have no single durable model-memory batch");
425
- }
419
+ // A logical turn can receive another atomic batch when an interrupted
420
+ // attempt resumes. Every update must retain its durable receipt, but the
421
+ // turn-wide query legitimately spans multiple history items. Canonical
422
+ // history owns their ordering and exactly-once inclusion below.
423
+ if (updates.some((update) => !update.deliveredHistoryItemId)) {
424
+ throw new Error("Delivered internal update has no durable model-memory batch");
426
425
  }
427
426
  const internalContext = joinInternalContext(
428
427
  options.recovering