@opengeni/worker-bundle 0.26.0-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/errors.d.ts +5 -4
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +4 -2
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-CTEMLTHG.js → activities-control-KS3ZPQVY.js} +202 -84
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/{activities-turn-S5IU347A.js → activities-turn-GHSFZB64.js} +176 -1077
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/{chunk-TAUE3GYT.js → chunk-O2LQWJIK.js} +3115 -1948
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/index.js +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +24 -2
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +1 -0
- package/src/activities/agent-turn/errors.ts +44 -11
- package/src/activities/agent-turn/failure-settlement.ts +21 -8
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +8 -2
- package/src/activities/agent-turn/run.ts +4 -0
- package/src/activities/agent-turn/stream-attempt.ts +20 -7
- package/src/activities/agent-turn/tool-environment.ts +12 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codex-capacity.ts +18 -0
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +229 -87
- package/src/activities/session-state.ts +3 -1
- package/src/activities/xai-quota.ts +35 -0
- package/src/sandbox-routing.ts +115 -0
- package/src/workflows/session.ts +29 -1
- package/dist/activities-control-CTEMLTHG.js.map +0 -1
- package/dist/activities-turn-S5IU347A.js.map +0 -1
- package/dist/chunk-TAUE3GYT.js.map +0 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.26.0-canary.
|
|
3
|
+
"version": "0.26.0-canary.1",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -54,22 +54,22 @@
|
|
|
54
54
|
},
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@llamaindex/liteparse": "2.14.2",
|
|
57
|
-
"@opengeni/agent-proto": "^0.5.1-canary.
|
|
58
|
-
"@opengeni/capabilities": "^0.3.2-canary.
|
|
59
|
-
"@opengeni/codemode": "^0.4.27-canary.
|
|
60
|
-
"@opengeni/codex": "^0.2.21-canary.
|
|
61
|
-
"@opengeni/config": "^1.0.0-canary.
|
|
62
|
-
"@opengeni/contracts": "^2.13.0-canary.
|
|
63
|
-
"@opengeni/core": "^2.7.5-canary.
|
|
64
|
-
"@opengeni/db": "^4.0.0-canary.
|
|
65
|
-
"@opengeni/documents": "^0.8.19-canary.
|
|
66
|
-
"@opengeni/events": "^0.4.17-canary.
|
|
67
|
-
"@opengeni/github": "^0.7.3-canary.
|
|
68
|
-
"@opengeni/network": "^0.3.0-canary.
|
|
69
|
-
"@opengeni/observability": "^0.8.19-canary.
|
|
70
|
-
"@opengeni/runtime": "^2.3.0-canary.
|
|
71
|
-
"@opengeni/storage": "^0.2.120-canary.
|
|
72
|
-
"@opengeni/xai-subscription": "^0.1.3-canary.
|
|
57
|
+
"@opengeni/agent-proto": "^0.5.1-canary.10",
|
|
58
|
+
"@opengeni/capabilities": "^0.3.2-canary.1",
|
|
59
|
+
"@opengeni/codemode": "^0.4.27-canary.1",
|
|
60
|
+
"@opengeni/codex": "^0.2.21-canary.1",
|
|
61
|
+
"@opengeni/config": "^1.0.0-canary.1",
|
|
62
|
+
"@opengeni/contracts": "^2.13.0-canary.1",
|
|
63
|
+
"@opengeni/core": "^2.7.5-canary.1",
|
|
64
|
+
"@opengeni/db": "^4.0.0-canary.1",
|
|
65
|
+
"@opengeni/documents": "^0.8.19-canary.1",
|
|
66
|
+
"@opengeni/events": "^0.4.17-canary.1",
|
|
67
|
+
"@opengeni/github": "^0.7.3-canary.1",
|
|
68
|
+
"@opengeni/network": "^0.3.0-canary.1",
|
|
69
|
+
"@opengeni/observability": "^0.8.19-canary.1",
|
|
70
|
+
"@opengeni/runtime": "^2.3.0-canary.1",
|
|
71
|
+
"@opengeni/storage": "^0.2.120-canary.1",
|
|
72
|
+
"@opengeni/xai-subscription": "^0.1.3-canary.1",
|
|
73
73
|
"@temporalio/activity": "^1.17.0",
|
|
74
74
|
"@temporalio/client": "^1.17.0",
|
|
75
75
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -599,6 +599,7 @@ export async function buildTurnAgent(deps: BuildTurnAgentDeps) {
|
|
|
599
599
|
let agentConstructionOutcome: "completed" | "failed" = "completed";
|
|
600
600
|
try {
|
|
601
601
|
return runtime.buildAgent(eventing.modelRunSettings, runtimeResources, {
|
|
602
|
+
...(preparedTools.inputWaitYield ? { inputWaitYield: preparedTools.inputWaitYield } : {}),
|
|
602
603
|
reasoningEffort: turn.reasoningEffort,
|
|
603
604
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
604
605
|
...(serviceTier ? { serviceTier } : {}),
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import {
|
|
2
2
|
ActiveSessionHistoryLimitExceededError,
|
|
3
3
|
ApprovalRunStateLimitExceededError,
|
|
4
|
+
nestedPostgresSqlState,
|
|
5
|
+
safeDatabaseErrorFacts,
|
|
4
6
|
isRetryableDatabaseTransportFailure,
|
|
5
7
|
isSessionEventPersistenceError,
|
|
6
8
|
SandboxLeaseTransitionError,
|
|
@@ -266,22 +268,21 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
|
|
|
266
268
|
* Convert the atomic claim transaction's failure into a small, stable
|
|
267
269
|
* Temporal wire contract. The original error remains in activity diagnostics,
|
|
268
270
|
* but SQL text, parameters, and arbitrary invariant messages never enter
|
|
269
|
-
* workflow history.
|
|
270
|
-
*
|
|
271
|
-
*
|
|
272
|
-
*
|
|
271
|
+
* workflow history. Every database failure is safe to re-read after backoff
|
|
272
|
+
* because the claim transaction contains no model/tool effects. Database
|
|
273
|
+
* constraints and authorization guards can also be repaired by a deployment or
|
|
274
|
+
* authority refresh, so they must preserve pending work. Only a non-database
|
|
275
|
+
* claim invariant is terminal.
|
|
273
276
|
*/
|
|
274
277
|
export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
|
|
275
278
|
const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
|
|
276
279
|
const retryableCode = retryableDatabaseFailureCode(error);
|
|
277
|
-
// The
|
|
278
|
-
//
|
|
279
|
-
//
|
|
280
|
-
//
|
|
281
|
-
// they commonly represent a lost connection; known constraint, auth, and
|
|
282
|
-
// application SQLSTATEs are permanent and must not create an infinite loop.
|
|
280
|
+
// The workflow retries after a durable re-read and bounded delay. A
|
|
281
|
+
// SessionEventPersistenceError proves the failure happened inside the atomic
|
|
282
|
+
// claim boundary, before provider or tool work; retaining it is safer than
|
|
283
|
+
// destroying accepted machine input on an application SQLSTATE.
|
|
283
284
|
const detail: PreClaimFailureDetail = {
|
|
284
|
-
disposition: retryableCode ? "retryable" : "permanent",
|
|
285
|
+
disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
|
|
285
286
|
code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
|
|
286
287
|
};
|
|
287
288
|
return ApplicationFailure.create({
|
|
@@ -826,6 +827,26 @@ export function classifyXaiCredentialFailure(error: unknown): XaiCredentialFailu
|
|
|
826
827
|
return null;
|
|
827
828
|
}
|
|
828
829
|
|
|
830
|
+
// The generic turn-failure boundary also receives application/provider errors.
|
|
831
|
+
// A five-character code or generic severity alone does not establish a driver error.
|
|
832
|
+
function findPostgresDriverError(error: unknown): Record<string, unknown> | null {
|
|
833
|
+
const queue: unknown[] = [error];
|
|
834
|
+
const seen = new Set<unknown>();
|
|
835
|
+
for (let index = 0; index < queue.length && index < 64; index += 1) {
|
|
836
|
+
const current = queue[index];
|
|
837
|
+
if (!current || typeof current !== "object" || seen.has(current)) continue;
|
|
838
|
+
seen.add(current);
|
|
839
|
+
const record = current as Record<string, unknown>;
|
|
840
|
+
if (record.name === "PostgresError") return record;
|
|
841
|
+
for (const key of ["cause", "original", "driverError", "error", "errors"]) {
|
|
842
|
+
const nested = record[key];
|
|
843
|
+
if (Array.isArray(nested)) queue.push(...nested.slice(0, 64));
|
|
844
|
+
else if (nested !== undefined) queue.push(nested);
|
|
845
|
+
}
|
|
846
|
+
}
|
|
847
|
+
return null;
|
|
848
|
+
}
|
|
849
|
+
|
|
829
850
|
export function agentRunFailurePayload(
|
|
830
851
|
error: unknown,
|
|
831
852
|
options: { isCodexTurn?: boolean } = {},
|
|
@@ -1066,6 +1087,18 @@ export function agentRunFailurePayload(
|
|
|
1066
1087
|
}
|
|
1067
1088
|
return { error: message, code: "provider_unavailable", retryable: true };
|
|
1068
1089
|
}
|
|
1090
|
+
const postgresDriverError = findPostgresDriverError(error);
|
|
1091
|
+
if (postgresDriverError) {
|
|
1092
|
+
const database = safeDatabaseErrorFacts(postgresDriverError);
|
|
1093
|
+
const sqlState = nestedPostgresSqlState(postgresDriverError);
|
|
1094
|
+
if (sqlState !== null || Object.keys(database).length > 0) {
|
|
1095
|
+
return {
|
|
1096
|
+
error: message,
|
|
1097
|
+
sqlState,
|
|
1098
|
+
...(Object.keys(database).length > 0 ? { database } : {}),
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1069
1102
|
return { error: message };
|
|
1070
1103
|
}
|
|
1071
1104
|
|
|
@@ -263,9 +263,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
263
263
|
// finished yet.
|
|
264
264
|
const recoveryTurnId = attempt.turnId;
|
|
265
265
|
// A true epoch supersession and a provider lifecycle transition are both
|
|
266
|
-
// recoverable control-plane states, never session failures.
|
|
267
|
-
//
|
|
268
|
-
//
|
|
266
|
+
// recoverable control-plane states, never session failures. A rotation
|
|
267
|
+
// persists an exact group/epoch wait marker so the workflow parks before
|
|
268
|
+
// another turn-worker dispatch; shorter non-rotation transitions retain
|
|
269
|
+
// their existing paced retry.
|
|
269
270
|
const lifecycleTransition = sandboxLifecycleTransitionDiagnostic(error);
|
|
270
271
|
const leaseControlError =
|
|
271
272
|
error instanceof SandboxLeaseSupersededError ? error : lifecycleTransition;
|
|
@@ -282,6 +283,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
282
283
|
const transitionPending = lifecycleTransition !== null || rotationPending;
|
|
283
284
|
const deadlineRotationPending =
|
|
284
285
|
rotationPending && fencedLease?.rotationReason === "provider_deadline";
|
|
286
|
+
const sandboxLifecycleWait = rotationPending
|
|
287
|
+
? {
|
|
288
|
+
version: 1 as const,
|
|
289
|
+
sandboxGroupId: leaseControlError.sandboxGroupId,
|
|
290
|
+
leaseEpoch: fencedLease?.leaseEpoch ?? leaseControlError.leaseEpoch,
|
|
291
|
+
reason: "rotation_in_progress" as const,
|
|
292
|
+
}
|
|
293
|
+
: undefined;
|
|
285
294
|
const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
|
|
286
295
|
sessionId: input.sessionId,
|
|
287
296
|
turnId: recoveryTurnId,
|
|
@@ -306,6 +315,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
306
315
|
},
|
|
307
316
|
}
|
|
308
317
|
: {}),
|
|
318
|
+
...(sandboxLifecycleWait ? { sandboxLifecycleWait } : {}),
|
|
309
319
|
});
|
|
310
320
|
if (recovery.action === "stale") {
|
|
311
321
|
acknowledgeLostAttemptOwnership();
|
|
@@ -319,7 +329,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
319
329
|
control.turnMetricOutcome = "recovering";
|
|
320
330
|
return claimedResult({
|
|
321
331
|
status: "recovering",
|
|
322
|
-
...(transitionPending
|
|
332
|
+
...(transitionPending && !sandboxLifecycleWait
|
|
323
333
|
? {
|
|
324
334
|
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
325
335
|
}
|
|
@@ -393,6 +403,12 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
393
403
|
sandboxGroupId: rotation.sandboxGroupId,
|
|
394
404
|
leaseEpoch: rotation.leaseEpoch,
|
|
395
405
|
},
|
|
406
|
+
sandboxLifecycleWait: {
|
|
407
|
+
version: 1,
|
|
408
|
+
sandboxGroupId: rotation.sandboxGroupId,
|
|
409
|
+
leaseEpoch: rotation.leaseEpoch,
|
|
410
|
+
reason: "rotation_in_progress",
|
|
411
|
+
},
|
|
396
412
|
});
|
|
397
413
|
if (recovery.action === "stale") {
|
|
398
414
|
acknowledgeLostAttemptOwnership();
|
|
@@ -404,10 +420,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
|
|
|
404
420
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
|
|
405
421
|
control.activityStatus = "recovering";
|
|
406
422
|
control.turnMetricOutcome = "recovering";
|
|
407
|
-
return claimedResult({
|
|
408
|
-
status: "recovering",
|
|
409
|
-
continueDelayMs: sandboxDeadlineRotationRecoveryDelayMs(settings),
|
|
410
|
-
});
|
|
423
|
+
return claimedResult({ status: "recovering" });
|
|
411
424
|
} catch (recoveryError) {
|
|
412
425
|
console.error(
|
|
413
426
|
"sandbox deadline rotation recovery failed",
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
|
|
3
|
+
|
|
4
|
+
/** A position is an append cursor only while the items before it stay identical. */
|
|
5
|
+
export class HistoryPrefixGuard {
|
|
6
|
+
private persisted: string[] = [];
|
|
7
|
+
|
|
8
|
+
seed(items: Array<Record<string, unknown>>, count: number, truncationTokens?: number) {
|
|
9
|
+
const keys = this.keys(items, truncationTokens);
|
|
10
|
+
if (keys.length < count)
|
|
11
|
+
throw new Error("Conversation history seed is shorter than its durable prefix");
|
|
12
|
+
this.persisted = keys.slice(0, count);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
verify(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
|
|
16
|
+
const keys = this.keys(items, truncationTokens);
|
|
17
|
+
for (let i = 0; i < this.persisted.length; i++) {
|
|
18
|
+
if (keys[i] !== this.persisted[i]) {
|
|
19
|
+
// No content in diagnostics: this may include private user/tool data.
|
|
20
|
+
throw new Error(
|
|
21
|
+
`Conversation history durable prefix changed at item ${i}; refusing to skip unsaved work`,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return keys;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
acknowledge(keys: string[], count: number) {
|
|
29
|
+
this.persisted = keys.slice(0, count);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private keys(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
|
|
33
|
+
return sanitizeHistoryItemsForModel(items, truncationTokens).map((item) =>
|
|
34
|
+
createHash("sha256")
|
|
35
|
+
.update(JSON.stringify(sorted(normalizeProtocolJsonValue(item))))
|
|
36
|
+
.digest("hex"),
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function sorted(value: unknown): unknown {
|
|
42
|
+
if (Array.isArray(value)) return value.map(sorted);
|
|
43
|
+
if (value !== null && typeof value === "object") {
|
|
44
|
+
return Object.fromEntries(
|
|
45
|
+
Object.entries(value)
|
|
46
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
47
|
+
.map(([key, entry]) => [key, sorted(entry)]),
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
return value;
|
|
51
|
+
}
|
|
@@ -10,6 +10,7 @@ import { safeErrorDiagnostic } from "./errors";
|
|
|
10
10
|
import { historyRowsToAppend, isModelOrToolProgressHistoryItem } from "./history";
|
|
11
11
|
import { runMandatoryHistoryPersistenceStep } from "./quiescence";
|
|
12
12
|
import type { TurnMediaArtifacts } from "./media-artifacts";
|
|
13
|
+
import { HistoryPrefixGuard } from "./history-prefix";
|
|
13
14
|
|
|
14
15
|
export type TurnHistorySinkDeps = {
|
|
15
16
|
db: SharedActivityServices["db"];
|
|
@@ -28,8 +29,8 @@ export type TurnHistorySinkDeps = {
|
|
|
28
29
|
* Dual-write of conversation truth (issue #35): completed items are reconciled
|
|
29
30
|
* into session_history_items after every model response and at every turn-end
|
|
30
31
|
* path (idempotent on position), and the sandbox recovery envelope is upserted
|
|
31
|
-
* alongside.
|
|
32
|
-
*
|
|
32
|
+
* alongside. Provider dispatch and settlement require durable persistence;
|
|
33
|
+
* the prefix guard rejects SDK history removal/reordering before slicing.
|
|
33
34
|
*
|
|
34
35
|
* Orphaned-tool-output guard: `stream.state.history` is NOT a plain
|
|
35
36
|
* append-only array — it is a computed getter
|
|
@@ -48,6 +49,23 @@ export type TurnHistorySinkDeps = {
|
|
|
48
49
|
* other reconcile.
|
|
49
50
|
*/
|
|
50
51
|
export class TurnHistorySink {
|
|
52
|
+
private readonly prefix = new HistoryPrefixGuard();
|
|
53
|
+
|
|
54
|
+
seedHistory(input: string | readonly unknown[] | { readonly history: unknown[] }, count: number) {
|
|
55
|
+
const items =
|
|
56
|
+
typeof input === "string" && count === 0
|
|
57
|
+
? []
|
|
58
|
+
: Array.isArray(input)
|
|
59
|
+
? input
|
|
60
|
+
: (input as { history?: unknown[] })?.history;
|
|
61
|
+
if (!Array.isArray(items)) throw new Error("Conversation history seed is unavailable");
|
|
62
|
+
this.prefix.seed(
|
|
63
|
+
items as Array<Record<string, unknown>>,
|
|
64
|
+
count,
|
|
65
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
|
|
66
|
+
);
|
|
67
|
+
this.persistedHistoryCount = count;
|
|
68
|
+
}
|
|
51
69
|
persistedHistoryCount = 0;
|
|
52
70
|
nextHistoryPosition = 0;
|
|
53
71
|
providerArtifactCandidates: Awaited<ReturnType<typeof turnInput>>["providerArtifactCandidates"] =
|
|
@@ -72,6 +90,10 @@ export class TurnHistorySink {
|
|
|
72
90
|
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
73
91
|
if (Array.isArray(rawHistory)) {
|
|
74
92
|
const typedHistory = rawHistory as Array<Record<string, unknown>>;
|
|
93
|
+
const verifiedKeys = this.prefix.verify(
|
|
94
|
+
typedHistory,
|
|
95
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
|
|
96
|
+
);
|
|
75
97
|
await media.retainNativeGeneratedImagesFromHistory(typedHistory);
|
|
76
98
|
const durableHistory = compactGeneratedImageHistory(
|
|
77
99
|
compactRetainedScreenshotHistory(typedHistory, media.retainedScreenshotReceiptsByCallId),
|
|
@@ -109,9 +131,14 @@ export class TurnHistorySink {
|
|
|
109
131
|
});
|
|
110
132
|
}
|
|
111
133
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
134
|
+
this.prefix.acknowledge(verifiedKeys, nextWatermark);
|
|
112
135
|
this.persistedHistoryCount = nextWatermark;
|
|
113
136
|
this.nextHistoryPosition = nextPosition;
|
|
114
137
|
}
|
|
138
|
+
} else if (options.requireDurable) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
"SDK conversation history is unavailable at a mandatory persistence boundary",
|
|
141
|
+
);
|
|
115
142
|
}
|
|
116
143
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
117
144
|
if (envelope) {
|
|
@@ -125,10 +152,7 @@ export class TurnHistorySink {
|
|
|
125
152
|
);
|
|
126
153
|
}
|
|
127
154
|
} catch (persistError) {
|
|
128
|
-
console.error(
|
|
129
|
-
"session history dual-write failed (run unaffected)",
|
|
130
|
-
safeErrorDiagnostic(persistError),
|
|
131
|
-
);
|
|
155
|
+
console.error("session history persistence failed", safeErrorDiagnostic(persistError));
|
|
132
156
|
if (options.requireDurable) throw persistError;
|
|
133
157
|
}
|
|
134
158
|
};
|
|
@@ -346,8 +346,14 @@ export function assertAgentStreamNotCancelled(cancelled: boolean): void {
|
|
|
346
346
|
}
|
|
347
347
|
|
|
348
348
|
/** Interruption streams do not produce a normal final output. Call this only
|
|
349
|
-
* after the interruption branch has settled any required action.
|
|
350
|
-
|
|
349
|
+
* after the interruption branch has settled any required action. A successful
|
|
350
|
+
* runtime-owned input wait is a normal terminal result without assistant text;
|
|
351
|
+
* its authority comes from prepared tools, never the provider's output. */
|
|
352
|
+
export function requireAgentStreamFinalOutput(
|
|
353
|
+
finalOutput: unknown | undefined,
|
|
354
|
+
inputWaitYielded = false,
|
|
355
|
+
): unknown {
|
|
356
|
+
if (inputWaitYielded) return "";
|
|
351
357
|
if (finalOutput === undefined) {
|
|
352
358
|
throw new IncompleteAgentStreamError();
|
|
353
359
|
}
|
|
@@ -1571,6 +1571,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1571
1571
|
videoGenerationAcceptancesByCallId,
|
|
1572
1572
|
});
|
|
1573
1573
|
} catch (error) {
|
|
1574
|
+
// Stop new external wait mutations before asynchronous failure
|
|
1575
|
+
// persistence. Do not drain here: the original failure/cancellation and
|
|
1576
|
+
// durable attempt fence retain authority over already-running calls.
|
|
1577
|
+
eventing.preparedTools?.inputWaitYield?.closeAdmission();
|
|
1574
1578
|
return await settleTurnFailure({
|
|
1575
1579
|
error,
|
|
1576
1580
|
input,
|
|
@@ -471,7 +471,7 @@ export async function runTurnStreamAttempt(
|
|
|
471
471
|
// prepareInput already sanitized the exact durable prefix represented
|
|
472
472
|
// by state.history. Carry its count forward instead of loading and
|
|
473
473
|
// retaining the full active transcript a second time beside runInput.
|
|
474
|
-
historySink.
|
|
474
|
+
historySink.seedHistory(prepared.input.input, prepared.persistedHistoryCount);
|
|
475
475
|
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
476
476
|
const historyPositionStartedAt = performance.now();
|
|
477
477
|
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
@@ -961,6 +961,7 @@ export async function runTurnStreamAttempt(
|
|
|
961
961
|
]);
|
|
962
962
|
};
|
|
963
963
|
const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
|
|
964
|
+
const closeStreamWaitAdmission = eventing.preparedTools?.inputWaitYield?.captureStreamClose();
|
|
964
965
|
let streamDone = false;
|
|
965
966
|
try {
|
|
966
967
|
while (true) {
|
|
@@ -1330,6 +1331,10 @@ export async function runTurnStreamAttempt(
|
|
|
1330
1331
|
}
|
|
1331
1332
|
}
|
|
1332
1333
|
} catch (error) {
|
|
1334
|
+
// Event processing can fail while SDK completion is still pending.
|
|
1335
|
+
// Close this stream before any failure publication; a legitimate
|
|
1336
|
+
// compaction retry may start a new, independently fenced generation.
|
|
1337
|
+
closeStreamWaitAdmission?.();
|
|
1333
1338
|
if (fallbackProviderRequestLifecycleStartedAt !== null) {
|
|
1334
1339
|
const durationMs = Math.max(
|
|
1335
1340
|
0,
|
|
@@ -1399,8 +1404,13 @@ export async function runTurnStreamAttempt(
|
|
|
1399
1404
|
temporalCancellationSignal: cancellationSignal,
|
|
1400
1405
|
runtimeCancellationSignal,
|
|
1401
1406
|
});
|
|
1407
|
+
// External Codemode stays reachable until finalization. Close wait
|
|
1408
|
+
// admission before any terminal output/history decision, and drain an
|
|
1409
|
+
// already-admitted wait before consulting the actual runner-yield latch.
|
|
1410
|
+
await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
|
|
1402
1411
|
if (
|
|
1403
1412
|
options.requireTerminalModelResponse &&
|
|
1413
|
+
!eventing.preparedTools?.inputWaitYield?.yielded &&
|
|
1404
1414
|
eventing.stream.interruptions.length === 0 &&
|
|
1405
1415
|
modelResponseState.responseCount === responseCountBeforeStream
|
|
1406
1416
|
) {
|
|
@@ -1409,7 +1419,8 @@ export async function runTurnStreamAttempt(
|
|
|
1409
1419
|
// is not a completed logical turn: accepting finalOutput's undefined ->
|
|
1410
1420
|
// empty-string fallback would release the queue and start newer user
|
|
1411
1421
|
// work. Cancellation retains priority; otherwise checkpoint and recover
|
|
1412
|
-
// this exact turn from the durable compacted history.
|
|
1422
|
+
// this exact turn from the durable compacted history. An actual runtime
|
|
1423
|
+
// wait yield deliberately needs no subsequent provider response.
|
|
1413
1424
|
throwIfWorkerShuttingDown();
|
|
1414
1425
|
throwIfTurnCancelled();
|
|
1415
1426
|
throw new PostCompactionContinuationEmptyError();
|
|
@@ -1652,7 +1663,10 @@ export async function runTurnStreamAttempt(
|
|
|
1652
1663
|
return claimedResult({ status: "requires_action" });
|
|
1653
1664
|
}
|
|
1654
1665
|
|
|
1655
|
-
const
|
|
1666
|
+
const inputWaitYielded = eventing.preparedTools?.inputWaitYield?.yielded === true;
|
|
1667
|
+
const finalOutput = String(
|
|
1668
|
+
requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded),
|
|
1669
|
+
);
|
|
1656
1670
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1657
1671
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
1658
1672
|
// history store (a redispatch would NOT re-execute them), so this
|
|
@@ -1665,10 +1679,9 @@ export async function runTurnStreamAttempt(
|
|
|
1665
1679
|
if (
|
|
1666
1680
|
!(await eventing.settle!({
|
|
1667
1681
|
events: [
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
payload: { text: finalOutput },
|
|
1671
|
-
},
|
|
1682
|
+
...(inputWaitYielded
|
|
1683
|
+
? []
|
|
1684
|
+
: [{ type: "agent.message.completed" as const, payload: { text: finalOutput } }]),
|
|
1672
1685
|
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1673
1686
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1674
1687
|
],
|
|
@@ -757,6 +757,18 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
757
757
|
resolveCredential,
|
|
758
758
|
onAuthNeeded: publishToolAuthNeeded,
|
|
759
759
|
materializeConnectorAttachments,
|
|
760
|
+
refreshOwnedCommand: async (commandId) => {
|
|
761
|
+
throwIfWorkerShuttingDown();
|
|
762
|
+
throwIfTurnCancelled();
|
|
763
|
+
// Read only the existing attempt-owned object. A retained read must
|
|
764
|
+
// not provision a sandbox or follow an active-pointer change.
|
|
765
|
+
const owned = (sandboxState.lazyOwnedSandbox?.session ??
|
|
766
|
+
sandboxState.resolvedSandbox?.established.session ??
|
|
767
|
+
media.sdkOwnedSandboxSession) as {
|
|
768
|
+
refreshOwnedCommand?: (id: string) => Promise<boolean>;
|
|
769
|
+
} | null;
|
|
770
|
+
return (await owned?.refreshOwnedCommand?.(commandId)) ?? false;
|
|
771
|
+
},
|
|
760
772
|
spillOversizedModelToolResult: async ({ operationId, result }) =>
|
|
761
773
|
await toolResultSpill.spill({ operationId, result }),
|
|
762
774
|
localMcpServers,
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
11
11
|
|
|
12
12
|
import type { CapacityPhaseDeps, CapacityPhaseOutcome } from "./codex-capacity";
|
|
13
|
+
import { refreshExhaustedXaiQuota } from "../xai-quota";
|
|
13
14
|
|
|
14
15
|
export async function selectXaiTurnCapacity(
|
|
15
16
|
deps: CapacityPhaseDeps,
|
|
@@ -43,6 +44,16 @@ export async function selectXaiTurnCapacity(
|
|
|
43
44
|
authoritySnapshot,
|
|
44
45
|
});
|
|
45
46
|
const leaseStartedAtMs = performance.now();
|
|
47
|
+
await refreshExhaustedXaiQuota({
|
|
48
|
+
db,
|
|
49
|
+
settings: deps.settings,
|
|
50
|
+
accountId: input.accountId,
|
|
51
|
+
workspaceId: input.workspaceId,
|
|
52
|
+
subjectId,
|
|
53
|
+
sessionId: input.sessionId,
|
|
54
|
+
turnId: turn.id,
|
|
55
|
+
authoritySnapshot,
|
|
56
|
+
});
|
|
46
57
|
const leased = await acquireXaiCredentialLease(db, {
|
|
47
58
|
accountId: input.accountId,
|
|
48
59
|
workspaceId: input.workspaceId,
|
|
@@ -3,6 +3,7 @@ import {
|
|
|
3
3
|
fetchCodexUsageForAccount,
|
|
4
4
|
getCodexCapacityWaitForSession,
|
|
5
5
|
getXaiCapacityWaitForSession,
|
|
6
|
+
resolveXaiWaiterSubject,
|
|
6
7
|
listCodexAccountStatuses,
|
|
7
8
|
listPendingCodexCapacityWakeTargets,
|
|
8
9
|
reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
|
|
@@ -11,6 +12,7 @@ import {
|
|
|
11
12
|
type CodexCapacitySelectionContext,
|
|
12
13
|
} from "@opengeni/db";
|
|
13
14
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
15
|
+
import { refreshExhaustedXaiQuota } from "./xai-quota";
|
|
14
16
|
import {
|
|
15
17
|
authoritativeCodexCapacityResetAt,
|
|
16
18
|
codexAccountNeedsLiveCapacityRefresh,
|
|
@@ -278,6 +280,22 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
278
280
|
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
279
281
|
return { action: "stale" };
|
|
280
282
|
}
|
|
283
|
+
const authority = await resolveXaiWaiterSubject(
|
|
284
|
+
resolved.db,
|
|
285
|
+
input.workspaceId,
|
|
286
|
+
input.sessionId,
|
|
287
|
+
);
|
|
288
|
+
if (authority)
|
|
289
|
+
await refreshExhaustedXaiQuota({
|
|
290
|
+
db: resolved.db,
|
|
291
|
+
settings: resolved.settings,
|
|
292
|
+
accountId: input.accountId,
|
|
293
|
+
workspaceId: input.workspaceId,
|
|
294
|
+
sessionId: input.sessionId,
|
|
295
|
+
turnId: authority.turnId,
|
|
296
|
+
subjectId: authority.subjectId,
|
|
297
|
+
authoritySnapshot: authority.snapshot,
|
|
298
|
+
});
|
|
281
299
|
const result = await reconcileXaiCapacityWaitDb(resolved.db, {
|
|
282
300
|
accountId: input.accountId,
|
|
283
301
|
workspaceId: input.workspaceId,
|
|
@@ -416,13 +416,12 @@ export async function turnInput(
|
|
|
416
416
|
options.turnId,
|
|
417
417
|
),
|
|
418
418
|
);
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
}
|
|
419
|
+
// A logical turn can receive another atomic batch when an interrupted
|
|
420
|
+
// attempt resumes. Every update must retain its durable receipt, but the
|
|
421
|
+
// turn-wide query legitimately spans multiple history items. Canonical
|
|
422
|
+
// history owns their ordering and exactly-once inclusion below.
|
|
423
|
+
if (updates.some((update) => !update.deliveredHistoryItemId)) {
|
|
424
|
+
throw new Error("Delivered internal update has no durable model-memory batch");
|
|
426
425
|
}
|
|
427
426
|
const internalContext = joinInternalContext(
|
|
428
427
|
options.recovering
|