@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { normalizeProtocolJsonValue, sanitizeHistoryItemsForModel } from "@opengeni/runtime";
|
|
3
|
+
|
|
4
|
+
/** A position is an append cursor only while the items before it stay identical. */
|
|
5
|
+
export class HistoryPrefixGuard {
|
|
6
|
+
private persisted: string[] = [];
|
|
7
|
+
|
|
8
|
+
seed(items: Array<Record<string, unknown>>, count: number, truncationTokens?: number) {
|
|
9
|
+
const keys = this.keys(items, truncationTokens);
|
|
10
|
+
if (keys.length < count)
|
|
11
|
+
throw new Error("Conversation history seed is shorter than its durable prefix");
|
|
12
|
+
this.persisted = keys.slice(0, count);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
verify(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
|
|
16
|
+
const keys = this.keys(items, truncationTokens);
|
|
17
|
+
for (let i = 0; i < this.persisted.length; i++) {
|
|
18
|
+
if (keys[i] !== this.persisted[i]) {
|
|
19
|
+
// No content in diagnostics: this may include private user/tool data.
|
|
20
|
+
throw new Error(
|
|
21
|
+
`Conversation history durable prefix changed at item ${i}; refusing to skip unsaved work`,
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return keys;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
acknowledge(keys: string[], count: number) {
|
|
29
|
+
this.persisted = keys.slice(0, count);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private keys(items: Array<Record<string, unknown>>, truncationTokens?: number): string[] {
|
|
33
|
+
return sanitizeHistoryItemsForModel(items, truncationTokens).map((item) =>
|
|
34
|
+
createHash("sha256")
|
|
35
|
+
.update(JSON.stringify(sorted(normalizeProtocolJsonValue(item))))
|
|
36
|
+
.digest("hex"),
|
|
37
|
+
);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function sorted(value: unknown): unknown {
|
|
42
|
+
if (Array.isArray(value)) return value.map(sorted);
|
|
43
|
+
if (value !== null && typeof value === "object") {
|
|
44
|
+
return Object.fromEntries(
|
|
45
|
+
Object.entries(value)
|
|
46
|
+
.sort(([a], [b]) => a.localeCompare(b))
|
|
47
|
+
.map(([key, entry]) => [key, sorted(entry)]),
|
|
48
|
+
);
|
|
49
|
+
}
|
|
50
|
+
return value;
|
|
51
|
+
}
|
|
@@ -10,6 +10,7 @@ import { safeErrorDiagnostic } from "./errors";
|
|
|
10
10
|
import { historyRowsToAppend, isModelOrToolProgressHistoryItem } from "./history";
|
|
11
11
|
import { runMandatoryHistoryPersistenceStep } from "./quiescence";
|
|
12
12
|
import type { TurnMediaArtifacts } from "./media-artifacts";
|
|
13
|
+
import { HistoryPrefixGuard } from "./history-prefix";
|
|
13
14
|
|
|
14
15
|
export type TurnHistorySinkDeps = {
|
|
15
16
|
db: SharedActivityServices["db"];
|
|
@@ -28,8 +29,8 @@ export type TurnHistorySinkDeps = {
|
|
|
28
29
|
* Dual-write of conversation truth (issue #35): completed items are reconciled
|
|
29
30
|
* into session_history_items after every model response and at every turn-end
|
|
30
31
|
* path (idempotent on position), and the sandbox recovery envelope is upserted
|
|
31
|
-
* alongside.
|
|
32
|
-
*
|
|
32
|
+
* alongside. Provider dispatch and settlement require durable persistence;
|
|
33
|
+
* the prefix guard rejects SDK history removal/reordering before slicing.
|
|
33
34
|
*
|
|
34
35
|
* Orphaned-tool-output guard: `stream.state.history` is NOT a plain
|
|
35
36
|
* append-only array — it is a computed getter
|
|
@@ -48,6 +49,23 @@ export type TurnHistorySinkDeps = {
|
|
|
48
49
|
* other reconcile.
|
|
49
50
|
*/
|
|
50
51
|
export class TurnHistorySink {
|
|
52
|
+
private readonly prefix = new HistoryPrefixGuard();
|
|
53
|
+
|
|
54
|
+
seedHistory(input: string | readonly unknown[] | { readonly history: unknown[] }, count: number) {
|
|
55
|
+
const items =
|
|
56
|
+
typeof input === "string" && count === 0
|
|
57
|
+
? []
|
|
58
|
+
: Array.isArray(input)
|
|
59
|
+
? input
|
|
60
|
+
: (input as { history?: unknown[] })?.history;
|
|
61
|
+
if (!Array.isArray(items)) throw new Error("Conversation history seed is unavailable");
|
|
62
|
+
this.prefix.seed(
|
|
63
|
+
items as Array<Record<string, unknown>>,
|
|
64
|
+
count,
|
|
65
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
|
|
66
|
+
);
|
|
67
|
+
this.persistedHistoryCount = count;
|
|
68
|
+
}
|
|
51
69
|
persistedHistoryCount = 0;
|
|
52
70
|
nextHistoryPosition = 0;
|
|
53
71
|
providerArtifactCandidates: Awaited<ReturnType<typeof turnInput>>["providerArtifactCandidates"] =
|
|
@@ -72,6 +90,10 @@ export class TurnHistorySink {
|
|
|
72
90
|
const rawHistory = (stream.state as { history?: unknown[] }).history;
|
|
73
91
|
if (Array.isArray(rawHistory)) {
|
|
74
92
|
const typedHistory = rawHistory as Array<Record<string, unknown>>;
|
|
93
|
+
const verifiedKeys = this.prefix.verify(
|
|
94
|
+
typedHistory,
|
|
95
|
+
this.deps.getModelRunSettings().modelToolOutputTruncationTokens,
|
|
96
|
+
);
|
|
75
97
|
await media.retainNativeGeneratedImagesFromHistory(typedHistory);
|
|
76
98
|
const durableHistory = compactGeneratedImageHistory(
|
|
77
99
|
compactRetainedScreenshotHistory(typedHistory, media.retainedScreenshotReceiptsByCallId),
|
|
@@ -109,9 +131,14 @@ export class TurnHistorySink {
|
|
|
109
131
|
});
|
|
110
132
|
}
|
|
111
133
|
if (shouldAppendRows || !options.skipInputOnlyRows) {
|
|
134
|
+
this.prefix.acknowledge(verifiedKeys, nextWatermark);
|
|
112
135
|
this.persistedHistoryCount = nextWatermark;
|
|
113
136
|
this.nextHistoryPosition = nextPosition;
|
|
114
137
|
}
|
|
138
|
+
} else if (options.requireDurable) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
"SDK conversation history is unavailable at a mandatory persistence boundary",
|
|
141
|
+
);
|
|
115
142
|
}
|
|
116
143
|
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
117
144
|
if (envelope) {
|
|
@@ -125,10 +152,7 @@ export class TurnHistorySink {
|
|
|
125
152
|
);
|
|
126
153
|
}
|
|
127
154
|
} catch (persistError) {
|
|
128
|
-
console.error(
|
|
129
|
-
"session history dual-write failed (run unaffected)",
|
|
130
|
-
safeErrorDiagnostic(persistError),
|
|
131
|
-
);
|
|
155
|
+
console.error("session history persistence failed", safeErrorDiagnostic(persistError));
|
|
132
156
|
if (options.requireDurable) throw persistError;
|
|
133
157
|
}
|
|
134
158
|
};
|
|
@@ -89,6 +89,7 @@ export function assertSessionAttemptQuiescenceRecoveryDurable(input: {
|
|
|
89
89
|
export const QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS = 250;
|
|
90
90
|
export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
|
|
91
91
|
export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
|
|
92
|
+
export const TURN_FINALIZER_STEP_TIMEOUT_MS = 30_000;
|
|
92
93
|
|
|
93
94
|
/** A fenced activity must never heartbeat forever while its physical writer
|
|
94
95
|
* drain is stuck. The last-resort worker exit stops every in-process writer;
|
|
@@ -241,35 +242,40 @@ export async function releaseTurnSandboxAfterWriterDrain(
|
|
|
241
242
|
}
|
|
242
243
|
|
|
243
244
|
/**
|
|
244
|
-
* Await
|
|
245
|
-
* execution window.
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
245
|
+
* Await bounded, idempotent finalizer housekeeping only while this Temporal
|
|
246
|
+
* activity still owns its execution window. A completed turn must never keep
|
|
247
|
+
* heartbeating forever because a provider/tool close promise did not settle:
|
|
248
|
+
* that strands every later durable prompt behind an already-terminal attempt.
|
|
249
|
+
* Cancellation or the hard timeout detaches the operation with a rejection
|
|
250
|
+
* handler; the operation may still finish its attempt-scoped cleanup, but it
|
|
251
|
+
* cannot pin activity terminalization.
|
|
249
252
|
*/
|
|
250
253
|
export async function waitForTurnFinalizerStep<T>(
|
|
251
254
|
operation: Promise<T>,
|
|
252
255
|
signal: AbortSignal | undefined,
|
|
256
|
+
timeoutMs = TURN_FINALIZER_STEP_TIMEOUT_MS,
|
|
253
257
|
): Promise<T | undefined> {
|
|
254
|
-
if (
|
|
255
|
-
if (signal.aborted) {
|
|
258
|
+
if (signal?.aborted) {
|
|
256
259
|
void operation.catch(() => undefined);
|
|
257
260
|
return undefined;
|
|
258
261
|
}
|
|
259
262
|
|
|
260
|
-
let
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
+
let resolveDetached: (() => void) | undefined;
|
|
264
|
+
const detached = new Promise<undefined>((resolve) => {
|
|
265
|
+
resolveDetached = () => resolve(undefined);
|
|
263
266
|
});
|
|
264
|
-
const
|
|
267
|
+
const detach = (): void => {
|
|
265
268
|
void operation.catch(() => undefined);
|
|
266
|
-
|
|
269
|
+
resolveDetached?.();
|
|
267
270
|
};
|
|
268
|
-
signal
|
|
271
|
+
signal?.addEventListener("abort", detach, { once: true });
|
|
272
|
+
const timeout = setTimeout(detach, Math.max(1, timeoutMs));
|
|
273
|
+
timeout.unref?.();
|
|
269
274
|
try {
|
|
270
|
-
return await Promise.race([operation,
|
|
275
|
+
return await Promise.race([operation, detached]);
|
|
271
276
|
} finally {
|
|
272
|
-
|
|
277
|
+
clearTimeout(timeout);
|
|
278
|
+
signal?.removeEventListener("abort", detach);
|
|
273
279
|
}
|
|
274
280
|
}
|
|
275
281
|
|
|
@@ -340,8 +346,14 @@ export function assertAgentStreamNotCancelled(cancelled: boolean): void {
|
|
|
340
346
|
}
|
|
341
347
|
|
|
342
348
|
/** Interruption streams do not produce a normal final output. Call this only
|
|
343
|
-
* after the interruption branch has settled any required action.
|
|
344
|
-
|
|
349
|
+
* after the interruption branch has settled any required action. A successful
|
|
350
|
+
* runtime-owned input wait is a normal terminal result without assistant text;
|
|
351
|
+
* its authority comes from prepared tools, never the provider's output. */
|
|
352
|
+
export function requireAgentStreamFinalOutput(
|
|
353
|
+
finalOutput: unknown | undefined,
|
|
354
|
+
inputWaitYielded = false,
|
|
355
|
+
): unknown {
|
|
356
|
+
if (inputWaitYielded) return "";
|
|
345
357
|
if (finalOutput === undefined) {
|
|
346
358
|
throw new IncompleteAgentStreamError();
|
|
347
359
|
}
|
|
@@ -98,6 +98,9 @@ export function sessionTitleCodexRequestContext(
|
|
|
98
98
|
refresh: context.refresh,
|
|
99
99
|
resolveModel: context.resolveModel,
|
|
100
100
|
...(context.onUsageHeaders ? { onUsageHeaders: context.onUsageHeaders } : {}),
|
|
101
|
+
...(context.beforeProviderDispatch
|
|
102
|
+
? { beforeProviderDispatch: context.beforeProviderDispatch }
|
|
103
|
+
: {}),
|
|
101
104
|
...(context.responseTimeoutPolicy
|
|
102
105
|
? { responseTimeoutPolicy: context.responseTimeoutPolicy }
|
|
103
106
|
: {}),
|
|
@@ -514,6 +517,13 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
514
517
|
input.workspaceId,
|
|
515
518
|
providerTurn.effectiveCodexCredentialId ?? "",
|
|
516
519
|
);
|
|
520
|
+
const resolveTrackedToken = async (
|
|
521
|
+
resolve: () => ReturnType<typeof resolver.getToken>,
|
|
522
|
+
) => {
|
|
523
|
+
const token = await resolve();
|
|
524
|
+
providerTurn.effectiveCodexCredentialVersion = token.credentialVersion;
|
|
525
|
+
return token;
|
|
526
|
+
};
|
|
517
527
|
return {
|
|
518
528
|
clientVersion: CODEX_CLIENT_VERSION,
|
|
519
529
|
// Backend sticky cache-routing key — the SAME id as the body's
|
|
@@ -523,8 +533,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
523
533
|
// (per-request shard lottery = prod's measured 48.6% on sol);
|
|
524
534
|
// with it, resends pin to the warm shard (Codex CLI parity).
|
|
525
535
|
sessionId: input.sessionId,
|
|
526
|
-
getToken: () => resolver.getToken
|
|
527
|
-
refresh: () => resolver.refresh
|
|
536
|
+
getToken: () => resolveTrackedToken(resolver.getToken),
|
|
537
|
+
refresh: () => resolveTrackedToken(resolver.refresh),
|
|
528
538
|
resolveModel: buildModelResolver(
|
|
529
539
|
CODEX_FALLBACK_MODEL_SLUGS,
|
|
530
540
|
CODEX_FALLBACK_MODEL_SLUGS[0],
|
|
@@ -532,6 +542,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
532
542
|
onUsageHeaders: (snapshot) => {
|
|
533
543
|
providerTurn.latestCodexUsage = snapshot;
|
|
534
544
|
}, // latest wins; flushed once in finally
|
|
545
|
+
beforeProviderDispatch: () => {
|
|
546
|
+
leases.codex.assertUsable();
|
|
547
|
+
},
|
|
535
548
|
onRequestPreparationDiagnostic: (phase) => {
|
|
536
549
|
if (
|
|
537
550
|
eventing.firstModelRequestCheckpointAt === null ||
|
|
@@ -1171,7 +1184,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1171
1184
|
});
|
|
1172
1185
|
const {
|
|
1173
1186
|
attemptConnectorActionBindings,
|
|
1174
|
-
|
|
1187
|
+
connectorActionPolicy,
|
|
1175
1188
|
generateSessionTitleInParallel,
|
|
1176
1189
|
postToolPreparationStartedAt,
|
|
1177
1190
|
preparationIndependentToolNames,
|
|
@@ -1226,7 +1239,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1226
1239
|
sandboxCodemodeToken,
|
|
1227
1240
|
fileResourceDownloads,
|
|
1228
1241
|
attemptConnectorActionBindings,
|
|
1229
|
-
|
|
1242
|
+
connectorActionPolicy,
|
|
1243
|
+
trigger,
|
|
1230
1244
|
preparationIndependentToolNames,
|
|
1231
1245
|
videoGenerationAcceptancesByCallId,
|
|
1232
1246
|
activeSandboxBackend,
|
|
@@ -1557,6 +1571,10 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1557
1571
|
videoGenerationAcceptancesByCallId,
|
|
1558
1572
|
});
|
|
1559
1573
|
} catch (error) {
|
|
1574
|
+
// Stop new external wait mutations before asynchronous failure
|
|
1575
|
+
// persistence. Do not drain here: the original failure/cancellation and
|
|
1576
|
+
// durable attempt fence retain authority over already-running calls.
|
|
1577
|
+
eventing.preparedTools?.inputWaitYield?.closeAdmission();
|
|
1560
1578
|
return await settleTurnFailure({
|
|
1561
1579
|
error,
|
|
1562
1580
|
input,
|
|
@@ -1565,7 +1583,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1565
1583
|
bus,
|
|
1566
1584
|
observability,
|
|
1567
1585
|
wakeSessionWorkflow,
|
|
1568
|
-
signalCodexCapacityWorkflow,
|
|
1569
1586
|
cancellationSignal,
|
|
1570
1587
|
sandboxRotationController,
|
|
1571
1588
|
noteCancellationRequested,
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
clearDurablePendingSessionToolCalls,
|
|
11
11
|
isSessionCompactionRequested,
|
|
12
12
|
nextSessionHistoryPosition,
|
|
13
|
+
persistModelContextSnapshot,
|
|
13
14
|
updateSessionTitleWithEvent,
|
|
14
15
|
} from "@opengeni/db";
|
|
15
16
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
@@ -470,7 +471,7 @@ export async function runTurnStreamAttempt(
|
|
|
470
471
|
// prepareInput already sanitized the exact durable prefix represented
|
|
471
472
|
// by state.history. Carry its count forward instead of loading and
|
|
472
473
|
// retaining the full active transcript a second time beside runInput.
|
|
473
|
-
historySink.
|
|
474
|
+
historySink.seedHistory(prepared.input.input, prepared.persistedHistoryCount);
|
|
474
475
|
preparedHistoryCount = prepared.persistedHistoryCount;
|
|
475
476
|
const historyPositionStartedAt = performance.now();
|
|
476
477
|
let historyPositionOutcome: "completed" | "failed" = "completed";
|
|
@@ -777,6 +778,17 @@ export async function runTurnStreamAttempt(
|
|
|
777
778
|
return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
|
|
778
779
|
signal: runtimeCancellationSignal,
|
|
779
780
|
sandboxEnvironment,
|
|
781
|
+
onModelVisibleContext: async (snapshot) => {
|
|
782
|
+
await persistModelContextSnapshot(db, {
|
|
783
|
+
accountId: input.accountId,
|
|
784
|
+
workspaceId: input.workspaceId,
|
|
785
|
+
sessionId: input.sessionId,
|
|
786
|
+
turnId: activeTurnId,
|
|
787
|
+
attemptId: input.attemptId,
|
|
788
|
+
executionGeneration: attempt.executionGeneration,
|
|
789
|
+
snapshot,
|
|
790
|
+
});
|
|
791
|
+
},
|
|
780
792
|
onRuntimeEvent: async (event) => {
|
|
781
793
|
await leases.renewServing("runtime_event");
|
|
782
794
|
if (leases.servingLost()) {
|
|
@@ -890,7 +902,9 @@ export async function runTurnStreamAttempt(
|
|
|
890
902
|
});
|
|
891
903
|
}
|
|
892
904
|
};
|
|
893
|
-
if (
|
|
905
|
+
if (billingState.isCodexTurn) {
|
|
906
|
+
leases.codex.assertUsable();
|
|
907
|
+
} else if (leases.codex.lost) {
|
|
894
908
|
throw new Error("Codex credential lease expired before the model run");
|
|
895
909
|
}
|
|
896
910
|
if (leases.xai.lost) {
|
|
@@ -947,6 +961,7 @@ export async function runTurnStreamAttempt(
|
|
|
947
961
|
]);
|
|
948
962
|
};
|
|
949
963
|
const iterator = eventing.stream.toStream()[Symbol.asyncIterator]();
|
|
964
|
+
const closeStreamWaitAdmission = eventing.preparedTools?.inputWaitYield?.captureStreamClose();
|
|
950
965
|
let streamDone = false;
|
|
951
966
|
try {
|
|
952
967
|
while (true) {
|
|
@@ -1316,6 +1331,10 @@ export async function runTurnStreamAttempt(
|
|
|
1316
1331
|
}
|
|
1317
1332
|
}
|
|
1318
1333
|
} catch (error) {
|
|
1334
|
+
// Event processing can fail while SDK completion is still pending.
|
|
1335
|
+
// Close this stream before any failure publication; a legitimate
|
|
1336
|
+
// compaction retry may start a new, independently fenced generation.
|
|
1337
|
+
closeStreamWaitAdmission?.();
|
|
1319
1338
|
if (fallbackProviderRequestLifecycleStartedAt !== null) {
|
|
1320
1339
|
const durationMs = Math.max(
|
|
1321
1340
|
0,
|
|
@@ -1385,8 +1404,13 @@ export async function runTurnStreamAttempt(
|
|
|
1385
1404
|
temporalCancellationSignal: cancellationSignal,
|
|
1386
1405
|
runtimeCancellationSignal,
|
|
1387
1406
|
});
|
|
1407
|
+
// External Codemode stays reachable until finalization. Close wait
|
|
1408
|
+
// admission before any terminal output/history decision, and drain an
|
|
1409
|
+
// already-admitted wait before consulting the actual runner-yield latch.
|
|
1410
|
+
await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
|
|
1388
1411
|
if (
|
|
1389
1412
|
options.requireTerminalModelResponse &&
|
|
1413
|
+
!eventing.preparedTools?.inputWaitYield?.yielded &&
|
|
1390
1414
|
eventing.stream.interruptions.length === 0 &&
|
|
1391
1415
|
modelResponseState.responseCount === responseCountBeforeStream
|
|
1392
1416
|
) {
|
|
@@ -1395,7 +1419,8 @@ export async function runTurnStreamAttempt(
|
|
|
1395
1419
|
// is not a completed logical turn: accepting finalOutput's undefined ->
|
|
1396
1420
|
// empty-string fallback would release the queue and start newer user
|
|
1397
1421
|
// work. Cancellation retains priority; otherwise checkpoint and recover
|
|
1398
|
-
// this exact turn from the durable compacted history.
|
|
1422
|
+
// this exact turn from the durable compacted history. An actual runtime
|
|
1423
|
+
// wait yield deliberately needs no subsequent provider response.
|
|
1399
1424
|
throwIfWorkerShuttingDown();
|
|
1400
1425
|
throwIfTurnCancelled();
|
|
1401
1426
|
throw new PostCompactionContinuationEmptyError();
|
|
@@ -1638,7 +1663,10 @@ export async function runTurnStreamAttempt(
|
|
|
1638
1663
|
return claimedResult({ status: "requires_action" });
|
|
1639
1664
|
}
|
|
1640
1665
|
|
|
1641
|
-
const
|
|
1666
|
+
const inputWaitYielded = eventing.preparedTools?.inputWaitYield?.yielded === true;
|
|
1667
|
+
const finalOutput = String(
|
|
1668
|
+
requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded),
|
|
1669
|
+
);
|
|
1642
1670
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1643
1671
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
1644
1672
|
// history store (a redispatch would NOT re-execute them), so this
|
|
@@ -1651,10 +1679,9 @@ export async function runTurnStreamAttempt(
|
|
|
1651
1679
|
if (
|
|
1652
1680
|
!(await eventing.settle!({
|
|
1653
1681
|
events: [
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
payload: { text: finalOutput },
|
|
1657
|
-
},
|
|
1682
|
+
...(inputWaitYielded
|
|
1683
|
+
? []
|
|
1684
|
+
: [{ type: "agent.message.completed" as const, payload: { text: finalOutput } }]),
|
|
1658
1685
|
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
1659
1686
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1660
1687
|
],
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
|
+
beginConnectorActionExecution,
|
|
3
|
+
completeConnectorActionExecution,
|
|
2
4
|
getScheduledVariableSetExpectedGenerationForAttempt,
|
|
3
5
|
getWorkspaceModelPolicy,
|
|
4
6
|
listWorkspaceGatewayCustomModels,
|
|
@@ -6,6 +8,8 @@ import {
|
|
|
6
8
|
listOrganizationModelProviderCustomModelsForWorkspace,
|
|
7
9
|
organizationModelProviderConnectionActiveForWorkspace,
|
|
8
10
|
persistAttemptToolCatalog,
|
|
11
|
+
prepareConnectorActionApproval,
|
|
12
|
+
previewConnectorActionApproval,
|
|
9
13
|
namedSubjectHasLiveWorkspaceAuthority,
|
|
10
14
|
updateSessionTitleWithEvent,
|
|
11
15
|
withCodexAppsRequestAuthorization,
|
|
@@ -19,16 +23,15 @@ import {
|
|
|
19
23
|
type OpenGeniRuntime,
|
|
20
24
|
type AttemptConnectorActionBinding,
|
|
21
25
|
type ConnectorAttachmentMaterializationRequest,
|
|
26
|
+
type ConnectorActionPolicyHooks,
|
|
22
27
|
createFirstPartyInteractionAttemptToolDefinitions,
|
|
23
28
|
} from "@opengeni/runtime";
|
|
24
29
|
import {
|
|
25
|
-
authorizeGoogleDrivePublicationAttempt,
|
|
26
30
|
createGoogleDrivePublicationAttemptTool,
|
|
27
31
|
googleDrivePublicationConnectorCall,
|
|
28
32
|
resolveGoogleDrivePublicationTarget,
|
|
29
33
|
} from "../google-drive-publication";
|
|
30
34
|
import { connectionTokenResolverForTurn } from "../mcp-credentials";
|
|
31
|
-
import { buildApiIntegrationServersForTurn } from "../api-integrations";
|
|
32
35
|
import { buildGitHubRestMcpForTurn } from "../../github-rest-mcp";
|
|
33
36
|
import { materializeConnectorAttachmentsInChannel } from "../connector-attachments";
|
|
34
37
|
import { allowedFirstPartyMcpToolsForSession, type Settings } from "@opengeni/config";
|
|
@@ -40,6 +43,7 @@ import {
|
|
|
40
43
|
defaultSessionMcpServerIds,
|
|
41
44
|
loadRigDefaultVariableSetEnvironment,
|
|
42
45
|
mergeRigDefaultVariableSetEnvironment,
|
|
46
|
+
buildApiIntegrationMcpServers,
|
|
43
47
|
resolveCatalogSettings,
|
|
44
48
|
resolveWorkspaceModelSelection,
|
|
45
49
|
withFrozenPersonalConnectionDelegations,
|
|
@@ -439,7 +443,7 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
439
443
|
);
|
|
440
444
|
};
|
|
441
445
|
const selectedApiIntegrationServerIds = new Set(turnTools.map((tool) => tool.id));
|
|
442
|
-
const apiIntegrationMcpServers =
|
|
446
|
+
const apiIntegrationMcpServers = buildApiIntegrationMcpServers({
|
|
443
447
|
settings: runSettings,
|
|
444
448
|
integrations: installedApiIntegrations.filter((integration) =>
|
|
445
449
|
selectedApiIntegrationServerIds.has(integration.serverId),
|
|
@@ -531,6 +535,25 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
531
535
|
},
|
|
532
536
|
]
|
|
533
537
|
: [];
|
|
538
|
+
const attemptConnectorActionBindings = [
|
|
539
|
+
...googleDriveConnectorBindings,
|
|
540
|
+
...githubRestMcp.connectorBindings,
|
|
541
|
+
];
|
|
542
|
+
const connectorActionPolicy: ConnectorActionPolicyHooks = {
|
|
543
|
+
preview: async (call) =>
|
|
544
|
+
await previewConnectorActionApproval(db, connectorActionIdentity, call),
|
|
545
|
+
prepare: async (call) =>
|
|
546
|
+
await prepareConnectorActionApproval(db, connectorActionIdentity, call),
|
|
547
|
+
begin: async (call) => await beginConnectorActionExecution(db, connectorActionIdentity, call),
|
|
548
|
+
complete: async ({ requestId, outcome }) =>
|
|
549
|
+
await completeConnectorActionExecution(db, {
|
|
550
|
+
accountId: input.accountId,
|
|
551
|
+
workspaceId: input.workspaceId,
|
|
552
|
+
requestId,
|
|
553
|
+
attemptId: input.attemptId,
|
|
554
|
+
outcome,
|
|
555
|
+
}),
|
|
556
|
+
};
|
|
534
557
|
const attemptToolDefinitions = [
|
|
535
558
|
createListModelsAttemptToolDefinition({
|
|
536
559
|
currentModelId: turnExecutionPolicy.productModelId,
|
|
@@ -734,6 +757,18 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
734
757
|
resolveCredential,
|
|
735
758
|
onAuthNeeded: publishToolAuthNeeded,
|
|
736
759
|
materializeConnectorAttachments,
|
|
760
|
+
refreshOwnedCommand: async (commandId) => {
|
|
761
|
+
throwIfWorkerShuttingDown();
|
|
762
|
+
throwIfTurnCancelled();
|
|
763
|
+
// Read only the existing attempt-owned object. A retained read must
|
|
764
|
+
// not provision a sandbox or follow an active-pointer change.
|
|
765
|
+
const owned = (sandboxState.lazyOwnedSandbox?.session ??
|
|
766
|
+
sandboxState.resolvedSandbox?.established.session ??
|
|
767
|
+
media.sdkOwnedSandboxSession) as {
|
|
768
|
+
refreshOwnedCommand?: (id: string) => Promise<boolean>;
|
|
769
|
+
} | null;
|
|
770
|
+
return (await owned?.refreshOwnedCommand?.(commandId)) ?? false;
|
|
771
|
+
},
|
|
737
772
|
spillOversizedModelToolResult: async ({ operationId, result }) =>
|
|
738
773
|
await toolResultSpill.spill({ operationId, result }),
|
|
739
774
|
localMcpServers,
|
|
@@ -760,30 +795,8 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
760
795
|
nestedAgentDepth: session.nestedAgentDepth,
|
|
761
796
|
effectiveMaxNestedAgentDepth: session.effectiveMaxNestedAgentDepth,
|
|
762
797
|
attemptToolDefinitions,
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
? {
|
|
766
|
-
attemptToolAuthorize: async (authorization) => {
|
|
767
|
-
const { call } = authorization;
|
|
768
|
-
if (
|
|
769
|
-
googleDrivePublicationTarget &&
|
|
770
|
-
googleDrivePublicationAllowed &&
|
|
771
|
-
call.caller.kind === "codemode" &&
|
|
772
|
-
call.identity.serverId === "google-drive-publishing" &&
|
|
773
|
-
call.identity.toolName === "google_drive_publish_file"
|
|
774
|
-
) {
|
|
775
|
-
await authorizeGoogleDrivePublicationAttempt({
|
|
776
|
-
db,
|
|
777
|
-
identity: connectorActionIdentity,
|
|
778
|
-
target: googleDrivePublicationTarget,
|
|
779
|
-
approvalId: call.operationId,
|
|
780
|
-
arguments: call.arguments,
|
|
781
|
-
});
|
|
782
|
-
}
|
|
783
|
-
await githubRestMcp.authorizeCodemodeCall?.(authorization);
|
|
784
|
-
},
|
|
785
|
-
}
|
|
786
|
-
: {}),
|
|
798
|
+
connectorActionPolicy,
|
|
799
|
+
attemptConnectorActionBindings,
|
|
787
800
|
}),
|
|
788
801
|
cancellationSignal,
|
|
789
802
|
async (latePreparedTools) => await latePreparedTools.close().catch(() => undefined),
|
|
@@ -855,11 +868,8 @@ export async function prepareTurnToolRuntime(deps: PrepareTurnToolRuntimeDeps) {
|
|
|
855
868
|
activatePreparedToolEnvironment(eventing.preparedTools);
|
|
856
869
|
}
|
|
857
870
|
return {
|
|
858
|
-
attemptConnectorActionBindings
|
|
859
|
-
|
|
860
|
-
...githubRestMcp.connectorBindings,
|
|
861
|
-
],
|
|
862
|
-
connectorActionIdentity,
|
|
871
|
+
attemptConnectorActionBindings,
|
|
872
|
+
connectorActionPolicy,
|
|
863
873
|
generateSessionTitleInParallel: titleToolPlan.generateTitleInParallel,
|
|
864
874
|
postToolPreparationStartedAt,
|
|
865
875
|
preparationIndependentToolNames: titleToolPlan.preparationIndependentToolNames,
|
|
@@ -64,17 +64,23 @@ export function computerToolModeForTurn(
|
|
|
64
64
|
/** Chat wires and Gateway models do not advertise OpenAI's hosted sandbox tool types. */
|
|
65
65
|
export function structuredToolTransportForTurn(
|
|
66
66
|
resolvedModel: {
|
|
67
|
-
provider: {
|
|
67
|
+
provider: {
|
|
68
|
+
id: string;
|
|
69
|
+
kind: ResolvedModelProvider["kind"];
|
|
70
|
+
api: ModelProviderApi;
|
|
71
|
+
wireProfile: "openai" | "azure-openai";
|
|
72
|
+
builtin: boolean;
|
|
73
|
+
baseUrl?: string | undefined;
|
|
74
|
+
};
|
|
68
75
|
} | null,
|
|
69
76
|
): boolean {
|
|
70
77
|
if (!resolvedModel) return true;
|
|
71
|
-
|
|
72
|
-
return
|
|
73
|
-
|
|
74
|
-
"
|
|
75
|
-
"
|
|
76
|
-
|
|
77
|
-
].includes(resolvedModel.provider.kind);
|
|
78
|
+
const provider = resolvedModel.provider;
|
|
79
|
+
if (provider.api === "chat" || provider.kind === "codex-subscription") return false;
|
|
80
|
+
return (
|
|
81
|
+
provider.wireProfile === "azure-openai" ||
|
|
82
|
+
(provider.builtin && provider.id === "openai" && provider.baseUrl === undefined)
|
|
83
|
+
);
|
|
78
84
|
}
|
|
79
85
|
|
|
80
86
|
/**
|
|
@@ -2,6 +2,7 @@ import type { Settings } from "@opengeni/config";
|
|
|
2
2
|
import type { CodexUsageHeaderSnapshot } from "@opengeni/codex";
|
|
3
3
|
import type { AppendEventInput, ApplySessionTurnSettlementInput } from "@opengeni/db";
|
|
4
4
|
import type {
|
|
5
|
+
CodexCredentialPolicySnapshotV1,
|
|
5
6
|
ModelContextContributionSummary,
|
|
6
7
|
SessionStatus,
|
|
7
8
|
XaiProviderAccountAuthoritySnapshotV1,
|
|
@@ -45,6 +46,7 @@ export type TurnSettleFn = (input: {
|
|
|
45
46
|
| "requires_action";
|
|
46
47
|
sessionStatus: SessionStatus;
|
|
47
48
|
activeTurnId: string | null;
|
|
49
|
+
suppressGoalContinuation?: boolean;
|
|
48
50
|
consumeRequestedCompactionFailure?: boolean;
|
|
49
51
|
runState?: ApplySessionTurnSettlementInput["runState"];
|
|
50
52
|
}) => Promise<boolean>;
|
|
@@ -59,6 +61,7 @@ export type TurnControlState = {
|
|
|
59
61
|
|
|
60
62
|
export type AttemptIdentityState = {
|
|
61
63
|
turnId: string | undefined;
|
|
64
|
+
dispatchId: string;
|
|
62
65
|
triggerEventId: string | undefined;
|
|
63
66
|
executionGeneration: number;
|
|
64
67
|
providerRecoveryCount: number;
|
|
@@ -150,6 +153,12 @@ export type ProviderTurnState = {
|
|
|
150
153
|
// The Codex account this turn runs on (pin > workspace active), resolved once
|
|
151
154
|
// a codex-billed turn is confirmed and threaded into the token resolver.
|
|
152
155
|
effectiveCodexCredentialId: string | null;
|
|
156
|
+
/** Exact row version returned by the resolver for the latest provider request. */
|
|
157
|
+
effectiveCodexCredentialVersion: number | null;
|
|
158
|
+
/** Frozen alternate-account ceiling observed by the fenced allocator. */
|
|
159
|
+
codexCredentialFailoverLimit: number;
|
|
160
|
+
/** Accepted Codex allocator policy captured with the first durable lease. */
|
|
161
|
+
codexPolicySnapshot: CodexCredentialPolicySnapshotV1 | null;
|
|
153
162
|
effectiveXaiCredentialId: string | null;
|
|
154
163
|
xaiRotationEnabled: boolean;
|
|
155
164
|
xaiAuthoritySnapshot: XaiProviderAccountAuthoritySnapshotV1 | null;
|
|
@@ -196,6 +205,7 @@ export function createTurnContext(input: {
|
|
|
196
205
|
},
|
|
197
206
|
attempt: {
|
|
198
207
|
turnId: undefined,
|
|
208
|
+
dispatchId: "",
|
|
199
209
|
triggerEventId: undefined,
|
|
200
210
|
executionGeneration: 0,
|
|
201
211
|
providerRecoveryCount: 0,
|
|
@@ -268,6 +278,9 @@ export function createTurnContext(input: {
|
|
|
268
278
|
},
|
|
269
279
|
providerTurn: {
|
|
270
280
|
effectiveCodexCredentialId: null,
|
|
281
|
+
effectiveCodexCredentialVersion: null,
|
|
282
|
+
codexCredentialFailoverLimit: 1,
|
|
283
|
+
codexPolicySnapshot: null,
|
|
271
284
|
effectiveXaiCredentialId: null,
|
|
272
285
|
xaiRotationEnabled: false,
|
|
273
286
|
xaiAuthoritySnapshot: null,
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
11
11
|
|
|
12
12
|
import type { CapacityPhaseDeps, CapacityPhaseOutcome } from "./codex-capacity";
|
|
13
|
+
import { refreshExhaustedXaiQuota } from "../xai-quota";
|
|
13
14
|
|
|
14
15
|
export async function selectXaiTurnCapacity(
|
|
15
16
|
deps: CapacityPhaseDeps,
|
|
@@ -43,6 +44,16 @@ export async function selectXaiTurnCapacity(
|
|
|
43
44
|
authoritySnapshot,
|
|
44
45
|
});
|
|
45
46
|
const leaseStartedAtMs = performance.now();
|
|
47
|
+
await refreshExhaustedXaiQuota({
|
|
48
|
+
db,
|
|
49
|
+
settings: deps.settings,
|
|
50
|
+
accountId: input.accountId,
|
|
51
|
+
workspaceId: input.workspaceId,
|
|
52
|
+
subjectId,
|
|
53
|
+
sessionId: input.sessionId,
|
|
54
|
+
turnId: turn.id,
|
|
55
|
+
authoritySnapshot,
|
|
56
|
+
});
|
|
46
57
|
const leased = await acquireXaiCredentialLease(db, {
|
|
47
58
|
accountId: input.accountId,
|
|
48
59
|
workspaceId: input.workspaceId,
|