@opengeni/worker-bundle 0.24.2-canary.0 → 0.26.0-canary.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +7 -0
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +6 -5
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/history-prefix.d.ts +8 -0
- package/dist/activities/agent-turn/history-sink.d.ts +6 -2
- package/dist/activities/agent-turn/quiescence.d.ts +13 -8
- package/dist/activities/agent-turn/tool-environment.d.ts +2 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +4 -0
- package/dist/activities/agent-turn/turn-context.d.ts +9 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/goals.d.ts +1 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/sandbox-lease.d.ts +7 -2
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +11 -1
- package/dist/activities/xai-quota.d.ts +3 -0
- package/dist/{activities-control-ZWH7J4KZ.js → activities-control-KS3ZPQVY.js} +233 -88
- package/dist/activities-control-KS3ZPQVY.js.map +1 -0
- package/dist/activities-control.d.ts +2 -0
- package/dist/{activities-turn-I4OPLQLS.js → activities-turn-GHSFZB64.js} +1109 -1865
- package/dist/activities-turn-GHSFZB64.js.map +1 -0
- package/dist/activities.d.ts +2 -0
- package/dist/{chunk-65JEWP2Q.js → chunk-D4ZEFOYH.js} +1 -1
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-RQNH6CGR.js → chunk-O2LQWJIK.js} +3250 -2061
- package/dist/chunk-O2LQWJIK.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +1 -1
- package/dist/sandbox-routing.d.ts +6 -0
- package/dist/workflow-bundle.js +94 -9
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-turn/agent-build.ts +21 -26
- package/src/activities/agent-turn/claim.ts +26 -1
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +73 -12
- package/src/activities/agent-turn/failure-settlement.ts +472 -331
- package/src/activities/agent-turn/history-prefix.ts +51 -0
- package/src/activities/agent-turn/history-sink.ts +30 -6
- package/src/activities/agent-turn/quiescence.ts +29 -17
- package/src/activities/agent-turn/run.ts +22 -5
- package/src/activities/agent-turn/stream-attempt.ts +35 -8
- package/src/activities/agent-turn/tool-environment.ts +42 -32
- package/src/activities/agent-turn/tool-policy.ts +14 -8
- package/src/activities/agent-turn/turn-context.ts +13 -0
- package/src/activities/agent-turn/xai-capacity.ts +11 -0
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +107 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/goals.ts +9 -9
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/run-input.ts +6 -7
- package/src/activities/sandbox-lease.ts +246 -87
- package/src/activities/session-state.ts +20 -1
- package/src/activities/types.ts +16 -8
- package/src/activities/xai-quota.ts +35 -0
- package/src/activities.ts +1 -0
- package/src/github-rest-mcp.ts +0 -82
- package/src/observability-metrics.ts +4 -0
- package/src/sandbox-routing.ts +186 -16
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/session.ts +114 -8
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-ZWH7J4KZ.js.map +0 -1
- package/dist/activities-turn-I4OPLQLS.js.map +0 -1
- package/dist/chunk-65JEWP2Q.js.map +0 -1
- package/dist/chunk-RQNH6CGR.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -151
|
@@ -2,21 +2,27 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import {
|
|
3
3
|
AttemptToolApprovalRequiredError,
|
|
4
4
|
AttemptToolCatalogStaleError,
|
|
5
|
+
AttemptToolInputValidationError,
|
|
5
6
|
AttemptToolNotFoundError,
|
|
6
7
|
codemodeDispatchSubject,
|
|
7
8
|
decodeCodemodeDispatchRequest,
|
|
8
9
|
encodeCodemodeDispatchAck,
|
|
10
|
+
type AttemptToolCatalogEntry,
|
|
9
11
|
type AttemptToolEnvironment,
|
|
10
12
|
} from "@opengeni/codemode";
|
|
11
13
|
import {
|
|
14
|
+
CODEMODE_CLAIM_LEASE_MS,
|
|
12
15
|
CODEMODE_CLAIM_HEARTBEAT_MS,
|
|
13
16
|
CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
|
|
14
17
|
type CodemodeOperation,
|
|
18
|
+
type SessionEvent,
|
|
15
19
|
} from "@opengeni/contracts";
|
|
16
20
|
import {
|
|
17
21
|
cancelQueuedCodemodeOperationsForAttempt,
|
|
18
22
|
claimCodemodeOperation,
|
|
19
23
|
failCodemodeOperation,
|
|
24
|
+
getSessionEventByClientEventId,
|
|
25
|
+
getSessionToolCallCreatedEventByOperationId,
|
|
20
26
|
markCodemodeOperationExecutionStarted,
|
|
21
27
|
renewCodemodeOperationClaim,
|
|
22
28
|
settleCodemodeOperationWithOutput,
|
|
@@ -37,6 +43,11 @@ export type CodemodeDispatcherScope = {
|
|
|
37
43
|
executionGeneration: number;
|
|
38
44
|
};
|
|
39
45
|
|
|
46
|
+
export type CodemodeDispatcherTimings = {
|
|
47
|
+
claimLeaseMs?: number;
|
|
48
|
+
claimHeartbeatMs?: number;
|
|
49
|
+
};
|
|
50
|
+
|
|
40
51
|
/**
|
|
41
52
|
* Owns the only Codemode execution edge for one exact attempt. NATS carries a
|
|
42
53
|
* wake-up only; the durable row is claimed before any side effect and the call
|
|
@@ -48,6 +59,8 @@ export class CodemodeAttemptDispatcher {
|
|
|
48
59
|
private pendingClaims = 0;
|
|
49
60
|
private unsubscribe: (() => void) | null = null;
|
|
50
61
|
private closing = false;
|
|
62
|
+
private readonly claimLeaseMs: number;
|
|
63
|
+
private readonly claimHeartbeatMs: number;
|
|
51
64
|
|
|
52
65
|
constructor(
|
|
53
66
|
private readonly db: Database,
|
|
@@ -56,6 +69,7 @@ export class CodemodeAttemptDispatcher {
|
|
|
56
69
|
private readonly scope: CodemodeDispatcherScope,
|
|
57
70
|
private readonly turnSignal?: AbortSignal,
|
|
58
71
|
private readonly maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
|
|
72
|
+
timings: CodemodeDispatcherTimings = {},
|
|
59
73
|
) {
|
|
60
74
|
if (
|
|
61
75
|
environment.catalog.accountId !== scope.accountId ||
|
|
@@ -70,6 +84,22 @@ export class CodemodeAttemptDispatcher {
|
|
|
70
84
|
if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
|
|
71
85
|
throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
|
|
72
86
|
}
|
|
87
|
+
this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
|
|
88
|
+
this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
|
|
89
|
+
if (
|
|
90
|
+
!Number.isSafeInteger(this.claimLeaseMs) ||
|
|
91
|
+
this.claimLeaseMs < 1_000 ||
|
|
92
|
+
this.claimLeaseMs > 10 * 60_000
|
|
93
|
+
) {
|
|
94
|
+
throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
|
|
95
|
+
}
|
|
96
|
+
if (
|
|
97
|
+
!Number.isSafeInteger(this.claimHeartbeatMs) ||
|
|
98
|
+
this.claimHeartbeatMs < 1 ||
|
|
99
|
+
this.claimHeartbeatMs >= this.claimLeaseMs
|
|
100
|
+
) {
|
|
101
|
+
throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
|
|
102
|
+
}
|
|
73
103
|
}
|
|
74
104
|
|
|
75
105
|
start(): void {
|
|
@@ -103,11 +133,14 @@ export class CodemodeAttemptDispatcher {
|
|
|
103
133
|
catalogDigest: request.catalogDigest,
|
|
104
134
|
operationId: request.operationId,
|
|
105
135
|
claimId: randomUUID(),
|
|
136
|
+
claimLeaseMs: this.claimLeaseMs,
|
|
106
137
|
});
|
|
107
138
|
if (claim.status === "claimed") {
|
|
108
|
-
const execution = this.execute(claim.operation, claim.claimId).finally(
|
|
109
|
-
|
|
110
|
-
|
|
139
|
+
const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
|
|
140
|
+
() => {
|
|
141
|
+
this.inFlight.delete(request.operationId);
|
|
142
|
+
},
|
|
143
|
+
);
|
|
111
144
|
this.inFlight.set(request.operationId, execution);
|
|
112
145
|
return encodeCodemodeDispatchAck({
|
|
113
146
|
version: 1,
|
|
@@ -115,6 +148,19 @@ export class CodemodeAttemptDispatcher {
|
|
|
115
148
|
status: "accepted",
|
|
116
149
|
});
|
|
117
150
|
}
|
|
151
|
+
if (claim.status === "execution_owner_lost") {
|
|
152
|
+
await this.settleWithOutput(claim.operation, claim.claimId, {
|
|
153
|
+
state: "outcome_unknown",
|
|
154
|
+
errorCode: "worker_lost_during_execution",
|
|
155
|
+
errorMessage:
|
|
156
|
+
"The execution owner disappeared after the tool call began. Inspect actual state before retrying.",
|
|
157
|
+
});
|
|
158
|
+
return encodeCodemodeDispatchAck({
|
|
159
|
+
version: 1,
|
|
160
|
+
operationId: request.operationId,
|
|
161
|
+
status: "terminal",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
118
164
|
return encodeCodemodeDispatchAck({
|
|
119
165
|
version: 1,
|
|
120
166
|
operationId: request.operationId,
|
|
@@ -145,7 +191,39 @@ export class CodemodeAttemptDispatcher {
|
|
|
145
191
|
await Promise.allSettled([...this.inFlight.values()]);
|
|
146
192
|
}
|
|
147
193
|
|
|
148
|
-
private async execute(
|
|
194
|
+
private async execute(
|
|
195
|
+
operation: CodemodeOperation,
|
|
196
|
+
claimId: string,
|
|
197
|
+
reclaimed: boolean,
|
|
198
|
+
): Promise<void> {
|
|
199
|
+
const claimAuthority = {
|
|
200
|
+
accountId: this.scope.accountId,
|
|
201
|
+
workspaceId: this.scope.workspaceId,
|
|
202
|
+
attemptId: this.scope.attemptId,
|
|
203
|
+
operationId: operation.operationId,
|
|
204
|
+
claimId,
|
|
205
|
+
claimLeaseMs: this.claimLeaseMs,
|
|
206
|
+
};
|
|
207
|
+
// Preparation may include provider preflight and policy resolution. Keep the
|
|
208
|
+
// durable claim alive from the moment execution is accepted, not only after
|
|
209
|
+
// the provider side-effect marker, so a slow preflight cannot be reclaimed.
|
|
210
|
+
const heartbeat = setInterval(() => {
|
|
211
|
+
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
|
|
212
|
+
}, this.claimHeartbeatMs);
|
|
213
|
+
heartbeat.unref?.();
|
|
214
|
+
try {
|
|
215
|
+
await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
|
|
216
|
+
} finally {
|
|
217
|
+
clearInterval(heartbeat);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
private async executeClaimed(
|
|
222
|
+
operation: CodemodeOperation,
|
|
223
|
+
claimId: string,
|
|
224
|
+
reclaimed: boolean,
|
|
225
|
+
claimAuthority: Parameters<typeof markCodemodeOperationExecutionStarted>[1],
|
|
226
|
+
): Promise<void> {
|
|
149
227
|
const entry = this.environment.catalog.entries.find(
|
|
150
228
|
(candidate) =>
|
|
151
229
|
candidate.identity.serverId === operation.identity.serverId &&
|
|
@@ -170,38 +248,7 @@ export class CodemodeAttemptDispatcher {
|
|
|
170
248
|
);
|
|
171
249
|
return;
|
|
172
250
|
}
|
|
173
|
-
|
|
174
|
-
this.db,
|
|
175
|
-
this.bus,
|
|
176
|
-
this.scope.workspaceId,
|
|
177
|
-
this.scope.sessionId,
|
|
178
|
-
this.scope.turnId,
|
|
179
|
-
this.scope.executionGeneration,
|
|
180
|
-
this.scope.attemptId,
|
|
181
|
-
[
|
|
182
|
-
{
|
|
183
|
-
type: "agent.toolCall.created",
|
|
184
|
-
turnId: this.scope.turnId,
|
|
185
|
-
turnGeneration: this.scope.executionGeneration,
|
|
186
|
-
turnAttemptId: this.scope.attemptId,
|
|
187
|
-
producerId: operation.caller.subjectId,
|
|
188
|
-
payload: {
|
|
189
|
-
id: operation.operationId,
|
|
190
|
-
name: entry.modelName,
|
|
191
|
-
arguments: operation.arguments,
|
|
192
|
-
origin: "codemode",
|
|
193
|
-
subjectId: operation.caller.subjectId,
|
|
194
|
-
raw: {
|
|
195
|
-
type: "codemode_call",
|
|
196
|
-
serverId: operation.identity.serverId,
|
|
197
|
-
toolName: operation.identity.toolName,
|
|
198
|
-
catalogDigest: operation.catalogDigest,
|
|
199
|
-
},
|
|
200
|
-
},
|
|
201
|
-
},
|
|
202
|
-
],
|
|
203
|
-
).catch(() => ({ accepted: false }));
|
|
204
|
-
if (!created.accepted) {
|
|
251
|
+
if (!(await this.ensureToolCallCreated(operation, entry, reclaimed))) {
|
|
205
252
|
await this.failBeforeExecution(
|
|
206
253
|
operation.operationId,
|
|
207
254
|
claimId,
|
|
@@ -219,24 +266,9 @@ export class CodemodeAttemptDispatcher {
|
|
|
219
266
|
return;
|
|
220
267
|
}
|
|
221
268
|
|
|
222
|
-
|
|
223
|
-
accountId: this.scope.accountId,
|
|
224
|
-
workspaceId: this.scope.workspaceId,
|
|
225
|
-
attemptId: this.scope.attemptId,
|
|
226
|
-
operationId: operation.operationId,
|
|
227
|
-
claimId,
|
|
228
|
-
};
|
|
229
|
-
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
230
|
-
this.db,
|
|
231
|
-
claimAuthority,
|
|
232
|
-
);
|
|
233
|
-
if (!crossedExecutionBoundary) return;
|
|
234
|
-
const heartbeat = setInterval(() => {
|
|
235
|
-
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
|
|
236
|
-
}, CODEMODE_CLAIM_HEARTBEAT_MS);
|
|
237
|
-
heartbeat.unref?.();
|
|
269
|
+
let preparedCall: Awaited<ReturnType<AttemptToolEnvironment["prepareCall"]>>;
|
|
238
270
|
try {
|
|
239
|
-
|
|
271
|
+
preparedCall = await this.environment.prepareCall(
|
|
240
272
|
{
|
|
241
273
|
operationId: operation.operationId,
|
|
242
274
|
catalogDigest: operation.catalogDigest,
|
|
@@ -246,7 +278,34 @@ export class CodemodeAttemptDispatcher {
|
|
|
246
278
|
},
|
|
247
279
|
{ signal },
|
|
248
280
|
);
|
|
249
|
-
|
|
281
|
+
} catch (error) {
|
|
282
|
+
await this.settleWithOutput(operation, claimId, {
|
|
283
|
+
state: "failed",
|
|
284
|
+
errorCode: errorCode(error, signal, false),
|
|
285
|
+
errorMessage: safeErrorMessage(error),
|
|
286
|
+
});
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
if (signal.aborted) {
|
|
290
|
+
await this.settleWithOutput(operation, claimId, {
|
|
291
|
+
state: "failed",
|
|
292
|
+
errorCode: "attempt_cancelled",
|
|
293
|
+
errorMessage: "Execution attempt ended before the Codemode call started",
|
|
294
|
+
});
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
299
|
+
this.db,
|
|
300
|
+
claimAuthority,
|
|
301
|
+
);
|
|
302
|
+
if (!crossedExecutionBoundary) return;
|
|
303
|
+
try {
|
|
304
|
+
const result = await preparedCall.execute();
|
|
305
|
+
await this.settleWithOutput(operation, claimId, {
|
|
306
|
+
state: "completed",
|
|
307
|
+
result,
|
|
308
|
+
});
|
|
250
309
|
} catch (error) {
|
|
251
310
|
const knownPreExecution =
|
|
252
311
|
error instanceof AttemptToolApprovalRequiredError ||
|
|
@@ -255,13 +314,90 @@ export class CodemodeAttemptDispatcher {
|
|
|
255
314
|
connectorActionWasNotExecuted(error);
|
|
256
315
|
await this.settleWithOutput(operation, claimId, {
|
|
257
316
|
state: knownPreExecution ? "failed" : "outcome_unknown",
|
|
258
|
-
errorCode: errorCode(error, signal),
|
|
317
|
+
errorCode: errorCode(error, signal, true),
|
|
259
318
|
errorMessage: !knownPreExecution
|
|
260
319
|
? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying."
|
|
261
320
|
: safeErrorMessage(error),
|
|
262
321
|
});
|
|
263
|
-
}
|
|
264
|
-
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private async ensureToolCallCreated(
|
|
326
|
+
operation: CodemodeOperation,
|
|
327
|
+
entry: AttemptToolCatalogEntry,
|
|
328
|
+
reclaimed: boolean,
|
|
329
|
+
): Promise<boolean> {
|
|
330
|
+
const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
|
|
331
|
+
if (reclaimed) {
|
|
332
|
+
const existing = await getSessionEventByClientEventId(
|
|
333
|
+
this.db,
|
|
334
|
+
this.scope.workspaceId,
|
|
335
|
+
this.scope.sessionId,
|
|
336
|
+
clientEventId,
|
|
337
|
+
).catch(() => null);
|
|
338
|
+
if (existing) {
|
|
339
|
+
assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
|
|
343
|
+
workspaceId: this.scope.workspaceId,
|
|
344
|
+
sessionId: this.scope.sessionId,
|
|
345
|
+
turnId: this.scope.turnId,
|
|
346
|
+
attemptId: this.scope.attemptId,
|
|
347
|
+
operationId: operation.operationId,
|
|
348
|
+
}).catch(() => null);
|
|
349
|
+
if (legacyCreated) {
|
|
350
|
+
assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
|
|
351
|
+
return true;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
const created = await appendAndPublishTurnEventsFenced(
|
|
356
|
+
this.db,
|
|
357
|
+
this.bus,
|
|
358
|
+
this.scope.workspaceId,
|
|
359
|
+
this.scope.sessionId,
|
|
360
|
+
this.scope.turnId,
|
|
361
|
+
this.scope.executionGeneration,
|
|
362
|
+
this.scope.attemptId,
|
|
363
|
+
[
|
|
364
|
+
{
|
|
365
|
+
type: "agent.toolCall.created",
|
|
366
|
+
clientEventId,
|
|
367
|
+
turnId: this.scope.turnId,
|
|
368
|
+
turnGeneration: this.scope.executionGeneration,
|
|
369
|
+
turnAttemptId: this.scope.attemptId,
|
|
370
|
+
producerId: operation.caller.subjectId,
|
|
371
|
+
payload: {
|
|
372
|
+
id: operation.operationId,
|
|
373
|
+
name: entry.modelName,
|
|
374
|
+
arguments: operation.arguments,
|
|
375
|
+
origin: "codemode",
|
|
376
|
+
subjectId: operation.caller.subjectId,
|
|
377
|
+
raw: {
|
|
378
|
+
type: "codemode_call",
|
|
379
|
+
serverId: operation.identity.serverId,
|
|
380
|
+
toolName: operation.identity.toolName,
|
|
381
|
+
catalogDigest: operation.catalogDigest,
|
|
382
|
+
},
|
|
383
|
+
},
|
|
384
|
+
},
|
|
385
|
+
],
|
|
386
|
+
);
|
|
387
|
+
return created.accepted;
|
|
388
|
+
} catch {
|
|
389
|
+
// A concurrent reclaimed claimant can win the unique client-event insert,
|
|
390
|
+
// or a committed append can lose its response. Reconcile durable truth
|
|
391
|
+
// before treating the attempt fence as closed.
|
|
392
|
+
const raced = await getSessionEventByClientEventId(
|
|
393
|
+
this.db,
|
|
394
|
+
this.scope.workspaceId,
|
|
395
|
+
this.scope.sessionId,
|
|
396
|
+
clientEventId,
|
|
397
|
+
).catch(() => null);
|
|
398
|
+
if (!raced) return false;
|
|
399
|
+
assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
|
|
400
|
+
return true;
|
|
265
401
|
}
|
|
266
402
|
}
|
|
267
403
|
|
|
@@ -311,6 +447,43 @@ export class CodemodeAttemptDispatcher {
|
|
|
311
447
|
}
|
|
312
448
|
}
|
|
313
449
|
|
|
450
|
+
export function codemodeToolCallCreatedClientEventId(operationId: string): string {
|
|
451
|
+
return `opengeni:codemode-tool-call-created:${operationId}`;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function assertMatchingCodemodeToolCallCreated(
|
|
455
|
+
event: SessionEvent,
|
|
456
|
+
operation: CodemodeOperation,
|
|
457
|
+
entry: AttemptToolCatalogEntry,
|
|
458
|
+
scope: CodemodeDispatcherScope,
|
|
459
|
+
): void {
|
|
460
|
+
const payload = recordValue(event.payload);
|
|
461
|
+
const raw = recordValue(payload?.raw);
|
|
462
|
+
if (
|
|
463
|
+
event.type !== "agent.toolCall.created" ||
|
|
464
|
+
event.turnId !== scope.turnId ||
|
|
465
|
+
event.turnGeneration !== scope.executionGeneration ||
|
|
466
|
+
event.turnAttemptId !== scope.attemptId ||
|
|
467
|
+
event.turnAssociation !== "current" ||
|
|
468
|
+
payload?.id !== operation.operationId ||
|
|
469
|
+
payload?.name !== entry.modelName ||
|
|
470
|
+
payload?.origin !== "codemode" ||
|
|
471
|
+
payload?.subjectId !== operation.caller.subjectId ||
|
|
472
|
+
raw?.type !== "codemode_call" ||
|
|
473
|
+
raw?.serverId !== operation.identity.serverId ||
|
|
474
|
+
raw?.toolName !== operation.identity.toolName ||
|
|
475
|
+
raw?.catalogDigest !== operation.catalogDigest
|
|
476
|
+
) {
|
|
477
|
+
throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function recordValue(value: unknown): Record<string, unknown> | null {
|
|
482
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
483
|
+
? (value as Record<string, unknown>)
|
|
484
|
+
: null;
|
|
485
|
+
}
|
|
486
|
+
|
|
314
487
|
function connectorActionWasNotExecuted(error: unknown): boolean {
|
|
315
488
|
return Boolean(
|
|
316
489
|
error &&
|
|
@@ -324,8 +497,10 @@ function combinedSignal(primary: AbortSignal, secondary?: AbortSignal): AbortSig
|
|
|
324
497
|
return secondary ? AbortSignal.any([primary, secondary]) : primary;
|
|
325
498
|
}
|
|
326
499
|
|
|
327
|
-
function errorCode(error: unknown, signal: AbortSignal): string {
|
|
328
|
-
if (signal.aborted)
|
|
500
|
+
function errorCode(error: unknown, signal: AbortSignal, executionStarted: boolean): string {
|
|
501
|
+
if (signal.aborted) {
|
|
502
|
+
return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
|
|
503
|
+
}
|
|
329
504
|
if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
|
|
330
505
|
if (typeof error === "object" && error !== null && "code" in error) {
|
|
331
506
|
const code = (error as { code?: unknown }).code;
|
|
@@ -338,6 +513,7 @@ function safeErrorMessage(error: unknown): string {
|
|
|
338
513
|
if (
|
|
339
514
|
error instanceof AttemptToolApprovalRequiredError ||
|
|
340
515
|
error instanceof AttemptToolCatalogStaleError ||
|
|
516
|
+
error instanceof AttemptToolInputValidationError ||
|
|
341
517
|
error instanceof AttemptToolNotFoundError
|
|
342
518
|
) {
|
|
343
519
|
return error.message;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
|
+
armCodexCapacityWait,
|
|
2
3
|
fetchCodexUsageForAccount,
|
|
3
4
|
getCodexCapacityWaitForSession,
|
|
4
5
|
getXaiCapacityWaitForSession,
|
|
6
|
+
resolveXaiWaiterSubject,
|
|
5
7
|
listCodexAccountStatuses,
|
|
6
8
|
listPendingCodexCapacityWakeTargets,
|
|
7
9
|
reconcileCodexCapacityWait as reconcileCodexCapacityWaitDb,
|
|
@@ -9,7 +11,8 @@ import {
|
|
|
9
11
|
type CodexCapacityWakeTarget,
|
|
10
12
|
type CodexCapacitySelectionContext,
|
|
11
13
|
} from "@opengeni/db";
|
|
12
|
-
import
|
|
14
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
15
|
+
import { refreshExhaustedXaiQuota } from "./xai-quota";
|
|
13
16
|
import {
|
|
14
17
|
authoritativeCodexCapacityResetAt,
|
|
15
18
|
codexAccountNeedsLiveCapacityRefresh,
|
|
@@ -85,14 +88,12 @@ export async function signalPendingCodexCapacityWakeTargets(
|
|
|
85
88
|
await signalCodexCapacityWakeTargets(services, targets);
|
|
86
89
|
}
|
|
87
90
|
|
|
88
|
-
export function codexCapacityDecision(
|
|
89
|
-
context: CodexCapacitySelectionContext,
|
|
90
|
-
settings: Settings,
|
|
91
|
+
export function codexCapacityDecision<TPolicyScope = never, TUnavailableDiagnostic = never>(
|
|
92
|
+
context: CodexCapacitySelectionContext<TPolicyScope, TUnavailableDiagnostic>,
|
|
91
93
|
): ReturnType<Parameters<typeof reconcileCodexCapacityWaitDb>[2]> {
|
|
92
94
|
const now = new Date();
|
|
93
95
|
const selected = selectCodexCredentialLeaseForTurn({
|
|
94
96
|
context,
|
|
95
|
-
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
96
97
|
sessionId: context.sessionId,
|
|
97
98
|
sessionPinnedCredentialId: context.sessionPinnedCredentialId,
|
|
98
99
|
sessionPinSource: context.sessionPinSource,
|
|
@@ -118,8 +119,17 @@ export function codexCapacityDecision(
|
|
|
118
119
|
},
|
|
119
120
|
};
|
|
120
121
|
}
|
|
121
|
-
const
|
|
122
|
-
|
|
122
|
+
const policyCredentialId =
|
|
123
|
+
context.sessionPinSource === "manual" && context.sessionPinnedCredentialId
|
|
124
|
+
? context.sessionPinnedCredentialId
|
|
125
|
+
: !context.rotationEnabled
|
|
126
|
+
? context.activeCredentialId
|
|
127
|
+
: null;
|
|
128
|
+
const capacityAccounts = policyCredentialId
|
|
129
|
+
? context.accounts.filter((account) => account.id === policyCredentialId)
|
|
130
|
+
: context.accounts;
|
|
131
|
+
const authoritativeReset = authoritativeCodexCapacityResetAt(capacityAccounts, now);
|
|
132
|
+
const hasReconcilableQuotaCooldown = capacityAccounts.some(
|
|
123
133
|
(account) =>
|
|
124
134
|
account.status === "active" &&
|
|
125
135
|
account.allocatorEnabled &&
|
|
@@ -127,19 +137,85 @@ export function codexCapacityDecision(
|
|
|
127
137
|
account.exhaustedUntil !== null &&
|
|
128
138
|
account.exhaustedUntil > now,
|
|
129
139
|
);
|
|
140
|
+
const policyAccount = capacityAccounts[0] ?? null;
|
|
141
|
+
const mutationOnlyStatusBlock =
|
|
142
|
+
(policyAccount != null &&
|
|
143
|
+
(!policyAccount.allocatorEnabled || policyAccount.status !== "active")) ||
|
|
144
|
+
(authoritativeReset === null &&
|
|
145
|
+
capacityAccounts.length > 0 &&
|
|
146
|
+
capacityAccounts.every(
|
|
147
|
+
(account) => !account.allocatorEnabled || account.status !== "active",
|
|
148
|
+
));
|
|
149
|
+
const noneReason =
|
|
150
|
+
selected.decision.kind === "none"
|
|
151
|
+
? context.sessionPinSource === "manual" && context.sessionPinnedCredentialId !== null
|
|
152
|
+
? "manual_pin_missing"
|
|
153
|
+
: !context.rotationEnabled && context.activeCredentialId === null
|
|
154
|
+
? "rotation_off_active_pointer_missing"
|
|
155
|
+
: context.policyScope !== null && context.accounts.length === 0
|
|
156
|
+
? "policy_filtered_pool_empty"
|
|
157
|
+
: context.accounts.length === 0
|
|
158
|
+
? "no_connected_credentials"
|
|
159
|
+
: "no_eligible_credential"
|
|
160
|
+
: null;
|
|
130
161
|
return {
|
|
131
162
|
kind: "unavailable",
|
|
132
163
|
earliestResetAt: authoritativeReset,
|
|
133
164
|
resetKind:
|
|
134
|
-
|
|
165
|
+
selected.decision.kind === "none" ||
|
|
166
|
+
selected.decision.kind === "allocatorDisabled" ||
|
|
167
|
+
mutationOnlyStatusBlock
|
|
168
|
+
? "mutation_only"
|
|
169
|
+
: authoritativeReset && !hasReconcilableQuotaCooldown
|
|
170
|
+
? "authoritative"
|
|
171
|
+
: "bounded_refresh",
|
|
135
172
|
diagnostic: {
|
|
136
173
|
connectedCount: context.accounts.length,
|
|
137
174
|
allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
|
|
138
175
|
policyHash: context.policyHash,
|
|
176
|
+
...(noneReason ? { reason: noneReason } : {}),
|
|
139
177
|
},
|
|
140
178
|
};
|
|
141
179
|
}
|
|
142
180
|
|
|
181
|
+
/**
|
|
182
|
+
* Arm a durable Codex waiter and immediately re-evaluate it under the
|
|
183
|
+
* allocator lock. A capacity mutation that commits just before the waiter is
|
|
184
|
+
* inserted cannot signal a row that does not exist yet, so every arm site must
|
|
185
|
+
* close that edge before returning an hours-away reset timer to the workflow.
|
|
186
|
+
* Mutations after the arm commit still advance the waiter's wake revision.
|
|
187
|
+
*/
|
|
188
|
+
export async function armAndReconcileCodexCapacityWait(
|
|
189
|
+
services: Pick<ControlActivityServices, "db" | "bus">,
|
|
190
|
+
input: Parameters<typeof armCodexCapacityWait>[1],
|
|
191
|
+
options: { onArmed?: () => void } = {},
|
|
192
|
+
) {
|
|
193
|
+
const armed = await armCodexCapacityWait(services.db, input);
|
|
194
|
+
if (armed.action !== "waiting") return armed;
|
|
195
|
+
|
|
196
|
+
options.onArmed?.();
|
|
197
|
+
await publishDurableSessionEvents(services.bus, input.workspaceId, input.sessionId, armed.events);
|
|
198
|
+
const evaluated = await reconcileCodexCapacityWaitDb(
|
|
199
|
+
services.db,
|
|
200
|
+
{
|
|
201
|
+
accountId: input.accountId,
|
|
202
|
+
workspaceId: input.workspaceId,
|
|
203
|
+
sessionId: input.sessionId,
|
|
204
|
+
waiterId: armed.waiter.id,
|
|
205
|
+
generation: armed.waiter.generation,
|
|
206
|
+
...(input.now ? { now: input.now } : {}),
|
|
207
|
+
},
|
|
208
|
+
(context) => codexCapacityDecision(context),
|
|
209
|
+
);
|
|
210
|
+
await publishDurableSessionEvents(
|
|
211
|
+
services.bus,
|
|
212
|
+
input.workspaceId,
|
|
213
|
+
input.sessionId,
|
|
214
|
+
evaluated.events,
|
|
215
|
+
);
|
|
216
|
+
return evaluated;
|
|
217
|
+
}
|
|
218
|
+
|
|
143
219
|
async function refreshCapacityMetadata(
|
|
144
220
|
services: ControlActivityServices,
|
|
145
221
|
workspaceId: string,
|
|
@@ -204,6 +280,22 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
204
280
|
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
205
281
|
return { action: "stale" };
|
|
206
282
|
}
|
|
283
|
+
const authority = await resolveXaiWaiterSubject(
|
|
284
|
+
resolved.db,
|
|
285
|
+
input.workspaceId,
|
|
286
|
+
input.sessionId,
|
|
287
|
+
);
|
|
288
|
+
if (authority)
|
|
289
|
+
await refreshExhaustedXaiQuota({
|
|
290
|
+
db: resolved.db,
|
|
291
|
+
settings: resolved.settings,
|
|
292
|
+
accountId: input.accountId,
|
|
293
|
+
workspaceId: input.workspaceId,
|
|
294
|
+
sessionId: input.sessionId,
|
|
295
|
+
turnId: authority.turnId,
|
|
296
|
+
subjectId: authority.subjectId,
|
|
297
|
+
authoritySnapshot: authority.snapshot,
|
|
298
|
+
});
|
|
207
299
|
const result = await reconcileXaiCapacityWaitDb(resolved.db, {
|
|
208
300
|
accountId: input.accountId,
|
|
209
301
|
workspaceId: input.workspaceId,
|
|
@@ -239,7 +331,11 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
239
331
|
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
240
332
|
return { action: "stale" };
|
|
241
333
|
}
|
|
242
|
-
|
|
334
|
+
const boundedRefreshAttempted =
|
|
335
|
+
current.resetKind === "bounded_refresh" &&
|
|
336
|
+
input.cause === "timer" &&
|
|
337
|
+
current.nextCheckAt.getTime() <= Date.now();
|
|
338
|
+
if (boundedRefreshAttempted) {
|
|
243
339
|
// This is a bounded secret-safe control-plane quota refresh. It creates no
|
|
244
340
|
// turn, model call, user message, schedule, or entitlement action.
|
|
245
341
|
await refreshCapacityMetadata(resolved, input.workspaceId);
|
|
@@ -252,8 +348,9 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
252
348
|
sessionId: input.sessionId,
|
|
253
349
|
waiterId: input.waiterId,
|
|
254
350
|
generation: input.generation,
|
|
351
|
+
boundedRefreshAttempted,
|
|
255
352
|
},
|
|
256
|
-
(context) => codexCapacityDecision(context
|
|
353
|
+
(context) => codexCapacityDecision(context),
|
|
257
354
|
);
|
|
258
355
|
if (result.events.length > 0) {
|
|
259
356
|
try {
|
|
@@ -16,7 +16,14 @@ export const CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1_024;
|
|
|
16
16
|
export type CodexFleetActualDecisionV1 = {
|
|
17
17
|
outcome: "selected" | "waiting" | "none";
|
|
18
18
|
candidateKey: string | null;
|
|
19
|
-
reason:
|
|
19
|
+
reason:
|
|
20
|
+
| "lease_reused"
|
|
21
|
+
| "pin"
|
|
22
|
+
| "rotation"
|
|
23
|
+
| "active"
|
|
24
|
+
| "all_capped"
|
|
25
|
+
| "allocator_disabled"
|
|
26
|
+
| "none";
|
|
20
27
|
};
|
|
21
28
|
|
|
22
29
|
export type CodexFleetShadowComparisonV1 =
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
import type { Database } from "@opengeni/db";
|
|
7
7
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
8
8
|
import type { GeneratedImageReceipt } from "./generated-images";
|
|
9
|
+
import { CodexCredentialLeaseLostError } from "./agent-turn/credential-leases";
|
|
9
10
|
import {
|
|
10
11
|
executeImageGenerationOperation,
|
|
11
12
|
imageProviderBindingHash,
|
|
@@ -27,22 +28,38 @@ export async function executeCodexImageGeneration(input: {
|
|
|
27
28
|
prompt: string;
|
|
28
29
|
references?: readonly ResolvedImageGenerationReference[];
|
|
29
30
|
credentialId: string;
|
|
30
|
-
codexContext: Pick<
|
|
31
|
+
codexContext: Pick<
|
|
32
|
+
CodexRequestContext,
|
|
33
|
+
"clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
|
|
34
|
+
>;
|
|
31
35
|
abortSignal?: AbortSignal;
|
|
32
36
|
}): Promise<GeneratedImageReceipt> {
|
|
33
37
|
const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID, input.credentialId);
|
|
38
|
+
let providerDispatchAdmitted = false;
|
|
39
|
+
const codexContext: Pick<
|
|
40
|
+
CodexRequestContext,
|
|
41
|
+
"clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
|
|
42
|
+
> = {
|
|
43
|
+
...input.codexContext,
|
|
44
|
+
beforeProviderDispatch: async () => {
|
|
45
|
+
await input.codexContext.beforeProviderDispatch?.();
|
|
46
|
+
providerDispatchAdmitted = true;
|
|
47
|
+
},
|
|
48
|
+
};
|
|
34
49
|
return await executeImageGenerationOperation({
|
|
35
50
|
...input,
|
|
36
51
|
providerId: CODEX_PROVIDER_ID,
|
|
37
52
|
providerBindingHash,
|
|
38
53
|
modelId: CODEX_IMAGE_MODEL,
|
|
39
54
|
...(input.references ? { referenceDigests: input.references } : {}),
|
|
55
|
+
isProviderDispatchRejected: (error) =>
|
|
56
|
+
!providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
|
|
40
57
|
generate: async () => {
|
|
41
58
|
const generated = await generateCodexSubscriptionImage({
|
|
42
59
|
prompt: input.prompt,
|
|
43
60
|
...(input.references ? { references: input.references } : {}),
|
|
44
61
|
turnId: input.turnId,
|
|
45
|
-
context:
|
|
62
|
+
context: codexContext,
|
|
46
63
|
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
|
47
64
|
});
|
|
48
65
|
return {
|