@opengeni/worker-bundle 0.23.4 → 0.26.0-canary.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-turn/admission.d.ts +17 -2
- package/dist/activities/agent-turn/agent-build.d.ts +3 -10
- package/dist/activities/agent-turn/claim.d.ts +8 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -2
- package/dist/activities/agent-turn/credential-leases.d.ts +8 -0
- package/dist/activities/agent-turn/errors.d.ts +1 -1
- package/dist/activities/agent-turn/failure-settlement.d.ts +43 -1
- package/dist/activities/agent-turn/list-models.d.ts +16 -0
- package/dist/activities/agent-turn/model-usage.d.ts +15 -1
- package/dist/activities/agent-turn/quiescence.d.ts +11 -6
- package/dist/activities/agent-turn/run.d.ts +9 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +15 -0
- package/dist/activities/agent-turn/session-title.d.ts +21 -0
- package/dist/activities/agent-turn/stream-attempt.d.ts +4 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +4 -13
- package/dist/activities/agent-turn/tool-policy.d.ts +40 -5
- package/dist/activities/agent-turn/turn-context.d.ts +13 -2
- package/dist/activities/agent-turn.d.ts +1 -1
- package/dist/activities/automations.d.ts +1 -0
- package/dist/activities/capabilities.d.ts +3 -1
- package/dist/activities/codemode-dispatcher.d.ts +10 -1
- package/dist/activities/codex-capacity.d.ts +32 -3
- package/dist/activities/codex-fleet-shadow.d.ts +1 -1
- package/dist/activities/codex-image-generation.d.ts +1 -1
- package/dist/activities/codex-rotation.d.ts +8 -67
- package/dist/activities/context-compaction.d.ts +2 -0
- package/dist/activities/goals.d.ts +12 -1
- package/dist/activities/google-drive-publication.d.ts +2 -13
- package/dist/activities/image-generation-operation.d.ts +13 -4
- package/dist/activities/image-generation-references.d.ts +2 -1
- package/dist/activities/scheduled-tasks.d.ts +1 -0
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/types.d.ts +19 -1
- package/dist/{activities-control-52W5IDWM.js → activities-control-CTEMLTHG.js} +339 -195
- package/dist/activities-control-CTEMLTHG.js.map +1 -0
- package/dist/activities-control.d.ts +4 -0
- package/dist/{activities-turn-A2G2KLSY.js → activities-turn-S5IU347A.js} +1846 -1136
- package/dist/activities-turn-S5IU347A.js.map +1 -0
- package/dist/activities.d.ts +4 -0
- package/dist/{chunk-GFNXVZVQ.js → chunk-D4ZEFOYH.js} +107 -7
- package/dist/chunk-D4ZEFOYH.js.map +1 -0
- package/dist/{chunk-GRFQTTSQ.js → chunk-GDCLDKSH.js} +7 -3
- package/dist/{chunk-GRFQTTSQ.js.map → chunk-GDCLDKSH.js.map} +1 -1
- package/dist/{chunk-IPVMCAW5.js → chunk-TAUE3GYT.js} +225 -154
- package/dist/chunk-TAUE3GYT.js.map +1 -0
- package/dist/github-rest-mcp.d.ts +0 -2
- package/dist/index.d.ts +2 -0
- package/dist/index.js +43 -7
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +35 -8
- package/dist/sandbox-reaper-timeout.d.ts +2 -2
- package/dist/sandbox-resume.d.ts +6 -1
- package/dist/workflow-bundle.js +96 -10
- package/dist/workflows/activities.d.ts +6 -1
- package/dist/workflows/automations.d.ts +5 -0
- package/dist/workflows/session.d.ts +3 -0
- package/package.json +17 -17
- package/src/activities/agent-run-admission.ts +7 -1
- package/src/activities/agent-turn/admission.ts +47 -6
- package/src/activities/agent-turn/agent-build.ts +20 -26
- package/src/activities/agent-turn/claim.ts +59 -6
- package/src/activities/agent-turn/codex-capacity.ts +311 -234
- package/src/activities/agent-turn/codex.ts +2 -2
- package/src/activities/agent-turn/compaction-prep.ts +8 -1
- package/src/activities/agent-turn/credential-leases.ts +36 -0
- package/src/activities/agent-turn/errors.ts +29 -1
- package/src/activities/agent-turn/failure-settlement.ts +451 -323
- package/src/activities/agent-turn/finalization.ts +31 -17
- package/src/activities/agent-turn/list-models.ts +101 -0
- package/src/activities/agent-turn/model-usage.ts +120 -45
- package/src/activities/agent-turn/quiescence.ts +32 -17
- package/src/activities/agent-turn/run.ts +98 -9
- package/src/activities/agent-turn/sandbox-runtime.ts +78 -81
- package/src/activities/agent-turn/session-title.ts +55 -2
- package/src/activities/agent-turn/stream-attempt.ts +281 -136
- package/src/activities/agent-turn/tool-environment.ts +132 -39
- package/src/activities/agent-turn/tool-policy.ts +71 -12
- package/src/activities/agent-turn/turn-context.ts +21 -2
- package/src/activities/agent-turn.ts +5 -1
- package/src/activities/automations.ts +122 -67
- package/src/activities/capabilities.ts +129 -2
- package/src/activities/codemode-dispatcher.ts +235 -59
- package/src/activities/codex-capacity.ts +89 -10
- package/src/activities/codex-fleet-shadow.ts +8 -1
- package/src/activities/codex-image-generation.ts +19 -2
- package/src/activities/codex-rotation.ts +45 -142
- package/src/activities/context-compaction.ts +3 -0
- package/src/activities/goals.ts +116 -39
- package/src/activities/google-drive-publication.ts +4 -81
- package/src/activities/image-generation-operation.ts +28 -3
- package/src/activities/image-generation-references.ts +9 -9
- package/src/activities/mcp-credentials.ts +53 -0
- package/src/activities/rig-verification.ts +5 -3
- package/src/activities/sandbox-lease.ts +29 -12
- package/src/activities/scheduled-tasks.ts +217 -110
- package/src/activities/session-state.ts +17 -0
- package/src/activities/site-auth-maintenance.ts +12 -12
- package/src/activities/types.ts +22 -8
- package/src/activities.ts +2 -0
- package/src/activity-services.ts +4 -1
- package/src/github-rest-mcp.ts +2 -83
- package/src/index.ts +34 -4
- package/src/observability-metrics.ts +158 -14
- package/src/sandbox-reaper-timeout.ts +10 -4
- package/src/sandbox-resume.ts +58 -8
- package/src/sandbox-routing.ts +71 -16
- package/src/workflows/activities.ts +16 -0
- package/src/workflows/automations.ts +21 -2
- package/src/workflows/session.ts +85 -7
- package/dist/activities/api-integrations.d.ts +0 -21
- package/dist/activities-control-52W5IDWM.js.map +0 -1
- package/dist/activities-turn-A2G2KLSY.js.map +0 -1
- package/dist/chunk-GFNXVZVQ.js.map +0 -1
- package/dist/chunk-IPVMCAW5.js.map +0 -1
- package/src/activities/api-integrations.ts +0 -141
|
@@ -2,21 +2,27 @@ import { randomUUID } from "node:crypto";
|
|
|
2
2
|
import {
|
|
3
3
|
AttemptToolApprovalRequiredError,
|
|
4
4
|
AttemptToolCatalogStaleError,
|
|
5
|
+
AttemptToolInputValidationError,
|
|
5
6
|
AttemptToolNotFoundError,
|
|
6
7
|
codemodeDispatchSubject,
|
|
7
8
|
decodeCodemodeDispatchRequest,
|
|
8
9
|
encodeCodemodeDispatchAck,
|
|
10
|
+
type AttemptToolCatalogEntry,
|
|
9
11
|
type AttemptToolEnvironment,
|
|
10
12
|
} from "@opengeni/codemode";
|
|
11
13
|
import {
|
|
14
|
+
CODEMODE_CLAIM_LEASE_MS,
|
|
12
15
|
CODEMODE_CLAIM_HEARTBEAT_MS,
|
|
13
16
|
CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
|
|
14
17
|
type CodemodeOperation,
|
|
18
|
+
type SessionEvent,
|
|
15
19
|
} from "@opengeni/contracts";
|
|
16
20
|
import {
|
|
17
21
|
cancelQueuedCodemodeOperationsForAttempt,
|
|
18
22
|
claimCodemodeOperation,
|
|
19
23
|
failCodemodeOperation,
|
|
24
|
+
getSessionEventByClientEventId,
|
|
25
|
+
getSessionToolCallCreatedEventByOperationId,
|
|
20
26
|
markCodemodeOperationExecutionStarted,
|
|
21
27
|
renewCodemodeOperationClaim,
|
|
22
28
|
settleCodemodeOperationWithOutput,
|
|
@@ -37,6 +43,11 @@ export type CodemodeDispatcherScope = {
|
|
|
37
43
|
executionGeneration: number;
|
|
38
44
|
};
|
|
39
45
|
|
|
46
|
+
export type CodemodeDispatcherTimings = {
|
|
47
|
+
claimLeaseMs?: number;
|
|
48
|
+
claimHeartbeatMs?: number;
|
|
49
|
+
};
|
|
50
|
+
|
|
40
51
|
/**
|
|
41
52
|
* Owns the only Codemode execution edge for one exact attempt. NATS carries a
|
|
42
53
|
* wake-up only; the durable row is claimed before any side effect and the call
|
|
@@ -48,6 +59,8 @@ export class CodemodeAttemptDispatcher {
|
|
|
48
59
|
private pendingClaims = 0;
|
|
49
60
|
private unsubscribe: (() => void) | null = null;
|
|
50
61
|
private closing = false;
|
|
62
|
+
private readonly claimLeaseMs: number;
|
|
63
|
+
private readonly claimHeartbeatMs: number;
|
|
51
64
|
|
|
52
65
|
constructor(
|
|
53
66
|
private readonly db: Database,
|
|
@@ -56,6 +69,7 @@ export class CodemodeAttemptDispatcher {
|
|
|
56
69
|
private readonly scope: CodemodeDispatcherScope,
|
|
57
70
|
private readonly turnSignal?: AbortSignal,
|
|
58
71
|
private readonly maxConcurrentCalls = CODEMODE_MAX_CONCURRENT_CALLS_PER_ATTEMPT,
|
|
72
|
+
timings: CodemodeDispatcherTimings = {},
|
|
59
73
|
) {
|
|
60
74
|
if (
|
|
61
75
|
environment.catalog.accountId !== scope.accountId ||
|
|
@@ -70,6 +84,22 @@ export class CodemodeAttemptDispatcher {
|
|
|
70
84
|
if (!Number.isSafeInteger(maxConcurrentCalls) || maxConcurrentCalls < 1) {
|
|
71
85
|
throw new Error("Codemode dispatcher concurrency must be a positive safe integer");
|
|
72
86
|
}
|
|
87
|
+
this.claimLeaseMs = timings.claimLeaseMs ?? CODEMODE_CLAIM_LEASE_MS;
|
|
88
|
+
this.claimHeartbeatMs = timings.claimHeartbeatMs ?? CODEMODE_CLAIM_HEARTBEAT_MS;
|
|
89
|
+
if (
|
|
90
|
+
!Number.isSafeInteger(this.claimLeaseMs) ||
|
|
91
|
+
this.claimLeaseMs < 1_000 ||
|
|
92
|
+
this.claimLeaseMs > 10 * 60_000
|
|
93
|
+
) {
|
|
94
|
+
throw new Error("Codemode dispatcher claim lease must be between 1 second and 10 minutes");
|
|
95
|
+
}
|
|
96
|
+
if (
|
|
97
|
+
!Number.isSafeInteger(this.claimHeartbeatMs) ||
|
|
98
|
+
this.claimHeartbeatMs < 1 ||
|
|
99
|
+
this.claimHeartbeatMs >= this.claimLeaseMs
|
|
100
|
+
) {
|
|
101
|
+
throw new Error("Codemode dispatcher heartbeat must be positive and shorter than its lease");
|
|
102
|
+
}
|
|
73
103
|
}
|
|
74
104
|
|
|
75
105
|
start(): void {
|
|
@@ -103,11 +133,14 @@ export class CodemodeAttemptDispatcher {
|
|
|
103
133
|
catalogDigest: request.catalogDigest,
|
|
104
134
|
operationId: request.operationId,
|
|
105
135
|
claimId: randomUUID(),
|
|
136
|
+
claimLeaseMs: this.claimLeaseMs,
|
|
106
137
|
});
|
|
107
138
|
if (claim.status === "claimed") {
|
|
108
|
-
const execution = this.execute(claim.operation, claim.claimId).finally(
|
|
109
|
-
|
|
110
|
-
|
|
139
|
+
const execution = this.execute(claim.operation, claim.claimId, claim.reclaimed).finally(
|
|
140
|
+
() => {
|
|
141
|
+
this.inFlight.delete(request.operationId);
|
|
142
|
+
},
|
|
143
|
+
);
|
|
111
144
|
this.inFlight.set(request.operationId, execution);
|
|
112
145
|
return encodeCodemodeDispatchAck({
|
|
113
146
|
version: 1,
|
|
@@ -115,6 +148,19 @@ export class CodemodeAttemptDispatcher {
|
|
|
115
148
|
status: "accepted",
|
|
116
149
|
});
|
|
117
150
|
}
|
|
151
|
+
if (claim.status === "execution_owner_lost") {
|
|
152
|
+
await this.settleWithOutput(claim.operation, claim.claimId, {
|
|
153
|
+
state: "outcome_unknown",
|
|
154
|
+
errorCode: "worker_lost_during_execution",
|
|
155
|
+
errorMessage:
|
|
156
|
+
"The execution owner disappeared after the tool call began. Inspect actual state before retrying.",
|
|
157
|
+
});
|
|
158
|
+
return encodeCodemodeDispatchAck({
|
|
159
|
+
version: 1,
|
|
160
|
+
operationId: request.operationId,
|
|
161
|
+
status: "terminal",
|
|
162
|
+
});
|
|
163
|
+
}
|
|
118
164
|
return encodeCodemodeDispatchAck({
|
|
119
165
|
version: 1,
|
|
120
166
|
operationId: request.operationId,
|
|
@@ -145,7 +191,39 @@ export class CodemodeAttemptDispatcher {
|
|
|
145
191
|
await Promise.allSettled([...this.inFlight.values()]);
|
|
146
192
|
}
|
|
147
193
|
|
|
148
|
-
private async execute(
|
|
194
|
+
private async execute(
|
|
195
|
+
operation: CodemodeOperation,
|
|
196
|
+
claimId: string,
|
|
197
|
+
reclaimed: boolean,
|
|
198
|
+
): Promise<void> {
|
|
199
|
+
const claimAuthority = {
|
|
200
|
+
accountId: this.scope.accountId,
|
|
201
|
+
workspaceId: this.scope.workspaceId,
|
|
202
|
+
attemptId: this.scope.attemptId,
|
|
203
|
+
operationId: operation.operationId,
|
|
204
|
+
claimId,
|
|
205
|
+
claimLeaseMs: this.claimLeaseMs,
|
|
206
|
+
};
|
|
207
|
+
// Preparation may include provider preflight and policy resolution. Keep the
|
|
208
|
+
// durable claim alive from the moment execution is accepted, not only after
|
|
209
|
+
// the provider side-effect marker, so a slow preflight cannot be reclaimed.
|
|
210
|
+
const heartbeat = setInterval(() => {
|
|
211
|
+
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
|
|
212
|
+
}, this.claimHeartbeatMs);
|
|
213
|
+
heartbeat.unref?.();
|
|
214
|
+
try {
|
|
215
|
+
await this.executeClaimed(operation, claimId, reclaimed, claimAuthority);
|
|
216
|
+
} finally {
|
|
217
|
+
clearInterval(heartbeat);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
private async executeClaimed(
|
|
222
|
+
operation: CodemodeOperation,
|
|
223
|
+
claimId: string,
|
|
224
|
+
reclaimed: boolean,
|
|
225
|
+
claimAuthority: Parameters<typeof markCodemodeOperationExecutionStarted>[1],
|
|
226
|
+
): Promise<void> {
|
|
149
227
|
const entry = this.environment.catalog.entries.find(
|
|
150
228
|
(candidate) =>
|
|
151
229
|
candidate.identity.serverId === operation.identity.serverId &&
|
|
@@ -170,38 +248,7 @@ export class CodemodeAttemptDispatcher {
|
|
|
170
248
|
);
|
|
171
249
|
return;
|
|
172
250
|
}
|
|
173
|
-
|
|
174
|
-
this.db,
|
|
175
|
-
this.bus,
|
|
176
|
-
this.scope.workspaceId,
|
|
177
|
-
this.scope.sessionId,
|
|
178
|
-
this.scope.turnId,
|
|
179
|
-
this.scope.executionGeneration,
|
|
180
|
-
this.scope.attemptId,
|
|
181
|
-
[
|
|
182
|
-
{
|
|
183
|
-
type: "agent.toolCall.created",
|
|
184
|
-
turnId: this.scope.turnId,
|
|
185
|
-
turnGeneration: this.scope.executionGeneration,
|
|
186
|
-
turnAttemptId: this.scope.attemptId,
|
|
187
|
-
producerId: operation.caller.subjectId,
|
|
188
|
-
payload: {
|
|
189
|
-
id: operation.operationId,
|
|
190
|
-
name: entry.modelName,
|
|
191
|
-
arguments: operation.arguments,
|
|
192
|
-
origin: "codemode",
|
|
193
|
-
subjectId: operation.caller.subjectId,
|
|
194
|
-
raw: {
|
|
195
|
-
type: "codemode_call",
|
|
196
|
-
serverId: operation.identity.serverId,
|
|
197
|
-
toolName: operation.identity.toolName,
|
|
198
|
-
catalogDigest: operation.catalogDigest,
|
|
199
|
-
},
|
|
200
|
-
},
|
|
201
|
-
},
|
|
202
|
-
],
|
|
203
|
-
).catch(() => ({ accepted: false }));
|
|
204
|
-
if (!created.accepted) {
|
|
251
|
+
if (!(await this.ensureToolCallCreated(operation, entry, reclaimed))) {
|
|
205
252
|
await this.failBeforeExecution(
|
|
206
253
|
operation.operationId,
|
|
207
254
|
claimId,
|
|
@@ -219,24 +266,9 @@ export class CodemodeAttemptDispatcher {
|
|
|
219
266
|
return;
|
|
220
267
|
}
|
|
221
268
|
|
|
222
|
-
|
|
223
|
-
accountId: this.scope.accountId,
|
|
224
|
-
workspaceId: this.scope.workspaceId,
|
|
225
|
-
attemptId: this.scope.attemptId,
|
|
226
|
-
operationId: operation.operationId,
|
|
227
|
-
claimId,
|
|
228
|
-
};
|
|
229
|
-
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
230
|
-
this.db,
|
|
231
|
-
claimAuthority,
|
|
232
|
-
);
|
|
233
|
-
if (!crossedExecutionBoundary) return;
|
|
234
|
-
const heartbeat = setInterval(() => {
|
|
235
|
-
void renewCodemodeOperationClaim(this.db, claimAuthority).catch(() => undefined);
|
|
236
|
-
}, CODEMODE_CLAIM_HEARTBEAT_MS);
|
|
237
|
-
heartbeat.unref?.();
|
|
269
|
+
let preparedCall: Awaited<ReturnType<AttemptToolEnvironment["prepareCall"]>>;
|
|
238
270
|
try {
|
|
239
|
-
|
|
271
|
+
preparedCall = await this.environment.prepareCall(
|
|
240
272
|
{
|
|
241
273
|
operationId: operation.operationId,
|
|
242
274
|
catalogDigest: operation.catalogDigest,
|
|
@@ -246,7 +278,34 @@ export class CodemodeAttemptDispatcher {
|
|
|
246
278
|
},
|
|
247
279
|
{ signal },
|
|
248
280
|
);
|
|
249
|
-
|
|
281
|
+
} catch (error) {
|
|
282
|
+
await this.settleWithOutput(operation, claimId, {
|
|
283
|
+
state: "failed",
|
|
284
|
+
errorCode: errorCode(error, signal, false),
|
|
285
|
+
errorMessage: safeErrorMessage(error),
|
|
286
|
+
});
|
|
287
|
+
return;
|
|
288
|
+
}
|
|
289
|
+
if (signal.aborted) {
|
|
290
|
+
await this.settleWithOutput(operation, claimId, {
|
|
291
|
+
state: "failed",
|
|
292
|
+
errorCode: "attempt_cancelled",
|
|
293
|
+
errorMessage: "Execution attempt ended before the Codemode call started",
|
|
294
|
+
});
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const crossedExecutionBoundary = await markCodemodeOperationExecutionStarted(
|
|
299
|
+
this.db,
|
|
300
|
+
claimAuthority,
|
|
301
|
+
);
|
|
302
|
+
if (!crossedExecutionBoundary) return;
|
|
303
|
+
try {
|
|
304
|
+
const result = await preparedCall.execute();
|
|
305
|
+
await this.settleWithOutput(operation, claimId, {
|
|
306
|
+
state: "completed",
|
|
307
|
+
result,
|
|
308
|
+
});
|
|
250
309
|
} catch (error) {
|
|
251
310
|
const knownPreExecution =
|
|
252
311
|
error instanceof AttemptToolApprovalRequiredError ||
|
|
@@ -255,13 +314,90 @@ export class CodemodeAttemptDispatcher {
|
|
|
255
314
|
connectorActionWasNotExecuted(error);
|
|
256
315
|
await this.settleWithOutput(operation, claimId, {
|
|
257
316
|
state: knownPreExecution ? "failed" : "outcome_unknown",
|
|
258
|
-
errorCode: errorCode(error, signal),
|
|
317
|
+
errorCode: errorCode(error, signal, true),
|
|
259
318
|
errorMessage: !knownPreExecution
|
|
260
319
|
? "Tool execution ended without a durable result. Its side-effect outcome is unknown; inspect actual state before retrying."
|
|
261
320
|
: safeErrorMessage(error),
|
|
262
321
|
});
|
|
263
|
-
}
|
|
264
|
-
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
private async ensureToolCallCreated(
|
|
326
|
+
operation: CodemodeOperation,
|
|
327
|
+
entry: AttemptToolCatalogEntry,
|
|
328
|
+
reclaimed: boolean,
|
|
329
|
+
): Promise<boolean> {
|
|
330
|
+
const clientEventId = codemodeToolCallCreatedClientEventId(operation.operationId);
|
|
331
|
+
if (reclaimed) {
|
|
332
|
+
const existing = await getSessionEventByClientEventId(
|
|
333
|
+
this.db,
|
|
334
|
+
this.scope.workspaceId,
|
|
335
|
+
this.scope.sessionId,
|
|
336
|
+
clientEventId,
|
|
337
|
+
).catch(() => null);
|
|
338
|
+
if (existing) {
|
|
339
|
+
assertMatchingCodemodeToolCallCreated(existing, operation, entry, this.scope);
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
const legacyCreated = await getSessionToolCallCreatedEventByOperationId(this.db, {
|
|
343
|
+
workspaceId: this.scope.workspaceId,
|
|
344
|
+
sessionId: this.scope.sessionId,
|
|
345
|
+
turnId: this.scope.turnId,
|
|
346
|
+
attemptId: this.scope.attemptId,
|
|
347
|
+
operationId: operation.operationId,
|
|
348
|
+
}).catch(() => null);
|
|
349
|
+
if (legacyCreated) {
|
|
350
|
+
assertMatchingCodemodeToolCallCreated(legacyCreated, operation, entry, this.scope);
|
|
351
|
+
return true;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
try {
|
|
355
|
+
const created = await appendAndPublishTurnEventsFenced(
|
|
356
|
+
this.db,
|
|
357
|
+
this.bus,
|
|
358
|
+
this.scope.workspaceId,
|
|
359
|
+
this.scope.sessionId,
|
|
360
|
+
this.scope.turnId,
|
|
361
|
+
this.scope.executionGeneration,
|
|
362
|
+
this.scope.attemptId,
|
|
363
|
+
[
|
|
364
|
+
{
|
|
365
|
+
type: "agent.toolCall.created",
|
|
366
|
+
clientEventId,
|
|
367
|
+
turnId: this.scope.turnId,
|
|
368
|
+
turnGeneration: this.scope.executionGeneration,
|
|
369
|
+
turnAttemptId: this.scope.attemptId,
|
|
370
|
+
producerId: operation.caller.subjectId,
|
|
371
|
+
payload: {
|
|
372
|
+
id: operation.operationId,
|
|
373
|
+
name: entry.modelName,
|
|
374
|
+
arguments: operation.arguments,
|
|
375
|
+
origin: "codemode",
|
|
376
|
+
subjectId: operation.caller.subjectId,
|
|
377
|
+
raw: {
|
|
378
|
+
type: "codemode_call",
|
|
379
|
+
serverId: operation.identity.serverId,
|
|
380
|
+
toolName: operation.identity.toolName,
|
|
381
|
+
catalogDigest: operation.catalogDigest,
|
|
382
|
+
},
|
|
383
|
+
},
|
|
384
|
+
},
|
|
385
|
+
],
|
|
386
|
+
);
|
|
387
|
+
return created.accepted;
|
|
388
|
+
} catch {
|
|
389
|
+
// A concurrent reclaimed claimant can win the unique client-event insert,
|
|
390
|
+
// or a committed append can lose its response. Reconcile durable truth
|
|
391
|
+
// before treating the attempt fence as closed.
|
|
392
|
+
const raced = await getSessionEventByClientEventId(
|
|
393
|
+
this.db,
|
|
394
|
+
this.scope.workspaceId,
|
|
395
|
+
this.scope.sessionId,
|
|
396
|
+
clientEventId,
|
|
397
|
+
).catch(() => null);
|
|
398
|
+
if (!raced) return false;
|
|
399
|
+
assertMatchingCodemodeToolCallCreated(raced, operation, entry, this.scope);
|
|
400
|
+
return true;
|
|
265
401
|
}
|
|
266
402
|
}
|
|
267
403
|
|
|
@@ -311,6 +447,43 @@ export class CodemodeAttemptDispatcher {
|
|
|
311
447
|
}
|
|
312
448
|
}
|
|
313
449
|
|
|
450
|
+
export function codemodeToolCallCreatedClientEventId(operationId: string): string {
|
|
451
|
+
return `opengeni:codemode-tool-call-created:${operationId}`;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function assertMatchingCodemodeToolCallCreated(
|
|
455
|
+
event: SessionEvent,
|
|
456
|
+
operation: CodemodeOperation,
|
|
457
|
+
entry: AttemptToolCatalogEntry,
|
|
458
|
+
scope: CodemodeDispatcherScope,
|
|
459
|
+
): void {
|
|
460
|
+
const payload = recordValue(event.payload);
|
|
461
|
+
const raw = recordValue(payload?.raw);
|
|
462
|
+
if (
|
|
463
|
+
event.type !== "agent.toolCall.created" ||
|
|
464
|
+
event.turnId !== scope.turnId ||
|
|
465
|
+
event.turnGeneration !== scope.executionGeneration ||
|
|
466
|
+
event.turnAttemptId !== scope.attemptId ||
|
|
467
|
+
event.turnAssociation !== "current" ||
|
|
468
|
+
payload?.id !== operation.operationId ||
|
|
469
|
+
payload?.name !== entry.modelName ||
|
|
470
|
+
payload?.origin !== "codemode" ||
|
|
471
|
+
payload?.subjectId !== operation.caller.subjectId ||
|
|
472
|
+
raw?.type !== "codemode_call" ||
|
|
473
|
+
raw?.serverId !== operation.identity.serverId ||
|
|
474
|
+
raw?.toolName !== operation.identity.toolName ||
|
|
475
|
+
raw?.catalogDigest !== operation.catalogDigest
|
|
476
|
+
) {
|
|
477
|
+
throw new Error("Codemode tool-call creation id is bound to incompatible durable evidence");
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
function recordValue(value: unknown): Record<string, unknown> | null {
|
|
482
|
+
return value !== null && typeof value === "object" && !Array.isArray(value)
|
|
483
|
+
? (value as Record<string, unknown>)
|
|
484
|
+
: null;
|
|
485
|
+
}
|
|
486
|
+
|
|
314
487
|
function connectorActionWasNotExecuted(error: unknown): boolean {
|
|
315
488
|
return Boolean(
|
|
316
489
|
error &&
|
|
@@ -324,8 +497,10 @@ function combinedSignal(primary: AbortSignal, secondary?: AbortSignal): AbortSig
|
|
|
324
497
|
return secondary ? AbortSignal.any([primary, secondary]) : primary;
|
|
325
498
|
}
|
|
326
499
|
|
|
327
|
-
function errorCode(error: unknown, signal: AbortSignal): string {
|
|
328
|
-
if (signal.aborted)
|
|
500
|
+
function errorCode(error: unknown, signal: AbortSignal, executionStarted: boolean): string {
|
|
501
|
+
if (signal.aborted) {
|
|
502
|
+
return executionStarted ? "attempt_cancelled_during_execution" : "attempt_cancelled";
|
|
503
|
+
}
|
|
329
504
|
if (connectorActionWasNotExecuted(error)) return "connector_action_not_executed";
|
|
330
505
|
if (typeof error === "object" && error !== null && "code" in error) {
|
|
331
506
|
const code = (error as { code?: unknown }).code;
|
|
@@ -338,6 +513,7 @@ function safeErrorMessage(error: unknown): string {
|
|
|
338
513
|
if (
|
|
339
514
|
error instanceof AttemptToolApprovalRequiredError ||
|
|
340
515
|
error instanceof AttemptToolCatalogStaleError ||
|
|
516
|
+
error instanceof AttemptToolInputValidationError ||
|
|
341
517
|
error instanceof AttemptToolNotFoundError
|
|
342
518
|
) {
|
|
343
519
|
return error.message;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
armCodexCapacityWait,
|
|
2
3
|
fetchCodexUsageForAccount,
|
|
3
4
|
getCodexCapacityWaitForSession,
|
|
4
5
|
getXaiCapacityWaitForSession,
|
|
@@ -9,7 +10,7 @@ import {
|
|
|
9
10
|
type CodexCapacityWakeTarget,
|
|
10
11
|
type CodexCapacitySelectionContext,
|
|
11
12
|
} from "@opengeni/db";
|
|
12
|
-
import
|
|
13
|
+
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
13
14
|
import {
|
|
14
15
|
authoritativeCodexCapacityResetAt,
|
|
15
16
|
codexAccountNeedsLiveCapacityRefresh,
|
|
@@ -85,14 +86,12 @@ export async function signalPendingCodexCapacityWakeTargets(
|
|
|
85
86
|
await signalCodexCapacityWakeTargets(services, targets);
|
|
86
87
|
}
|
|
87
88
|
|
|
88
|
-
export function codexCapacityDecision(
|
|
89
|
-
context: CodexCapacitySelectionContext,
|
|
90
|
-
settings: Settings,
|
|
89
|
+
export function codexCapacityDecision<TPolicyScope = never, TUnavailableDiagnostic = never>(
|
|
90
|
+
context: CodexCapacitySelectionContext<TPolicyScope, TUnavailableDiagnostic>,
|
|
91
91
|
): ReturnType<Parameters<typeof reconcileCodexCapacityWaitDb>[2]> {
|
|
92
92
|
const now = new Date();
|
|
93
93
|
const selected = selectCodexCredentialLeaseForTurn({
|
|
94
94
|
context,
|
|
95
|
-
leasingEnabled: settings.codexCredentialLeasingEnabled,
|
|
96
95
|
sessionId: context.sessionId,
|
|
97
96
|
sessionPinnedCredentialId: context.sessionPinnedCredentialId,
|
|
98
97
|
sessionPinSource: context.sessionPinSource,
|
|
@@ -118,8 +117,17 @@ export function codexCapacityDecision(
|
|
|
118
117
|
},
|
|
119
118
|
};
|
|
120
119
|
}
|
|
121
|
-
const
|
|
122
|
-
|
|
120
|
+
const policyCredentialId =
|
|
121
|
+
context.sessionPinSource === "manual" && context.sessionPinnedCredentialId
|
|
122
|
+
? context.sessionPinnedCredentialId
|
|
123
|
+
: !context.rotationEnabled
|
|
124
|
+
? context.activeCredentialId
|
|
125
|
+
: null;
|
|
126
|
+
const capacityAccounts = policyCredentialId
|
|
127
|
+
? context.accounts.filter((account) => account.id === policyCredentialId)
|
|
128
|
+
: context.accounts;
|
|
129
|
+
const authoritativeReset = authoritativeCodexCapacityResetAt(capacityAccounts, now);
|
|
130
|
+
const hasReconcilableQuotaCooldown = capacityAccounts.some(
|
|
123
131
|
(account) =>
|
|
124
132
|
account.status === "active" &&
|
|
125
133
|
account.allocatorEnabled &&
|
|
@@ -127,19 +135,85 @@ export function codexCapacityDecision(
|
|
|
127
135
|
account.exhaustedUntil !== null &&
|
|
128
136
|
account.exhaustedUntil > now,
|
|
129
137
|
);
|
|
138
|
+
const policyAccount = capacityAccounts[0] ?? null;
|
|
139
|
+
const mutationOnlyStatusBlock =
|
|
140
|
+
(policyAccount != null &&
|
|
141
|
+
(!policyAccount.allocatorEnabled || policyAccount.status !== "active")) ||
|
|
142
|
+
(authoritativeReset === null &&
|
|
143
|
+
capacityAccounts.length > 0 &&
|
|
144
|
+
capacityAccounts.every(
|
|
145
|
+
(account) => !account.allocatorEnabled || account.status !== "active",
|
|
146
|
+
));
|
|
147
|
+
const noneReason =
|
|
148
|
+
selected.decision.kind === "none"
|
|
149
|
+
? context.sessionPinSource === "manual" && context.sessionPinnedCredentialId !== null
|
|
150
|
+
? "manual_pin_missing"
|
|
151
|
+
: !context.rotationEnabled && context.activeCredentialId === null
|
|
152
|
+
? "rotation_off_active_pointer_missing"
|
|
153
|
+
: context.policyScope !== null && context.accounts.length === 0
|
|
154
|
+
? "policy_filtered_pool_empty"
|
|
155
|
+
: context.accounts.length === 0
|
|
156
|
+
? "no_connected_credentials"
|
|
157
|
+
: "no_eligible_credential"
|
|
158
|
+
: null;
|
|
130
159
|
return {
|
|
131
160
|
kind: "unavailable",
|
|
132
161
|
earliestResetAt: authoritativeReset,
|
|
133
162
|
resetKind:
|
|
134
|
-
|
|
163
|
+
selected.decision.kind === "none" ||
|
|
164
|
+
selected.decision.kind === "allocatorDisabled" ||
|
|
165
|
+
mutationOnlyStatusBlock
|
|
166
|
+
? "mutation_only"
|
|
167
|
+
: authoritativeReset && !hasReconcilableQuotaCooldown
|
|
168
|
+
? "authoritative"
|
|
169
|
+
: "bounded_refresh",
|
|
135
170
|
diagnostic: {
|
|
136
171
|
connectedCount: context.accounts.length,
|
|
137
172
|
allocatorEnabledCount: context.accounts.filter((account) => account.allocatorEnabled).length,
|
|
138
173
|
policyHash: context.policyHash,
|
|
174
|
+
...(noneReason ? { reason: noneReason } : {}),
|
|
139
175
|
},
|
|
140
176
|
};
|
|
141
177
|
}
|
|
142
178
|
|
|
179
|
+
/**
|
|
180
|
+
* Arm a durable Codex waiter and immediately re-evaluate it under the
|
|
181
|
+
* allocator lock. A capacity mutation that commits just before the waiter is
|
|
182
|
+
* inserted cannot signal a row that does not exist yet, so every arm site must
|
|
183
|
+
* close that edge before returning an hours-away reset timer to the workflow.
|
|
184
|
+
* Mutations after the arm commit still advance the waiter's wake revision.
|
|
185
|
+
*/
|
|
186
|
+
export async function armAndReconcileCodexCapacityWait(
|
|
187
|
+
services: Pick<ControlActivityServices, "db" | "bus">,
|
|
188
|
+
input: Parameters<typeof armCodexCapacityWait>[1],
|
|
189
|
+
options: { onArmed?: () => void } = {},
|
|
190
|
+
) {
|
|
191
|
+
const armed = await armCodexCapacityWait(services.db, input);
|
|
192
|
+
if (armed.action !== "waiting") return armed;
|
|
193
|
+
|
|
194
|
+
options.onArmed?.();
|
|
195
|
+
await publishDurableSessionEvents(services.bus, input.workspaceId, input.sessionId, armed.events);
|
|
196
|
+
const evaluated = await reconcileCodexCapacityWaitDb(
|
|
197
|
+
services.db,
|
|
198
|
+
{
|
|
199
|
+
accountId: input.accountId,
|
|
200
|
+
workspaceId: input.workspaceId,
|
|
201
|
+
sessionId: input.sessionId,
|
|
202
|
+
waiterId: armed.waiter.id,
|
|
203
|
+
generation: armed.waiter.generation,
|
|
204
|
+
...(input.now ? { now: input.now } : {}),
|
|
205
|
+
},
|
|
206
|
+
(context) => codexCapacityDecision(context),
|
|
207
|
+
);
|
|
208
|
+
await publishDurableSessionEvents(
|
|
209
|
+
services.bus,
|
|
210
|
+
input.workspaceId,
|
|
211
|
+
input.sessionId,
|
|
212
|
+
evaluated.events,
|
|
213
|
+
);
|
|
214
|
+
return evaluated;
|
|
215
|
+
}
|
|
216
|
+
|
|
143
217
|
async function refreshCapacityMetadata(
|
|
144
218
|
services: ControlActivityServices,
|
|
145
219
|
workspaceId: string,
|
|
@@ -239,7 +313,11 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
239
313
|
if (!current || current.id !== input.waiterId || current.generation !== input.generation) {
|
|
240
314
|
return { action: "stale" };
|
|
241
315
|
}
|
|
242
|
-
|
|
316
|
+
const boundedRefreshAttempted =
|
|
317
|
+
current.resetKind === "bounded_refresh" &&
|
|
318
|
+
input.cause === "timer" &&
|
|
319
|
+
current.nextCheckAt.getTime() <= Date.now();
|
|
320
|
+
if (boundedRefreshAttempted) {
|
|
243
321
|
// This is a bounded secret-safe control-plane quota refresh. It creates no
|
|
244
322
|
// turn, model call, user message, schedule, or entitlement action.
|
|
245
323
|
await refreshCapacityMetadata(resolved, input.workspaceId);
|
|
@@ -252,8 +330,9 @@ export function createCodexCapacityActivities(services: () => Promise<ControlAct
|
|
|
252
330
|
sessionId: input.sessionId,
|
|
253
331
|
waiterId: input.waiterId,
|
|
254
332
|
generation: input.generation,
|
|
333
|
+
boundedRefreshAttempted,
|
|
255
334
|
},
|
|
256
|
-
(context) => codexCapacityDecision(context
|
|
335
|
+
(context) => codexCapacityDecision(context),
|
|
257
336
|
);
|
|
258
337
|
if (result.events.length > 0) {
|
|
259
338
|
try {
|
|
@@ -16,7 +16,14 @@ export const CODEX_FLEET_SHADOW_MAX_PAYLOAD_BYTES = 34 * 1_024;
|
|
|
16
16
|
export type CodexFleetActualDecisionV1 = {
|
|
17
17
|
outcome: "selected" | "waiting" | "none";
|
|
18
18
|
candidateKey: string | null;
|
|
19
|
-
reason:
|
|
19
|
+
reason:
|
|
20
|
+
| "lease_reused"
|
|
21
|
+
| "pin"
|
|
22
|
+
| "rotation"
|
|
23
|
+
| "active"
|
|
24
|
+
| "all_capped"
|
|
25
|
+
| "allocator_disabled"
|
|
26
|
+
| "none";
|
|
20
27
|
};
|
|
21
28
|
|
|
22
29
|
export type CodexFleetShadowComparisonV1 =
|
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
import type { Database } from "@opengeni/db";
|
|
7
7
|
import type { ObjectStorage } from "@opengeni/storage";
|
|
8
8
|
import type { GeneratedImageReceipt } from "./generated-images";
|
|
9
|
+
import { CodexCredentialLeaseLostError } from "./agent-turn/credential-leases";
|
|
9
10
|
import {
|
|
10
11
|
executeImageGenerationOperation,
|
|
11
12
|
imageProviderBindingHash,
|
|
@@ -27,22 +28,38 @@ export async function executeCodexImageGeneration(input: {
|
|
|
27
28
|
prompt: string;
|
|
28
29
|
references?: readonly ResolvedImageGenerationReference[];
|
|
29
30
|
credentialId: string;
|
|
30
|
-
codexContext: Pick<
|
|
31
|
+
codexContext: Pick<
|
|
32
|
+
CodexRequestContext,
|
|
33
|
+
"clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
|
|
34
|
+
>;
|
|
31
35
|
abortSignal?: AbortSignal;
|
|
32
36
|
}): Promise<GeneratedImageReceipt> {
|
|
33
37
|
const providerBindingHash = imageProviderBindingHash(CODEX_PROVIDER_ID, input.credentialId);
|
|
38
|
+
let providerDispatchAdmitted = false;
|
|
39
|
+
const codexContext: Pick<
|
|
40
|
+
CodexRequestContext,
|
|
41
|
+
"clientVersion" | "getToken" | "refresh" | "beforeProviderDispatch"
|
|
42
|
+
> = {
|
|
43
|
+
...input.codexContext,
|
|
44
|
+
beforeProviderDispatch: async () => {
|
|
45
|
+
await input.codexContext.beforeProviderDispatch?.();
|
|
46
|
+
providerDispatchAdmitted = true;
|
|
47
|
+
},
|
|
48
|
+
};
|
|
34
49
|
return await executeImageGenerationOperation({
|
|
35
50
|
...input,
|
|
36
51
|
providerId: CODEX_PROVIDER_ID,
|
|
37
52
|
providerBindingHash,
|
|
38
53
|
modelId: CODEX_IMAGE_MODEL,
|
|
39
54
|
...(input.references ? { referenceDigests: input.references } : {}),
|
|
55
|
+
isProviderDispatchRejected: (error) =>
|
|
56
|
+
!providerDispatchAdmitted && error instanceof CodexCredentialLeaseLostError,
|
|
40
57
|
generate: async () => {
|
|
41
58
|
const generated = await generateCodexSubscriptionImage({
|
|
42
59
|
prompt: input.prompt,
|
|
43
60
|
...(input.references ? { references: input.references } : {}),
|
|
44
61
|
turnId: input.turnId,
|
|
45
|
-
context:
|
|
62
|
+
context: codexContext,
|
|
46
63
|
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
|
47
64
|
});
|
|
48
65
|
return {
|