@opengeni/worker-bundle 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,3960 @@
1
+ // src/index.ts
2
+ import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
3
+ import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
4
+ import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
5
+ import { NativeConnection, Worker } from "@temporalio/worker";
6
+
7
+ // src/activities.ts
8
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
9
+ import { createDb } from "@opengeni/db";
10
+ import { createDocumentServices } from "@opengeni/documents";
11
+ import { createNatsEventBus } from "@opengeni/events";
12
+ import { createObservability } from "@opengeni/observability";
13
+ import { createProductionAgentRuntime } from "@opengeni/runtime";
14
+ import { createObjectStorage as createObjectStorage2 } from "@opengeni/storage";
15
+
16
+ // src/activities/agent-turn.ts
17
+ import {
18
+ claimNextQueuedTurn as claimNextQueuedTurnDb,
19
+ createTurn,
20
+ applyCreditDebitUpToBalance,
21
+ finishTurn,
22
+ getBillingBalance as getBillingBalance2,
23
+ getSandbox as getSandbox2,
24
+ readActiveSandbox as readActiveSandbox2,
25
+ requireFile as requireFile2,
26
+ getSessionEvent as getSessionEvent3,
27
+ getSessionGoal as getSessionGoal3,
28
+ getSessionTurn,
29
+ isCodexBilledTurn as isCodexBilledTurn2,
30
+ workspaceCodexSubscriptionActive as workspaceCodexSubscriptionActive2,
31
+ getCodexRotationSettings,
32
+ listCodexAccountStatuses,
33
+ fetchCodexUsageForAccount,
34
+ getSessionCodexState,
35
+ recordSessionActiveCodexCredential,
36
+ recordCodexAccountUsage,
37
+ recordCodexAccountConnectors,
38
+ setActiveCodexCredential,
39
+ setCodexCredentialExhausted,
40
+ countConsecutiveReactiveRotations,
41
+ requireSession as requireSession2,
42
+ recordUsageEvent as recordUsageEvent2,
43
+ appendSessionHistoryItems,
44
+ consumeSessionCompactionRequest,
45
+ countSessionHistoryItems,
46
+ getActiveSessionHistoryItems as getActiveSessionHistoryItems3,
47
+ nextSessionHistoryPosition,
48
+ requeuePreemptedTurn,
49
+ saveRunState,
50
+ upsertSandboxSessionEnvelope,
51
+ setSessionStatus,
52
+ setSessionLastInputTokens,
53
+ sumUsageQuantity as sumUsageQuantity2,
54
+ heartbeatLeaseHolder,
55
+ accrueWarmSeconds,
56
+ SandboxLeaseSupersededError as SandboxLeaseSupersededError2
57
+ } from "@opengeni/db";
58
+ import { appendAndPublishEvents as appendAndPublishEvents2 } from "@opengeni/events";
59
+ import {
60
+ sandboxStateEntryFromRunState,
61
+ agentsErrorRunState,
62
+ maxTurnsExceededRunState,
63
+ modelResponseUsageFromSdkEvent,
64
+ normalizeSdkEvent,
65
+ sanitizeHistoryItemsForModel,
66
+ summarizeForCompaction as summarizeForCompaction2
67
+ } from "@opengeni/runtime";
68
+ import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits as configuredStaticUsageLimits2, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
69
+ import { CancelledFailure } from "@temporalio/activity";
70
+
71
+ // src/activities/capabilities.ts
72
+ import { environmentsEncryptionKeyBytes, parseModelProvidersJson } from "@opengeni/config";
73
+ import {
74
+ CODEX_APPS_MCP_SERVER_ID,
75
+ CODEX_APPS_MCP_SERVER_NAME,
76
+ CODEX_APPS_MCP_URL,
77
+ CODEX_APPS_STARTUP_TIMEOUT_MS,
78
+ CODEX_FALLBACK_MODEL_SLUGS,
79
+ CODEX_MODEL_ID_PREFIX,
80
+ CODEX_PROVIDER_BASE_URL,
81
+ CODEX_PROVIDER_ID
82
+ } from "@opengeni/codex";
83
+ import {
84
+ decryptedCapabilityHeaders,
85
+ listEnabledMcpCapabilityServers,
86
+ workspaceCodexSubscriptionActive
87
+ } from "@opengeni/db";
88
+ async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings) {
89
+ const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
90
+ return settingsWithMcpCapabilityServers(settings, enabled);
91
+ }
92
+ async function settingsWithCodexCredential(db, workspaceId, settings, activeOverride) {
93
+ const active = activeOverride ?? await workspaceCodexSubscriptionActive(db, settings, workspaceId);
94
+ if (!active) {
95
+ return settings;
96
+ }
97
+ const withProvider = withCodexProvider(settings);
98
+ return withCodexAppsMcpServer(withProvider);
99
+ }
100
+ function withCodexAppsMcpServer(settings) {
101
+ if (settings.mcpServers.some((server) => server.id === CODEX_APPS_MCP_SERVER_ID)) {
102
+ return settings;
103
+ }
104
+ return {
105
+ ...settings,
106
+ mcpServers: [
107
+ ...settings.mcpServers,
108
+ {
109
+ id: CODEX_APPS_MCP_SERVER_ID,
110
+ name: CODEX_APPS_MCP_SERVER_NAME,
111
+ url: CODEX_APPS_MCP_URL,
112
+ timeoutMs: CODEX_APPS_STARTUP_TIMEOUT_MS,
113
+ // Connector availability is per-credential and must re-discover each
114
+ // run; never poison a process-global tools-list cache.
115
+ cacheToolsList: false
116
+ // deliberately NO `headers` — the refreshing bearer is dynamic
117
+ }
118
+ ]
119
+ };
120
+ }
121
+ function withCodexProvider(settings) {
122
+ const providers = parseModelProvidersJson(settings.modelProvidersJson);
123
+ if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
124
+ return settings;
125
+ }
126
+ const codexProvider = {
127
+ kind: "codex-subscription",
128
+ id: CODEX_PROVIDER_ID,
129
+ label: "Codex (ChatGPT subscription)",
130
+ api: "responses",
131
+ baseUrl: CODEX_PROVIDER_BASE_URL,
132
+ models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({ id: `${CODEX_MODEL_ID_PREFIX}${slug}`, label: slug, reasoningEffort: true }))
133
+ };
134
+ return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
135
+ }
136
+ function settingsWithMcpCapabilityServers(settings, enabled) {
137
+ if (enabled.length === 0) {
138
+ return settings;
139
+ }
140
+ const encryptionKey = environmentsEncryptionKeyBytes(settings);
141
+ const existingIds = new Set(settings.mcpServers.map((server) => server.id));
142
+ const dynamicServers = enabled.filter((server) => !existingIds.has(server.id)).flatMap((server) => {
143
+ const headers = decryptedCapabilityHeaders(server, encryptionKey);
144
+ if (headers === "unavailable") {
145
+ return [];
146
+ }
147
+ return [{
148
+ id: server.id,
149
+ name: server.name,
150
+ url: server.url,
151
+ ...server.allowedTools ? { allowedTools: server.allowedTools } : {},
152
+ ...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
153
+ cacheToolsList: server.cacheToolsList ?? false,
154
+ ...headers ? { headers } : {}
155
+ }];
156
+ });
157
+ return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
158
+ }
159
+
160
+ // src/activities/codex-rotation.ts
161
+ var MIN_IDLE_MS = 6e4;
162
+ var DEFAULT_RESET_COOLDOWN_MS = 6e4;
163
+ function bindingUsedPct(acct) {
164
+ return Math.max(acct.primaryUsedPercent ?? 0, acct.secondaryUsedPercent ?? 0);
165
+ }
166
+ function bindingRemaining(acct) {
167
+ const primaryRemaining = 100 - (acct.primaryUsedPercent ?? 0);
168
+ const secondaryRemaining = 100 - (acct.secondaryUsedPercent ?? 0);
169
+ return Math.min(primaryRemaining, secondaryRemaining);
170
+ }
171
+ function cooling(acct, now) {
172
+ return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
173
+ }
174
+ function covers(acct, usedConnectors) {
175
+ if (usedConnectors.length === 0) {
176
+ return true;
177
+ }
178
+ const owned = acct.connectorNamespaces;
179
+ if (owned === null) {
180
+ return false;
181
+ }
182
+ const ownedSet = new Set(owned);
183
+ return usedConnectors.every((connector) => ownedSet.has(connector));
184
+ }
185
+ function droppedConnectorsFor(acct, usedConnectors) {
186
+ if (usedConnectors.length === 0) {
187
+ return [];
188
+ }
189
+ const owned = acct.connectorNamespaces;
190
+ if (owned === null) {
191
+ return [...usedConnectors];
192
+ }
193
+ const ownedSet = new Set(owned);
194
+ return usedConnectors.filter((connector) => !ownedSet.has(connector));
195
+ }
196
+ function eligible(acct, nearExhaustionPct, now) {
197
+ return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct) < nearExhaustionPct;
198
+ }
199
+ function availableAt(acct, nearExhaustionPct, now) {
200
+ const nowMs = now.getTime();
201
+ const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
202
+ const candidates = [];
203
+ if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
204
+ candidates.push(acct.exhaustedUntil);
205
+ }
206
+ const windowClear = (over, resetAt) => {
207
+ if (!over) return;
208
+ candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
209
+ };
210
+ windowClear((acct.primaryUsedPercent ?? 0) >= nearExhaustionPct, acct.primaryResetAt);
211
+ windowClear((acct.secondaryUsedPercent ?? 0) >= nearExhaustionPct, acct.secondaryResetAt);
212
+ if (candidates.length === 0) {
213
+ return defaultClear;
214
+ }
215
+ return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
216
+ }
217
+ function earliestReset(accounts, nearExhaustionPct, now) {
218
+ return accounts.map((acct) => availableAt(acct, nearExhaustionPct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
219
+ }
220
+ function computeIdleDelayMs(earliestResetAt, now, maxMs) {
221
+ const delta = earliestResetAt.getTime() - now.getTime();
222
+ return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
223
+ }
224
+ var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
225
+ var REACTIVE_ROTATION_MARGIN = 2;
226
+ var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
227
+ function computeReactiveRotationResume(args) {
228
+ const streak = args.priorConsecutiveRotations + 1;
229
+ const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
230
+ if (streak > bound) {
231
+ return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
232
+ }
233
+ if (!args.cooldownPersisted) {
234
+ return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
235
+ }
236
+ return { continueDelayMs: 0, idleUntilReset: false };
237
+ }
238
+ function chooseRotationActive(args) {
239
+ const { rotationStrategy, activeCredentialId, priorCredentialId, accounts, nearExhaustionPct, now } = args;
240
+ const usedConnectors = args.usedConnectors ?? [];
241
+ if (accounts.length === 0) {
242
+ return { kind: "none" };
243
+ }
244
+ const eligibles = accounts.filter((acct) => eligible(acct, nearExhaustionPct, now));
245
+ const activeRow = activeCredentialId ? accounts.find((acct) => acct.id === activeCredentialId) ?? null : null;
246
+ const decide = (chosen2) => {
247
+ if (!chosen2) {
248
+ return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
249
+ }
250
+ const dropped = droppedConnectorsFor(chosen2, usedConnectors);
251
+ return {
252
+ kind: "active",
253
+ credentialId: chosen2.id,
254
+ moved: chosen2.id !== activeCredentialId,
255
+ ...dropped.length > 0 ? { droppedConnectors: dropped } : {}
256
+ };
257
+ };
258
+ if (rotationStrategy === "round_robin") {
259
+ if (eligibles.length === 0) {
260
+ return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
261
+ }
262
+ const priorIdx = priorCredentialId ? accounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
263
+ const ordered = priorIdx >= 0 ? [...accounts.slice(priorIdx + 1), ...accounts.slice(0, priorIdx + 1)] : accounts;
264
+ const chosen2 = ordered.find((acct) => eligible(acct, nearExhaustionPct, now));
265
+ return decide(chosen2);
266
+ }
267
+ if (rotationStrategy === "drain_then_next") {
268
+ const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
269
+ if (priorRow && eligible(priorRow, nearExhaustionPct, now)) {
270
+ return decide(priorRow);
271
+ }
272
+ return decide(eligibles[0]);
273
+ }
274
+ if (activeRow && eligible(activeRow, nearExhaustionPct, now)) {
275
+ return { kind: "active", credentialId: activeRow.id, moved: false };
276
+ }
277
+ const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
278
+ const pool = covering.length > 0 ? covering : eligibles;
279
+ const chosen = pool.reduce((best, acct) => {
280
+ if (!best) {
281
+ return acct;
282
+ }
283
+ return bindingRemaining(acct) > bindingRemaining(best) ? acct : best;
284
+ }, void 0);
285
+ return decide(chosen);
286
+ }
287
+
288
+ // src/activities/codex-auth.ts
289
+ import { buildCodexTokenResolver } from "@opengeni/db";
290
+
291
+ // src/activities/agent-turn.ts
292
+ import {
293
+ buildModelResolver,
294
+ CODEX_CLIENT_VERSION,
295
+ CODEX_FALLBACK_MODEL_SLUGS as CODEX_FALLBACK_MODEL_SLUGS2,
296
+ classifyCodexUsageLimitError,
297
+ codexRequestStorage
298
+ } from "@opengeni/codex";
299
+
300
+ // src/activities/common.ts
301
+ import {
302
+ mergeResourceRefs,
303
+ mergeToolRefs,
304
+ reasoningEffortForMetadata
305
+ } from "@opengeni/contracts";
306
+ function assertReusableSessionRevivable(status) {
307
+ if (status === "cancelled") {
308
+ throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
309
+ }
310
+ }
311
+ function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
312
+ return {
313
+ text: prompt,
314
+ scheduledTaskId: taskId,
315
+ scheduledTaskRunId: runId,
316
+ ...resources.length ? { resources } : {},
317
+ ...tools.length ? { tools } : {}
318
+ };
319
+ }
320
+ function workflowIdForSession(sessionId) {
321
+ return `session-${sessionId}`;
322
+ }
323
+
324
+ // src/activities/context-compaction.ts
325
+ import {
326
+ applyContextCompaction,
327
+ getActiveSessionHistoryItems
328
+ } from "@opengeni/db";
329
+ import {
330
+ buildCompactionMessages,
331
+ buildSummaryItem,
332
+ planCompaction,
333
+ summarizeForCompaction
334
+ } from "@opengeni/runtime";
335
+ import {
336
+ contextInputBudgetTokens,
337
+ resolveContextCompactionMode
338
+ } from "@opengeni/config";
339
+ async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: s.contextSummaryMaxTokens }), options = {}) {
340
+ if (resolveContextCompactionMode(settings) !== "client") {
341
+ return { compacted: false, reason: "mode_not_client" };
342
+ }
343
+ const active = await getActiveSessionHistoryItems(db, scope.workspaceId, scope.sessionId);
344
+ if (active.length === 0) {
345
+ return { compacted: false, reason: "no_history" };
346
+ }
347
+ const items = active.map((row) => row.item);
348
+ const plan = planCompaction({
349
+ items,
350
+ lastInputTokens,
351
+ inputBudgetTokens: contextInputBudgetTokens(settings),
352
+ softFraction: settings.contextCompactSoftFraction,
353
+ hardFraction: settings.contextCompactHardFraction,
354
+ keepRecentTokens: settings.contextKeepRecentTokens,
355
+ ...options.force ? { force: true } : {}
356
+ });
357
+ if (!plan.shouldCompact) {
358
+ return { compacted: false, reason: plan.reason };
359
+ }
360
+ if (plan.hardForced) {
361
+ console.warn(
362
+ `context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`
363
+ );
364
+ }
365
+ const messages = buildCompactionMessages(plan);
366
+ const summaryBody = await summarize(settings, messages);
367
+ if (!summaryBody) {
368
+ return { compacted: false, reason: "summarize_failed" };
369
+ }
370
+ const boundaryRow = active[plan.boundaryIndex];
371
+ if (!boundaryRow) {
372
+ return { compacted: false, reason: "boundary_out_of_range" };
373
+ }
374
+ const boundaryPosition = boundaryRow.position;
375
+ if (boundaryPosition <= 0) {
376
+ return { compacted: false, reason: "boundary_at_origin" };
377
+ }
378
+ const summaryPosition = boundaryPosition - 0.5;
379
+ await applyContextCompaction(db, {
380
+ accountId: scope.accountId,
381
+ workspaceId: scope.workspaceId,
382
+ sessionId: scope.sessionId,
383
+ turnId: scope.turnId ?? null,
384
+ boundaryPosition,
385
+ summaryPosition,
386
+ summaryItem: buildSummaryItem(summaryBody)
387
+ });
388
+ return { compacted: true, supersededFrom: boundaryPosition, summaryPosition, hardForced: plan.hardForced };
389
+ }
390
+
391
+ // src/activities/environment.ts
392
+ import {
393
+ applyGitAuthPointerEnvironment,
394
+ stableSandboxEnvironmentForRun
395
+ } from "@opengeni/config";
396
+ import {
397
+ loadWorkspaceEnvironmentForRun as loadWorkspaceEnvironmentForRunFromDb
398
+ } from "@opengeni/db";
399
+ import {
400
+ createGitHubAppInstallationToken,
401
+ githubAppBotIdentity
402
+ } from "@opengeni/github";
403
+ async function loadWorkspaceEnvironmentForRunWithCredentials(db, settings, scope, environmentId, sandboxSecrets) {
404
+ if (!sandboxSecrets) {
405
+ return loadWorkspaceEnvironmentForRunFromDb(db, settings, scope.workspaceId, environmentId);
406
+ }
407
+ if (!environmentId) {
408
+ return null;
409
+ }
410
+ const secrets = await sandboxSecrets({
411
+ accountId: scope.accountId,
412
+ workspaceId: scope.workspaceId,
413
+ environmentId
414
+ });
415
+ assertWorkspaceEcho("sandboxSecrets", scope, secrets.workspaceId);
416
+ return {
417
+ id: secrets.id ?? environmentId,
418
+ name: secrets.name ?? environmentId,
419
+ description: secrets.description ?? null,
420
+ values: secrets.values
421
+ };
422
+ }
423
+ function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
424
+ if (echoedWorkspaceId !== scope.workspaceId) {
425
+ throw new Error(
426
+ `connection-credential provider (${kind}) scoped to workspace ${echoedWorkspaceId} but the run is workspace ${scope.workspaceId}`
427
+ );
428
+ }
429
+ }
430
+ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
431
+ const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment);
432
+ const selection = githubRepositorySelection(resources);
433
+ if (!selection || options.skipGitHubToken) {
434
+ return { environment };
435
+ }
436
+ let token;
437
+ let identity;
438
+ if (options?.gitCredentials && options.scope) {
439
+ const minted = await options.gitCredentials({
440
+ accountId: options.scope.accountId,
441
+ workspaceId: options.scope.workspaceId,
442
+ installationId: selection.installationId,
443
+ repositoryIds: selection.repositoryIds
444
+ });
445
+ assertWorkspaceEcho("gitCredentials", options.scope, minted.workspaceId);
446
+ token = minted.token;
447
+ identity = minted.identity ?? githubAppBotIdentity(settings);
448
+ } else {
449
+ token = await createGitHubAppInstallationToken(settings, {
450
+ installationId: selection.installationId,
451
+ repositoryIds: selection.repositoryIds
452
+ });
453
+ identity = githubAppBotIdentity(settings);
454
+ }
455
+ applyGitAuthPointerEnvironment(environment, identity);
456
+ return { environment, gitToken: token };
457
+ }
458
+ function githubRepositorySelection(resources) {
459
+ const selected = resources.flatMap((resource) => {
460
+ if (resource.kind !== "repository") {
461
+ return [];
462
+ }
463
+ const installationId2 = positiveInteger(resource.githubInstallationId);
464
+ const repositoryId = positiveInteger(resource.githubRepositoryId);
465
+ return installationId2 && repositoryId ? [{ installationId: installationId2, repositoryId }] : [];
466
+ });
467
+ if (selected.length === 0) {
468
+ return null;
469
+ }
470
+ const installationId = selected[0].installationId;
471
+ if (selected.some((item) => item.installationId !== installationId)) {
472
+ throw new Error("GitHub App repository resources must belong to one installation");
473
+ }
474
+ return {
475
+ installationId,
476
+ repositoryIds: selected.map((item) => item.repositoryId)
477
+ };
478
+ }
479
+ function positiveInteger(value) {
480
+ if (typeof value === "number" && Number.isInteger(value) && value > 0) {
481
+ return value;
482
+ }
483
+ if (typeof value === "string" && /^\d+$/.test(value) && Number(value) > 0) {
484
+ return Number(value);
485
+ }
486
+ return null;
487
+ }
488
+
489
+ // src/activities/goals.ts
490
+ import { configuredStaticUsageLimits } from "@opengeni/config";
491
+ import {
492
+ mergeToolRefs as mergeToolRefs2,
493
+ reasoningEffortForMetadata as reasoningEffortForMetadata2
494
+ } from "@opengeni/contracts";
495
+ import {
496
+ enqueueSessionTurn,
497
+ evaluateGoalContinuation,
498
+ getBillingBalance,
499
+ getSessionEvent,
500
+ getSessionGoal,
501
+ isCodexBilledTurn,
502
+ recordUsageEvent,
503
+ requireSession,
504
+ setSessionGoalLastContinuationTurn,
505
+ setSessionGoalStatus,
506
+ sumUsageQuantity
507
+ } from "@opengeni/db";
508
+ import { appendAndPublishEvents } from "@opengeni/events";
509
+ function createGoalActivities(services) {
510
+ async function maybeContinueGoal2(input) {
511
+ const { settings, db, bus } = await services();
512
+ const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
513
+ if (!existingGoal || existingGoal.status !== "active") {
514
+ return { action: "none" };
515
+ }
516
+ const session = await requireSession(db, input.workspaceId, input.sessionId);
517
+ const isCodexRun = await isCodexBilledTurn({ db, settings, workspaceId: input.workspaceId, model: session.model });
518
+ const budgetBlocked = await goalRunBudgetBlocked(settings, db, input.accountId, input.workspaceId, isCodexRun);
519
+ const decision = await evaluateGoalContinuation(db, {
520
+ workspaceId: input.workspaceId,
521
+ sessionId: input.sessionId,
522
+ defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
523
+ noProgressLimit: settings.goalNoProgressLimit,
524
+ budgetBlocked
525
+ });
526
+ if (decision.decision === "none" || decision.decision === "queue") {
527
+ return { action: decision.decision };
528
+ }
529
+ if (decision.decision === "paused") {
530
+ await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
531
+ type: "goal.paused",
532
+ payload: {
533
+ goalId: decision.goal.id,
534
+ actor: "system",
535
+ reason: decision.reason,
536
+ ...decision.goal.rationale ? { rationale: decision.goal.rationale } : {},
537
+ autoContinuations: decision.goal.autoContinuations,
538
+ noProgressStreak: decision.goal.noProgressStreak
539
+ }
540
+ }]);
541
+ return { action: "paused" };
542
+ }
543
+ const recheck = await getSessionGoal(db, input.workspaceId, input.sessionId);
544
+ if (!recheck || recheck.status !== "active" || recheck.version !== decision.goal.version) {
545
+ return { action: "none" };
546
+ }
547
+ const prompt = goalContinuationPrompt(decision.goal, decision.autoContinuation, decision.cap);
548
+ const [continuationEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
549
+ type: "goal.continuation",
550
+ payload: {
551
+ goalId: decision.goal.id,
552
+ text: prompt,
553
+ autoContinuation: decision.autoContinuation,
554
+ maxAutoContinuations: decision.cap,
555
+ goalVersion: decision.goal.version
556
+ }
557
+ }]);
558
+ if (!continuationEvent) {
559
+ throw new Error("failed to append goal continuation trigger event");
560
+ }
561
+ const turn = await enqueueSessionTurn(db, {
562
+ accountId: input.accountId,
563
+ workspaceId: input.workspaceId,
564
+ sessionId: input.sessionId,
565
+ triggerEventId: continuationEvent.id,
566
+ temporalWorkflowId: input.workflowId,
567
+ source: "goal",
568
+ prompt,
569
+ resources: [],
570
+ // Continuations keep the session tool surface and force the first-party
571
+ // server so the goal_complete/goal_pause escape hatches stay reachable.
572
+ tools: withFirstPartyTools(settings, session.tools),
573
+ model: session.model,
574
+ reasoningEffort: reasoningEffortForMetadata2(session.metadata, settings.openaiReasoningEffort),
575
+ sandboxBackend: session.sandboxBackend,
576
+ metadata: { goalId: decision.goal.id, autoContinuation: decision.autoContinuation }
577
+ });
578
+ await setSessionGoalLastContinuationTurn(db, input.workspaceId, input.sessionId, turn.id);
579
+ await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
580
+ type: "turn.queued",
581
+ turnId: turn.id,
582
+ payload: { turnId: turn.id, triggerEventId: continuationEvent.id, source: turn.source }
583
+ }]);
584
+ await recordUsageEvent(db, {
585
+ accountId: input.accountId,
586
+ workspaceId: input.workspaceId,
587
+ eventType: "agent_run.created",
588
+ quantity: 1,
589
+ unit: "run",
590
+ sourceResourceType: "session_turn",
591
+ sourceResourceId: turn.id,
592
+ idempotencyKey: `agent_run.created:goal:${input.workspaceId}:${turn.id}`
593
+ });
594
+ return { action: "continue" };
595
+ }
596
+ async function pauseGoalForInterrupt2(input) {
597
+ const { db, bus } = await services();
598
+ if (input.triggerEventId) {
599
+ const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
600
+ if (isSteerInterrupt(trigger)) {
601
+ return;
602
+ }
603
+ }
604
+ await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
605
+ }
606
+ return {
607
+ maybeContinueGoal: maybeContinueGoal2,
608
+ pauseGoalForInterrupt: pauseGoalForInterrupt2
609
+ };
610
+ }
611
+ function isSteerInterrupt(trigger) {
612
+ if (!trigger || trigger.type !== "user.interrupt") {
613
+ return false;
614
+ }
615
+ const payload = trigger.payload;
616
+ return typeof payload === "object" && payload !== null && payload.reason === "steer";
617
+ }
618
+ async function pauseActiveGoalOnInterrupt(db, bus, workspaceId, sessionId) {
619
+ const goal = await getSessionGoal(db, workspaceId, sessionId);
620
+ if (!goal || goal.status !== "active") {
621
+ return;
622
+ }
623
+ const { goal: paused, changed } = await setSessionGoalStatus(db, workspaceId, sessionId, {
624
+ status: "paused",
625
+ pausedReason: "user_interrupt"
626
+ });
627
+ if (changed) {
628
+ await appendAndPublishEvents(db, bus, workspaceId, sessionId, [{
629
+ type: "goal.paused",
630
+ payload: {
631
+ goalId: paused.id,
632
+ actor: "user",
633
+ reason: "user_interrupt",
634
+ autoContinuations: paused.autoContinuations,
635
+ noProgressStreak: paused.noProgressStreak
636
+ }
637
+ }]);
638
+ }
639
+ }
640
+ function goalContinuationPrompt(goal, autoContinuation, cap) {
641
+ const counter = cap === null ? `${autoContinuation}` : `${autoContinuation}/${cap}`;
642
+ return [
643
+ `[GOAL CONTINUATION ${counter}] The session goal is not done. Goal: ${goal.text}.`,
644
+ `Success criteria: ${goal.successCriteria ?? "none specified"}.`,
645
+ "Continue working toward the goal now. If it is actually complete, call opengeni__goal_complete with concrete evidence.",
646
+ "If you are blocked or continuing is not productive, call opengeni__goal_pause with your rationale.",
647
+ "You may revise the goal with opengeni__goal_update. Do not stop without one of these explicit actions."
648
+ ].join("\n");
649
+ }
650
+ function withFirstPartyTools(settings, tools) {
651
+ if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
652
+ return tools;
653
+ }
654
+ return mergeToolRefs2(tools, [{ kind: "mcp", id: "opengeni" }]);
655
+ }
656
+ function withCodexAppsTool(settings, tools) {
657
+ if (!settings.mcpServers.some((server) => server.id === "codex_apps")) {
658
+ return tools;
659
+ }
660
+ return mergeToolRefs2(tools, [{ kind: "mcp", id: "codex_apps" }]);
661
+ }
662
+ async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
663
+ if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
664
+ const balance = await getBillingBalance(db, accountId);
665
+ if (balance.balanceMicros <= 0) {
666
+ return "insufficient OpenGeni credits";
667
+ }
668
+ }
669
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
670
+ const limits = configuredStaticUsageLimits(settings);
671
+ if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
672
+ const used = await sumUsageQuantity(db, {
673
+ accountId,
674
+ eventType: "model.cost",
675
+ since: startOfUtcMonth()
676
+ });
677
+ if (used >= limits.maxMonthlyCostMicrosPerAccount) {
678
+ return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
679
+ }
680
+ }
681
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
682
+ const used = await sumUsageQuantity(db, {
683
+ workspaceId,
684
+ eventType: "agent_run.created",
685
+ since: startOfUtcMonth()
686
+ });
687
+ if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
688
+ return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
689
+ }
690
+ }
691
+ }
692
+ return null;
693
+ }
694
+ function startOfUtcMonth() {
695
+ const now = /* @__PURE__ */ new Date();
696
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
697
+ }
698
+
699
+ // src/activities/packs.ts
700
+ import { CapabilityPack } from "@opengeni/contracts";
701
+ import { getWorkspace, getWorkspacePack, listPackInstallations } from "@opengeni/db";
702
+ var emptyPackRuntime = { sandboxImage: null, skills: [] };
703
+ async function resolveWorkspacePackRuntime(db, workspaceId) {
704
+ const installations = await listPackInstallations(db, workspaceId);
705
+ const active = installations.filter((installation) => installation.status === "active");
706
+ if (active.length === 0) {
707
+ return emptyPackRuntime;
708
+ }
709
+ const packs = [];
710
+ for (const installation of active) {
711
+ const registration = await getWorkspacePack(db, workspaceId, installation.packId);
712
+ if (!registration) {
713
+ continue;
714
+ }
715
+ const parsed = CapabilityPack.safeParse(registration.pack);
716
+ if (parsed.success) {
717
+ packs.push(parsed.data);
718
+ }
719
+ }
720
+ return workspacePackRuntimeFromPacks(packs);
721
+ }
722
+ function workspacePackRuntimeFromPacks(packs) {
723
+ const imagePacks = packs.filter((pack) => typeof pack.sandboxImage === "string" && pack.sandboxImage.trim().length > 0);
724
+ if (imagePacks.length > 1) {
725
+ const ids = imagePacks.map((pack) => pack.id).sort().join(", ");
726
+ throw new Error(
727
+ `Multiple enabled packs declare a sandbox image (${ids}). Only one enabled pack per workspace may declare sandboxImage; disable the others and retry.`
728
+ );
729
+ }
730
+ const skills = [];
731
+ const skillOwners = /* @__PURE__ */ new Map();
732
+ for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
733
+ for (const skill of pack.skills) {
734
+ const key = skill.name.toLowerCase();
735
+ const existingOwner = skillOwners.get(key);
736
+ if (existingOwner !== void 0 && existingOwner !== pack.id) {
737
+ throw new Error(
738
+ `Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
739
+ );
740
+ }
741
+ skillOwners.set(key, pack.id);
742
+ skills.push({
743
+ name: skill.name,
744
+ description: skill.description ?? null,
745
+ files: skill.files.map((file) => ({ path: file.path, content: file.content }))
746
+ });
747
+ }
748
+ }
749
+ return {
750
+ sandboxImage: imagePacks[0]?.sandboxImage?.trim() ?? null,
751
+ skills
752
+ };
753
+ }
754
+ async function resolveWorkspaceAgentInstructions(db, workspaceId) {
755
+ const workspace = await getWorkspace(db, workspaceId);
756
+ return workspace?.agentInstructions ?? null;
757
+ }
758
+ function settingsWithPackSandboxImage(settings, sandboxImage) {
759
+ if (!sandboxImage) {
760
+ return settings;
761
+ }
762
+ return {
763
+ ...settings,
764
+ dockerImage: sandboxImage,
765
+ modalImageRef: sandboxImage
766
+ };
767
+ }
768
+
769
+ // src/activities/parent-wake.ts
770
+ import { getSession, getSessionGoal as getSessionGoal2, wakeParentSessionForChildCompletion } from "@opengeni/db";
771
+ async function notifyParentOfChildTerminal(svc, workspaceId, childSessionId, terminalStatus, episodeKey) {
772
+ try {
773
+ const child = await getSession(svc.db, workspaceId, childSessionId);
774
+ if (!child || !child.parentSessionId) {
775
+ return;
776
+ }
777
+ const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
778
+ const clientEventId = `child-completion:${childSessionId}:${episodeKey ?? child.lastSequence}`;
779
+ const result = await wakeParentSessionForChildCompletion(svc.db, {
780
+ workspaceId,
781
+ parentSessionId: child.parentSessionId,
782
+ clientEventId,
783
+ text: childCompletionWakeText(child, goal, terminalStatus),
784
+ childCompletion: childCompletionPayload(child, goal, terminalStatus),
785
+ reasoningEffortFallback: svc.settings.openaiReasoningEffort
786
+ });
787
+ if (!result.delivered) {
788
+ return;
789
+ }
790
+ await svc.bus.publish(workspaceId, child.parentSessionId, result.events);
791
+ if (svc.wakeSessionWorkflow) {
792
+ await svc.wakeSessionWorkflow({
793
+ accountId: child.accountId,
794
+ workspaceId,
795
+ sessionId: child.parentSessionId,
796
+ workflowId: result.temporalWorkflowId
797
+ });
798
+ }
799
+ svc.observability.info("Woke parent session on worker completion", {
800
+ childSessionId,
801
+ parentSessionId: child.parentSessionId,
802
+ terminalStatus
803
+ });
804
+ } catch (error) {
805
+ svc.observability.error("Failed to wake parent session on worker completion", {
806
+ childSessionId,
807
+ error: error instanceof Error ? error.message : String(error)
808
+ });
809
+ }
810
+ }
811
+ function childCompletionPayload(child, goal, terminalStatus) {
812
+ return {
813
+ childSessionId: child.id,
814
+ status: terminalStatus,
815
+ ...goal ? {
816
+ goal: {
817
+ status: goal.status,
818
+ text: goal.text,
819
+ ...goal.evidence ? { evidence: goal.evidence } : {},
820
+ ...goal.rationale ? { rationale: goal.rationale } : {},
821
+ ...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
822
+ }
823
+ } : {}
824
+ };
825
+ }
826
+ function childCompletionWakeText(child, goal, terminalStatus) {
827
+ const lines = [];
828
+ if (terminalStatus === "failed") {
829
+ lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
830
+ } else if (goal?.status === "completed") {
831
+ lines.push(`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`);
832
+ } else if (goal?.status === "paused") {
833
+ lines.push(`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`);
834
+ } else {
835
+ lines.push(`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`);
836
+ }
837
+ if (goal) {
838
+ lines.push(`Worker goal: ${goal.text}`);
839
+ if (goal.status === "completed" && goal.evidence) {
840
+ lines.push(`Completion evidence: ${goal.evidence}`);
841
+ }
842
+ if (goal.status === "paused" && goal.rationale) {
843
+ lines.push(`Pause rationale: ${goal.rationale}`);
844
+ }
845
+ }
846
+ lines.push("Read the worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting this worker, resume it now.");
847
+ return lines.join("\n");
848
+ }
849
+
850
+ // src/activities/redaction.ts
851
+ var MIN_REDACTABLE_VALUE_LENGTH = 6;
852
+ function createSecretRedactor(secrets) {
853
+ const usable = secrets.filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH).sort((a, b) => b.value.length - a.value.length);
854
+ if (usable.length === 0) {
855
+ return identityRedactor;
856
+ }
857
+ const redact = (payload) => {
858
+ if (typeof payload === "string") {
859
+ let out = payload;
860
+ for (const secret of usable) {
861
+ if (out.includes(secret.value)) {
862
+ out = out.split(secret.value).join(`[redacted:${secret.name}]`);
863
+ }
864
+ }
865
+ return out;
866
+ }
867
+ if (Array.isArray(payload)) {
868
+ return payload.map(redact);
869
+ }
870
+ if (isPlainObject(payload)) {
871
+ return Object.fromEntries(Object.entries(payload).map(([key, value]) => [key, redact(value)]));
872
+ }
873
+ return payload;
874
+ };
875
+ return redact;
876
+ }
877
+ function identityRedactor(payload) {
878
+ return payload;
879
+ }
880
+ function isPlainObject(value) {
881
+ if (!value || typeof value !== "object") {
882
+ return false;
883
+ }
884
+ const proto = Object.getPrototypeOf(value);
885
+ return proto === Object.prototype || proto === null;
886
+ }
887
+
888
+ // src/activities/run-input.ts
889
+ import { contextInputBudgetTokens as contextInputBudgetTokens2, resolveContextCompactionMode as resolveContextCompactionMode2 } from "@opengeni/config";
890
+ import {
891
+ getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
892
+ getLatestRunState,
893
+ getSandboxSessionEnvelope,
894
+ requireFile
895
+ } from "@opengeni/db";
896
+ import { stripReasoningEncryptedContent, stripReasoningIdentityFromSerializedRunState, neutralizeToolSearchItemsInSerializedRunState } from "@opengeni/runtime";
897
+ var NON_CODEX_TURN = { currentCodexCredentialId: null };
898
+ function applyCodexHistoryStrip(rows, current) {
899
+ const out = [];
900
+ for (const row of rows) {
901
+ if (row.producerCodexCredentialId === current.currentCodexCredentialId) {
902
+ out.push(row.item);
903
+ continue;
904
+ }
905
+ const type = typeof row.item.type === "string" ? row.item.type : void 0;
906
+ if (type === "reasoning") {
907
+ continue;
908
+ }
909
+ if (type === "tool_search_call" || type === "tool_search_output") {
910
+ continue;
911
+ }
912
+ if (type === "compaction") {
913
+ out.push(stripReasoningEncryptedContent(row.item));
914
+ continue;
915
+ }
916
+ out.push(row.item);
917
+ }
918
+ return out;
919
+ }
920
+ function resumeRunStateForCodexAccount(state, current) {
921
+ if (state.frozenCodexCredentialId === current.currentCodexCredentialId) {
922
+ return state.serializedRunState;
923
+ }
924
+ return neutralizeToolSearchItemsInSerializedRunState(stripReasoningIdentityFromSerializedRunState(state.serializedRunState));
925
+ }
926
+ async function turnInput(db, runtime, agent, trigger, settings, current = NON_CODEX_TURN) {
927
+ if (!trigger) {
928
+ throw new Error("Missing trigger event");
929
+ }
930
+ if (trigger.type === "user.message") {
931
+ const payload = trigger.payload;
932
+ if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
933
+ throw new Error("user.message payload is missing text");
934
+ }
935
+ const text = await userMessageTextWithAttachments(
936
+ db,
937
+ trigger.workspaceId,
938
+ payload.text,
939
+ Array.isArray(payload.resources) ? payload.resources : []
940
+ );
941
+ return await messageInput(db, runtime, agent, trigger, text, settings, current);
942
+ }
943
+ if (trigger.type === "goal.continuation") {
944
+ const payload = trigger.payload;
945
+ if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
946
+ throw new Error("goal.continuation payload is missing text");
947
+ }
948
+ return await messageInput(db, runtime, agent, trigger, payload.text, settings, current);
949
+ }
950
+ if (trigger.type === "turn.preempted") {
951
+ const payload = trigger.payload;
952
+ if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
953
+ throw new Error("turn.preempted payload is missing text");
954
+ }
955
+ return await messageInput(db, runtime, agent, trigger, payload.text, settings, current);
956
+ }
957
+ if (trigger.type === "user.approvalDecision") {
958
+ const payload = trigger.payload;
959
+ const state = await getLatestRunState(db, trigger.workspaceId, trigger.sessionId);
960
+ if (!state) {
961
+ throw new Error("No saved run state is available for approval decision");
962
+ }
963
+ return {
964
+ input: await runtime.prepareInput(agent, {
965
+ kind: "approval",
966
+ // Cross-account run-state strip (HOLE C): if the account resuming this
967
+ // frozen approval differs from the one that froze it, neutralize the
968
+ // blob's account-bound reasoning before replay (else byte-for-byte).
969
+ serializedRunState: resumeRunStateForCodexAccount(state, current),
970
+ approvalId: String(payload.approvalId ?? ""),
971
+ decision: payload.decision === "approve" ? "approve" : "reject",
972
+ ...typeof payload.message === "string" ? { message: payload.message } : {}
973
+ }),
974
+ // Model seeded from the run-state BLOB (neutralize-in-place), NOT stripped
975
+ // items: the reconcile watermark must NOT apply the cross-account strip
976
+ // (HOLE E) — else a cross-account approval resume re-appends K
977
+ // already-persisted items at fresh positions.
978
+ modelHistoryFromItems: false
979
+ };
980
+ }
981
+ throw new Error(`Unsupported trigger event type: ${trigger.type}`);
982
+ }
983
+ async function messageInput(db, runtime, agent, trigger, text, settings, current = NON_CODEX_TURN) {
984
+ const inputBudgetTokens = readPathBudgetTokens(settings);
985
+ if (settings?.sessionHistorySource === "items") {
986
+ const stored = await getActiveSessionHistoryItems2(db, trigger.workspaceId, trigger.sessionId);
987
+ if (stored.length > 0) {
988
+ const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
989
+ const historyItems = applyCodexHistoryStrip(stored, current);
990
+ return {
991
+ input: await runtime.prepareInput(
992
+ agent,
993
+ {
994
+ kind: "message",
995
+ text,
996
+ historyItems,
997
+ sandboxEnvelope: envelope
998
+ },
999
+ inputBudgetTokens ? { inputBudgetTokens } : {}
1000
+ ),
1001
+ // state.history seeded from the cross-account-STRIPPED active items: the
1002
+ // reconcile watermark must apply the SAME strip (HOLE D).
1003
+ modelHistoryFromItems: true
1004
+ };
1005
+ }
1006
+ }
1007
+ const latestState = await getLatestRunState(db, trigger.workspaceId, trigger.sessionId);
1008
+ return {
1009
+ input: await runtime.prepareInput(
1010
+ agent,
1011
+ {
1012
+ kind: "message",
1013
+ text,
1014
+ // Cross-account run-state strip (HOLE C): the items-mode fallback replays
1015
+ // the RunState blob when no history rows exist yet. If the resuming turn's
1016
+ // codex account differs from the one that froze the blob, neutralize its
1017
+ // account-bound reasoning before replay (else byte-for-byte).
1018
+ serializedRunState: latestState ? resumeRunStateForCodexAccount(latestState, current) : null
1019
+ },
1020
+ inputBudgetTokens ? { inputBudgetTokens } : {}
1021
+ ),
1022
+ // state.history seeded from the run-state BLOB (or empty): NOT the stripped
1023
+ // items, so the reconcile watermark must NOT apply the cross-account strip
1024
+ // (HOLE E). On this fallback the active rows are empty anyway (the read above
1025
+ // took the items branch when stored.length > 0), so strip-or-not both yield 0.
1026
+ modelHistoryFromItems: false
1027
+ };
1028
+ }
1029
+ function readPathBudgetTokens(settings) {
1030
+ if (!settings || resolveContextCompactionMode2(settings) !== "client") {
1031
+ return void 0;
1032
+ }
1033
+ const budget = contextInputBudgetTokens2(settings);
1034
+ return budget > 0 ? budget : void 0;
1035
+ }
1036
+ async function userMessageTextWithAttachments(db, workspaceId, text, resources) {
1037
+ const attachedFiles = [];
1038
+ for (const resource of resources) {
1039
+ if (resource.kind !== "file") {
1040
+ continue;
1041
+ }
1042
+ const file = await requireFile(db, workspaceId, resource.fileId);
1043
+ attachedFiles.push(`- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`);
1044
+ }
1045
+ if (attachedFiles.length === 0) {
1046
+ return text;
1047
+ }
1048
+ return [
1049
+ text,
1050
+ "",
1051
+ "Attached files are available in the sandbox:",
1052
+ ...attachedFiles
1053
+ ].join("\n");
1054
+ }
1055
+ function sandboxFilePath(resource, file) {
1056
+ return `/workspace/${resource.mountPath ?? `files/${file.id}`}/${file.safeFilename}`;
1057
+ }
1058
+
1059
+ // src/activities/streaming.ts
1060
+ import { Context } from "@temporalio/activity";
1061
+ function createRuntimeBatcher(flushEvents) {
1062
+ let pending = [];
1063
+ let lastFlush = Date.now();
1064
+ const structural = /* @__PURE__ */ new Set([
1065
+ "agent.message.delta",
1066
+ "agent.reasoning.delta",
1067
+ "sandbox.command.output.delta",
1068
+ // Channel-A PTY bytes flush promptly (P4.4) — a terminal stream is useless
1069
+ // batched 33ms behind; the structural set bypasses the batcher's coalesce.
1070
+ "terminal.pty.output.delta",
1071
+ "agent.toolCall.created",
1072
+ "agent.toolCall.output",
1073
+ "agent.message.completed",
1074
+ "session.requiresAction",
1075
+ "turn.completed",
1076
+ "turn.failed",
1077
+ "turn.cancelled"
1078
+ ]);
1079
+ return {
1080
+ push: async (event) => {
1081
+ pending.push({ type: event.type, payload: event.payload });
1082
+ const elapsed = Date.now() - lastFlush;
1083
+ if (pending.length >= 50 || elapsed >= 33 || structural.has(event.type)) {
1084
+ await flush();
1085
+ }
1086
+ },
1087
+ flush
1088
+ };
1089
+ async function flush() {
1090
+ if (pending.length === 0) {
1091
+ return;
1092
+ }
1093
+ const events = pending;
1094
+ pending = [];
1095
+ lastFlush = Date.now();
1096
+ await flushEvents(events);
1097
+ }
1098
+ }
1099
+ function currentActivityContext() {
1100
+ try {
1101
+ return Context.current();
1102
+ } catch {
1103
+ return null;
1104
+ }
1105
+ }
1106
+ function startActivityHeartbeat(context, details) {
1107
+ if (!context) {
1108
+ return null;
1109
+ }
1110
+ const timer = setInterval(() => {
1111
+ context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
1112
+ }, 1e4);
1113
+ if ("unref" in timer && typeof timer.unref === "function") {
1114
+ timer.unref();
1115
+ }
1116
+ return timer;
1117
+ }
1118
+ async function nextStreamEvent(iterator, context) {
1119
+ if (!context) {
1120
+ return await iterator.next();
1121
+ }
1122
+ return await Promise.race([
1123
+ iterator.next(),
1124
+ context.cancelled
1125
+ ]);
1126
+ }
1127
+
1128
+ // src/sandbox-resume.ts
1129
+ import {
1130
+ acquireLease,
1131
+ commitWarmingToWarm,
1132
+ failWarmingToCold,
1133
+ getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
1134
+ readLease,
1135
+ releaseLeaseHolder,
1136
+ SandboxLeaseSupersededError
1137
+ } from "@opengeni/db";
1138
+ import {
1139
+ establishSandboxSessionFromEnvelope,
1140
+ ensureDisplayStack as ensureDisplayStackOnBox,
1141
+ desktopCapableBackend,
1142
+ serializeEstablishedSandboxEnvelope,
1143
+ DisplayStackError,
1144
+ DisplayStackUnsupportedError,
1145
+ buildStreamUrl,
1146
+ StreamPortUnavailableError
1147
+ } from "@opengeni/runtime";
1148
+ import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
1149
+ var WARMING_POLL_INTERVAL_MS = 250;
1150
+ async function sleep(ms) {
1151
+ await new Promise((resolve) => setTimeout(resolve, ms));
1152
+ }
1153
+ async function resumeBoxForTurn(services, ids, kind, holderId) {
1154
+ const { db, settings } = services;
1155
+ const os = ids.os ?? "linux";
1156
+ const leaseTtlMs = settings.sandboxLeaseTtlMs;
1157
+ let released = false;
1158
+ const release = async () => {
1159
+ if (released) {
1160
+ return;
1161
+ }
1162
+ released = true;
1163
+ await releaseLeaseHolder(db, {
1164
+ accountId: ids.accountId,
1165
+ workspaceId: ids.workspaceId,
1166
+ sandboxGroupId: ids.sandboxGroupId,
1167
+ kind,
1168
+ holderId,
1169
+ idleGraceMs: settings.sandboxIdleGraceMs
1170
+ });
1171
+ };
1172
+ const acquired = await acquireLease(db, {
1173
+ accountId: ids.accountId,
1174
+ workspaceId: ids.workspaceId,
1175
+ sandboxGroupId: ids.sandboxGroupId,
1176
+ kind,
1177
+ holderId,
1178
+ subjectId: ids.sessionId,
1179
+ backend: ids.backend,
1180
+ os,
1181
+ // IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
1182
+ // conflicts on a live box already running a different image. A SandboxImageConflictError
1183
+ // propagates to the turn activity (an actionable error); a solo image change is handled
1184
+ // by acquireLease recreating the box cold on the new image.
1185
+ ...ids.image ? { image: ids.image } : {},
1186
+ leaseTtlMs
1187
+ });
1188
+ if (acquired.role === "fenced") {
1189
+ await release();
1190
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
1191
+ }
1192
+ if (acquired.role === "spawner") {
1193
+ const expectedEpoch = acquired.lease.leaseEpoch;
1194
+ try {
1195
+ const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1196
+ const spawnEnvelope = acquired.lease.resumeState ?? envelope;
1197
+ const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
1198
+ sessionId: ids.sessionId,
1199
+ backendOverride: ids.backend,
1200
+ ...ids.environment ? { environment: ids.environment } : {}
1201
+ });
1202
+ await ensureDisplayStack(settings, established);
1203
+ const endpoint = await exposeStreamPort(settings, established);
1204
+ const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
1205
+ const committed = await commitWarmingToWarm(db, {
1206
+ accountId: ids.accountId,
1207
+ workspaceId: ids.workspaceId,
1208
+ sandboxGroupId: ids.sandboxGroupId,
1209
+ expectedEpoch,
1210
+ instanceId: established.instanceId,
1211
+ dataPlaneUrl: endpoint?.url ?? null,
1212
+ resumeBackendId: established.backendId,
1213
+ resumeState: resumeEnvelope,
1214
+ leaseTtlMs
1215
+ });
1216
+ if (!committed.committed || !committed.lease) {
1217
+ await release();
1218
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1219
+ }
1220
+ return { established, leaseEpoch: committed.lease.leaseEpoch, release };
1221
+ } catch (error) {
1222
+ if (error instanceof SandboxLeaseSupersededError) {
1223
+ throw error;
1224
+ }
1225
+ await failWarmingToCold(db, {
1226
+ accountId: ids.accountId,
1227
+ workspaceId: ids.workspaceId,
1228
+ sandboxGroupId: ids.sandboxGroupId,
1229
+ expectedEpoch
1230
+ });
1231
+ await release();
1232
+ throw error;
1233
+ }
1234
+ }
1235
+ let leaseEpoch = acquired.lease.leaseEpoch;
1236
+ if (acquired.lease.liveness === "warming") {
1237
+ leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
1238
+ }
1239
+ try {
1240
+ const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1241
+ const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
1242
+ const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
1243
+ sessionId: ids.sessionId,
1244
+ backendOverride: ids.backend,
1245
+ ...ids.environment ? { environment: ids.environment } : {}
1246
+ });
1247
+ await ensureDisplayStack(settings, established);
1248
+ return { established, leaseEpoch, release };
1249
+ } catch (error) {
1250
+ await release();
1251
+ throw error;
1252
+ }
1253
+ }
1254
+ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
1255
+ const { db, settings } = services;
1256
+ const leaseTtlMs = settings.sandboxLeaseTtlMs;
1257
+ let released = false;
1258
+ const release = async () => {
1259
+ if (released) {
1260
+ return;
1261
+ }
1262
+ released = true;
1263
+ await releaseLeaseHolder(db, {
1264
+ accountId: ids.accountId,
1265
+ workspaceId: ids.workspaceId,
1266
+ sandboxGroupId: ids.sandboxGroupId,
1267
+ kind,
1268
+ holderId,
1269
+ idleGraceMs: settings.sandboxIdleGraceMs
1270
+ });
1271
+ };
1272
+ const acquired = await acquireLease(db, {
1273
+ accountId: ids.accountId,
1274
+ workspaceId: ids.workspaceId,
1275
+ sandboxGroupId: ids.sandboxGroupId,
1276
+ kind,
1277
+ holderId,
1278
+ subjectId: ids.sessionId,
1279
+ backend: "selfhosted",
1280
+ os: "linux",
1281
+ leaseTtlMs
1282
+ });
1283
+ if (acquired.role === "fenced") {
1284
+ await release();
1285
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
1286
+ }
1287
+ if (acquired.role === "spawner") {
1288
+ const expectedEpoch = acquired.lease.leaseEpoch;
1289
+ const committed = await commitWarmingToWarm(db, {
1290
+ accountId: ids.accountId,
1291
+ workspaceId: ids.workspaceId,
1292
+ sandboxGroupId: ids.sandboxGroupId,
1293
+ expectedEpoch,
1294
+ // No provider box id; stamp a clearly non-box marker for diagnostics.
1295
+ instanceId: `selfhosted:${ids.sessionId}`,
1296
+ resumeBackendId: "selfhosted",
1297
+ resumeState: null,
1298
+ leaseTtlMs
1299
+ });
1300
+ if (!committed.committed || !committed.lease) {
1301
+ await release();
1302
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
1303
+ }
1304
+ return { leaseEpoch: committed.lease.leaseEpoch, release };
1305
+ }
1306
+ return { leaseEpoch: acquired.lease.leaseEpoch, release };
1307
+ }
1308
+ async function waitForWarmOrReacquire(services, ids, kind, holderId) {
1309
+ const { db, settings } = services;
1310
+ const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
1311
+ while (Date.now() < deadline) {
1312
+ await sleep(WARMING_POLL_INTERVAL_MS);
1313
+ const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
1314
+ if (!lease) {
1315
+ break;
1316
+ }
1317
+ if (lease.liveness === "warm" || lease.liveness === "draining") {
1318
+ return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
1319
+ }
1320
+ if (lease.liveness === "cold") {
1321
+ break;
1322
+ }
1323
+ }
1324
+ const reacquired = await acquireLease(db, {
1325
+ accountId: ids.accountId,
1326
+ workspaceId: ids.workspaceId,
1327
+ sandboxGroupId: ids.sandboxGroupId,
1328
+ kind,
1329
+ holderId,
1330
+ subjectId: ids.sessionId,
1331
+ backend: ids.backend,
1332
+ os: ids.os ?? "linux",
1333
+ leaseTtlMs: settings.sandboxLeaseTtlMs
1334
+ });
1335
+ if (reacquired.role === "fenced") {
1336
+ throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
1337
+ }
1338
+ return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
1339
+ }
1340
+ async function ensureDisplayStack(settings, established) {
1341
+ if (!settings.sandboxDesktopEnabled) {
1342
+ return;
1343
+ }
1344
+ if (!desktopCapableBackend(established.backendId)) {
1345
+ return;
1346
+ }
1347
+ try {
1348
+ await ensureDisplayStackOnBox(established.session);
1349
+ } catch (error) {
1350
+ if (error instanceof DisplayStackUnsupportedError || error instanceof DisplayStackError) {
1351
+ console.warn(
1352
+ `[sandbox-resume] ensureDisplayStack degraded to Channel-A-only (turn continues): ${error instanceof Error ? error.message : String(error)}`
1353
+ );
1354
+ return;
1355
+ }
1356
+ throw error;
1357
+ }
1358
+ }
1359
+ async function exposeStreamPort(settings, established) {
1360
+ if (!settings.sandboxDesktopEnabled) {
1361
+ return null;
1362
+ }
1363
+ if (!desktopCapableBackend(established.backendId)) {
1364
+ return null;
1365
+ }
1366
+ const session = established.session;
1367
+ if (typeof session?.resolveExposedPort !== "function") {
1368
+ return null;
1369
+ }
1370
+ try {
1371
+ const endpoint = await session.resolveExposedPort(DESKTOP_STREAM_PORT);
1372
+ const url = buildStreamUrl(endpoint);
1373
+ return { url, expiresAt: new Date(Date.now() + settings.sandboxLeaseTtlMs) };
1374
+ } catch (error) {
1375
+ if (error instanceof StreamPortUnavailableError) {
1376
+ return null;
1377
+ }
1378
+ return null;
1379
+ }
1380
+ }
1381
+
1382
+ // src/sandbox-routing.ts
1383
+ import { getSandbox, readActiveSandbox } from "@opengeni/db";
1384
+ import {
1385
+ buildSelfhostedBackendSession,
1386
+ makeActiveBackendResolver,
1387
+ NatsControlRpc,
1388
+ RoutingSandboxSession
1389
+ } from "@opengeni/runtime";
1390
+ function relayConfigFromSettings(settings) {
1391
+ const raw = settings.selfhostedRelayUrl?.trim();
1392
+ if (!raw) {
1393
+ return { host: "relay.opengeni.local", port: 443, tls: true };
1394
+ }
1395
+ try {
1396
+ const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
1397
+ const tls = url.protocol === "wss:" || url.protocol === "https:";
1398
+ const port = url.port ? Number(url.port) : tls ? 443 : 80;
1399
+ return { host: url.hostname, port, tls };
1400
+ } catch {
1401
+ return { host: raw, port: 443, tls: true };
1402
+ }
1403
+ }
1404
+ function controlRpcFactory(bus) {
1405
+ return () => new NatsControlRpc(async () => {
1406
+ if (!bus) {
1407
+ return null;
1408
+ }
1409
+ return bus.getRequestConnection();
1410
+ });
1411
+ }
1412
+ function wrapTurnBoxWithRouting(services, ids, established) {
1413
+ const { db, settings, bus } = services;
1414
+ const resolver = makeActiveBackendResolver({
1415
+ workspaceId: ids.workspaceId,
1416
+ defaultBackend: established.session,
1417
+ defaultKind: established.backendId,
1418
+ getSandbox: async (sandboxId) => {
1419
+ const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
1420
+ return sandbox ? { id: sandbox.id, kind: sandbox.kind, name: sandbox.name, enrollmentId: sandbox.enrollmentId } : null;
1421
+ },
1422
+ controlRpcFactory: controlRpcFactory(bus),
1423
+ relay: relayConfigFromSettings(settings),
1424
+ // The turn's declared environment → a selfhosted swap target's manifest, so the
1425
+ // SDK's per-turn manifest-env delta is empty (no "cannot change manifest
1426
+ // environment variables" throw when the turn pins to a vm). Mirrors the group
1427
+ // box, which is created WITH this same environment (resumeBoxForTurn).
1428
+ ...ids.environment !== void 0 ? { environment: ids.environment } : {},
1429
+ // Stage D machine-primary: pin THIS established SelfhostedSession for the machine
1430
+ // pointer so the resolver returns the SAME instance (no two-instance manifest
1431
+ // divergence). `established.session` is the SelfhostedSession the establish branch
1432
+ // bound to (sandboxId, epoch).
1433
+ ...ids.pinnedSelfhosted ? {
1434
+ pinnedSelfhosted: {
1435
+ sandboxId: ids.pinnedSelfhosted.sandboxId,
1436
+ epoch: ids.pinnedSelfhosted.epoch,
1437
+ session: established.session
1438
+ }
1439
+ } : {}
1440
+ // A modal swap target in the turn path would need its own lease resume-by-id;
1441
+ // that is a future cross-group-box concern. Until then a modal swap target is
1442
+ // unresolvable (the swap tool validates liveness, so this only triggers if a
1443
+ // session points at a sibling modal box the turn cannot resume here) and the
1444
+ // op surfaces unresolvable — never a silent wrong-box landing.
1445
+ });
1446
+ const proxy = new RoutingSandboxSession({
1447
+ // Seed the DEFAULT backend (the established group box) at construction so
1448
+ // `session.state` is the real backend's state object BEFORE the first op. The
1449
+ // SDK reads `session.state.manifest` at turn START (and writes it back); an
1450
+ // empty `{}` there crashes serializeManifestEnvironment /
1451
+ // validateProvidedSessionManifestUpdate. This is byte-identical to what the
1452
+ // resolver returns for the default pointer (`activeSandboxId === null`).
1453
+ defaultResolved: {
1454
+ session: established.session,
1455
+ sandboxId: null,
1456
+ kind: established.backendId
1457
+ },
1458
+ readPointer: async () => {
1459
+ const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
1460
+ return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
1461
+ },
1462
+ resolveActiveBackend: resolver
1463
+ });
1464
+ return { ...established, session: proxy };
1465
+ }
1466
+ async function establishSelfhostedTurnSession(services, args) {
1467
+ const { settings, bus } = services;
1468
+ const { client, session } = await buildSelfhostedBackendSession({
1469
+ workspaceId: args.workspaceId,
1470
+ agentId: args.agentId,
1471
+ relay: relayConfigFromSettings(settings),
1472
+ controlRpcFactory: controlRpcFactory(bus),
1473
+ epoch: args.epoch,
1474
+ environment: args.environment,
1475
+ workingDir: args.workingDir
1476
+ });
1477
+ return {
1478
+ client,
1479
+ session,
1480
+ sessionState: { agentId: args.agentId },
1481
+ instanceId: args.agentId,
1482
+ backendId: "selfhosted"
1483
+ };
1484
+ }
1485
+ function routingEnabled(settings) {
1486
+ return settings.sandboxSelfhostedEnabled === true;
1487
+ }
1488
+
1489
+ // src/activities/recording.ts
1490
+ import { deleteRecording, updateRecording } from "@opengeni/db";
1491
+ import {
1492
+ contentTypeForCodec,
1493
+ deleteRecordingArtifacts,
1494
+ readRecordingBytes,
1495
+ RecordingError,
1496
+ recordingStorageKey,
1497
+ startRecording as startRecordingOnBox,
1498
+ stopRecording as stopRecordingOnBox
1499
+ } from "@opengeni/runtime";
1500
+ import { DOWNLOAD_URL_TTL_SECONDS } from "@opengeni/storage";
1501
+ async function beginRecording(args) {
1502
+ const { settings, db } = args;
1503
+ const codec = settings.recordingDefaultCodec;
1504
+ const dimensions = [settings.streamResolutionWidth, settings.streamResolutionHeight];
1505
+ const framerate = settings.recordingFramerate;
1506
+ await import("@opengeni/db").then(({ insertRecording }) => insertRecording(db, {
1507
+ id: args.recordingId,
1508
+ accountId: args.accountId,
1509
+ workspaceId: args.workspaceId,
1510
+ sessionId: args.sessionId,
1511
+ turnId: args.turnId,
1512
+ mode: args.mode,
1513
+ codec,
1514
+ width: dimensions[0],
1515
+ height: dimensions[1],
1516
+ reason: scrubFreeText(args.reason)
1517
+ }));
1518
+ const proc = await startRecordingOnBox(args.session, {
1519
+ recordingId: args.recordingId,
1520
+ codec,
1521
+ framerate,
1522
+ maxSeconds: settings.recordingMaxSeconds,
1523
+ dimensions,
1524
+ ...args.runAs ? { runAs: args.runAs } : {}
1525
+ });
1526
+ const active = {
1527
+ recordingId: args.recordingId,
1528
+ turnId: args.turnId,
1529
+ mode: args.mode,
1530
+ proc,
1531
+ dimensions,
1532
+ framerate
1533
+ };
1534
+ const started = {
1535
+ recordingId: args.recordingId,
1536
+ turnId: args.turnId,
1537
+ mode: args.mode,
1538
+ codec,
1539
+ dimensions,
1540
+ framerate,
1541
+ startedAt: (/* @__PURE__ */ new Date()).toISOString(),
1542
+ reason: scrubFreeText(args.reason)
1543
+ };
1544
+ return { active, started };
1545
+ }
1546
+ async function finalizeRecording(args) {
1547
+ const { settings, db, objectStorage, active } = args;
1548
+ const codec = settings.recordingDefaultCodec;
1549
+ const fail = async (reason, detail) => {
1550
+ await updateRecording(db, {
1551
+ accountId: args.accountId,
1552
+ workspaceId: args.workspaceId,
1553
+ recordingId: active.recordingId,
1554
+ state: "failed",
1555
+ reason: scrubFreeText(detail)
1556
+ }).catch(() => void 0);
1557
+ return { ok: false, reason, detail: scrubFreeText(detail) };
1558
+ };
1559
+ try {
1560
+ await updateRecording(db, {
1561
+ accountId: args.accountId,
1562
+ workspaceId: args.workspaceId,
1563
+ recordingId: active.recordingId,
1564
+ state: "finalizing"
1565
+ }).catch(() => void 0);
1566
+ await stopRecordingOnBox(args.session, active.proc);
1567
+ const finalized = await readRecordingBytes(args.session, active.proc, settings.recordingMaxBytes);
1568
+ if (!objectStorage) {
1569
+ return await fail("upload-failed", "object storage is not configured");
1570
+ }
1571
+ const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
1572
+ try {
1573
+ await objectStorage.putObject({ key, contentType: finalized.contentType, body: finalized.bytes.slice() });
1574
+ } catch (uploadError) {
1575
+ return await fail("upload-failed", uploadError instanceof Error ? uploadError.message : String(uploadError));
1576
+ }
1577
+ await updateRecording(db, {
1578
+ accountId: args.accountId,
1579
+ workspaceId: args.workspaceId,
1580
+ recordingId: active.recordingId,
1581
+ state: "available",
1582
+ storageKey: key,
1583
+ sizeBytes: finalized.sizeBytes,
1584
+ durationSeconds: finalized.durationSeconds
1585
+ });
1586
+ await deleteRecordingArtifacts(args.session, active.proc);
1587
+ const available = {
1588
+ recordingId: active.recordingId,
1589
+ turnId: active.turnId,
1590
+ codec,
1591
+ contentType: contentTypeForCodec(codec),
1592
+ storageKey: key,
1593
+ durationSeconds: finalized.durationSeconds,
1594
+ sizeBytes: finalized.sizeBytes,
1595
+ dimensions: active.dimensions
1596
+ };
1597
+ return { ok: true, available };
1598
+ } catch (error) {
1599
+ const reason = error instanceof RecordingError ? error.reason : "ffmpeg-error";
1600
+ return await fail(reason, error instanceof Error ? error.message : String(error));
1601
+ }
1602
+ }
1603
+ async function discardRecording(args) {
1604
+ const { db, active } = args;
1605
+ try {
1606
+ await stopRecordingOnBox(args.session, active.proc).catch(() => void 0);
1607
+ await deleteRecordingArtifacts(args.session, active.proc).catch(() => void 0);
1608
+ } finally {
1609
+ await deleteRecording(db, {
1610
+ accountId: args.accountId,
1611
+ workspaceId: args.workspaceId,
1612
+ recordingId: active.recordingId
1613
+ }).catch(() => void 0);
1614
+ }
1615
+ }
1616
+ function scrubFreeText(value) {
1617
+ if (value === null || value === void 0) return null;
1618
+ const trimmed = String(value).slice(0, 2e3);
1619
+ return trimmed.length === 0 ? null : trimmed;
1620
+ }
1621
+
1622
+ // src/activities/agent-turn.ts
1623
+ import { createObjectStorage } from "@opengeni/storage";
1624
+ import { desktopCapableBackend as desktopCapableBackend2, sandboxRunAs } from "@opengeni/runtime";
1625
+ import { CAPABILITY_DESCRIPTORS } from "@opengeni/contracts";
1626
+ import { randomUUID } from "crypto";
1627
+ var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
1628
+ function selectCodexCredentialForTurn(args) {
1629
+ const { sessionPinnedCredentialId: pin, activeCredentialId: active, connectedIds } = args;
1630
+ if (pin && connectedIds.has(pin)) {
1631
+ return pin;
1632
+ }
1633
+ if (active && connectedIds.has(active)) {
1634
+ return active;
1635
+ }
1636
+ return null;
1637
+ }
1638
+ var WORKER_SHUTDOWN_RESUME_TEXT = [
1639
+ "[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
1640
+ "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
1641
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
1642
+ ].join("\n");
1643
+ var WORKER_DEATH_RESUME_TEXT = [
1644
+ "[TURN RESUMED AFTER WORKER CRASH] The platform worker running this turn died before the turn finished.",
1645
+ "Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; anything in flight after that checkpoint was lost.",
1646
+ "Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
1647
+ ].join("\n");
1648
+ function isWorkerShutdownCancellation(error) {
1649
+ return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
1650
+ }
1651
+ function modelUsageSourceKey(input) {
1652
+ if (input.responseId) {
1653
+ return input.responseId;
1654
+ }
1655
+ return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
1656
+ }
1657
+ function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount) {
1658
+ const sanitized = sanitizeHistoryItemsForModel(rawHistory);
1659
+ if (sanitized.length <= persistedHistoryCount) {
1660
+ return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
1661
+ }
1662
+ const rows = sanitized.slice(persistedHistoryCount).map((item, offset) => ({
1663
+ position: nextPosition + offset,
1664
+ item
1665
+ }));
1666
+ return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
1667
+ }
1668
+ function reconcileSeedCount(activeSeedRows, modelHistoryFromItems, current) {
1669
+ return sanitizeHistoryItemsForModel(
1670
+ modelHistoryFromItems ? applyCodexHistoryStrip(activeSeedRows, current) : activeSeedRows.map((row) => row.item)
1671
+ ).length;
1672
+ }
1673
+ async function resolveActiveSandboxBackend(routingOn, loadPointer, loadSandboxKind) {
1674
+ if (!routingOn) {
1675
+ return void 0;
1676
+ }
1677
+ try {
1678
+ const pointer = await loadPointer();
1679
+ if (!pointer?.activeSandboxId) {
1680
+ return void 0;
1681
+ }
1682
+ const kind = await loadSandboxKind(pointer.activeSandboxId);
1683
+ return kind === "selfhosted" ? "selfhosted" : void 0;
1684
+ } catch (error) {
1685
+ console.error("active sandbox backend resolution failed (turn proceeds on home backend)", error);
1686
+ return void 0;
1687
+ }
1688
+ }
1689
+ async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts) {
1690
+ const nearPct = settings.codexRotationNearExhaustionPct;
1691
+ const stale = accounts.filter(
1692
+ (a) => a.status === "active" && ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct)
1693
+ );
1694
+ if (stale.length === 0) {
1695
+ return accounts;
1696
+ }
1697
+ await Promise.all(stale.map((a) => fetchCodexUsageForAccount(db, settings, workspaceId, a.id).catch(() => void 0)));
1698
+ return listCodexAccountStatuses(db, workspaceId).catch(() => accounts);
1699
+ }
1700
+ function createRunAgentTurnActivity(services) {
1701
+ return async function runAgentTurn2(input) {
1702
+ const { settings, db, bus, runtime, objectStorage, observability, wakeSessionWorkflow, entitlements, connectionCredentials } = await services();
1703
+ const activityStarted = performance.now();
1704
+ const activitySpan = observability.startSpan("worker.run_agent_segment", {
1705
+ "opengeni.session_id": input.sessionId,
1706
+ "opengeni.workflow_id": input.workflowId,
1707
+ "opengeni.trigger_event_id": input.triggerEventId
1708
+ });
1709
+ let activityStatus = "unknown";
1710
+ let activityError;
1711
+ let turnId;
1712
+ let heartbeatTimer;
1713
+ let resolvedSandbox = null;
1714
+ let setupBoxSession = null;
1715
+ let sandboxHolderId = null;
1716
+ let sandboxGroupId = null;
1717
+ let leaseHeartbeatTimer;
1718
+ let activeRecording = null;
1719
+ let didComputerUse = false;
1720
+ let batcher = null;
1721
+ let preparedTools = null;
1722
+ let publish = null;
1723
+ let turnStartedPublished = false;
1724
+ let stream;
1725
+ let persistedHistoryCount = 0;
1726
+ let historyCountAtTurnStart = 0;
1727
+ let nextHistoryPosition = 0;
1728
+ const reconcileConversationTruth = async () => {
1729
+ if (!stream || !turnId) {
1730
+ return;
1731
+ }
1732
+ try {
1733
+ const rawHistory = stream.state.history;
1734
+ if (Array.isArray(rawHistory)) {
1735
+ const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
1736
+ rawHistory,
1737
+ persistedHistoryCount,
1738
+ nextHistoryPosition
1739
+ );
1740
+ if (rows.length > 0) {
1741
+ await appendSessionHistoryItems(db, {
1742
+ accountId: input.accountId,
1743
+ workspaceId: input.workspaceId,
1744
+ sessionId: input.sessionId,
1745
+ turnId,
1746
+ // Tag each row with the codex account that produced it (null on the
1747
+ // non-codex path). Resolved at line ~504 before any reconcile pass
1748
+ // runs, so this is the turn's effective account. The read path uses
1749
+ // it to strip cross-account reasoning.encrypted_content next turn.
1750
+ producerCodexCredentialId: effectiveCodexCredentialId,
1751
+ items: rows
1752
+ });
1753
+ }
1754
+ persistedHistoryCount = nextWatermark;
1755
+ nextHistoryPosition = nextPosition;
1756
+ }
1757
+ const envelope = sandboxStateEntryFromRunState(stream.state);
1758
+ if (envelope) {
1759
+ await upsertSandboxSessionEnvelope(db, {
1760
+ accountId: input.accountId,
1761
+ workspaceId: input.workspaceId,
1762
+ sessionId: input.sessionId,
1763
+ envelope
1764
+ });
1765
+ }
1766
+ } catch (persistError) {
1767
+ console.error("session history dual-write failed (run unaffected)", persistError);
1768
+ }
1769
+ };
1770
+ let redact = identityRedactor;
1771
+ let environmentId = "";
1772
+ let effectiveCodexCredentialId = null;
1773
+ let latestCodexUsage = null;
1774
+ let triggerType = null;
1775
+ try {
1776
+ const mcpSettings = await settingsWithEnabledCapabilityMcpServers(db, input.workspaceId, settings);
1777
+ const codexSubscriptionActive = await workspaceCodexSubscriptionActive2(db, mcpSettings, input.workspaceId);
1778
+ const capabilitySettings = await settingsWithCodexCredential(db, input.workspaceId, mcpSettings, codexSubscriptionActive);
1779
+ runtime.configure(capabilitySettings);
1780
+ const session = await requireSession2(db, input.workspaceId, input.sessionId);
1781
+ const trigger = await getSessionEvent3(db, input.workspaceId, input.triggerEventId);
1782
+ if (!trigger) {
1783
+ throw new Error(`Trigger event not found: ${input.triggerEventId}`);
1784
+ }
1785
+ triggerType = trigger.type;
1786
+ let turn = input.turnId ? await getSessionTurn(db, input.workspaceId, input.turnId) : await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
1787
+ if (!turn && !input.turnId) {
1788
+ const createdTurnId = await createTurn(db, {
1789
+ workspaceId: input.workspaceId,
1790
+ sessionId: input.sessionId,
1791
+ temporalWorkflowId: input.workflowId,
1792
+ triggerEventId: input.triggerEventId
1793
+ });
1794
+ turn = await getSessionTurn(db, input.workspaceId, createdTurnId);
1795
+ }
1796
+ if (!turn) {
1797
+ throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
1798
+ }
1799
+ turnId = turn.id;
1800
+ const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
1801
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
1802
+ const activityContext = currentActivityContext();
1803
+ const throwIfWorkerShuttingDown = () => {
1804
+ const reason = activityContext?.cancellationSignal.reason;
1805
+ if (isWorkerShutdownCancellation(reason)) {
1806
+ throw reason;
1807
+ }
1808
+ };
1809
+ heartbeatTimer = startActivityHeartbeat(activityContext, {
1810
+ phase: "running",
1811
+ sessionId: input.sessionId,
1812
+ turnId
1813
+ });
1814
+ let producerSeq = 0;
1815
+ const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
1816
+ const dispatchId = activityContext?.info.activityId ?? null;
1817
+ publish = async (events, immediate = false) => {
1818
+ const inputs = events.map((event) => ({
1819
+ ...event,
1820
+ payload: redact(event.payload),
1821
+ turnId,
1822
+ producerId,
1823
+ producerSeq: ++producerSeq
1824
+ }));
1825
+ await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, inputs);
1826
+ activityContext?.heartbeat({ phase: "events_published", sessionId: input.sessionId, turnId, producerSeq });
1827
+ if (immediate) {
1828
+ await Bun.sleep(0);
1829
+ }
1830
+ };
1831
+ activityContext?.heartbeat({ phase: "turn_started", sessionId: input.sessionId, turnId });
1832
+ throwIfWorkerShuttingDown();
1833
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "running", turnId);
1834
+ await publish([
1835
+ { type: "session.status.changed", payload: { status: "running" } },
1836
+ { type: "turn.started", payload: { triggerEventId: input.triggerEventId } }
1837
+ ], true);
1838
+ turnStartedPublished = true;
1839
+ if (isCodexTurn) {
1840
+ const [rotation, accounts, sessionCodex] = await Promise.all([
1841
+ getCodexRotationSettings(db, input.workspaceId),
1842
+ listCodexAccountStatuses(db, input.workspaceId),
1843
+ getSessionCodexState(db, input.workspaceId, input.sessionId)
1844
+ ]);
1845
+ const connectedIds = new Set(accounts.map((account) => account.id));
1846
+ const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
1847
+ let chosenActive = rotation?.activeCredentialId ?? null;
1848
+ let rotationDecision = null;
1849
+ if (rotation?.rotationEnabled && sessionPin == null) {
1850
+ let rankAccounts = accounts;
1851
+ rotationDecision = chooseRotationActive({
1852
+ rotationStrategy: rotation.rotationStrategy,
1853
+ activeCredentialId: rotation.activeCredentialId,
1854
+ priorCredentialId: sessionCodex?.lastCredentialId ?? null,
1855
+ accounts: rankAccounts,
1856
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
1857
+ now: /* @__PURE__ */ new Date(),
1858
+ // P4: the leaving (active) account's cached connector set is the proxy
1859
+ // for "what this session has access to" — prefer a covering target.
1860
+ usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? []
1861
+ });
1862
+ if (rotationDecision.kind === "allCapped") {
1863
+ rankAccounts = await refreshCappedCodexUsageRows(db, settings, input.workspaceId, rankAccounts);
1864
+ rotationDecision = chooseRotationActive({
1865
+ rotationStrategy: rotation.rotationStrategy,
1866
+ activeCredentialId: rotation.activeCredentialId,
1867
+ priorCredentialId: sessionCodex?.lastCredentialId ?? null,
1868
+ accounts: rankAccounts,
1869
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
1870
+ now: /* @__PURE__ */ new Date(),
1871
+ // P4: leaving (active) account's connector set (refreshCappedCodexUsageRows
1872
+ // only touches usage columns, so connectorNamespaces is preserved here).
1873
+ usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? []
1874
+ });
1875
+ }
1876
+ if (rotationDecision.kind === "active") {
1877
+ if (rotationDecision.moved) {
1878
+ await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
1879
+ }
1880
+ chosenActive = rotationDecision.credentialId;
1881
+ } else if (rotationDecision.kind === "allCapped" && publish && turnId) {
1882
+ const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
1883
+ const goalActive = Boolean(goal && goal.status === "active");
1884
+ const resumeMs = computeIdleDelayMs(rotationDecision.earliestResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS);
1885
+ const failurePayload = codexUsageLimitFailurePayload(
1886
+ { resetsInSeconds: Math.ceil(resumeMs / 1e3) },
1887
+ "all connected Codex subscriptions are rate-limited",
1888
+ { allAccounts: true }
1889
+ );
1890
+ await publish([
1891
+ // `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
1892
+ // rotation-wait state as the reactive all-capped path, so it must freeze
1893
+ // autoContinuations identically (evaluateGoalContinuation reads this marker)
1894
+ // — a goal waiting out a long reset must not burn its continuation budget on
1895
+ // the proactive path while the reactive path spares it.
1896
+ { type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved: false, rotated: true } },
1897
+ { type: "session.status.changed", payload: { status: "idle" } }
1898
+ ], true);
1899
+ await finishTurn(db, input.workspaceId, turnId, "failed");
1900
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
1901
+ activityStatus = "idle";
1902
+ return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
1903
+ }
1904
+ }
1905
+ effectiveCodexCredentialId = selectCodexCredentialForTurn({
1906
+ sessionPinnedCredentialId: sessionPin,
1907
+ // pin still wins, structurally
1908
+ activeCredentialId: chosenActive,
1909
+ // rotation-choice OR today's active
1910
+ connectedIds
1911
+ });
1912
+ if (effectiveCodexCredentialId) {
1913
+ const priorAccountId = sessionCodex?.lastCredentialId ?? null;
1914
+ await recordSessionActiveCodexCredential(db, input.workspaceId, input.sessionId, effectiveCodexCredentialId);
1915
+ if (priorAccountId !== effectiveCodexCredentialId) {
1916
+ const rotated = rotationDecision?.kind === "active" && rotationDecision.moved;
1917
+ const droppedConnectors = rotationDecision?.kind === "active" ? rotationDecision.droppedConnectors : void 0;
1918
+ await publish([{
1919
+ type: "codex.account.switched",
1920
+ payload: {
1921
+ fromAccountId: priorAccountId,
1922
+ toAccountId: effectiveCodexCredentialId,
1923
+ reason: rotated ? "rotation" : "manual",
1924
+ ...droppedConnectors && droppedConnectors.length > 0 ? { droppedConnectors } : {}
1925
+ }
1926
+ }]);
1927
+ }
1928
+ }
1929
+ }
1930
+ const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
1931
+ const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
1932
+ const runSettings = {
1933
+ ...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
1934
+ openaiModel: turn.model,
1935
+ openaiReasoningEffort: turn.reasoningEffort,
1936
+ sandboxBackend: turn.sandboxBackend
1937
+ };
1938
+ const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
1939
+ const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
1940
+ const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
1941
+ return {
1942
+ clientVersion: CODEX_CLIENT_VERSION,
1943
+ getToken: resolver.getToken,
1944
+ refresh: resolver.refresh,
1945
+ resolveModel: buildModelResolver(CODEX_FALLBACK_MODEL_SLUGS2, CODEX_FALLBACK_MODEL_SLUGS2[0]),
1946
+ onUsageHeaders: (snapshot) => {
1947
+ latestCodexUsage = snapshot;
1948
+ }
1949
+ // latest wins; flushed once in finally
1950
+ };
1951
+ })() : null;
1952
+ const withCodex = (fn) => codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
1953
+ const turnResources = mergeResourceRefs(session.resources, turn.resources);
1954
+ const turnTools = withCodexAppsTool(runSettings, withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)));
1955
+ const connectionScope = { accountId: input.accountId, workspaceId: input.workspaceId };
1956
+ const workspaceEnvironment = await loadWorkspaceEnvironmentForRunWithCredentials(
1957
+ db,
1958
+ runSettings,
1959
+ connectionScope,
1960
+ session.environmentId,
1961
+ connectionCredentials?.sandboxSecrets
1962
+ );
1963
+ environmentId = workspaceEnvironment?.id ?? "";
1964
+ redact = createSecretRedactor(
1965
+ Object.entries(workspaceEnvironment?.values ?? {}).map(([name, value]) => ({ name, value }))
1966
+ );
1967
+ const routingOn = routingEnabled(settings);
1968
+ const activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
1969
+ const activeSandboxRecord = routingOn && activeSandboxPointer?.activeSandboxId ? await getSandbox2(db, input.workspaceId, activeSandboxPointer.activeSandboxId).catch(() => null) : null;
1970
+ const activeSandboxBackend = await resolveActiveSandboxBackend(
1971
+ routingOn,
1972
+ async () => activeSandboxPointer,
1973
+ async () => activeSandboxRecord?.kind ?? null
1974
+ );
1975
+ const machinePrimary = activeSandboxBackend === "selfhosted" && Boolean(activeSandboxPointer?.activeSandboxId) && Boolean(activeSandboxRecord?.enrollmentId);
1976
+ const { environment: sandboxEnvironment, gitToken: sandboxGitToken } = await sandboxEnvironmentForRun(
1977
+ runSettings,
1978
+ turnResources,
1979
+ workspaceEnvironment?.values ?? {},
1980
+ {
1981
+ skipGitHubToken: activeSandboxBackend === "selfhosted",
1982
+ scope: connectionScope,
1983
+ gitCredentials: connectionCredentials?.gitCredentials
1984
+ }
1985
+ );
1986
+ if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
1987
+ sandboxHolderId = dispatchId ?? `turn:${turnId}`;
1988
+ sandboxGroupId = session.sandboxGroupId;
1989
+ const groupBoxBackend = turn.sandboxBackend === "selfhosted" && !machinePrimary ? settings.sandboxBackend : turn.sandboxBackend;
1990
+ if (machinePrimary) {
1991
+ const established = await establishSelfhostedTurnSession(
1992
+ { db, settings, bus },
1993
+ {
1994
+ workspaceId: input.workspaceId,
1995
+ agentId: activeSandboxRecord.enrollmentId,
1996
+ epoch: activeSandboxPointer.activeEpoch,
1997
+ environment: sandboxEnvironment,
1998
+ workingDir: activeSandboxPointer.workingDir
1999
+ }
2000
+ );
2001
+ const lease = await acquireSelfhostedLeaseForTurn(
2002
+ { db, settings },
2003
+ {
2004
+ accountId: input.accountId,
2005
+ workspaceId: input.workspaceId,
2006
+ sandboxGroupId: session.sandboxGroupId,
2007
+ sessionId: input.sessionId
2008
+ },
2009
+ "turn",
2010
+ sandboxHolderId
2011
+ );
2012
+ setupBoxSession = established.session;
2013
+ resolvedSandbox = {
2014
+ // Wrap in the SAME routing proxy so a mid-turn swap (to another machine
2015
+ // or back to the group box) still re-routes per op. PIN this established
2016
+ // SelfhostedSession for the machine pointer so the turn-start manifest
2017
+ // write (via the proxy's `state` getter) and the per-op reads hit ONE
2018
+ // instance — no two-instance manifest divergence.
2019
+ established: wrapTurnBoxWithRouting(
2020
+ { db, settings, bus },
2021
+ {
2022
+ workspaceId: input.workspaceId,
2023
+ sessionId: input.sessionId,
2024
+ environment: sandboxEnvironment,
2025
+ pinnedSelfhosted: {
2026
+ sandboxId: activeSandboxPointer.activeSandboxId,
2027
+ epoch: activeSandboxPointer.activeEpoch
2028
+ }
2029
+ },
2030
+ established
2031
+ ),
2032
+ leaseEpoch: lease.leaseEpoch,
2033
+ release: lease.release
2034
+ };
2035
+ } else {
2036
+ resolvedSandbox = await resumeBoxForTurn(
2037
+ { db, settings },
2038
+ {
2039
+ accountId: input.accountId,
2040
+ workspaceId: input.workspaceId,
2041
+ sandboxGroupId: session.sandboxGroupId,
2042
+ sessionId: input.sessionId,
2043
+ // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
2044
+ // is not machine-primary resumes a real cloud group box (the
2045
+ // deployment default), never a "selfhosted" box (which would throw
2046
+ // for lack of a bound agentId).
2047
+ backend: groupBoxBackend,
2048
+ os: session.sandboxOs,
2049
+ environment: sandboxEnvironment,
2050
+ // IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
2051
+ // this run resolves. The lease stamps it + conflicts on a live shared box
2052
+ // running a DIFFERENT image (solo → recreate on the new image; N-holders →
2053
+ // SandboxImageConflictError surfaced as an actionable turn error). Prefer the
2054
+ // explicit Modal image ref, else the docker image. The selfhosted branch
2055
+ // (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
2056
+ // an image — B3 lives only on this Modal else-branch.
2057
+ ...runSettings.modalImageRef ?? runSettings.dockerImage ? { image: runSettings.modalImageRef ?? runSettings.dockerImage } : {}
2058
+ },
2059
+ "turn",
2060
+ sandboxHolderId
2061
+ );
2062
+ setupBoxSession = resolvedSandbox.established.session;
2063
+ if (routingEnabled(settings)) {
2064
+ resolvedSandbox = {
2065
+ ...resolvedSandbox,
2066
+ established: wrapTurnBoxWithRouting(
2067
+ { db, settings, bus },
2068
+ // Thread the SAME declared environment the group box was created with
2069
+ // (resumeBoxForTurn, above) so a selfhosted swap target's manifest
2070
+ // carries it too — the SDK's per-turn manifest-env delta stays empty
2071
+ // (no "cannot change manifest environment variables" throw).
2072
+ { workspaceId: input.workspaceId, sessionId: input.sessionId, environment: sandboxEnvironment },
2073
+ resolvedSandbox.established
2074
+ )
2075
+ };
2076
+ }
2077
+ }
2078
+ const heartbeatEpoch = resolvedSandbox.leaseEpoch;
2079
+ const heartbeatHolderId = sandboxHolderId;
2080
+ const heartbeatGroupId = sandboxGroupId;
2081
+ const warmRate = sandboxWarmRateMicrosPerSecond(settings, activeSandboxBackend ?? groupBoxBackend);
2082
+ leaseHeartbeatTimer = setInterval(() => {
2083
+ void heartbeatLeaseHolder(db, {
2084
+ accountId: input.accountId,
2085
+ workspaceId: input.workspaceId,
2086
+ sandboxGroupId: heartbeatGroupId,
2087
+ kind: "turn",
2088
+ holderId: heartbeatHolderId,
2089
+ leaseTtlMs: settings.sandboxLeaseTtlMs,
2090
+ expectedEpoch: heartbeatEpoch
2091
+ }).catch(() => void 0);
2092
+ void accrueWarmSeconds(db, {
2093
+ accountId: input.accountId,
2094
+ workspaceId: input.workspaceId,
2095
+ sandboxGroupId: heartbeatGroupId,
2096
+ expectedEpoch: heartbeatEpoch,
2097
+ warmRateMicrosPerSecond: warmRate,
2098
+ subjectId: input.sessionId
2099
+ }).catch(() => void 0);
2100
+ }, 1e4);
2101
+ if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
2102
+ leaseHeartbeatTimer.unref();
2103
+ }
2104
+ }
2105
+ if (resolvedSandbox && settings.sandboxDesktopEnabled && settings.recordingEnabled && desktopCapableBackend2(resolvedSandbox.established.backendId)) {
2106
+ try {
2107
+ const begun = await beginRecording({
2108
+ settings,
2109
+ db,
2110
+ accountId: input.accountId,
2111
+ workspaceId: input.workspaceId,
2112
+ sessionId: input.sessionId,
2113
+ turnId,
2114
+ recordingId: randomUUID(),
2115
+ mode: "on-turn",
2116
+ session: resolvedSandbox.established.session,
2117
+ runAs: sandboxRunAs(settings),
2118
+ reason: null
2119
+ });
2120
+ activeRecording = begun.active;
2121
+ await publish([{ type: "recording.started", payload: begun.started }]);
2122
+ } catch (recordingError) {
2123
+ activeRecording = null;
2124
+ console.error("on-turn recording start failed (turn outcome unaffected)", recordingError);
2125
+ }
2126
+ }
2127
+ const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
2128
+ throwIfWorkerShuttingDown();
2129
+ preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
2130
+ accountId: input.accountId,
2131
+ workspaceId: input.workspaceId,
2132
+ sessionId: input.sessionId,
2133
+ subjectId: "worker:first-party-mcp",
2134
+ subjectLabel: "OpenGeni worker",
2135
+ // Manager-style sessions carry a creation-validated permission set
2136
+ // for their first-party MCP token; null keeps the fixed default.
2137
+ ...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}
2138
+ }));
2139
+ const isGenesisTurn = triggerType === "user.message" && await countSessionHistoryItems(db, input.workspaceId, input.sessionId) === 0;
2140
+ const agent = runtime.buildAgent(runSettings, turnResources, {
2141
+ reasoningEffort: turn.reasoningEffort,
2142
+ genesisTitleHint: isGenesisTurn,
2143
+ sandboxEnvironment,
2144
+ // TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
2145
+ // seed. ONLY when the effective backend is NOT selfhosted (the connected
2146
+ // machine uses its own git creds — mirrors the skipGitHubToken gate above)
2147
+ // AND the mint actually produced a token (repo resources present). The runtime
2148
+ // seeds it to the box's token file before the repository-clone runs; it never
2149
+ // touches the box/agent manifest env.
2150
+ ...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
2151
+ ...activeSandboxBackend ? { activeSandboxBackend } : {},
2152
+ fileResourceDownloads,
2153
+ mcpServers: preparedTools.mcpServers,
2154
+ // LIVE by-reference connector namespaces (fills during this turn's
2155
+ // codex_apps tools/list): the codex tool_search description reads it per
2156
+ // model call so the model sees the account's real connected sources.
2157
+ codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
2158
+ // Resolved-model routing + gating (legacy defaults when null). The model
2159
+ // is passed as the model *string* (agent.model = runSettings.openaiModel),
2160
+ // NOT a Model instance: an instance only survives the in-process
2161
+ // ("none") run, whereas the SandboxAgent/Modal path drops it and
2162
+ // re-resolves the model *name* through the global MultiProviderModelProvider
2163
+ // configureOpenAI installed — so registry models (Fireworks GLM) route to
2164
+ // their own client instead of 404ing against the built-in Azure/OpenAI
2165
+ // client. The gating still comes from the resolved provider: server-side
2166
+ // store/compaction follow the provider's compaction mode (registry
2167
+ // providers resolve to "client"); encrypted reasoning is only
2168
+ // round-tripped on the Responses wire API; hosted web search is attached
2169
+ // only when the model opts in; the effective context window drives the
2170
+ // compaction threshold.
2171
+ ...resolvedModel ? {
2172
+ compactionMode: resolvedModel.provider.compactionMode,
2173
+ hostedWebSearch: resolvedModel.configured.hostedWebSearch,
2174
+ encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
2175
+ contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
2176
+ // The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
2177
+ // the `apply_patch` tool type ("Unsupported tool type: apply_patch")
2178
+ // and structured tool output — which the OpenAIResponsesModel the SDK
2179
+ // binds would otherwise select. Tell buildAgent to emit the function
2180
+ // `apply_patch` + text `view_image` variants the backend accepts. Only
2181
+ // the codex-subscription provider needs this; every other backend
2182
+ // (built-in OpenAI/Azure = real hosted support; registry "chat"
2183
+ // providers = the SDK's own ChatCompletions detection) keeps the SDK
2184
+ // default.
2185
+ structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription"
2186
+ } : {},
2187
+ ...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
2188
+ ...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
2189
+ ...workspaceEnvironment ? {
2190
+ workspaceEnvironment: {
2191
+ name: workspaceEnvironment.name,
2192
+ description: workspaceEnvironment.description,
2193
+ variableNames: Object.keys(workspaceEnvironment.values)
2194
+ }
2195
+ } : {}
2196
+ });
2197
+ if (triggerType === "user.message" || triggerType === "goal.continuation") {
2198
+ try {
2199
+ const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
2200
+ const outcome = await maybeCompactContext(
2201
+ db,
2202
+ runSettings,
2203
+ { accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId },
2204
+ session.lastInputTokens,
2205
+ // Provider-aware summarizer: when the turn's model resolved to a
2206
+ // registry provider, summarize on THAT provider's client + wire API
2207
+ // (a chat provider can't summarize through OpenAI/Azure). Null
2208
+ // resolution keeps the default built-in Responses summarizer.
2209
+ resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
2210
+ client: resolvedModel.client,
2211
+ api: resolvedModel.provider.api,
2212
+ model: resolvedModel.configured.id,
2213
+ maxOutputTokens: s.contextSummaryMaxTokens
2214
+ })) : void 0,
2215
+ forced ? { force: true } : {}
2216
+ );
2217
+ if (outcome.compacted) {
2218
+ const trigger2 = forced ? "operator" : outcome.hardForced ? "hard" : void 0;
2219
+ await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...trigger2 ? { trigger: trigger2 } : {} } }]);
2220
+ }
2221
+ } catch (compactError) {
2222
+ console.error("context compaction failed (turn proceeds un-compacted)", compactError);
2223
+ }
2224
+ }
2225
+ const { input: runInput, modelHistoryFromItems } = await turnInput(
2226
+ db,
2227
+ runtime,
2228
+ agent,
2229
+ trigger,
2230
+ runSettings,
2231
+ { currentCodexCredentialId: effectiveCodexCredentialId }
2232
+ );
2233
+ const activeSeedRows = await getActiveSessionHistoryItems3(db, input.workspaceId, input.sessionId);
2234
+ persistedHistoryCount = reconcileSeedCount(
2235
+ activeSeedRows,
2236
+ modelHistoryFromItems,
2237
+ { currentCodexCredentialId: effectiveCodexCredentialId }
2238
+ );
2239
+ historyCountAtTurnStart = persistedHistoryCount;
2240
+ nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
2241
+ let responseUsageCount = 0;
2242
+ let lastInputTokensObserved = null;
2243
+ throwIfWorkerShuttingDown();
2244
+ const runStreamOnce = () => runtime.runStream(agent, runInput, runSettings, {
2245
+ sandboxEnvironment,
2246
+ onRuntimeEvent: async (event) => {
2247
+ await publish([{ type: event.type, payload: event.payload }], true);
2248
+ },
2249
+ // P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
2250
+ // keystone). Absent when the flag is off -> legacy build-and-discard.
2251
+ ...resolvedSandbox ? {
2252
+ ownedSandbox: {
2253
+ client: resolvedSandbox.established.client,
2254
+ session: resolvedSandbox.established.session,
2255
+ sessionState: resolvedSandbox.established.sessionState,
2256
+ // Pin platform setup (hooks + file materialization) to the un-proxied
2257
+ // established box — never through the routing proxy, which would
2258
+ // re-route those execs onto a machine swapped in mid-turn.
2259
+ ...setupBoxSession ? { setupSession: setupBoxSession } : {}
2260
+ }
2261
+ } : {}
2262
+ });
2263
+ stream = await withCodex(runStreamOnce);
2264
+ batcher = createRuntimeBatcher(async (events) => {
2265
+ await publish(events);
2266
+ });
2267
+ const iterator = stream.toStream()[Symbol.asyncIterator]();
2268
+ let streamDone = false;
2269
+ try {
2270
+ while (true) {
2271
+ const next = await nextStreamEvent(iterator, activityContext);
2272
+ if (next.done) {
2273
+ streamDone = true;
2274
+ break;
2275
+ }
2276
+ const responseUsage = modelResponseUsageFromSdkEvent(next.value);
2277
+ if (responseUsage) {
2278
+ responseUsageCount += 1;
2279
+ const observed = responseUsage.usage?.inputTokens;
2280
+ if (typeof observed === "number" && observed > 0) {
2281
+ lastInputTokensObserved = observed;
2282
+ }
2283
+ await recordModelUsageAndDebitCredits(settings, db, {
2284
+ accountId: input.accountId,
2285
+ workspaceId: input.workspaceId,
2286
+ sessionId: input.sessionId,
2287
+ turnId,
2288
+ model: turn.model,
2289
+ isCodexTurn,
2290
+ usage: responseUsage.usage,
2291
+ sourceKey: modelUsageSourceKey({
2292
+ responseId: responseUsage.responseId,
2293
+ dispatchId,
2294
+ positionalKey: `response-${responseUsageCount}`
2295
+ })
2296
+ });
2297
+ await reconcileConversationTruth();
2298
+ try {
2299
+ await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
2300
+ } catch (limitError) {
2301
+ let serializedRunState = null;
2302
+ try {
2303
+ serializedRunState = stream.state.toString();
2304
+ } catch {
2305
+ serializedRunState = null;
2306
+ }
2307
+ throw new BudgetExhaustedError(
2308
+ limitError instanceof Error ? limitError.message : String(limitError),
2309
+ serializedRunState
2310
+ );
2311
+ }
2312
+ }
2313
+ if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
2314
+ didComputerUse = true;
2315
+ }
2316
+ const normalized = normalizeSdkEvent(next.value);
2317
+ for (const event of normalized) {
2318
+ await batcher.push(event);
2319
+ }
2320
+ }
2321
+ } finally {
2322
+ if (!streamDone) {
2323
+ await iterator.return?.();
2324
+ }
2325
+ }
2326
+ await batcher.flush();
2327
+ await stream.completed.catch(() => void 0);
2328
+ if (responseUsageCount === 0) {
2329
+ const aggregateInput = stream.state.usage?.inputTokens;
2330
+ if (typeof aggregateInput === "number" && aggregateInput > 0) {
2331
+ lastInputTokensObserved = aggregateInput;
2332
+ }
2333
+ await recordModelUsageAndDebitCredits(settings, db, {
2334
+ accountId: input.accountId,
2335
+ workspaceId: input.workspaceId,
2336
+ sessionId: input.sessionId,
2337
+ turnId,
2338
+ model: turn.model,
2339
+ isCodexTurn,
2340
+ usage: stream.state.usage,
2341
+ sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
2342
+ });
2343
+ }
2344
+ if (lastInputTokensObserved !== null) {
2345
+ await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
2346
+ }
2347
+ if (stream.interruptions.length > 0) {
2348
+ await reconcileConversationTruth();
2349
+ const approvals = runtime.serializeApprovals(stream.interruptions);
2350
+ await saveRunState(db, {
2351
+ accountId: input.accountId,
2352
+ workspaceId: input.workspaceId,
2353
+ sessionId: input.sessionId,
2354
+ turnId,
2355
+ serializedRunState: stream.state.toString(),
2356
+ pendingApprovals: approvals,
2357
+ // Record the account freezing this state so a resume on a DIFFERENT
2358
+ // codex account strips its account-bound reasoning before replay (HOLE C).
2359
+ frozenCodexCredentialId: effectiveCodexCredentialId
2360
+ });
2361
+ await publish([
2362
+ { type: "session.requiresAction", payload: { approvals } },
2363
+ { type: "session.status.changed", payload: { status: "requires_action" } }
2364
+ ], true);
2365
+ await finishTurn(db, input.workspaceId, turnId, "requires_action");
2366
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
2367
+ activityStatus = "requires_action";
2368
+ return { status: "requires_action" };
2369
+ }
2370
+ const finalOutput = String(stream.finalOutput ?? "");
2371
+ await reconcileConversationTruth();
2372
+ if (settings.sessionHistorySource !== "items") {
2373
+ await saveRunState(db, {
2374
+ accountId: input.accountId,
2375
+ workspaceId: input.workspaceId,
2376
+ sessionId: input.sessionId,
2377
+ turnId,
2378
+ serializedRunState: stream.state.toString(),
2379
+ pendingApprovals: [],
2380
+ frozenCodexCredentialId: effectiveCodexCredentialId
2381
+ });
2382
+ }
2383
+ await publish([
2384
+ { type: "agent.message.completed", payload: { text: finalOutput } },
2385
+ { type: "turn.completed", payload: { output: finalOutput } },
2386
+ { type: "session.status.changed", payload: { status: "idle" } }
2387
+ ], true);
2388
+ await finishTurn(db, input.workspaceId, turnId, "idle");
2389
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2390
+ await recordUsageEvent2(db, {
2391
+ accountId: input.accountId,
2392
+ workspaceId: input.workspaceId,
2393
+ eventType: "agent_run.completed",
2394
+ quantity: 1,
2395
+ unit: "run",
2396
+ sourceResourceType: "session_turn",
2397
+ sourceResourceId: turnId,
2398
+ idempotencyKey: `usage:agent_run.completed:${turnId}`
2399
+ });
2400
+ activityStatus = "idle";
2401
+ return { status: "idle" };
2402
+ } catch (error) {
2403
+ const preemptTurnId = turnId ?? input.turnId;
2404
+ if (error instanceof SandboxLeaseSupersededError2 && preemptTurnId) {
2405
+ try {
2406
+ await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
2407
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2408
+ activityStatus = "preempted";
2409
+ return { status: "preempted" };
2410
+ } catch (requeueError) {
2411
+ console.error("sandbox lease supersession requeue failed; falling back", requeueError);
2412
+ }
2413
+ }
2414
+ if (isWorkerShutdownCancellation(error) && preemptTurnId) {
2415
+ try {
2416
+ await batcher?.flush().catch(() => void 0);
2417
+ await reconcileConversationTruth();
2418
+ const approvalRerun = triggerType === "user.approvalDecision";
2419
+ let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
2420
+ if (settings.sessionHistorySource !== "items" && stream) {
2421
+ try {
2422
+ await saveRunState(db, {
2423
+ accountId: input.accountId,
2424
+ workspaceId: input.workspaceId,
2425
+ sessionId: input.sessionId,
2426
+ turnId: preemptTurnId,
2427
+ serializedRunState: stream.state.toString(),
2428
+ pendingApprovals: runtime.serializeApprovals(stream.interruptions ?? []),
2429
+ frozenCodexCredentialId: effectiveCodexCredentialId
2430
+ });
2431
+ resumeWithNotice = true;
2432
+ } catch {
2433
+ resumeWithNotice = false;
2434
+ }
2435
+ }
2436
+ const [preemptedEvent] = await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, [
2437
+ {
2438
+ turnId: preemptTurnId,
2439
+ type: "turn.preempted",
2440
+ payload: {
2441
+ triggerEventId: input.triggerEventId,
2442
+ reason: "worker_shutdown",
2443
+ resumeWithNotice,
2444
+ ...resumeWithNotice ? { text: WORKER_SHUTDOWN_RESUME_TEXT } : {}
2445
+ }
2446
+ },
2447
+ {
2448
+ turnId: preemptTurnId,
2449
+ type: "session.status.changed",
2450
+ payload: { status: "queued" }
2451
+ }
2452
+ ]);
2453
+ await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
2454
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
2455
+ activityStatus = "preempted";
2456
+ return { status: "preempted" };
2457
+ } catch (preemptError) {
2458
+ console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
2459
+ }
2460
+ }
2461
+ if (error instanceof CancelledFailure) {
2462
+ activityStatus = "cancelled";
2463
+ activityError = error;
2464
+ await batcher?.flush().catch(() => void 0);
2465
+ if (preemptTurnId) {
2466
+ await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
2467
+ }
2468
+ throw error;
2469
+ }
2470
+ const maxTurns = maxTurnsExceededRunState(error);
2471
+ if (maxTurns && publish && turnId && turnStartedPublished) {
2472
+ await batcher?.flush().catch(() => void 0);
2473
+ await reconcileConversationTruth();
2474
+ const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
2475
+ if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
2476
+ await saveRunState(db, {
2477
+ accountId: input.accountId,
2478
+ workspaceId: input.workspaceId,
2479
+ sessionId: input.sessionId,
2480
+ turnId,
2481
+ serializedRunState: maxTurns.serializedRunState,
2482
+ pendingApprovals: [],
2483
+ frozenCodexCredentialId: effectiveCodexCredentialId
2484
+ });
2485
+ }
2486
+ await publish([
2487
+ { type: "turn.completed", payload: { output: "", segmentLimit: "max_turns", runStateSaved } },
2488
+ { type: "session.status.changed", payload: { status: "idle" } }
2489
+ ], true);
2490
+ await finishTurn(db, input.workspaceId, turnId, "idle");
2491
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2492
+ await recordUsageEvent2(db, {
2493
+ accountId: input.accountId,
2494
+ workspaceId: input.workspaceId,
2495
+ eventType: "agent_run.completed",
2496
+ quantity: 1,
2497
+ unit: "run",
2498
+ sourceResourceType: "session_turn",
2499
+ sourceResourceId: turnId,
2500
+ idempotencyKey: `usage:agent_run.completed:${turnId}`
2501
+ });
2502
+ activityStatus = "idle";
2503
+ return { status: "idle" };
2504
+ }
2505
+ const usageLimit = classifyCodexUsageLimitError(error);
2506
+ if (usageLimit && publish && turnId && turnStartedPublished) {
2507
+ const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
2508
+ const goalActive = Boolean(goal && goal.status === "active");
2509
+ await batcher?.flush().catch(() => void 0);
2510
+ await reconcileConversationTruth();
2511
+ const serializedRunState = agentsErrorRunState(error);
2512
+ const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
2513
+ if (serializedRunState && settings.sessionHistorySource !== "items") {
2514
+ await saveRunState(db, {
2515
+ accountId: input.accountId,
2516
+ workspaceId: input.workspaceId,
2517
+ sessionId: input.sessionId,
2518
+ turnId,
2519
+ serializedRunState,
2520
+ pendingApprovals: [],
2521
+ frozenCodexCredentialId: effectiveCodexCredentialId
2522
+ });
2523
+ }
2524
+ let rotated = false;
2525
+ let rotationResumeMs = null;
2526
+ let rotationResumeIdleUntilReset = false;
2527
+ let allCappedResetAt = null;
2528
+ if (effectiveCodexCredentialId) {
2529
+ const [rotation, sessionCodex] = await Promise.all([
2530
+ getCodexRotationSettings(db, input.workspaceId).catch(() => null),
2531
+ getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null)
2532
+ ]);
2533
+ const rotating = Boolean(rotation?.rotationEnabled) && sessionCodex?.pinnedCredentialId == null;
2534
+ if (rotating && rotation) {
2535
+ const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
2536
+ const serving = accounts.find((a) => a.id === effectiveCodexCredentialId) ?? null;
2537
+ const cachedReset = [serving?.primaryResetAt, serving?.secondaryResetAt].filter((d) => d instanceof Date && d.getTime() > Date.now()).sort((a, b) => a.getTime() - b.getTime())[0] ?? null;
2538
+ const until = usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : cachedReset ?? new Date(Date.now() + CODEX_USAGE_LIMIT_MAX_RESUME_MS);
2539
+ const cooldownPersisted = await setCodexCredentialExhausted(db, input.workspaceId, effectiveCodexCredentialId, until).catch(() => false);
2540
+ const fresh = accounts.map((a) => a.id === effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a);
2541
+ const decision = chooseRotationActive({
2542
+ rotationStrategy: rotation.rotationStrategy,
2543
+ activeCredentialId: rotation.activeCredentialId,
2544
+ priorCredentialId: effectiveCodexCredentialId,
2545
+ accounts: fresh,
2546
+ nearExhaustionPct: settings.codexRotationNearExhaustionPct,
2547
+ now: /* @__PURE__ */ new Date(),
2548
+ // P4: the just-capped serving account's connector set is the proxy for
2549
+ // "what this session has access to" — prefer a covering failover target.
2550
+ usedConnectors: serving?.connectorNamespaces ?? []
2551
+ });
2552
+ if (decision.kind === "active") {
2553
+ rotated = true;
2554
+ const priorConsecutiveRotations = await countConsecutiveReactiveRotations(db, input.workspaceId, input.sessionId).catch(() => 0);
2555
+ const resume = computeReactiveRotationResume({
2556
+ cooldownPersisted,
2557
+ priorConsecutiveRotations,
2558
+ connectedAccountCount: accounts.length
2559
+ });
2560
+ rotationResumeMs = resume.continueDelayMs;
2561
+ rotationResumeIdleUntilReset = resume.idleUntilReset;
2562
+ } else if (decision.kind === "allCapped") {
2563
+ rotated = true;
2564
+ allCappedResetAt = decision.earliestResetAt;
2565
+ }
2566
+ }
2567
+ }
2568
+ const failurePayload = allCappedResetAt ? codexUsageLimitFailurePayload(
2569
+ { resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1e3) },
2570
+ error instanceof Error ? error.message : String(error),
2571
+ { allAccounts: true }
2572
+ ) : codexUsageLimitFailurePayload(usageLimit, error instanceof Error ? error.message : String(error));
2573
+ await publish([
2574
+ // `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
2575
+ // freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
2576
+ { type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved, ...rotated ? { rotated: true } : {} } },
2577
+ { type: "session.status.changed", payload: { status: "idle" } }
2578
+ ], true);
2579
+ await finishTurn(db, input.workspaceId, turnId, "failed");
2580
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2581
+ activityStatus = "idle";
2582
+ activityError = error;
2583
+ if (goalActive) {
2584
+ if (rotationResumeMs !== null) {
2585
+ return { status: "idle", continueDelayMs: rotationResumeMs, ...rotationResumeIdleUntilReset ? { idleUntilReset: true } : {} };
2586
+ }
2587
+ const resumeMs = allCappedResetAt ? computeIdleDelayMs(allCappedResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS) : usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1e3, CODEX_USAGE_LIMIT_MAX_RESUME_MS) : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
2588
+ return { status: "idle", continueDelayMs: resumeMs, ...allCappedResetAt ? { idleUntilReset: true } : {} };
2589
+ }
2590
+ return { status: "idle" };
2591
+ }
2592
+ if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
2593
+ await batcher?.flush().catch(() => void 0);
2594
+ await reconcileConversationTruth();
2595
+ const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
2596
+ if (error.serializedRunState && settings.sessionHistorySource !== "items") {
2597
+ await saveRunState(db, {
2598
+ accountId: input.accountId,
2599
+ workspaceId: input.workspaceId,
2600
+ sessionId: input.sessionId,
2601
+ turnId,
2602
+ serializedRunState: error.serializedRunState,
2603
+ pendingApprovals: [],
2604
+ frozenCodexCredentialId: effectiveCodexCredentialId
2605
+ });
2606
+ }
2607
+ await publish([
2608
+ { type: "turn.completed", payload: { output: "", segmentLimit: "budget_exhausted", detail: error.message, runStateSaved } },
2609
+ { type: "session.status.changed", payload: { status: "idle" } }
2610
+ ], true);
2611
+ await finishTurn(db, input.workspaceId, turnId, "idle");
2612
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2613
+ await recordUsageEvent2(db, {
2614
+ accountId: input.accountId,
2615
+ workspaceId: input.workspaceId,
2616
+ eventType: "agent_run.completed",
2617
+ quantity: 1,
2618
+ unit: "run",
2619
+ sourceResourceType: "session_turn",
2620
+ sourceResourceId: turnId,
2621
+ idempotencyKey: `usage:agent_run.completed:${turnId}`
2622
+ });
2623
+ activityStatus = "idle";
2624
+ return { status: "idle" };
2625
+ }
2626
+ const failure = agentRunFailurePayload(error);
2627
+ if (failure.retryable && publish && turnId && turnStartedPublished) {
2628
+ const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
2629
+ const goalActive = Boolean(goal && goal.status === "active");
2630
+ await batcher?.flush().catch(() => void 0);
2631
+ await reconcileConversationTruth();
2632
+ const serializedRunState = agentsErrorRunState(error);
2633
+ const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
2634
+ if (serializedRunState && settings.sessionHistorySource !== "items") {
2635
+ await saveRunState(db, {
2636
+ accountId: input.accountId,
2637
+ workspaceId: input.workspaceId,
2638
+ sessionId: input.sessionId,
2639
+ turnId,
2640
+ serializedRunState,
2641
+ pendingApprovals: [],
2642
+ frozenCodexCredentialId: effectiveCodexCredentialId
2643
+ });
2644
+ }
2645
+ await publish([
2646
+ { type: "turn.failed", payload: { ...failure, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved } },
2647
+ { type: "session.status.changed", payload: { status: "idle" } }
2648
+ ], true);
2649
+ await finishTurn(db, input.workspaceId, turnId, "failed");
2650
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
2651
+ activityStatus = "idle";
2652
+ activityError = error;
2653
+ return goalActive ? { status: "idle", continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS } : { status: "idle" };
2654
+ }
2655
+ activityStatus = "failed";
2656
+ activityError = error;
2657
+ if (!publish || !turnId || !turnStartedPublished) {
2658
+ throw error;
2659
+ }
2660
+ await publish([
2661
+ { type: "turn.failed", payload: failure },
2662
+ { type: "session.status.changed", payload: { status: "failed" } }
2663
+ ], true);
2664
+ await finishTurn(db, input.workspaceId, turnId, "failed");
2665
+ await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
2666
+ await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
2667
+ return { status: "failed" };
2668
+ } finally {
2669
+ const durationSeconds = (performance.now() - activityStarted) / 1e3;
2670
+ observability.recordWorkerActivity({
2671
+ activity: "runAgentTurn",
2672
+ status: activityStatus,
2673
+ durationSeconds
2674
+ });
2675
+ activitySpan.end({
2676
+ attributes: {
2677
+ "opengeni.turn_id": turnId ?? "",
2678
+ "opengeni.status": activityStatus,
2679
+ "opengeni.environment_id": environmentId,
2680
+ "opengeni.duration_ms": Math.round(durationSeconds * 1e3)
2681
+ },
2682
+ error: activityError
2683
+ });
2684
+ if (effectiveCodexCredentialId) {
2685
+ if (latestCodexUsage) {
2686
+ await recordCodexAccountUsage(db, input.workspaceId, effectiveCodexCredentialId, latestCodexUsage).catch(() => void 0);
2687
+ }
2688
+ const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
2689
+ if (connectorNamespaces && connectorNamespaces.size > 0) {
2690
+ await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [...connectorNamespaces]).catch(() => void 0);
2691
+ }
2692
+ }
2693
+ await preparedTools?.close().catch(() => void 0);
2694
+ if (heartbeatTimer) {
2695
+ clearInterval(heartbeatTimer);
2696
+ }
2697
+ if (leaseHeartbeatTimer) {
2698
+ clearInterval(leaseHeartbeatTimer);
2699
+ }
2700
+ if (activeRecording && resolvedSandbox) {
2701
+ try {
2702
+ if (!didComputerUse) {
2703
+ await discardRecording({
2704
+ db,
2705
+ accountId: input.accountId,
2706
+ workspaceId: input.workspaceId,
2707
+ active: activeRecording,
2708
+ session: resolvedSandbox.established.session
2709
+ });
2710
+ } else {
2711
+ const outcome = await finalizeRecording({
2712
+ settings,
2713
+ db,
2714
+ objectStorage,
2715
+ accountId: input.accountId,
2716
+ workspaceId: input.workspaceId,
2717
+ sessionId: input.sessionId,
2718
+ active: activeRecording,
2719
+ session: resolvedSandbox.established.session,
2720
+ runAs: sandboxRunAs(settings)
2721
+ });
2722
+ if (publish) {
2723
+ await publish(outcome.ok ? [{ type: "recording.available", payload: outcome.available }] : [{ type: "recording.failed", payload: { recordingId: activeRecording.recordingId, turnId: activeRecording.turnId, reason: outcome.reason, detail: outcome.detail } }]);
2724
+ }
2725
+ }
2726
+ } catch (finalizeError) {
2727
+ console.error("recording finalize failed (turn outcome unaffected)", finalizeError);
2728
+ } finally {
2729
+ activeRecording = null;
2730
+ }
2731
+ }
2732
+ if (resolvedSandbox) {
2733
+ await resolvedSandbox.release().catch((releaseError) => {
2734
+ console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
2735
+ });
2736
+ resolvedSandbox = null;
2737
+ }
2738
+ }
2739
+ };
2740
+ }
2741
+ function agentRunFailurePayload(error) {
2742
+ const message = error instanceof Error ? error.message : String(error);
2743
+ const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
2744
+ const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
2745
+ const usageLimit = classifyCodexUsageLimitError(error);
2746
+ if (usageLimit) {
2747
+ return codexUsageLimitFailurePayload(usageLimit, message);
2748
+ }
2749
+ if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
2750
+ return {
2751
+ error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
2752
+ code: "provider_rate_limited",
2753
+ retryable: true,
2754
+ ...message && message !== "Too Many Requests" ? { detail: message } : {}
2755
+ };
2756
+ }
2757
+ return { error: message };
2758
+ }
2759
+ function humanizeResetWindow(resetsInSeconds) {
2760
+ if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
2761
+ return "shortly";
2762
+ }
2763
+ const total = Math.ceil(resetsInSeconds);
2764
+ if (total < 60) {
2765
+ return "in under a minute";
2766
+ }
2767
+ const hours = Math.floor(total / 3600);
2768
+ const minutes = Math.round(total % 3600 / 60);
2769
+ if (hours > 0) {
2770
+ return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
2771
+ }
2772
+ return `in about ${minutes}m`;
2773
+ }
2774
+ function codexUsageLimitFailurePayload(info, detail, opts) {
2775
+ const error = opts?.allAccounts ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for a subscription to reset.` : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
2776
+ return {
2777
+ error,
2778
+ code: "codex_usage_limit_reached",
2779
+ retryable: false,
2780
+ ...detail ? { detail } : {}
2781
+ };
2782
+ }
2783
+ var CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 6e4;
2784
+ function isComputerCallStreamEvent(event) {
2785
+ if (!event || typeof event !== "object") {
2786
+ return false;
2787
+ }
2788
+ if (event.type !== "run_item_stream_event") {
2789
+ return false;
2790
+ }
2791
+ const item = event.item;
2792
+ if (!item || item.type !== "tool_call_item") {
2793
+ return false;
2794
+ }
2795
+ return item.rawItem?.type === "computer_call";
2796
+ }
2797
+ var BudgetExhaustedError = class extends Error {
2798
+ constructor(message, serializedRunState) {
2799
+ super(message);
2800
+ this.serializedRunState = serializedRunState;
2801
+ this.name = "BudgetExhaustedError";
2802
+ }
2803
+ serializedRunState;
2804
+ };
2805
+ async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTurn, entitlements) {
2806
+ if (!isCodexTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
2807
+ const decision = await entitlements.admitRun({
2808
+ accountId,
2809
+ workspaceId,
2810
+ action: "agent_run:create",
2811
+ quantity: 1
2812
+ });
2813
+ if (!decision.allowed) {
2814
+ throw new Error(decision.reason || "insufficient OpenGeni credits");
2815
+ }
2816
+ } else if (!isCodexTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
2817
+ const balance = await getBillingBalance2(db, accountId);
2818
+ if (balance.balanceMicros <= 0) {
2819
+ throw new Error("insufficient OpenGeni credits");
2820
+ }
2821
+ }
2822
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
2823
+ const limits = configuredStaticUsageLimits2(settings);
2824
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
2825
+ const used = await sumUsageQuantity2(db, {
2826
+ workspaceId,
2827
+ eventType: "agent_run.created",
2828
+ since: startOfUtcMonth2()
2829
+ });
2830
+ if (used > limits.maxMonthlyAgentRunsPerWorkspace) {
2831
+ throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
2832
+ }
2833
+ }
2834
+ if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
2835
+ const used = await sumUsageQuantity2(db, {
2836
+ workspaceId,
2837
+ eventType: "model.tokens",
2838
+ since: startOfUtcMonth2()
2839
+ });
2840
+ if (used >= limits.maxMonthlyTokensPerWorkspace) {
2841
+ throw new Error(`monthly token limit reached (${limits.maxMonthlyTokensPerWorkspace})`);
2842
+ }
2843
+ }
2844
+ }
2845
+ }
2846
+ async function recordModelUsageAndDebitCredits(settings, db, input) {
2847
+ if (!input.usage) {
2848
+ return;
2849
+ }
2850
+ const inputTokens = positiveInt(input.usage.inputTokens);
2851
+ const outputTokens = positiveInt(input.usage.outputTokens);
2852
+ const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
2853
+ if (input.isCodexTurn) {
2854
+ await recordUsageEvent2(db, {
2855
+ accountId: input.accountId,
2856
+ workspaceId: input.workspaceId,
2857
+ eventType: "model.cost",
2858
+ quantity: 0,
2859
+ unit: "usd_micros",
2860
+ sourceResourceType: "model_response",
2861
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2862
+ idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
2863
+ });
2864
+ return;
2865
+ }
2866
+ if (totalTokens > 0) {
2867
+ await recordUsageEvent2(db, {
2868
+ accountId: input.accountId,
2869
+ workspaceId: input.workspaceId,
2870
+ eventType: "model.tokens",
2871
+ quantity: totalTokens,
2872
+ unit: "tokens",
2873
+ sourceResourceType: "model_response",
2874
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2875
+ idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`
2876
+ });
2877
+ }
2878
+ const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
2879
+ if (!shouldDebit || totalTokens === 0) {
2880
+ return;
2881
+ }
2882
+ if (!configuredModelPricing(settings)[input.model]) {
2883
+ throw new Error(`Missing model pricing for ${input.model}`);
2884
+ }
2885
+ const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
2886
+ await recordUsageEvent2(db, {
2887
+ accountId: input.accountId,
2888
+ workspaceId: input.workspaceId,
2889
+ eventType: "model.cost",
2890
+ quantity: costMicros,
2891
+ unit: "usd_micros",
2892
+ sourceResourceType: "model_response",
2893
+ sourceResourceId: `${input.turnId}:${input.sourceKey}`,
2894
+ idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
2895
+ });
2896
+ if (costMicros > 0) {
2897
+ await applyCreditDebitUpToBalance(db, {
2898
+ accountId: input.accountId,
2899
+ workspaceId: input.workspaceId,
2900
+ type: "model_usage_debit",
2901
+ requestedAmountMicros: costMicros,
2902
+ sourceType: "model_response",
2903
+ sourceId: `${input.turnId}:${input.sourceKey}`,
2904
+ idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,
2905
+ metadata: {
2906
+ model: input.model,
2907
+ sessionId: input.sessionId,
2908
+ turnId: input.turnId,
2909
+ sourceKey: input.sourceKey,
2910
+ inputTokens,
2911
+ outputTokens,
2912
+ totalTokens
2913
+ }
2914
+ });
2915
+ }
2916
+ }
2917
+ function positiveInt(value) {
2918
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
2919
+ }
2920
+ function startOfUtcMonth2() {
2921
+ const now = /* @__PURE__ */ new Date();
2922
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
2923
+ }
2924
+ async function sandboxFileDownloadsForRun(settings, db, objectStorage, workspaceId, resources) {
2925
+ if (settings.sandboxBackend === "none" || !requiresSignedFileResourceDownloads(settings)) {
2926
+ return [];
2927
+ }
2928
+ const fileResources = resources.filter((resource) => resource.kind === "file");
2929
+ if (fileResources.length === 0) {
2930
+ return [];
2931
+ }
2932
+ if (!objectStorage) {
2933
+ throw new Error(`${settings.objectStorageBackend} file resources require configured object storage`);
2934
+ }
2935
+ const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
2936
+ const downloads = [];
2937
+ for (const resource of fileResources) {
2938
+ const file = await requireFile2(db, workspaceId, resource.fileId);
2939
+ const url = await downloadStorage.createGetUrl({ key: file.objectKey });
2940
+ downloads.push({
2941
+ fileId: file.id,
2942
+ mountPath: resource.mountPath ?? `files/${file.id}`,
2943
+ filename: file.safeFilename,
2944
+ url: url.url,
2945
+ expiresAt: url.expiresAt,
2946
+ sizeBytes: file.sizeBytes
2947
+ });
2948
+ }
2949
+ return downloads;
2950
+ }
2951
+ function requiresSignedFileResourceDownloads(settings) {
2952
+ if (settings.sandboxBackend === "selfhosted") {
2953
+ return true;
2954
+ }
2955
+ const nativeBucketMount = CAPABILITY_DESCRIPTORS[settings.sandboxBackend].nativeBucketMount;
2956
+ return settings.sandboxBackend === "docker" && settings.objectStorageBackend === "s3-compatible" || settings.objectStorageBackend === "aws-s3" || settings.objectStorageBackend === "gcs" || nativeBucketMount && settings.objectStorageBackend === "azure-blob";
2957
+ }
2958
+ function objectStorageForSandboxDownloads(settings, objectStorage) {
2959
+ if (settings.objectStorageBackend !== "s3-compatible" || !settings.objectStorageSandboxEndpoint) {
2960
+ return objectStorage;
2961
+ }
2962
+ return createObjectStorage({
2963
+ ...settings,
2964
+ objectStorageEndpoint: settings.objectStorageSandboxEndpoint
2965
+ }) ?? objectStorage;
2966
+ }
2967
+
2968
+ // src/activities/documents.ts
2969
+ import { indexDocumentNow } from "@opengeni/documents";
2970
+ import { configuredStaticUsageLimits as configuredStaticUsageLimits3 } from "@opengeni/config";
2971
+ import {
2972
+ getBillingBalance as getBillingBalance3,
2973
+ recordUsageEvent as recordUsageEvent3,
2974
+ sumUsageQuantity as sumUsageQuantity3,
2975
+ withWorkspaceUsageLock
2976
+ } from "@opengeni/db";
2977
+ function createDocumentActivities(services) {
2978
+ return {
2979
+ indexDocument: async (input) => {
2980
+ const { settings, db, objectStorage, documentServices } = await services();
2981
+ if (!objectStorage) {
2982
+ throw new Error("object storage is not configured");
2983
+ }
2984
+ return await withWorkspaceUsageLock(db, input.workspaceId, async (lockedDb) => {
2985
+ const document = await indexDocumentNow(lockedDb, objectStorage, input.workspaceId, input.documentId, documentServices, {
2986
+ beforeEmbed: async ({ chunkCount }) => {
2987
+ if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
2988
+ const balance = await getBillingBalance3(lockedDb, input.accountId);
2989
+ if (balance.balanceMicros <= 0) {
2990
+ throw new Error("insufficient OpenGeni credits");
2991
+ }
2992
+ }
2993
+ if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
2994
+ return;
2995
+ }
2996
+ const limit = configuredStaticUsageLimits3(settings).maxDocumentIndexedChunksPerWorkspace;
2997
+ if (!limit) {
2998
+ return;
2999
+ }
3000
+ const used = await sumUsageQuantity3(lockedDb, {
3001
+ workspaceId: input.workspaceId,
3002
+ eventType: "document.indexed",
3003
+ since: startOfUtcMonth3()
3004
+ });
3005
+ if (used + chunkCount > limit) {
3006
+ throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
3007
+ }
3008
+ }
3009
+ });
3010
+ if (document.status === "ready") {
3011
+ await recordUsageEvent3(lockedDb, {
3012
+ accountId: input.accountId,
3013
+ workspaceId: input.workspaceId,
3014
+ eventType: "document.indexed",
3015
+ quantity: document.chunkCount,
3016
+ unit: "chunk",
3017
+ sourceResourceType: "document",
3018
+ sourceResourceId: document.id,
3019
+ idempotencyKey: `document.indexed:${input.workspaceId}:${document.id}:${document.updatedAt}`
3020
+ });
3021
+ }
3022
+ return document;
3023
+ });
3024
+ }
3025
+ };
3026
+ }
3027
+ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
3028
+ return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
3029
+ }
3030
+
3031
+ // src/activities/sandbox-lease.ts
3032
+ import {
3033
+ accrueWarmSeconds as accrueWarmSeconds2,
3034
+ confirmDrainCold,
3035
+ forceDrainOverLimitViewerOnlyBoxes,
3036
+ getBillingBalance as getBillingBalance4,
3037
+ listMeterableWarmLeases,
3038
+ persistDrainSnapshot,
3039
+ readLease as readLease2,
3040
+ reapStaleLeaseHoldersGlobal,
3041
+ rlsContextForWorkspace
3042
+ } from "@opengeni/db";
3043
+ import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
3044
+ import {
3045
+ createSandboxClientForBackend,
3046
+ deletePriorPersistedSnapshot,
3047
+ deserializeSandboxSessionStateEnvelope,
3048
+ isProviderSandboxNotFoundError
3049
+ } from "@opengeni/runtime";
3050
+ function createSandboxLeaseActivities(services, options = {}) {
3051
+ const terminateBox = options.terminateBox ?? terminateProviderBox;
3052
+ async function reapSandboxLeases2() {
3053
+ const { db, settings, observability } = await services();
3054
+ if (!settings.sandboxOwnershipEnabled) {
3055
+ return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
3056
+ }
3057
+ const metered = await accrueWarmTick(db, settings, observability);
3058
+ const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
3059
+ const drainable = await reapStaleLeaseHoldersGlobal(db, {
3060
+ viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
3061
+ idleGraceMs: settings.sandboxIdleGraceMs
3062
+ });
3063
+ let terminated = 0;
3064
+ let skipped = 0;
3065
+ for (const row of drainable) {
3066
+ try {
3067
+ const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
3068
+ if (drainedCold) {
3069
+ terminated += 1;
3070
+ } else {
3071
+ skipped += 1;
3072
+ }
3073
+ } catch (error) {
3074
+ skipped += 1;
3075
+ observability.warn("sandbox reaper: terminate failed for drainable lease", {
3076
+ workspaceId: row.workspaceId,
3077
+ sandboxGroupId: row.sandboxGroupId,
3078
+ leaseEpoch: row.leaseEpoch,
3079
+ error: error instanceof Error ? error.message : String(error)
3080
+ });
3081
+ }
3082
+ }
3083
+ if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
3084
+ observability.info("sandbox reaper swept", {
3085
+ drainable: drainable.length,
3086
+ terminated,
3087
+ skipped,
3088
+ metered: metered.accrued,
3089
+ forceDrained
3090
+ });
3091
+ }
3092
+ return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
3093
+ }
3094
+ return { reapSandboxLeases: reapSandboxLeases2 };
3095
+ }
3096
+ async function accrueWarmTick(db, settings, observability) {
3097
+ const workspaceIds = /* @__PURE__ */ new Set();
3098
+ let accrued = 0;
3099
+ let leases = [];
3100
+ try {
3101
+ leases = await listMeterableWarmLeases(db);
3102
+ } catch (error) {
3103
+ observability.warn("sandbox reaper: warm-lease read failed", {
3104
+ error: error instanceof Error ? error.message : String(error)
3105
+ });
3106
+ return { accrued, workspaceIds };
3107
+ }
3108
+ for (const lease of leases) {
3109
+ workspaceIds.add(lease.workspaceId);
3110
+ try {
3111
+ const rate = sandboxWarmRateMicrosPerSecond2(settings, lease.backend);
3112
+ const result = await accrueWarmSeconds2(db, {
3113
+ accountId: lease.accountId,
3114
+ workspaceId: lease.workspaceId,
3115
+ sandboxGroupId: lease.sandboxGroupId,
3116
+ expectedEpoch: lease.leaseEpoch,
3117
+ warmRateMicrosPerSecond: rate,
3118
+ subjectId: lease.sandboxGroupId
3119
+ });
3120
+ if (result.accrued) {
3121
+ accrued += 1;
3122
+ }
3123
+ } catch (error) {
3124
+ observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
3125
+ workspaceId: lease.workspaceId,
3126
+ sandboxGroupId: lease.sandboxGroupId,
3127
+ error: error instanceof Error ? error.message : String(error)
3128
+ });
3129
+ }
3130
+ }
3131
+ return { accrued, workspaceIds };
3132
+ }
3133
+ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
3134
+ const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
3135
+ const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
3136
+ if (!enforceBalance && cap <= 0) {
3137
+ return 0;
3138
+ }
3139
+ let forceDrained = 0;
3140
+ for (const workspaceId of workspaceIds) {
3141
+ try {
3142
+ const { accountId } = await rlsContextForWorkspace(db, workspaceId);
3143
+ const balance = enforceBalance ? await getBillingBalance4(db, accountId) : { balanceMicros: 1 };
3144
+ const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
3145
+ workspaceId,
3146
+ balanceMicros: balance.balanceMicros,
3147
+ enforceBalance,
3148
+ maxWarmSecondsPerWorkspace: cap,
3149
+ idleGraceMs: settings.sandboxIdleGraceMs
3150
+ });
3151
+ if (result.overLimit && result.drained.length > 0) {
3152
+ forceDrained += result.drained.length;
3153
+ observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
3154
+ workspaceId,
3155
+ reason: result.reason,
3156
+ drained: result.drained.length
3157
+ });
3158
+ }
3159
+ } catch (error) {
3160
+ observability.warn("sandbox reaper: force-drain check failed for workspace", {
3161
+ workspaceId,
3162
+ error: error instanceof Error ? error.message : String(error)
3163
+ });
3164
+ }
3165
+ }
3166
+ return forceDrained;
3167
+ }
3168
+ async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
3169
+ const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
3170
+ const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
3171
+ if (!lease) {
3172
+ return false;
3173
+ }
3174
+ if (lease.liveness !== "draining" || lease.refcount !== 0 || lease.leaseEpoch !== row.leaseEpoch) {
3175
+ return false;
3176
+ }
3177
+ const persistArchive = async (archiveBase64) => await persistDrainSnapshot(db, {
3178
+ accountId,
3179
+ workspaceId: row.workspaceId,
3180
+ sandboxGroupId: row.sandboxGroupId,
3181
+ expectedEpoch: row.leaseEpoch,
3182
+ workspaceArchive: archiveBase64
3183
+ });
3184
+ const terminated = await terminateBox(settings, lease, observability, persistArchive);
3185
+ if (!terminated) {
3186
+ return false;
3187
+ }
3188
+ const { wentCold } = await confirmDrainCold(db, {
3189
+ accountId,
3190
+ workspaceId: row.workspaceId,
3191
+ sandboxGroupId: row.sandboxGroupId,
3192
+ expectedEpoch: row.leaseEpoch
3193
+ });
3194
+ return wentCold;
3195
+ }
3196
+ async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
3197
+ const backend = lease.resumeBackendId ?? lease.backend;
3198
+ if (!backend || backend === "none") {
3199
+ return true;
3200
+ }
3201
+ if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
3202
+ observability.info("sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped \u2014 it is the user's machine)", {
3203
+ sandboxGroupId: lease.sandboxGroupId,
3204
+ backend
3205
+ });
3206
+ return true;
3207
+ }
3208
+ if (!lease.resumeState) {
3209
+ return true;
3210
+ }
3211
+ const client = createClientForBackend(backend, settings);
3212
+ if (!client) {
3213
+ return true;
3214
+ }
3215
+ let session;
3216
+ let sessionState;
3217
+ try {
3218
+ if (!client.resume || !client.deserializeSessionState) {
3219
+ return true;
3220
+ }
3221
+ const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
3222
+ const resumedState = await deserializeSandboxSessionStateEnvelope(client, envelopeSessionState);
3223
+ if (resumedState === void 0) {
3224
+ return true;
3225
+ }
3226
+ session = await client.resume(resumedState);
3227
+ sessionState = resumedState;
3228
+ } catch (error) {
3229
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
3230
+ observability.info("sandbox reaper: drainable box already gone (NotFound on resume) \u2014 proceeding to cold", {
3231
+ sandboxGroupId: lease.sandboxGroupId,
3232
+ backend
3233
+ });
3234
+ return true;
3235
+ }
3236
+ throw error;
3237
+ }
3238
+ let archiveBytes;
3239
+ try {
3240
+ archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : void 0;
3241
+ } catch (error) {
3242
+ if (isProviderSandboxNotFoundError(client.backendId, error)) {
3243
+ observability.info("sandbox reaper: box gone during persist (NotFound) \u2014 nothing to snapshot, proceeding to cold", {
3244
+ sandboxGroupId: lease.sandboxGroupId,
3245
+ backend
3246
+ });
3247
+ return true;
3248
+ }
3249
+ observability.warn("sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)", {
3250
+ sandboxGroupId: lease.sandboxGroupId,
3251
+ backend,
3252
+ error: error instanceof Error ? error.message : String(error)
3253
+ });
3254
+ throw error;
3255
+ }
3256
+ if (archiveBytes && archiveBytes.length > 0) {
3257
+ const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
3258
+ const { wrote, priorArchive } = await persistArchive(archiveBase64);
3259
+ if (!wrote) {
3260
+ observability.info("sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)", {
3261
+ sandboxGroupId: lease.sandboxGroupId,
3262
+ backend
3263
+ });
3264
+ return false;
3265
+ }
3266
+ const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
3267
+ if (deleted) {
3268
+ observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
3269
+ sandboxGroupId: lease.sandboxGroupId,
3270
+ backend,
3271
+ snapshotId: deleted
3272
+ });
3273
+ }
3274
+ } else {
3275
+ const { wrote } = await persistArchive(null);
3276
+ if (!wrote) {
3277
+ observability.info("sandbox reaper: lease re-armed during snapshot window (no-archive path) \u2014 leaving box RUNNING (no terminate)", {
3278
+ sandboxGroupId: lease.sandboxGroupId,
3279
+ backend
3280
+ });
3281
+ return false;
3282
+ }
3283
+ }
3284
+ try {
3285
+ if (client.delete && sessionState !== void 0) {
3286
+ await client.delete(sessionState);
3287
+ return true;
3288
+ }
3289
+ if (session?.kill) {
3290
+ await session.kill();
3291
+ return true;
3292
+ }
3293
+ if (session?.terminate) {
3294
+ await session.terminate();
3295
+ return true;
3296
+ }
3297
+ if (session?.close && !session.closed) {
3298
+ await session.close();
3299
+ }
3300
+ } catch (error) {
3301
+ observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
3302
+ sandboxGroupId: lease.sandboxGroupId,
3303
+ backend,
3304
+ error: error instanceof Error ? error.message : String(error)
3305
+ });
3306
+ }
3307
+ return true;
3308
+ }
3309
+
3310
+ // src/activities/scheduled-tasks.ts
3311
+ import {
3312
+ appendSessionEventsWithLockedSessionUpdate,
3313
+ createScheduledTaskRun,
3314
+ createSession,
3315
+ createSessionGoal,
3316
+ enqueueSessionTurn as enqueueSessionTurn2,
3317
+ getBillingBalance as getBillingBalance5,
3318
+ getWorkspaceEnvironment,
3319
+ isCodexBilledTurn as isCodexBilledTurn3,
3320
+ recordUsageEvent as recordUsageEvent4,
3321
+ requireScheduledTask,
3322
+ requireSession as requireSession3,
3323
+ setTemporalWorkflowId,
3324
+ sumUsageQuantity as sumUsageQuantity4,
3325
+ updateScheduledTask,
3326
+ updateScheduledTaskRun,
3327
+ upsertSessionGoal
3328
+ } from "@opengeni/db";
3329
+ import { appendAndPublishEvents as appendAndPublishEvents3 } from "@opengeni/events";
3330
+ import { configuredStaticUsageLimits as configuredStaticUsageLimits4 } from "@opengeni/config";
3331
+ function createScheduledTaskActivities(services) {
3332
+ return {
3333
+ dispatchScheduledTaskRun: async (input) => {
3334
+ const { settings, db, bus } = await services();
3335
+ const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
3336
+ const model = task.agentConfig.model ?? settings.openaiModel;
3337
+ const isCodexRun = await isCodexBilledTurn3({ db, settings, workspaceId: task.workspaceId, model });
3338
+ await ensureScheduledRunAllowed(settings, db, task.accountId, task.workspaceId, input.agentRunUsageIdempotencyKey ? 0 : 1, isCodexRun);
3339
+ const run = await createScheduledTaskRun(db, {
3340
+ workspaceId: task.workspaceId,
3341
+ taskId: task.id,
3342
+ triggerType: input.triggerType,
3343
+ scheduledAt: null
3344
+ });
3345
+ await recordUsageEvent4(db, {
3346
+ accountId: task.accountId,
3347
+ workspaceId: task.workspaceId,
3348
+ eventType: "scheduled_task.fired",
3349
+ quantity: 1,
3350
+ unit: "run",
3351
+ sourceResourceType: "scheduled_task_run",
3352
+ sourceResourceId: run.id,
3353
+ idempotencyKey: `usage:scheduled_task.fired:${run.id}`
3354
+ });
3355
+ let result;
3356
+ try {
3357
+ const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
3358
+ const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
3359
+ const goalSpec = task.agentConfig.goal ?? null;
3360
+ const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
3361
+ if (task.runMode === "new_session_per_run" || !task.reusableSessionId) {
3362
+ const environment = task.environmentId ? await getWorkspaceEnvironment(db, task.workspaceId, task.environmentId) : null;
3363
+ if (task.environmentId && !environment) {
3364
+ throw new Error(`workspace environment not found: ${task.environmentId}`);
3365
+ }
3366
+ const session = await createSession(db, {
3367
+ accountId: task.accountId,
3368
+ workspaceId: task.workspaceId,
3369
+ initialMessage: task.agentConfig.prompt,
3370
+ resources: task.agentConfig.resources,
3371
+ tools: taskTools,
3372
+ metadata: {
3373
+ ...task.agentConfig.metadata,
3374
+ model,
3375
+ reasoningEffort,
3376
+ scheduledTaskId: task.id,
3377
+ scheduledTaskRunId: run.id
3378
+ },
3379
+ model,
3380
+ sandboxBackend,
3381
+ environmentId: task.environmentId ?? null
3382
+ });
3383
+ const goal = goalSpec ? await createSessionGoal(db, {
3384
+ accountId: task.accountId,
3385
+ workspaceId: task.workspaceId,
3386
+ sessionId: session.id,
3387
+ text: goalSpec.text,
3388
+ successCriteria: goalSpec.successCriteria ?? null,
3389
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
3390
+ createdBy: "scheduled_task"
3391
+ }) : null;
3392
+ const workflowId = workflowIdForSession(session.id);
3393
+ await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
3394
+ if (task.runMode === "reusable_session") {
3395
+ await updateScheduledTask(db, task.workspaceId, task.id, { reusableSessionId: session.id });
3396
+ }
3397
+ const events = await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [
3398
+ {
3399
+ type: "session.created",
3400
+ payload: {
3401
+ status: "queued",
3402
+ scheduledTaskId: task.id,
3403
+ scheduledTaskRunId: run.id,
3404
+ // Names/ids only; never values.
3405
+ ...environment ? { environmentId: environment.id, environmentName: environment.name } : {}
3406
+ }
3407
+ },
3408
+ ...goal ? [{
3409
+ type: "goal.set",
3410
+ payload: {
3411
+ goalId: goal.id,
3412
+ text: goal.text,
3413
+ ...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
3414
+ version: goal.version,
3415
+ actor: "scheduled_task",
3416
+ replaced: false
3417
+ }
3418
+ }] : [],
3419
+ {
3420
+ type: "user.message",
3421
+ payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id)
3422
+ },
3423
+ { type: "session.status.changed", payload: { status: "queued" } }
3424
+ ]);
3425
+ const trigger = events.find((event) => event.type === "user.message");
3426
+ if (!trigger) {
3427
+ throw new Error("failed to append scheduled task trigger event");
3428
+ }
3429
+ const turn = await enqueueSessionTurn2(db, {
3430
+ accountId: task.accountId,
3431
+ workspaceId: task.workspaceId,
3432
+ sessionId: session.id,
3433
+ triggerEventId: trigger.id,
3434
+ temporalWorkflowId: workflowId,
3435
+ source: "scheduled_task",
3436
+ prompt: task.agentConfig.prompt,
3437
+ resources: task.agentConfig.resources,
3438
+ tools: taskTools,
3439
+ model,
3440
+ reasoningEffort,
3441
+ sandboxBackend,
3442
+ metadata: {
3443
+ scheduledTaskId: task.id,
3444
+ scheduledTaskRunId: run.id
3445
+ }
3446
+ });
3447
+ await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [{
3448
+ type: "turn.queued",
3449
+ turnId: turn.id,
3450
+ payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source }
3451
+ }]);
3452
+ await updateScheduledTaskRun(db, task.workspaceId, run.id, {
3453
+ status: "dispatched",
3454
+ sessionId: session.id,
3455
+ triggerEventId: trigger.id
3456
+ });
3457
+ result = {
3458
+ action: "start",
3459
+ accountId: task.accountId,
3460
+ workspaceId: task.workspaceId,
3461
+ sessionId: session.id,
3462
+ triggerEventId: trigger.id,
3463
+ workflowId
3464
+ };
3465
+ } else {
3466
+ const session = await requireSession3(db, task.workspaceId, task.reusableSessionId);
3467
+ assertReusableSessionRevivable(session.status);
3468
+ if ((session.environmentId ?? null) !== (task.environmentId ?? null)) {
3469
+ throw new Error("scheduled task environment attachment does not match its reusable session");
3470
+ }
3471
+ const reusableGoal = goalSpec ? await upsertSessionGoal(db, {
3472
+ accountId: task.accountId,
3473
+ workspaceId: task.workspaceId,
3474
+ sessionId: session.id,
3475
+ text: goalSpec.text,
3476
+ successCriteria: goalSpec.successCriteria ?? null,
3477
+ maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
3478
+ createdBy: "scheduled_task"
3479
+ }) : null;
3480
+ const events = await appendSessionEventsWithLockedSessionUpdate(db, task.workspaceId, session.id, (locked) => {
3481
+ assertReusableSessionRevivable(locked.status);
3482
+ return {
3483
+ events: [
3484
+ ...reusableGoal ? [{
3485
+ type: "goal.set",
3486
+ payload: {
3487
+ goalId: reusableGoal.goal.id,
3488
+ text: reusableGoal.goal.text,
3489
+ ...reusableGoal.goal.successCriteria ? { successCriteria: reusableGoal.goal.successCriteria } : {},
3490
+ version: reusableGoal.goal.version,
3491
+ actor: "scheduled_task",
3492
+ replaced: reusableGoal.replaced
3493
+ }
3494
+ }] : [],
3495
+ {
3496
+ type: "user.message",
3497
+ payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id)
3498
+ }
3499
+ ],
3500
+ update: {
3501
+ resources: mergeResourceRefs(locked.resources, task.agentConfig.resources),
3502
+ tools: mergeToolRefs(locked.tools, taskTools)
3503
+ }
3504
+ };
3505
+ });
3506
+ await bus.publish(task.workspaceId, session.id, events);
3507
+ const trigger = events.find((event) => event.type === "user.message");
3508
+ if (!trigger) {
3509
+ throw new Error("failed to append scheduled task trigger event");
3510
+ }
3511
+ const turn = await enqueueSessionTurn2(db, {
3512
+ accountId: task.accountId,
3513
+ workspaceId: task.workspaceId,
3514
+ sessionId: session.id,
3515
+ triggerEventId: trigger.id,
3516
+ temporalWorkflowId: workflowIdForSession(session.id),
3517
+ source: "scheduled_task",
3518
+ prompt: task.agentConfig.prompt,
3519
+ resources: task.agentConfig.resources,
3520
+ tools: taskTools,
3521
+ model,
3522
+ reasoningEffort,
3523
+ sandboxBackend,
3524
+ metadata: {
3525
+ scheduledTaskId: task.id,
3526
+ scheduledTaskRunId: run.id
3527
+ }
3528
+ });
3529
+ await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [{
3530
+ type: "turn.queued",
3531
+ turnId: turn.id,
3532
+ payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source }
3533
+ }]);
3534
+ await updateScheduledTaskRun(db, task.workspaceId, run.id, {
3535
+ status: "dispatched",
3536
+ sessionId: session.id,
3537
+ triggerEventId: trigger.id
3538
+ });
3539
+ result = {
3540
+ action: "signal",
3541
+ accountId: task.accountId,
3542
+ workspaceId: task.workspaceId,
3543
+ sessionId: session.id,
3544
+ triggerEventId: trigger.id,
3545
+ workflowId: workflowIdForSession(session.id)
3546
+ };
3547
+ }
3548
+ } catch (error) {
3549
+ await updateScheduledTaskRun(db, task.workspaceId, run.id, {
3550
+ status: "failed",
3551
+ error: error instanceof Error ? error.message : String(error)
3552
+ }).catch(() => void 0);
3553
+ throw error;
3554
+ }
3555
+ await recordUsageEvent4(db, {
3556
+ accountId: task.accountId,
3557
+ workspaceId: task.workspaceId,
3558
+ eventType: "agent_run.created",
3559
+ quantity: 1,
3560
+ unit: "run",
3561
+ sourceResourceType: "scheduled_task_run",
3562
+ sourceResourceId: run.id,
3563
+ idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
3564
+ });
3565
+ return result;
3566
+ }
3567
+ };
3568
+ }
3569
+ async function ensureScheduledRunAllowed(settings, db, accountId, workspaceId, requestedAgentRuns, isCodexRun) {
3570
+ if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
3571
+ const balance = await getBillingBalance5(db, accountId);
3572
+ if (balance.balanceMicros <= 0) {
3573
+ throw new Error("insufficient OpenGeni credits");
3574
+ }
3575
+ }
3576
+ if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
3577
+ const limits = configuredStaticUsageLimits4(settings);
3578
+ if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
3579
+ const used = await sumUsageQuantity4(db, {
3580
+ accountId,
3581
+ eventType: "model.cost",
3582
+ since: startOfUtcMonth4()
3583
+ });
3584
+ if (used >= limits.maxMonthlyCostMicrosPerAccount) {
3585
+ throw new Error(`monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`);
3586
+ }
3587
+ }
3588
+ if (limits.maxMonthlyAgentRunsPerWorkspace) {
3589
+ const used = await sumUsageQuantity4(db, {
3590
+ workspaceId,
3591
+ eventType: "agent_run.created",
3592
+ since: startOfUtcMonth4()
3593
+ });
3594
+ if (used + requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
3595
+ throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
3596
+ }
3597
+ }
3598
+ }
3599
+ }
3600
+ function startOfUtcMonth4() {
3601
+ const now = /* @__PURE__ */ new Date();
3602
+ return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
3603
+ }
3604
+
3605
+ // src/activities/session-state.ts
3606
+ import {
3607
+ claimNextQueuedTurn as claimNextQueuedTurnDb2,
3608
+ countTurnSessionHistoryItems,
3609
+ finishTurn as finishTurn2,
3610
+ getSessionEvent as getSessionEvent4,
3611
+ getSessionTurn as getSessionTurn2,
3612
+ incrementTurnWorkerDeathRedispatches,
3613
+ requeuePreemptedTurn as requeuePreemptedTurn2,
3614
+ requireSession as requireSession4,
3615
+ setSessionStatus as setSessionStatus2
3616
+ } from "@opengeni/db";
3617
+ import { appendAndPublishEvents as appendAndPublishEvents4 } from "@opengeni/events";
3618
+ var WORKER_DEATH_MAX_REDISPATCHES = 3;
3619
+ function createSessionStateActivities(services) {
3620
+ async function failSession2(input) {
3621
+ const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
3622
+ const session = await requireSession4(db, input.workspaceId, input.sessionId);
3623
+ if (session.status === "failed") {
3624
+ return;
3625
+ }
3626
+ const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
3627
+ const turnId = session.activeTurnId ?? null;
3628
+ await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
3629
+ {
3630
+ type: "turn.failed",
3631
+ turnId,
3632
+ payload: {
3633
+ triggerEventId: input.triggerEventId,
3634
+ trigger: trigger?.payload ?? null,
3635
+ error: input.error ?? "Agent activity failed before it could report a terminal state."
3636
+ }
3637
+ },
3638
+ {
3639
+ type: "session.status.changed",
3640
+ turnId,
3641
+ payload: { status: "failed" }
3642
+ }
3643
+ ]);
3644
+ if (turnId) {
3645
+ await finishTurn2(db, input.workspaceId, turnId, "failed");
3646
+ }
3647
+ await setSessionStatus2(db, input.workspaceId, input.sessionId, "failed", null);
3648
+ await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
3649
+ }
3650
+ async function interruptActiveTurn2(input) {
3651
+ const { db, bus } = await services();
3652
+ const session = await requireSession4(db, input.workspaceId, input.sessionId);
3653
+ const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
3654
+ if (!isSteerInterrupt(trigger)) {
3655
+ await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
3656
+ }
3657
+ if (!session.activeTurnId) {
3658
+ return;
3659
+ }
3660
+ await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
3661
+ {
3662
+ turnId: session.activeTurnId,
3663
+ type: "turn.cancelled",
3664
+ payload: { triggerEventId: input.triggerEventId, reason: trigger?.payload ?? null }
3665
+ },
3666
+ {
3667
+ turnId: session.activeTurnId,
3668
+ type: "session.status.changed",
3669
+ payload: { status: "queued" }
3670
+ }
3671
+ ]);
3672
+ await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
3673
+ await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
3674
+ }
3675
+ async function requeueTurnAfterWorkerDeath2(input) {
3676
+ const { settings, db, bus } = await services();
3677
+ const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
3678
+ if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
3679
+ return { action: "stale" };
3680
+ }
3681
+ const redispatches = await incrementTurnWorkerDeathRedispatches(db, input.workspaceId, input.turnId);
3682
+ if (redispatches > WORKER_DEATH_MAX_REDISPATCHES) {
3683
+ return { action: "exceeded", redispatches: redispatches - 1 };
3684
+ }
3685
+ const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
3686
+ const approvalRerun = trigger?.type === "user.approvalDecision";
3687
+ const resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && await countTurnSessionHistoryItems(db, input.workspaceId, input.turnId) > 0;
3688
+ const [preemptedEvent] = await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
3689
+ {
3690
+ turnId: turn.id,
3691
+ type: "turn.preempted",
3692
+ payload: {
3693
+ triggerEventId: input.triggerEventId,
3694
+ reason: "worker_death",
3695
+ redispatches,
3696
+ resumeWithNotice,
3697
+ ...resumeWithNotice ? { text: WORKER_DEATH_RESUME_TEXT } : {}
3698
+ }
3699
+ },
3700
+ {
3701
+ turnId: turn.id,
3702
+ type: "session.status.changed",
3703
+ payload: { status: "queued" }
3704
+ }
3705
+ ]);
3706
+ try {
3707
+ await requeuePreemptedTurn2(db, input.workspaceId, turn.id, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
3708
+ } catch (requeueError) {
3709
+ const current = await getSessionTurn2(db, input.workspaceId, input.turnId);
3710
+ if (current && current.status !== "running" && current.status !== "requires_action") {
3711
+ return { action: "stale" };
3712
+ }
3713
+ throw requeueError;
3714
+ }
3715
+ await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
3716
+ return { action: "requeued", redispatches };
3717
+ }
3718
+ async function claimNextQueuedTurn2(input) {
3719
+ const { db } = await services();
3720
+ return await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
3721
+ }
3722
+ async function markSessionIdle2(input) {
3723
+ const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
3724
+ const session = await requireSession4(db, input.workspaceId, input.sessionId);
3725
+ if (session.status === "queued" || session.status === "running") {
3726
+ await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
3727
+ }
3728
+ await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
3729
+ }
3730
+ return {
3731
+ failSession: failSession2,
3732
+ interruptActiveTurn: interruptActiveTurn2,
3733
+ requeueTurnAfterWorkerDeath: requeueTurnAfterWorkerDeath2,
3734
+ claimNextQueuedTurn: claimNextQueuedTurn2,
3735
+ markSessionIdle: markSessionIdle2
3736
+ };
3737
+ }
3738
+
3739
+ // src/activities.ts
3740
+ function createActivities(dependencies = {}) {
3741
+ let servicesPromise = null;
3742
+ async function services() {
3743
+ servicesPromise ??= (async () => {
3744
+ const settings = dependencies.settings ?? getSettings();
3745
+ const searchPath = dbSearchPath(settings);
3746
+ const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
3747
+ ...searchPath ? { searchPath } : {},
3748
+ rlsStrategy: settings.rlsStrategy
3749
+ });
3750
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
3751
+ return {
3752
+ settings,
3753
+ db: dependencies.db ?? dbClient.db,
3754
+ // §7 Step G — EventBus binding contract. `bus` is the INJECTED
3755
+ // live-fanout port; the ONE production impl is `createNatsEventBus`,
3756
+ // the default on BOTH processes (this worker edge + the API edge in
3757
+ // `apps/api/src/index.ts`). A host that embeds OpenGeni injects ONE
3758
+ // broker binding (the same `createNatsEventBus(natsUrl)`) here AND on
3759
+ // the mounted API, so the two SEPARATE processes share one broker and
3760
+ // derive the IDENTICAL `sessionSubject` — the only way live fanout
3761
+ // (worker emit → API SSE) works cross-process (SPIKE-1 F5/F6, proven).
3762
+ // NEVER default to an in-memory bus: it fans out intra-process only and
3763
+ // would silently break live SSE. unset → today's NATS default,
3764
+ // byte-for-byte. The bus is live-fanout ONLY — the durable Postgres
3765
+ // `session_events` log is source-of-truth (the API backfills missed
3766
+ // events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
3767
+ bus: dependencies.bus ?? await createNatsEventBus(
3768
+ settings.natsUrl,
3769
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
3770
+ ),
3771
+ runtime: dependencies.runtime ?? createProductionAgentRuntime(),
3772
+ objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
3773
+ documentServices: dependencies.documentServices ?? createDocumentServices(settings),
3774
+ observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
3775
+ wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
3776
+ // §7.5 P3 — host-entitlements port. No constructed default: standalone
3777
+ // has no host meter, so unset → null → `ensureRunAllowed` reads the
3778
+ // local ledger exactly as today (mirrors `wakeSessionWorkflow`'s
3779
+ // null-degrades-gracefully shape, not a `createX(settings)` default).
3780
+ entitlements: dependencies.entitlements ?? null,
3781
+ // §7.6 P4a — host connection-credential provider. No constructed
3782
+ // default: standalone owns its own GitHub App + encryption key, so unset
3783
+ // → null → the per-run credential mint self-mints from `settings`
3784
+ // (createGitHubAppInstallationToken + environmentsEncryptionKeyBytes)
3785
+ // exactly as today. Same null-degrades shape as `entitlements`.
3786
+ connectionCredentials: dependencies.connectionCredentials ?? null
3787
+ };
3788
+ })();
3789
+ return servicesPromise;
3790
+ }
3791
+ const runAgentTurn2 = createRunAgentTurnActivity(services);
3792
+ return {
3793
+ runAgentTurn: runAgentTurn2,
3794
+ // Legacy Temporal activity name. In-flight session workflows (which can
3795
+ // legitimately live for days) recorded ScheduleActivityTask events as
3796
+ // "runAgentSegment"; the name must keep resolving until those histories
3797
+ // drain. New workflow code must use runAgentTurn; migrate the session
3798
+ // workflow call site via patched() before removing this alias.
3799
+ runAgentSegment: runAgentTurn2,
3800
+ ...createDocumentActivities(services),
3801
+ ...createSessionStateActivities(services),
3802
+ ...createScheduledTaskActivities(services),
3803
+ ...createGoalActivities(services),
3804
+ // P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
3805
+ // *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
3806
+ ...createSandboxLeaseActivities(services)
3807
+ };
3808
+ }
3809
+ var defaultActivities = createActivities();
3810
+ var runAgentTurn = defaultActivities.runAgentTurn;
3811
+ var runAgentSegment = defaultActivities.runAgentSegment;
3812
+ var indexDocument = defaultActivities.indexDocument;
3813
+ var failSession = defaultActivities.failSession;
3814
+ var interruptActiveTurn = defaultActivities.interruptActiveTurn;
3815
+ var requeueTurnAfterWorkerDeath = defaultActivities.requeueTurnAfterWorkerDeath;
3816
+ var claimNextQueuedTurn = defaultActivities.claimNextQueuedTurn;
3817
+ var markSessionIdle = defaultActivities.markSessionIdle;
3818
+ var dispatchScheduledTaskRun = defaultActivities.dispatchScheduledTaskRun;
3819
+ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
3820
+ var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
3821
+ var reapSandboxLeases = defaultActivities.reapSandboxLeases;
3822
+
3823
+ // src/index.ts
3824
+ var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
3825
+ async function createOpenGeniWorker(options = {}) {
3826
+ const settings = options.settings ?? getSettings2();
3827
+ const observability = options.activityDependencies?.observability ?? createObservability2(settings, { component: "worker" });
3828
+ const connection = await retryStartupDependency(
3829
+ "Temporal",
3830
+ () => NativeConnection.connect({ address: settings.temporalHost }),
3831
+ {
3832
+ ...startupRetryOptions(settings),
3833
+ onRetry: (event) => logStartupDependencyRetry(observability, event)
3834
+ }
3835
+ );
3836
+ const activities = options.activities ?? createActivities({
3837
+ ...options.activityDependencies,
3838
+ settings,
3839
+ observability
3840
+ });
3841
+ const worker = await Worker.create({
3842
+ connection,
3843
+ namespace: settings.temporalNamespace,
3844
+ taskQueue: settings.temporalTaskQueue,
3845
+ workflowsPath: new URL("../src/workflows.ts", import.meta.url).pathname,
3846
+ activities
3847
+ });
3848
+ return { worker, connection };
3849
+ }
3850
+ async function createWorkerWorkflowSignaler(settings) {
3851
+ const connection = await Connection.connect({ address: settings.temporalHost });
3852
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
3853
+ return {
3854
+ wakeSessionWorkflow: async ({ accountId, workspaceId, sessionId, workflowId }) => {
3855
+ await temporal.workflow.signalWithStart("sessionWorkflow", {
3856
+ taskQueue: settings.temporalTaskQueue,
3857
+ workflowId,
3858
+ workflowIdReusePolicy: "ALLOW_DUPLICATE",
3859
+ args: [{ accountId, workspaceId, sessionId }],
3860
+ signal: "queueChanged"
3861
+ });
3862
+ },
3863
+ close: async () => {
3864
+ await connection.close();
3865
+ }
3866
+ };
3867
+ }
3868
+ async function registerSandboxReaperSchedule(settings, observability) {
3869
+ if (!settings.sandboxOwnershipEnabled) {
3870
+ return { registered: false, close: async () => {
3871
+ } };
3872
+ }
3873
+ const connection = await Connection.connect({ address: settings.temporalHost });
3874
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
3875
+ try {
3876
+ await temporal.schedule.create({
3877
+ scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
3878
+ spec: {
3879
+ // @every-style interval: fire once per reaper period. The boot invariant
3880
+ // (config) guarantees reaperPeriod < viewerHolderTTL and
3881
+ // reaperPeriod + idleGrace < providerLifetime.
3882
+ intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }]
3883
+ },
3884
+ action: {
3885
+ type: "startWorkflow",
3886
+ workflowType: "sandboxReaperWorkflow",
3887
+ taskQueue: settings.temporalTaskQueue,
3888
+ args: []
3889
+ },
3890
+ policies: {
3891
+ // A slow sweep must never overlap itself; the next fire is skipped.
3892
+ overlap: ScheduleOverlapPolicy.SKIP,
3893
+ catchupWindow: "1m",
3894
+ pauseOnFailure: false
3895
+ }
3896
+ });
3897
+ observability.info("Registered the global sandbox-lease reaper Schedule", {
3898
+ scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
3899
+ reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs
3900
+ });
3901
+ return { registered: true, close: async () => {
3902
+ await connection.close();
3903
+ } };
3904
+ } catch (error) {
3905
+ if (error instanceof ScheduleAlreadyRunning) {
3906
+ observability.info("Global sandbox-lease reaper Schedule already registered", {
3907
+ scheduleId: SANDBOX_REAPER_SCHEDULE_ID
3908
+ });
3909
+ return { registered: false, close: async () => {
3910
+ await connection.close();
3911
+ } };
3912
+ }
3913
+ await connection.close().catch(() => void 0);
3914
+ throw error;
3915
+ }
3916
+ }
3917
+ async function startWorker() {
3918
+ const settings = getSettings2();
3919
+ const observability = createObservability2(settings, { component: "worker" });
3920
+ const signaler = await retryStartupDependency(
3921
+ "Temporal client",
3922
+ () => createWorkerWorkflowSignaler(settings),
3923
+ {
3924
+ ...startupRetryOptions(settings),
3925
+ onRetry: (event) => logStartupDependencyRetry(observability, event)
3926
+ }
3927
+ );
3928
+ const { worker, connection } = await createOpenGeniWorker({
3929
+ settings,
3930
+ activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
3931
+ });
3932
+ const reaperSchedule = await retryStartupDependency(
3933
+ "Temporal schedule (sandbox reaper)",
3934
+ () => registerSandboxReaperSchedule(settings, observability),
3935
+ {
3936
+ ...startupRetryOptions(settings),
3937
+ onRetry: (event) => logStartupDependencyRetry(observability, event)
3938
+ }
3939
+ );
3940
+ observability.info("OpenGeni worker listening", {
3941
+ temporalTaskQueue: settings.temporalTaskQueue
3942
+ });
3943
+ try {
3944
+ await worker.run();
3945
+ } finally {
3946
+ await connection.close();
3947
+ await signaler.close();
3948
+ await reaperSchedule.close();
3949
+ }
3950
+ }
3951
+ if (import.meta.main) {
3952
+ await startWorker();
3953
+ }
3954
+ export {
3955
+ createOpenGeniWorker,
3956
+ createWorkerWorkflowSignaler,
3957
+ registerSandboxReaperSchedule,
3958
+ startWorker
3959
+ };
3960
+ //# sourceMappingURL=index.js.map