@opengeni/worker-bundle 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +222 -0
- package/dist/index.js +3960 -0
- package/dist/index.js.map +1 -0
- package/package.json +60 -0
- package/src/activities/agent-turn.ts +2290 -0
- package/src/activities/capabilities.ts +131 -0
- package/src/activities/codex-auth.ts +9 -0
- package/src/activities/codex-rotation.ts +326 -0
- package/src/activities/common.ts +37 -0
- package/src/activities/context-compaction.ts +125 -0
- package/src/activities/documents.ts +67 -0
- package/src/activities/environment.ts +209 -0
- package/src/activities/goals.ts +281 -0
- package/src/activities/packs.ts +116 -0
- package/src/activities/parent-wake.ts +129 -0
- package/src/activities/recording.ts +248 -0
- package/src/activities/redaction.ts +54 -0
- package/src/activities/run-input.ts +331 -0
- package/src/activities/sandbox-lease.ts +612 -0
- package/src/activities/scheduled-tasks.ts +346 -0
- package/src/activities/session-state.ts +201 -0
- package/src/activities/streaming.ts +74 -0
- package/src/activities/types.ts +165 -0
- package/src/activities.ts +124 -0
- package/src/index.ts +184 -0
- package/src/sandbox-resume.ts +558 -0
- package/src/sandbox-routing.ts +232 -0
- package/src/workflows/activities.ts +23 -0
- package/src/workflows/document-index.ts +11 -0
- package/src/workflows/sandbox-reaper.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +38 -0
- package/src/workflows/session.ts +376 -0
- package/src/workflows.ts +19 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,3960 @@
|
|
|
1
|
+
// src/index.ts
|
|
2
|
+
import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
3
|
+
import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
|
|
4
|
+
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
5
|
+
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
6
|
+
|
|
7
|
+
// src/activities.ts
|
|
8
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
9
|
+
import { createDb } from "@opengeni/db";
|
|
10
|
+
import { createDocumentServices } from "@opengeni/documents";
|
|
11
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
12
|
+
import { createObservability } from "@opengeni/observability";
|
|
13
|
+
import { createProductionAgentRuntime } from "@opengeni/runtime";
|
|
14
|
+
import { createObjectStorage as createObjectStorage2 } from "@opengeni/storage";
|
|
15
|
+
|
|
16
|
+
// src/activities/agent-turn.ts
|
|
17
|
+
import {
|
|
18
|
+
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
19
|
+
createTurn,
|
|
20
|
+
applyCreditDebitUpToBalance,
|
|
21
|
+
finishTurn,
|
|
22
|
+
getBillingBalance as getBillingBalance2,
|
|
23
|
+
getSandbox as getSandbox2,
|
|
24
|
+
readActiveSandbox as readActiveSandbox2,
|
|
25
|
+
requireFile as requireFile2,
|
|
26
|
+
getSessionEvent as getSessionEvent3,
|
|
27
|
+
getSessionGoal as getSessionGoal3,
|
|
28
|
+
getSessionTurn,
|
|
29
|
+
isCodexBilledTurn as isCodexBilledTurn2,
|
|
30
|
+
workspaceCodexSubscriptionActive as workspaceCodexSubscriptionActive2,
|
|
31
|
+
getCodexRotationSettings,
|
|
32
|
+
listCodexAccountStatuses,
|
|
33
|
+
fetchCodexUsageForAccount,
|
|
34
|
+
getSessionCodexState,
|
|
35
|
+
recordSessionActiveCodexCredential,
|
|
36
|
+
recordCodexAccountUsage,
|
|
37
|
+
recordCodexAccountConnectors,
|
|
38
|
+
setActiveCodexCredential,
|
|
39
|
+
setCodexCredentialExhausted,
|
|
40
|
+
countConsecutiveReactiveRotations,
|
|
41
|
+
requireSession as requireSession2,
|
|
42
|
+
recordUsageEvent as recordUsageEvent2,
|
|
43
|
+
appendSessionHistoryItems,
|
|
44
|
+
consumeSessionCompactionRequest,
|
|
45
|
+
countSessionHistoryItems,
|
|
46
|
+
getActiveSessionHistoryItems as getActiveSessionHistoryItems3,
|
|
47
|
+
nextSessionHistoryPosition,
|
|
48
|
+
requeuePreemptedTurn,
|
|
49
|
+
saveRunState,
|
|
50
|
+
upsertSandboxSessionEnvelope,
|
|
51
|
+
setSessionStatus,
|
|
52
|
+
setSessionLastInputTokens,
|
|
53
|
+
sumUsageQuantity as sumUsageQuantity2,
|
|
54
|
+
heartbeatLeaseHolder,
|
|
55
|
+
accrueWarmSeconds,
|
|
56
|
+
SandboxLeaseSupersededError as SandboxLeaseSupersededError2
|
|
57
|
+
} from "@opengeni/db";
|
|
58
|
+
import { appendAndPublishEvents as appendAndPublishEvents2 } from "@opengeni/events";
|
|
59
|
+
import {
|
|
60
|
+
sandboxStateEntryFromRunState,
|
|
61
|
+
agentsErrorRunState,
|
|
62
|
+
maxTurnsExceededRunState,
|
|
63
|
+
modelResponseUsageFromSdkEvent,
|
|
64
|
+
normalizeSdkEvent,
|
|
65
|
+
sanitizeHistoryItemsForModel,
|
|
66
|
+
summarizeForCompaction as summarizeForCompaction2
|
|
67
|
+
} from "@opengeni/runtime";
|
|
68
|
+
import { calculateModelUsageCostMicros, configuredModelPricing, configuredStaticUsageLimits as configuredStaticUsageLimits2, sandboxWarmRateMicrosPerSecond } from "@opengeni/config";
|
|
69
|
+
import { CancelledFailure } from "@temporalio/activity";
|
|
70
|
+
|
|
71
|
+
// src/activities/capabilities.ts
|
|
72
|
+
import { environmentsEncryptionKeyBytes, parseModelProvidersJson } from "@opengeni/config";
|
|
73
|
+
import {
|
|
74
|
+
CODEX_APPS_MCP_SERVER_ID,
|
|
75
|
+
CODEX_APPS_MCP_SERVER_NAME,
|
|
76
|
+
CODEX_APPS_MCP_URL,
|
|
77
|
+
CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
78
|
+
CODEX_FALLBACK_MODEL_SLUGS,
|
|
79
|
+
CODEX_MODEL_ID_PREFIX,
|
|
80
|
+
CODEX_PROVIDER_BASE_URL,
|
|
81
|
+
CODEX_PROVIDER_ID
|
|
82
|
+
} from "@opengeni/codex";
|
|
83
|
+
import {
|
|
84
|
+
decryptedCapabilityHeaders,
|
|
85
|
+
listEnabledMcpCapabilityServers,
|
|
86
|
+
workspaceCodexSubscriptionActive
|
|
87
|
+
} from "@opengeni/db";
|
|
88
|
+
async function settingsWithEnabledCapabilityMcpServers(db, workspaceId, settings) {
|
|
89
|
+
const enabled = await listEnabledMcpCapabilityServers(db, workspaceId);
|
|
90
|
+
return settingsWithMcpCapabilityServers(settings, enabled);
|
|
91
|
+
}
|
|
92
|
+
async function settingsWithCodexCredential(db, workspaceId, settings, activeOverride) {
|
|
93
|
+
const active = activeOverride ?? await workspaceCodexSubscriptionActive(db, settings, workspaceId);
|
|
94
|
+
if (!active) {
|
|
95
|
+
return settings;
|
|
96
|
+
}
|
|
97
|
+
const withProvider = withCodexProvider(settings);
|
|
98
|
+
return withCodexAppsMcpServer(withProvider);
|
|
99
|
+
}
|
|
100
|
+
function withCodexAppsMcpServer(settings) {
|
|
101
|
+
if (settings.mcpServers.some((server) => server.id === CODEX_APPS_MCP_SERVER_ID)) {
|
|
102
|
+
return settings;
|
|
103
|
+
}
|
|
104
|
+
return {
|
|
105
|
+
...settings,
|
|
106
|
+
mcpServers: [
|
|
107
|
+
...settings.mcpServers,
|
|
108
|
+
{
|
|
109
|
+
id: CODEX_APPS_MCP_SERVER_ID,
|
|
110
|
+
name: CODEX_APPS_MCP_SERVER_NAME,
|
|
111
|
+
url: CODEX_APPS_MCP_URL,
|
|
112
|
+
timeoutMs: CODEX_APPS_STARTUP_TIMEOUT_MS,
|
|
113
|
+
// Connector availability is per-credential and must re-discover each
|
|
114
|
+
// run; never poison a process-global tools-list cache.
|
|
115
|
+
cacheToolsList: false
|
|
116
|
+
// deliberately NO `headers` — the refreshing bearer is dynamic
|
|
117
|
+
}
|
|
118
|
+
]
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
function withCodexProvider(settings) {
|
|
122
|
+
const providers = parseModelProvidersJson(settings.modelProvidersJson);
|
|
123
|
+
if (providers.some((provider) => provider.id === CODEX_PROVIDER_ID)) {
|
|
124
|
+
return settings;
|
|
125
|
+
}
|
|
126
|
+
const codexProvider = {
|
|
127
|
+
kind: "codex-subscription",
|
|
128
|
+
id: CODEX_PROVIDER_ID,
|
|
129
|
+
label: "Codex (ChatGPT subscription)",
|
|
130
|
+
api: "responses",
|
|
131
|
+
baseUrl: CODEX_PROVIDER_BASE_URL,
|
|
132
|
+
models: CODEX_FALLBACK_MODEL_SLUGS.map((slug) => ({ id: `${CODEX_MODEL_ID_PREFIX}${slug}`, label: slug, reasoningEffort: true }))
|
|
133
|
+
};
|
|
134
|
+
return { ...settings, modelProvidersJson: JSON.stringify([...providers, codexProvider]) };
|
|
135
|
+
}
|
|
136
|
+
function settingsWithMcpCapabilityServers(settings, enabled) {
|
|
137
|
+
if (enabled.length === 0) {
|
|
138
|
+
return settings;
|
|
139
|
+
}
|
|
140
|
+
const encryptionKey = environmentsEncryptionKeyBytes(settings);
|
|
141
|
+
const existingIds = new Set(settings.mcpServers.map((server) => server.id));
|
|
142
|
+
const dynamicServers = enabled.filter((server) => !existingIds.has(server.id)).flatMap((server) => {
|
|
143
|
+
const headers = decryptedCapabilityHeaders(server, encryptionKey);
|
|
144
|
+
if (headers === "unavailable") {
|
|
145
|
+
return [];
|
|
146
|
+
}
|
|
147
|
+
return [{
|
|
148
|
+
id: server.id,
|
|
149
|
+
name: server.name,
|
|
150
|
+
url: server.url,
|
|
151
|
+
...server.allowedTools ? { allowedTools: server.allowedTools } : {},
|
|
152
|
+
...server.timeoutMs ? { timeoutMs: server.timeoutMs } : {},
|
|
153
|
+
cacheToolsList: server.cacheToolsList ?? false,
|
|
154
|
+
...headers ? { headers } : {}
|
|
155
|
+
}];
|
|
156
|
+
});
|
|
157
|
+
return dynamicServers.length ? { ...settings, mcpServers: [...settings.mcpServers, ...dynamicServers] } : settings;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// src/activities/codex-rotation.ts
|
|
161
|
+
var MIN_IDLE_MS = 6e4;
|
|
162
|
+
var DEFAULT_RESET_COOLDOWN_MS = 6e4;
|
|
163
|
+
function bindingUsedPct(acct) {
|
|
164
|
+
return Math.max(acct.primaryUsedPercent ?? 0, acct.secondaryUsedPercent ?? 0);
|
|
165
|
+
}
|
|
166
|
+
function bindingRemaining(acct) {
|
|
167
|
+
const primaryRemaining = 100 - (acct.primaryUsedPercent ?? 0);
|
|
168
|
+
const secondaryRemaining = 100 - (acct.secondaryUsedPercent ?? 0);
|
|
169
|
+
return Math.min(primaryRemaining, secondaryRemaining);
|
|
170
|
+
}
|
|
171
|
+
function cooling(acct, now) {
|
|
172
|
+
return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
|
|
173
|
+
}
|
|
174
|
+
function covers(acct, usedConnectors) {
|
|
175
|
+
if (usedConnectors.length === 0) {
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
const owned = acct.connectorNamespaces;
|
|
179
|
+
if (owned === null) {
|
|
180
|
+
return false;
|
|
181
|
+
}
|
|
182
|
+
const ownedSet = new Set(owned);
|
|
183
|
+
return usedConnectors.every((connector) => ownedSet.has(connector));
|
|
184
|
+
}
|
|
185
|
+
function droppedConnectorsFor(acct, usedConnectors) {
|
|
186
|
+
if (usedConnectors.length === 0) {
|
|
187
|
+
return [];
|
|
188
|
+
}
|
|
189
|
+
const owned = acct.connectorNamespaces;
|
|
190
|
+
if (owned === null) {
|
|
191
|
+
return [...usedConnectors];
|
|
192
|
+
}
|
|
193
|
+
const ownedSet = new Set(owned);
|
|
194
|
+
return usedConnectors.filter((connector) => !ownedSet.has(connector));
|
|
195
|
+
}
|
|
196
|
+
function eligible(acct, nearExhaustionPct, now) {
|
|
197
|
+
return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct) < nearExhaustionPct;
|
|
198
|
+
}
|
|
199
|
+
function availableAt(acct, nearExhaustionPct, now) {
|
|
200
|
+
const nowMs = now.getTime();
|
|
201
|
+
const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
|
|
202
|
+
const candidates = [];
|
|
203
|
+
if (acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > nowMs) {
|
|
204
|
+
candidates.push(acct.exhaustedUntil);
|
|
205
|
+
}
|
|
206
|
+
const windowClear = (over, resetAt) => {
|
|
207
|
+
if (!over) return;
|
|
208
|
+
candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
|
|
209
|
+
};
|
|
210
|
+
windowClear((acct.primaryUsedPercent ?? 0) >= nearExhaustionPct, acct.primaryResetAt);
|
|
211
|
+
windowClear((acct.secondaryUsedPercent ?? 0) >= nearExhaustionPct, acct.secondaryResetAt);
|
|
212
|
+
if (candidates.length === 0) {
|
|
213
|
+
return defaultClear;
|
|
214
|
+
}
|
|
215
|
+
return candidates.reduce((a, b) => b.getTime() > a.getTime() ? b : a);
|
|
216
|
+
}
|
|
217
|
+
function earliestReset(accounts, nearExhaustionPct, now) {
|
|
218
|
+
return accounts.map((acct) => availableAt(acct, nearExhaustionPct, now)).reduce((a, b) => b.getTime() < a.getTime() ? b : a);
|
|
219
|
+
}
|
|
220
|
+
function computeIdleDelayMs(earliestResetAt, now, maxMs) {
|
|
221
|
+
const delta = earliestResetAt.getTime() - now.getTime();
|
|
222
|
+
return Math.min(Math.max(delta, MIN_IDLE_MS), maxMs);
|
|
223
|
+
}
|
|
224
|
+
var REACTIVE_PERSISTENCE_FAULT_FLOOR_MS = 5e3;
|
|
225
|
+
var REACTIVE_ROTATION_MARGIN = 2;
|
|
226
|
+
var REACTIVE_CIRCUIT_BREAKER_IDLE_MS = MIN_IDLE_MS;
|
|
227
|
+
function computeReactiveRotationResume(args) {
|
|
228
|
+
const streak = args.priorConsecutiveRotations + 1;
|
|
229
|
+
const bound = args.connectedAccountCount + REACTIVE_ROTATION_MARGIN;
|
|
230
|
+
if (streak > bound) {
|
|
231
|
+
return { continueDelayMs: REACTIVE_CIRCUIT_BREAKER_IDLE_MS, idleUntilReset: true };
|
|
232
|
+
}
|
|
233
|
+
if (!args.cooldownPersisted) {
|
|
234
|
+
return { continueDelayMs: REACTIVE_PERSISTENCE_FAULT_FLOOR_MS, idleUntilReset: false };
|
|
235
|
+
}
|
|
236
|
+
return { continueDelayMs: 0, idleUntilReset: false };
|
|
237
|
+
}
|
|
238
|
+
function chooseRotationActive(args) {
|
|
239
|
+
const { rotationStrategy, activeCredentialId, priorCredentialId, accounts, nearExhaustionPct, now } = args;
|
|
240
|
+
const usedConnectors = args.usedConnectors ?? [];
|
|
241
|
+
if (accounts.length === 0) {
|
|
242
|
+
return { kind: "none" };
|
|
243
|
+
}
|
|
244
|
+
const eligibles = accounts.filter((acct) => eligible(acct, nearExhaustionPct, now));
|
|
245
|
+
const activeRow = activeCredentialId ? accounts.find((acct) => acct.id === activeCredentialId) ?? null : null;
|
|
246
|
+
const decide = (chosen2) => {
|
|
247
|
+
if (!chosen2) {
|
|
248
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
|
|
249
|
+
}
|
|
250
|
+
const dropped = droppedConnectorsFor(chosen2, usedConnectors);
|
|
251
|
+
return {
|
|
252
|
+
kind: "active",
|
|
253
|
+
credentialId: chosen2.id,
|
|
254
|
+
moved: chosen2.id !== activeCredentialId,
|
|
255
|
+
...dropped.length > 0 ? { droppedConnectors: dropped } : {}
|
|
256
|
+
};
|
|
257
|
+
};
|
|
258
|
+
if (rotationStrategy === "round_robin") {
|
|
259
|
+
if (eligibles.length === 0) {
|
|
260
|
+
return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
|
|
261
|
+
}
|
|
262
|
+
const priorIdx = priorCredentialId ? accounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
|
|
263
|
+
const ordered = priorIdx >= 0 ? [...accounts.slice(priorIdx + 1), ...accounts.slice(0, priorIdx + 1)] : accounts;
|
|
264
|
+
const chosen2 = ordered.find((acct) => eligible(acct, nearExhaustionPct, now));
|
|
265
|
+
return decide(chosen2);
|
|
266
|
+
}
|
|
267
|
+
if (rotationStrategy === "drain_then_next") {
|
|
268
|
+
const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : void 0;
|
|
269
|
+
if (priorRow && eligible(priorRow, nearExhaustionPct, now)) {
|
|
270
|
+
return decide(priorRow);
|
|
271
|
+
}
|
|
272
|
+
return decide(eligibles[0]);
|
|
273
|
+
}
|
|
274
|
+
if (activeRow && eligible(activeRow, nearExhaustionPct, now)) {
|
|
275
|
+
return { kind: "active", credentialId: activeRow.id, moved: false };
|
|
276
|
+
}
|
|
277
|
+
const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
|
|
278
|
+
const pool = covering.length > 0 ? covering : eligibles;
|
|
279
|
+
const chosen = pool.reduce((best, acct) => {
|
|
280
|
+
if (!best) {
|
|
281
|
+
return acct;
|
|
282
|
+
}
|
|
283
|
+
return bindingRemaining(acct) > bindingRemaining(best) ? acct : best;
|
|
284
|
+
}, void 0);
|
|
285
|
+
return decide(chosen);
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
// src/activities/codex-auth.ts
|
|
289
|
+
import { buildCodexTokenResolver } from "@opengeni/db";
|
|
290
|
+
|
|
291
|
+
// src/activities/agent-turn.ts
|
|
292
|
+
import {
|
|
293
|
+
buildModelResolver,
|
|
294
|
+
CODEX_CLIENT_VERSION,
|
|
295
|
+
CODEX_FALLBACK_MODEL_SLUGS as CODEX_FALLBACK_MODEL_SLUGS2,
|
|
296
|
+
classifyCodexUsageLimitError,
|
|
297
|
+
codexRequestStorage
|
|
298
|
+
} from "@opengeni/codex";
|
|
299
|
+
|
|
300
|
+
// src/activities/common.ts
|
|
301
|
+
import {
|
|
302
|
+
mergeResourceRefs,
|
|
303
|
+
mergeToolRefs,
|
|
304
|
+
reasoningEffortForMetadata
|
|
305
|
+
} from "@opengeni/contracts";
|
|
306
|
+
function assertReusableSessionRevivable(status) {
|
|
307
|
+
if (status === "cancelled") {
|
|
308
|
+
throw new Error("reusable session is cancelled; refusing to revive on scheduled fire");
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
function scheduledUserMessagePayload(prompt, resources, tools, taskId, runId) {
|
|
312
|
+
return {
|
|
313
|
+
text: prompt,
|
|
314
|
+
scheduledTaskId: taskId,
|
|
315
|
+
scheduledTaskRunId: runId,
|
|
316
|
+
...resources.length ? { resources } : {},
|
|
317
|
+
...tools.length ? { tools } : {}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
function workflowIdForSession(sessionId) {
|
|
321
|
+
return `session-${sessionId}`;
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// src/activities/context-compaction.ts
|
|
325
|
+
import {
|
|
326
|
+
applyContextCompaction,
|
|
327
|
+
getActiveSessionHistoryItems
|
|
328
|
+
} from "@opengeni/db";
|
|
329
|
+
import {
|
|
330
|
+
buildCompactionMessages,
|
|
331
|
+
buildSummaryItem,
|
|
332
|
+
planCompaction,
|
|
333
|
+
summarizeForCompaction
|
|
334
|
+
} from "@opengeni/runtime";
|
|
335
|
+
import {
|
|
336
|
+
contextInputBudgetTokens,
|
|
337
|
+
resolveContextCompactionMode
|
|
338
|
+
} from "@opengeni/config";
|
|
339
|
+
async function maybeCompactContext(db, settings, scope, lastInputTokens, summarize = (s, m) => summarizeForCompaction(s, m, { maxOutputTokens: s.contextSummaryMaxTokens }), options = {}) {
|
|
340
|
+
if (resolveContextCompactionMode(settings) !== "client") {
|
|
341
|
+
return { compacted: false, reason: "mode_not_client" };
|
|
342
|
+
}
|
|
343
|
+
const active = await getActiveSessionHistoryItems(db, scope.workspaceId, scope.sessionId);
|
|
344
|
+
if (active.length === 0) {
|
|
345
|
+
return { compacted: false, reason: "no_history" };
|
|
346
|
+
}
|
|
347
|
+
const items = active.map((row) => row.item);
|
|
348
|
+
const plan = planCompaction({
|
|
349
|
+
items,
|
|
350
|
+
lastInputTokens,
|
|
351
|
+
inputBudgetTokens: contextInputBudgetTokens(settings),
|
|
352
|
+
softFraction: settings.contextCompactSoftFraction,
|
|
353
|
+
hardFraction: settings.contextCompactHardFraction,
|
|
354
|
+
keepRecentTokens: settings.contextKeepRecentTokens,
|
|
355
|
+
...options.force ? { force: true } : {}
|
|
356
|
+
});
|
|
357
|
+
if (!plan.shouldCompact) {
|
|
358
|
+
return { compacted: false, reason: plan.reason };
|
|
359
|
+
}
|
|
360
|
+
if (plan.hardForced) {
|
|
361
|
+
console.warn(
|
|
362
|
+
`context compaction HARD-FORCED for session ${scope.sessionId}: signal ${plan.signalTokens} tokens at/over hard ceiling`
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
const messages = buildCompactionMessages(plan);
|
|
366
|
+
const summaryBody = await summarize(settings, messages);
|
|
367
|
+
if (!summaryBody) {
|
|
368
|
+
return { compacted: false, reason: "summarize_failed" };
|
|
369
|
+
}
|
|
370
|
+
const boundaryRow = active[plan.boundaryIndex];
|
|
371
|
+
if (!boundaryRow) {
|
|
372
|
+
return { compacted: false, reason: "boundary_out_of_range" };
|
|
373
|
+
}
|
|
374
|
+
const boundaryPosition = boundaryRow.position;
|
|
375
|
+
if (boundaryPosition <= 0) {
|
|
376
|
+
return { compacted: false, reason: "boundary_at_origin" };
|
|
377
|
+
}
|
|
378
|
+
const summaryPosition = boundaryPosition - 0.5;
|
|
379
|
+
await applyContextCompaction(db, {
|
|
380
|
+
accountId: scope.accountId,
|
|
381
|
+
workspaceId: scope.workspaceId,
|
|
382
|
+
sessionId: scope.sessionId,
|
|
383
|
+
turnId: scope.turnId ?? null,
|
|
384
|
+
boundaryPosition,
|
|
385
|
+
summaryPosition,
|
|
386
|
+
summaryItem: buildSummaryItem(summaryBody)
|
|
387
|
+
});
|
|
388
|
+
return { compacted: true, supersededFrom: boundaryPosition, summaryPosition, hardForced: plan.hardForced };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// src/activities/environment.ts
|
|
392
|
+
import {
|
|
393
|
+
applyGitAuthPointerEnvironment,
|
|
394
|
+
stableSandboxEnvironmentForRun
|
|
395
|
+
} from "@opengeni/config";
|
|
396
|
+
import {
|
|
397
|
+
loadWorkspaceEnvironmentForRun as loadWorkspaceEnvironmentForRunFromDb
|
|
398
|
+
} from "@opengeni/db";
|
|
399
|
+
import {
|
|
400
|
+
createGitHubAppInstallationToken,
|
|
401
|
+
githubAppBotIdentity
|
|
402
|
+
} from "@opengeni/github";
|
|
403
|
+
async function loadWorkspaceEnvironmentForRunWithCredentials(db, settings, scope, environmentId, sandboxSecrets) {
|
|
404
|
+
if (!sandboxSecrets) {
|
|
405
|
+
return loadWorkspaceEnvironmentForRunFromDb(db, settings, scope.workspaceId, environmentId);
|
|
406
|
+
}
|
|
407
|
+
if (!environmentId) {
|
|
408
|
+
return null;
|
|
409
|
+
}
|
|
410
|
+
const secrets = await sandboxSecrets({
|
|
411
|
+
accountId: scope.accountId,
|
|
412
|
+
workspaceId: scope.workspaceId,
|
|
413
|
+
environmentId
|
|
414
|
+
});
|
|
415
|
+
assertWorkspaceEcho("sandboxSecrets", scope, secrets.workspaceId);
|
|
416
|
+
return {
|
|
417
|
+
id: secrets.id ?? environmentId,
|
|
418
|
+
name: secrets.name ?? environmentId,
|
|
419
|
+
description: secrets.description ?? null,
|
|
420
|
+
values: secrets.values
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
function assertWorkspaceEcho(kind, scope, echoedWorkspaceId) {
|
|
424
|
+
if (echoedWorkspaceId !== scope.workspaceId) {
|
|
425
|
+
throw new Error(
|
|
426
|
+
`connection-credential provider (${kind}) scoped to workspace ${echoedWorkspaceId} but the run is workspace ${scope.workspaceId}`
|
|
427
|
+
);
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironment = {}, options = {}) {
|
|
431
|
+
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment);
|
|
432
|
+
const selection = githubRepositorySelection(resources);
|
|
433
|
+
if (!selection || options.skipGitHubToken) {
|
|
434
|
+
return { environment };
|
|
435
|
+
}
|
|
436
|
+
let token;
|
|
437
|
+
let identity;
|
|
438
|
+
if (options?.gitCredentials && options.scope) {
|
|
439
|
+
const minted = await options.gitCredentials({
|
|
440
|
+
accountId: options.scope.accountId,
|
|
441
|
+
workspaceId: options.scope.workspaceId,
|
|
442
|
+
installationId: selection.installationId,
|
|
443
|
+
repositoryIds: selection.repositoryIds
|
|
444
|
+
});
|
|
445
|
+
assertWorkspaceEcho("gitCredentials", options.scope, minted.workspaceId);
|
|
446
|
+
token = minted.token;
|
|
447
|
+
identity = minted.identity ?? githubAppBotIdentity(settings);
|
|
448
|
+
} else {
|
|
449
|
+
token = await createGitHubAppInstallationToken(settings, {
|
|
450
|
+
installationId: selection.installationId,
|
|
451
|
+
repositoryIds: selection.repositoryIds
|
|
452
|
+
});
|
|
453
|
+
identity = githubAppBotIdentity(settings);
|
|
454
|
+
}
|
|
455
|
+
applyGitAuthPointerEnvironment(environment, identity);
|
|
456
|
+
return { environment, gitToken: token };
|
|
457
|
+
}
|
|
458
|
+
function githubRepositorySelection(resources) {
|
|
459
|
+
const selected = resources.flatMap((resource) => {
|
|
460
|
+
if (resource.kind !== "repository") {
|
|
461
|
+
return [];
|
|
462
|
+
}
|
|
463
|
+
const installationId2 = positiveInteger(resource.githubInstallationId);
|
|
464
|
+
const repositoryId = positiveInteger(resource.githubRepositoryId);
|
|
465
|
+
return installationId2 && repositoryId ? [{ installationId: installationId2, repositoryId }] : [];
|
|
466
|
+
});
|
|
467
|
+
if (selected.length === 0) {
|
|
468
|
+
return null;
|
|
469
|
+
}
|
|
470
|
+
const installationId = selected[0].installationId;
|
|
471
|
+
if (selected.some((item) => item.installationId !== installationId)) {
|
|
472
|
+
throw new Error("GitHub App repository resources must belong to one installation");
|
|
473
|
+
}
|
|
474
|
+
return {
|
|
475
|
+
installationId,
|
|
476
|
+
repositoryIds: selected.map((item) => item.repositoryId)
|
|
477
|
+
};
|
|
478
|
+
}
|
|
479
|
+
function positiveInteger(value) {
|
|
480
|
+
if (typeof value === "number" && Number.isInteger(value) && value > 0) {
|
|
481
|
+
return value;
|
|
482
|
+
}
|
|
483
|
+
if (typeof value === "string" && /^\d+$/.test(value) && Number(value) > 0) {
|
|
484
|
+
return Number(value);
|
|
485
|
+
}
|
|
486
|
+
return null;
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// src/activities/goals.ts
|
|
490
|
+
import { configuredStaticUsageLimits } from "@opengeni/config";
|
|
491
|
+
import {
|
|
492
|
+
mergeToolRefs as mergeToolRefs2,
|
|
493
|
+
reasoningEffortForMetadata as reasoningEffortForMetadata2
|
|
494
|
+
} from "@opengeni/contracts";
|
|
495
|
+
import {
|
|
496
|
+
enqueueSessionTurn,
|
|
497
|
+
evaluateGoalContinuation,
|
|
498
|
+
getBillingBalance,
|
|
499
|
+
getSessionEvent,
|
|
500
|
+
getSessionGoal,
|
|
501
|
+
isCodexBilledTurn,
|
|
502
|
+
recordUsageEvent,
|
|
503
|
+
requireSession,
|
|
504
|
+
setSessionGoalLastContinuationTurn,
|
|
505
|
+
setSessionGoalStatus,
|
|
506
|
+
sumUsageQuantity
|
|
507
|
+
} from "@opengeni/db";
|
|
508
|
+
import { appendAndPublishEvents } from "@opengeni/events";
|
|
509
|
+
function createGoalActivities(services) {
|
|
510
|
+
async function maybeContinueGoal2(input) {
|
|
511
|
+
const { settings, db, bus } = await services();
|
|
512
|
+
const existingGoal = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
513
|
+
if (!existingGoal || existingGoal.status !== "active") {
|
|
514
|
+
return { action: "none" };
|
|
515
|
+
}
|
|
516
|
+
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
517
|
+
const isCodexRun = await isCodexBilledTurn({ db, settings, workspaceId: input.workspaceId, model: session.model });
|
|
518
|
+
const budgetBlocked = await goalRunBudgetBlocked(settings, db, input.accountId, input.workspaceId, isCodexRun);
|
|
519
|
+
const decision = await evaluateGoalContinuation(db, {
|
|
520
|
+
workspaceId: input.workspaceId,
|
|
521
|
+
sessionId: input.sessionId,
|
|
522
|
+
defaultMaxAutoContinuations: settings.goalMaxAutoContinuations ?? null,
|
|
523
|
+
noProgressLimit: settings.goalNoProgressLimit,
|
|
524
|
+
budgetBlocked
|
|
525
|
+
});
|
|
526
|
+
if (decision.decision === "none" || decision.decision === "queue") {
|
|
527
|
+
return { action: decision.decision };
|
|
528
|
+
}
|
|
529
|
+
if (decision.decision === "paused") {
|
|
530
|
+
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
|
|
531
|
+
type: "goal.paused",
|
|
532
|
+
payload: {
|
|
533
|
+
goalId: decision.goal.id,
|
|
534
|
+
actor: "system",
|
|
535
|
+
reason: decision.reason,
|
|
536
|
+
...decision.goal.rationale ? { rationale: decision.goal.rationale } : {},
|
|
537
|
+
autoContinuations: decision.goal.autoContinuations,
|
|
538
|
+
noProgressStreak: decision.goal.noProgressStreak
|
|
539
|
+
}
|
|
540
|
+
}]);
|
|
541
|
+
return { action: "paused" };
|
|
542
|
+
}
|
|
543
|
+
const recheck = await getSessionGoal(db, input.workspaceId, input.sessionId);
|
|
544
|
+
if (!recheck || recheck.status !== "active" || recheck.version !== decision.goal.version) {
|
|
545
|
+
return { action: "none" };
|
|
546
|
+
}
|
|
547
|
+
const prompt = goalContinuationPrompt(decision.goal, decision.autoContinuation, decision.cap);
|
|
548
|
+
const [continuationEvent] = await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
|
|
549
|
+
type: "goal.continuation",
|
|
550
|
+
payload: {
|
|
551
|
+
goalId: decision.goal.id,
|
|
552
|
+
text: prompt,
|
|
553
|
+
autoContinuation: decision.autoContinuation,
|
|
554
|
+
maxAutoContinuations: decision.cap,
|
|
555
|
+
goalVersion: decision.goal.version
|
|
556
|
+
}
|
|
557
|
+
}]);
|
|
558
|
+
if (!continuationEvent) {
|
|
559
|
+
throw new Error("failed to append goal continuation trigger event");
|
|
560
|
+
}
|
|
561
|
+
const turn = await enqueueSessionTurn(db, {
|
|
562
|
+
accountId: input.accountId,
|
|
563
|
+
workspaceId: input.workspaceId,
|
|
564
|
+
sessionId: input.sessionId,
|
|
565
|
+
triggerEventId: continuationEvent.id,
|
|
566
|
+
temporalWorkflowId: input.workflowId,
|
|
567
|
+
source: "goal",
|
|
568
|
+
prompt,
|
|
569
|
+
resources: [],
|
|
570
|
+
// Continuations keep the session tool surface and force the first-party
|
|
571
|
+
// server so the goal_complete/goal_pause escape hatches stay reachable.
|
|
572
|
+
tools: withFirstPartyTools(settings, session.tools),
|
|
573
|
+
model: session.model,
|
|
574
|
+
reasoningEffort: reasoningEffortForMetadata2(session.metadata, settings.openaiReasoningEffort),
|
|
575
|
+
sandboxBackend: session.sandboxBackend,
|
|
576
|
+
metadata: { goalId: decision.goal.id, autoContinuation: decision.autoContinuation }
|
|
577
|
+
});
|
|
578
|
+
await setSessionGoalLastContinuationTurn(db, input.workspaceId, input.sessionId, turn.id);
|
|
579
|
+
await appendAndPublishEvents(db, bus, input.workspaceId, input.sessionId, [{
|
|
580
|
+
type: "turn.queued",
|
|
581
|
+
turnId: turn.id,
|
|
582
|
+
payload: { turnId: turn.id, triggerEventId: continuationEvent.id, source: turn.source }
|
|
583
|
+
}]);
|
|
584
|
+
await recordUsageEvent(db, {
|
|
585
|
+
accountId: input.accountId,
|
|
586
|
+
workspaceId: input.workspaceId,
|
|
587
|
+
eventType: "agent_run.created",
|
|
588
|
+
quantity: 1,
|
|
589
|
+
unit: "run",
|
|
590
|
+
sourceResourceType: "session_turn",
|
|
591
|
+
sourceResourceId: turn.id,
|
|
592
|
+
idempotencyKey: `agent_run.created:goal:${input.workspaceId}:${turn.id}`
|
|
593
|
+
});
|
|
594
|
+
return { action: "continue" };
|
|
595
|
+
}
|
|
596
|
+
async function pauseGoalForInterrupt2(input) {
|
|
597
|
+
const { db, bus } = await services();
|
|
598
|
+
if (input.triggerEventId) {
|
|
599
|
+
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
600
|
+
if (isSteerInterrupt(trigger)) {
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
|
|
605
|
+
}
|
|
606
|
+
return {
|
|
607
|
+
maybeContinueGoal: maybeContinueGoal2,
|
|
608
|
+
pauseGoalForInterrupt: pauseGoalForInterrupt2
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
function isSteerInterrupt(trigger) {
|
|
612
|
+
if (!trigger || trigger.type !== "user.interrupt") {
|
|
613
|
+
return false;
|
|
614
|
+
}
|
|
615
|
+
const payload = trigger.payload;
|
|
616
|
+
return typeof payload === "object" && payload !== null && payload.reason === "steer";
|
|
617
|
+
}
|
|
618
|
+
async function pauseActiveGoalOnInterrupt(db, bus, workspaceId, sessionId) {
|
|
619
|
+
const goal = await getSessionGoal(db, workspaceId, sessionId);
|
|
620
|
+
if (!goal || goal.status !== "active") {
|
|
621
|
+
return;
|
|
622
|
+
}
|
|
623
|
+
const { goal: paused, changed } = await setSessionGoalStatus(db, workspaceId, sessionId, {
|
|
624
|
+
status: "paused",
|
|
625
|
+
pausedReason: "user_interrupt"
|
|
626
|
+
});
|
|
627
|
+
if (changed) {
|
|
628
|
+
await appendAndPublishEvents(db, bus, workspaceId, sessionId, [{
|
|
629
|
+
type: "goal.paused",
|
|
630
|
+
payload: {
|
|
631
|
+
goalId: paused.id,
|
|
632
|
+
actor: "user",
|
|
633
|
+
reason: "user_interrupt",
|
|
634
|
+
autoContinuations: paused.autoContinuations,
|
|
635
|
+
noProgressStreak: paused.noProgressStreak
|
|
636
|
+
}
|
|
637
|
+
}]);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
function goalContinuationPrompt(goal, autoContinuation, cap) {
|
|
641
|
+
const counter = cap === null ? `${autoContinuation}` : `${autoContinuation}/${cap}`;
|
|
642
|
+
return [
|
|
643
|
+
`[GOAL CONTINUATION ${counter}] The session goal is not done. Goal: ${goal.text}.`,
|
|
644
|
+
`Success criteria: ${goal.successCriteria ?? "none specified"}.`,
|
|
645
|
+
"Continue working toward the goal now. If it is actually complete, call opengeni__goal_complete with concrete evidence.",
|
|
646
|
+
"If you are blocked or continuing is not productive, call opengeni__goal_pause with your rationale.",
|
|
647
|
+
"You may revise the goal with opengeni__goal_update. Do not stop without one of these explicit actions."
|
|
648
|
+
].join("\n");
|
|
649
|
+
}
|
|
650
|
+
function withFirstPartyTools(settings, tools) {
|
|
651
|
+
if (!settings.mcpServers.some((server) => server.id === "opengeni")) {
|
|
652
|
+
return tools;
|
|
653
|
+
}
|
|
654
|
+
return mergeToolRefs2(tools, [{ kind: "mcp", id: "opengeni" }]);
|
|
655
|
+
}
|
|
656
|
+
function withCodexAppsTool(settings, tools) {
|
|
657
|
+
if (!settings.mcpServers.some((server) => server.id === "codex_apps")) {
|
|
658
|
+
return tools;
|
|
659
|
+
}
|
|
660
|
+
return mergeToolRefs2(tools, [{ kind: "mcp", id: "codex_apps" }]);
|
|
661
|
+
}
|
|
662
|
+
async function goalRunBudgetBlocked(settings, db, accountId, workspaceId, isCodexRun) {
|
|
663
|
+
if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
664
|
+
const balance = await getBillingBalance(db, accountId);
|
|
665
|
+
if (balance.balanceMicros <= 0) {
|
|
666
|
+
return "insufficient OpenGeni credits";
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
670
|
+
const limits = configuredStaticUsageLimits(settings);
|
|
671
|
+
if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
|
|
672
|
+
const used = await sumUsageQuantity(db, {
|
|
673
|
+
accountId,
|
|
674
|
+
eventType: "model.cost",
|
|
675
|
+
since: startOfUtcMonth()
|
|
676
|
+
});
|
|
677
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
678
|
+
return `monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`;
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
682
|
+
const used = await sumUsageQuantity(db, {
|
|
683
|
+
workspaceId,
|
|
684
|
+
eventType: "agent_run.created",
|
|
685
|
+
since: startOfUtcMonth()
|
|
686
|
+
});
|
|
687
|
+
if (used + 1 > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
688
|
+
return `monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
return null;
|
|
693
|
+
}
|
|
694
|
+
function startOfUtcMonth() {
|
|
695
|
+
const now = /* @__PURE__ */ new Date();
|
|
696
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// src/activities/packs.ts
|
|
700
|
+
import { CapabilityPack } from "@opengeni/contracts";
|
|
701
|
+
import { getWorkspace, getWorkspacePack, listPackInstallations } from "@opengeni/db";
|
|
702
|
+
var emptyPackRuntime = { sandboxImage: null, skills: [] };
|
|
703
|
+
async function resolveWorkspacePackRuntime(db, workspaceId) {
|
|
704
|
+
const installations = await listPackInstallations(db, workspaceId);
|
|
705
|
+
const active = installations.filter((installation) => installation.status === "active");
|
|
706
|
+
if (active.length === 0) {
|
|
707
|
+
return emptyPackRuntime;
|
|
708
|
+
}
|
|
709
|
+
const packs = [];
|
|
710
|
+
for (const installation of active) {
|
|
711
|
+
const registration = await getWorkspacePack(db, workspaceId, installation.packId);
|
|
712
|
+
if (!registration) {
|
|
713
|
+
continue;
|
|
714
|
+
}
|
|
715
|
+
const parsed = CapabilityPack.safeParse(registration.pack);
|
|
716
|
+
if (parsed.success) {
|
|
717
|
+
packs.push(parsed.data);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
return workspacePackRuntimeFromPacks(packs);
|
|
721
|
+
}
|
|
722
|
+
function workspacePackRuntimeFromPacks(packs) {
|
|
723
|
+
const imagePacks = packs.filter((pack) => typeof pack.sandboxImage === "string" && pack.sandboxImage.trim().length > 0);
|
|
724
|
+
if (imagePacks.length > 1) {
|
|
725
|
+
const ids = imagePacks.map((pack) => pack.id).sort().join(", ");
|
|
726
|
+
throw new Error(
|
|
727
|
+
`Multiple enabled packs declare a sandbox image (${ids}). Only one enabled pack per workspace may declare sandboxImage; disable the others and retry.`
|
|
728
|
+
);
|
|
729
|
+
}
|
|
730
|
+
const skills = [];
|
|
731
|
+
const skillOwners = /* @__PURE__ */ new Map();
|
|
732
|
+
for (const pack of [...packs].sort((a, b) => a.id.localeCompare(b.id))) {
|
|
733
|
+
for (const skill of pack.skills) {
|
|
734
|
+
const key = skill.name.toLowerCase();
|
|
735
|
+
const existingOwner = skillOwners.get(key);
|
|
736
|
+
if (existingOwner !== void 0 && existingOwner !== pack.id) {
|
|
737
|
+
throw new Error(
|
|
738
|
+
`Enabled packs ${existingOwner} and ${pack.id} both declare a skill named "${skill.name}". Pack skill names must be unique across enabled packs; disable one of the packs and retry.`
|
|
739
|
+
);
|
|
740
|
+
}
|
|
741
|
+
skillOwners.set(key, pack.id);
|
|
742
|
+
skills.push({
|
|
743
|
+
name: skill.name,
|
|
744
|
+
description: skill.description ?? null,
|
|
745
|
+
files: skill.files.map((file) => ({ path: file.path, content: file.content }))
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
return {
|
|
750
|
+
sandboxImage: imagePacks[0]?.sandboxImage?.trim() ?? null,
|
|
751
|
+
skills
|
|
752
|
+
};
|
|
753
|
+
}
|
|
754
|
+
async function resolveWorkspaceAgentInstructions(db, workspaceId) {
|
|
755
|
+
const workspace = await getWorkspace(db, workspaceId);
|
|
756
|
+
return workspace?.agentInstructions ?? null;
|
|
757
|
+
}
|
|
758
|
+
function settingsWithPackSandboxImage(settings, sandboxImage) {
|
|
759
|
+
if (!sandboxImage) {
|
|
760
|
+
return settings;
|
|
761
|
+
}
|
|
762
|
+
return {
|
|
763
|
+
...settings,
|
|
764
|
+
dockerImage: sandboxImage,
|
|
765
|
+
modalImageRef: sandboxImage
|
|
766
|
+
};
|
|
767
|
+
}
|
|
768
|
+
|
|
769
|
+
// src/activities/parent-wake.ts
|
|
770
|
+
import { getSession, getSessionGoal as getSessionGoal2, wakeParentSessionForChildCompletion } from "@opengeni/db";
|
|
771
|
+
async function notifyParentOfChildTerminal(svc, workspaceId, childSessionId, terminalStatus, episodeKey) {
|
|
772
|
+
try {
|
|
773
|
+
const child = await getSession(svc.db, workspaceId, childSessionId);
|
|
774
|
+
if (!child || !child.parentSessionId) {
|
|
775
|
+
return;
|
|
776
|
+
}
|
|
777
|
+
const goal = await getSessionGoal2(svc.db, workspaceId, childSessionId);
|
|
778
|
+
const clientEventId = `child-completion:${childSessionId}:${episodeKey ?? child.lastSequence}`;
|
|
779
|
+
const result = await wakeParentSessionForChildCompletion(svc.db, {
|
|
780
|
+
workspaceId,
|
|
781
|
+
parentSessionId: child.parentSessionId,
|
|
782
|
+
clientEventId,
|
|
783
|
+
text: childCompletionWakeText(child, goal, terminalStatus),
|
|
784
|
+
childCompletion: childCompletionPayload(child, goal, terminalStatus),
|
|
785
|
+
reasoningEffortFallback: svc.settings.openaiReasoningEffort
|
|
786
|
+
});
|
|
787
|
+
if (!result.delivered) {
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
790
|
+
await svc.bus.publish(workspaceId, child.parentSessionId, result.events);
|
|
791
|
+
if (svc.wakeSessionWorkflow) {
|
|
792
|
+
await svc.wakeSessionWorkflow({
|
|
793
|
+
accountId: child.accountId,
|
|
794
|
+
workspaceId,
|
|
795
|
+
sessionId: child.parentSessionId,
|
|
796
|
+
workflowId: result.temporalWorkflowId
|
|
797
|
+
});
|
|
798
|
+
}
|
|
799
|
+
svc.observability.info("Woke parent session on worker completion", {
|
|
800
|
+
childSessionId,
|
|
801
|
+
parentSessionId: child.parentSessionId,
|
|
802
|
+
terminalStatus
|
|
803
|
+
});
|
|
804
|
+
} catch (error) {
|
|
805
|
+
svc.observability.error("Failed to wake parent session on worker completion", {
|
|
806
|
+
childSessionId,
|
|
807
|
+
error: error instanceof Error ? error.message : String(error)
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
function childCompletionPayload(child, goal, terminalStatus) {
|
|
812
|
+
return {
|
|
813
|
+
childSessionId: child.id,
|
|
814
|
+
status: terminalStatus,
|
|
815
|
+
...goal ? {
|
|
816
|
+
goal: {
|
|
817
|
+
status: goal.status,
|
|
818
|
+
text: goal.text,
|
|
819
|
+
...goal.evidence ? { evidence: goal.evidence } : {},
|
|
820
|
+
...goal.rationale ? { rationale: goal.rationale } : {},
|
|
821
|
+
...goal.pausedReason ? { pausedReason: goal.pausedReason } : {}
|
|
822
|
+
}
|
|
823
|
+
} : {}
|
|
824
|
+
};
|
|
825
|
+
}
|
|
826
|
+
function childCompletionWakeText(child, goal, terminalStatus) {
|
|
827
|
+
const lines = [];
|
|
828
|
+
if (terminalStatus === "failed") {
|
|
829
|
+
lines.push(`A worker session you spawned has FAILED. Worker session id: ${child.id}.`);
|
|
830
|
+
} else if (goal?.status === "completed") {
|
|
831
|
+
lines.push(`A worker session you spawned has COMPLETED its goal. Worker session id: ${child.id}.`);
|
|
832
|
+
} else if (goal?.status === "paused") {
|
|
833
|
+
lines.push(`A worker session you spawned has PAUSED its goal and gone idle. Worker session id: ${child.id}.`);
|
|
834
|
+
} else {
|
|
835
|
+
lines.push(`A worker session you spawned has finished its work and gone idle. Worker session id: ${child.id}.`);
|
|
836
|
+
}
|
|
837
|
+
if (goal) {
|
|
838
|
+
lines.push(`Worker goal: ${goal.text}`);
|
|
839
|
+
if (goal.status === "completed" && goal.evidence) {
|
|
840
|
+
lines.push(`Completion evidence: ${goal.evidence}`);
|
|
841
|
+
}
|
|
842
|
+
if (goal.status === "paused" && goal.rationale) {
|
|
843
|
+
lines.push(`Pause rationale: ${goal.rationale}`);
|
|
844
|
+
}
|
|
845
|
+
}
|
|
846
|
+
lines.push("Read the worker's session events/notebook output for its result, then continue. If your own goal was paused awaiting this worker, resume it now.");
|
|
847
|
+
return lines.join("\n");
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
// src/activities/redaction.ts
|
|
851
|
+
var MIN_REDACTABLE_VALUE_LENGTH = 6;
|
|
852
|
+
function createSecretRedactor(secrets) {
|
|
853
|
+
const usable = secrets.filter((secret) => secret.value.length >= MIN_REDACTABLE_VALUE_LENGTH).sort((a, b) => b.value.length - a.value.length);
|
|
854
|
+
if (usable.length === 0) {
|
|
855
|
+
return identityRedactor;
|
|
856
|
+
}
|
|
857
|
+
const redact = (payload) => {
|
|
858
|
+
if (typeof payload === "string") {
|
|
859
|
+
let out = payload;
|
|
860
|
+
for (const secret of usable) {
|
|
861
|
+
if (out.includes(secret.value)) {
|
|
862
|
+
out = out.split(secret.value).join(`[redacted:${secret.name}]`);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
return out;
|
|
866
|
+
}
|
|
867
|
+
if (Array.isArray(payload)) {
|
|
868
|
+
return payload.map(redact);
|
|
869
|
+
}
|
|
870
|
+
if (isPlainObject(payload)) {
|
|
871
|
+
return Object.fromEntries(Object.entries(payload).map(([key, value]) => [key, redact(value)]));
|
|
872
|
+
}
|
|
873
|
+
return payload;
|
|
874
|
+
};
|
|
875
|
+
return redact;
|
|
876
|
+
}
|
|
877
|
+
function identityRedactor(payload) {
|
|
878
|
+
return payload;
|
|
879
|
+
}
|
|
880
|
+
function isPlainObject(value) {
|
|
881
|
+
if (!value || typeof value !== "object") {
|
|
882
|
+
return false;
|
|
883
|
+
}
|
|
884
|
+
const proto = Object.getPrototypeOf(value);
|
|
885
|
+
return proto === Object.prototype || proto === null;
|
|
886
|
+
}
|
|
887
|
+
|
|
888
|
+
// src/activities/run-input.ts
|
|
889
|
+
import { contextInputBudgetTokens as contextInputBudgetTokens2, resolveContextCompactionMode as resolveContextCompactionMode2 } from "@opengeni/config";
|
|
890
|
+
import {
|
|
891
|
+
getActiveSessionHistoryItems as getActiveSessionHistoryItems2,
|
|
892
|
+
getLatestRunState,
|
|
893
|
+
getSandboxSessionEnvelope,
|
|
894
|
+
requireFile
|
|
895
|
+
} from "@opengeni/db";
|
|
896
|
+
import { stripReasoningEncryptedContent, stripReasoningIdentityFromSerializedRunState, neutralizeToolSearchItemsInSerializedRunState } from "@opengeni/runtime";
|
|
897
|
+
var NON_CODEX_TURN = { currentCodexCredentialId: null };
|
|
898
|
+
function applyCodexHistoryStrip(rows, current) {
|
|
899
|
+
const out = [];
|
|
900
|
+
for (const row of rows) {
|
|
901
|
+
if (row.producerCodexCredentialId === current.currentCodexCredentialId) {
|
|
902
|
+
out.push(row.item);
|
|
903
|
+
continue;
|
|
904
|
+
}
|
|
905
|
+
const type = typeof row.item.type === "string" ? row.item.type : void 0;
|
|
906
|
+
if (type === "reasoning") {
|
|
907
|
+
continue;
|
|
908
|
+
}
|
|
909
|
+
if (type === "tool_search_call" || type === "tool_search_output") {
|
|
910
|
+
continue;
|
|
911
|
+
}
|
|
912
|
+
if (type === "compaction") {
|
|
913
|
+
out.push(stripReasoningEncryptedContent(row.item));
|
|
914
|
+
continue;
|
|
915
|
+
}
|
|
916
|
+
out.push(row.item);
|
|
917
|
+
}
|
|
918
|
+
return out;
|
|
919
|
+
}
|
|
920
|
+
function resumeRunStateForCodexAccount(state, current) {
|
|
921
|
+
if (state.frozenCodexCredentialId === current.currentCodexCredentialId) {
|
|
922
|
+
return state.serializedRunState;
|
|
923
|
+
}
|
|
924
|
+
return neutralizeToolSearchItemsInSerializedRunState(stripReasoningIdentityFromSerializedRunState(state.serializedRunState));
|
|
925
|
+
}
|
|
926
|
+
async function turnInput(db, runtime, agent, trigger, settings, current = NON_CODEX_TURN) {
|
|
927
|
+
if (!trigger) {
|
|
928
|
+
throw new Error("Missing trigger event");
|
|
929
|
+
}
|
|
930
|
+
if (trigger.type === "user.message") {
|
|
931
|
+
const payload = trigger.payload;
|
|
932
|
+
if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
|
|
933
|
+
throw new Error("user.message payload is missing text");
|
|
934
|
+
}
|
|
935
|
+
const text = await userMessageTextWithAttachments(
|
|
936
|
+
db,
|
|
937
|
+
trigger.workspaceId,
|
|
938
|
+
payload.text,
|
|
939
|
+
Array.isArray(payload.resources) ? payload.resources : []
|
|
940
|
+
);
|
|
941
|
+
return await messageInput(db, runtime, agent, trigger, text, settings, current);
|
|
942
|
+
}
|
|
943
|
+
if (trigger.type === "goal.continuation") {
|
|
944
|
+
const payload = trigger.payload;
|
|
945
|
+
if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
|
|
946
|
+
throw new Error("goal.continuation payload is missing text");
|
|
947
|
+
}
|
|
948
|
+
return await messageInput(db, runtime, agent, trigger, payload.text, settings, current);
|
|
949
|
+
}
|
|
950
|
+
if (trigger.type === "turn.preempted") {
|
|
951
|
+
const payload = trigger.payload;
|
|
952
|
+
if (typeof payload.text !== "string" || payload.text.trim().length === 0) {
|
|
953
|
+
throw new Error("turn.preempted payload is missing text");
|
|
954
|
+
}
|
|
955
|
+
return await messageInput(db, runtime, agent, trigger, payload.text, settings, current);
|
|
956
|
+
}
|
|
957
|
+
if (trigger.type === "user.approvalDecision") {
|
|
958
|
+
const payload = trigger.payload;
|
|
959
|
+
const state = await getLatestRunState(db, trigger.workspaceId, trigger.sessionId);
|
|
960
|
+
if (!state) {
|
|
961
|
+
throw new Error("No saved run state is available for approval decision");
|
|
962
|
+
}
|
|
963
|
+
return {
|
|
964
|
+
input: await runtime.prepareInput(agent, {
|
|
965
|
+
kind: "approval",
|
|
966
|
+
// Cross-account run-state strip (HOLE C): if the account resuming this
|
|
967
|
+
// frozen approval differs from the one that froze it, neutralize the
|
|
968
|
+
// blob's account-bound reasoning before replay (else byte-for-byte).
|
|
969
|
+
serializedRunState: resumeRunStateForCodexAccount(state, current),
|
|
970
|
+
approvalId: String(payload.approvalId ?? ""),
|
|
971
|
+
decision: payload.decision === "approve" ? "approve" : "reject",
|
|
972
|
+
...typeof payload.message === "string" ? { message: payload.message } : {}
|
|
973
|
+
}),
|
|
974
|
+
// Model seeded from the run-state BLOB (neutralize-in-place), NOT stripped
|
|
975
|
+
// items: the reconcile watermark must NOT apply the cross-account strip
|
|
976
|
+
// (HOLE E) — else a cross-account approval resume re-appends K
|
|
977
|
+
// already-persisted items at fresh positions.
|
|
978
|
+
modelHistoryFromItems: false
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
throw new Error(`Unsupported trigger event type: ${trigger.type}`);
|
|
982
|
+
}
|
|
983
|
+
async function messageInput(db, runtime, agent, trigger, text, settings, current = NON_CODEX_TURN) {
|
|
984
|
+
const inputBudgetTokens = readPathBudgetTokens(settings);
|
|
985
|
+
if (settings?.sessionHistorySource === "items") {
|
|
986
|
+
const stored = await getActiveSessionHistoryItems2(db, trigger.workspaceId, trigger.sessionId);
|
|
987
|
+
if (stored.length > 0) {
|
|
988
|
+
const envelope = await getSandboxSessionEnvelope(db, trigger.workspaceId, trigger.sessionId);
|
|
989
|
+
const historyItems = applyCodexHistoryStrip(stored, current);
|
|
990
|
+
return {
|
|
991
|
+
input: await runtime.prepareInput(
|
|
992
|
+
agent,
|
|
993
|
+
{
|
|
994
|
+
kind: "message",
|
|
995
|
+
text,
|
|
996
|
+
historyItems,
|
|
997
|
+
sandboxEnvelope: envelope
|
|
998
|
+
},
|
|
999
|
+
inputBudgetTokens ? { inputBudgetTokens } : {}
|
|
1000
|
+
),
|
|
1001
|
+
// state.history seeded from the cross-account-STRIPPED active items: the
|
|
1002
|
+
// reconcile watermark must apply the SAME strip (HOLE D).
|
|
1003
|
+
modelHistoryFromItems: true
|
|
1004
|
+
};
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
const latestState = await getLatestRunState(db, trigger.workspaceId, trigger.sessionId);
|
|
1008
|
+
return {
|
|
1009
|
+
input: await runtime.prepareInput(
|
|
1010
|
+
agent,
|
|
1011
|
+
{
|
|
1012
|
+
kind: "message",
|
|
1013
|
+
text,
|
|
1014
|
+
// Cross-account run-state strip (HOLE C): the items-mode fallback replays
|
|
1015
|
+
// the RunState blob when no history rows exist yet. If the resuming turn's
|
|
1016
|
+
// codex account differs from the one that froze the blob, neutralize its
|
|
1017
|
+
// account-bound reasoning before replay (else byte-for-byte).
|
|
1018
|
+
serializedRunState: latestState ? resumeRunStateForCodexAccount(latestState, current) : null
|
|
1019
|
+
},
|
|
1020
|
+
inputBudgetTokens ? { inputBudgetTokens } : {}
|
|
1021
|
+
),
|
|
1022
|
+
// state.history seeded from the run-state BLOB (or empty): NOT the stripped
|
|
1023
|
+
// items, so the reconcile watermark must NOT apply the cross-account strip
|
|
1024
|
+
// (HOLE E). On this fallback the active rows are empty anyway (the read above
|
|
1025
|
+
// took the items branch when stored.length > 0), so strip-or-not both yield 0.
|
|
1026
|
+
modelHistoryFromItems: false
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
function readPathBudgetTokens(settings) {
|
|
1030
|
+
if (!settings || resolveContextCompactionMode2(settings) !== "client") {
|
|
1031
|
+
return void 0;
|
|
1032
|
+
}
|
|
1033
|
+
const budget = contextInputBudgetTokens2(settings);
|
|
1034
|
+
return budget > 0 ? budget : void 0;
|
|
1035
|
+
}
|
|
1036
|
+
async function userMessageTextWithAttachments(db, workspaceId, text, resources) {
|
|
1037
|
+
const attachedFiles = [];
|
|
1038
|
+
for (const resource of resources) {
|
|
1039
|
+
if (resource.kind !== "file") {
|
|
1040
|
+
continue;
|
|
1041
|
+
}
|
|
1042
|
+
const file = await requireFile(db, workspaceId, resource.fileId);
|
|
1043
|
+
attachedFiles.push(`- ${file.filename} (${file.contentType}, ${file.sizeBytes} bytes): ${sandboxFilePath(resource, file)}`);
|
|
1044
|
+
}
|
|
1045
|
+
if (attachedFiles.length === 0) {
|
|
1046
|
+
return text;
|
|
1047
|
+
}
|
|
1048
|
+
return [
|
|
1049
|
+
text,
|
|
1050
|
+
"",
|
|
1051
|
+
"Attached files are available in the sandbox:",
|
|
1052
|
+
...attachedFiles
|
|
1053
|
+
].join("\n");
|
|
1054
|
+
}
|
|
1055
|
+
function sandboxFilePath(resource, file) {
|
|
1056
|
+
return `/workspace/${resource.mountPath ?? `files/${file.id}`}/${file.safeFilename}`;
|
|
1057
|
+
}
|
|
1058
|
+
|
|
1059
|
+
// src/activities/streaming.ts
|
|
1060
|
+
import { Context } from "@temporalio/activity";
|
|
1061
|
+
function createRuntimeBatcher(flushEvents) {
|
|
1062
|
+
let pending = [];
|
|
1063
|
+
let lastFlush = Date.now();
|
|
1064
|
+
const structural = /* @__PURE__ */ new Set([
|
|
1065
|
+
"agent.message.delta",
|
|
1066
|
+
"agent.reasoning.delta",
|
|
1067
|
+
"sandbox.command.output.delta",
|
|
1068
|
+
// Channel-A PTY bytes flush promptly (P4.4) — a terminal stream is useless
|
|
1069
|
+
// batched 33ms behind; the structural set bypasses the batcher's coalesce.
|
|
1070
|
+
"terminal.pty.output.delta",
|
|
1071
|
+
"agent.toolCall.created",
|
|
1072
|
+
"agent.toolCall.output",
|
|
1073
|
+
"agent.message.completed",
|
|
1074
|
+
"session.requiresAction",
|
|
1075
|
+
"turn.completed",
|
|
1076
|
+
"turn.failed",
|
|
1077
|
+
"turn.cancelled"
|
|
1078
|
+
]);
|
|
1079
|
+
return {
|
|
1080
|
+
push: async (event) => {
|
|
1081
|
+
pending.push({ type: event.type, payload: event.payload });
|
|
1082
|
+
const elapsed = Date.now() - lastFlush;
|
|
1083
|
+
if (pending.length >= 50 || elapsed >= 33 || structural.has(event.type)) {
|
|
1084
|
+
await flush();
|
|
1085
|
+
}
|
|
1086
|
+
},
|
|
1087
|
+
flush
|
|
1088
|
+
};
|
|
1089
|
+
async function flush() {
|
|
1090
|
+
if (pending.length === 0) {
|
|
1091
|
+
return;
|
|
1092
|
+
}
|
|
1093
|
+
const events = pending;
|
|
1094
|
+
pending = [];
|
|
1095
|
+
lastFlush = Date.now();
|
|
1096
|
+
await flushEvents(events);
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
function currentActivityContext() {
|
|
1100
|
+
try {
|
|
1101
|
+
return Context.current();
|
|
1102
|
+
} catch {
|
|
1103
|
+
return null;
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
function startActivityHeartbeat(context, details) {
|
|
1107
|
+
if (!context) {
|
|
1108
|
+
return null;
|
|
1109
|
+
}
|
|
1110
|
+
const timer = setInterval(() => {
|
|
1111
|
+
context.heartbeat({ ...details, at: (/* @__PURE__ */ new Date()).toISOString() });
|
|
1112
|
+
}, 1e4);
|
|
1113
|
+
if ("unref" in timer && typeof timer.unref === "function") {
|
|
1114
|
+
timer.unref();
|
|
1115
|
+
}
|
|
1116
|
+
return timer;
|
|
1117
|
+
}
|
|
1118
|
+
async function nextStreamEvent(iterator, context) {
|
|
1119
|
+
if (!context) {
|
|
1120
|
+
return await iterator.next();
|
|
1121
|
+
}
|
|
1122
|
+
return await Promise.race([
|
|
1123
|
+
iterator.next(),
|
|
1124
|
+
context.cancelled
|
|
1125
|
+
]);
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
// src/sandbox-resume.ts
|
|
1129
|
+
import {
|
|
1130
|
+
acquireLease,
|
|
1131
|
+
commitWarmingToWarm,
|
|
1132
|
+
failWarmingToCold,
|
|
1133
|
+
getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
|
|
1134
|
+
readLease,
|
|
1135
|
+
releaseLeaseHolder,
|
|
1136
|
+
SandboxLeaseSupersededError
|
|
1137
|
+
} from "@opengeni/db";
|
|
1138
|
+
import {
|
|
1139
|
+
establishSandboxSessionFromEnvelope,
|
|
1140
|
+
ensureDisplayStack as ensureDisplayStackOnBox,
|
|
1141
|
+
desktopCapableBackend,
|
|
1142
|
+
serializeEstablishedSandboxEnvelope,
|
|
1143
|
+
DisplayStackError,
|
|
1144
|
+
DisplayStackUnsupportedError,
|
|
1145
|
+
buildStreamUrl,
|
|
1146
|
+
StreamPortUnavailableError
|
|
1147
|
+
} from "@opengeni/runtime";
|
|
1148
|
+
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
1149
|
+
var WARMING_POLL_INTERVAL_MS = 250;
|
|
1150
|
+
async function sleep(ms) {
|
|
1151
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
1152
|
+
}
|
|
1153
|
+
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
1154
|
+
const { db, settings } = services;
|
|
1155
|
+
const os = ids.os ?? "linux";
|
|
1156
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
1157
|
+
let released = false;
|
|
1158
|
+
const release = async () => {
|
|
1159
|
+
if (released) {
|
|
1160
|
+
return;
|
|
1161
|
+
}
|
|
1162
|
+
released = true;
|
|
1163
|
+
await releaseLeaseHolder(db, {
|
|
1164
|
+
accountId: ids.accountId,
|
|
1165
|
+
workspaceId: ids.workspaceId,
|
|
1166
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1167
|
+
kind,
|
|
1168
|
+
holderId,
|
|
1169
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
1170
|
+
});
|
|
1171
|
+
};
|
|
1172
|
+
const acquired = await acquireLease(db, {
|
|
1173
|
+
accountId: ids.accountId,
|
|
1174
|
+
workspaceId: ids.workspaceId,
|
|
1175
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1176
|
+
kind,
|
|
1177
|
+
holderId,
|
|
1178
|
+
subjectId: ids.sessionId,
|
|
1179
|
+
backend: ids.backend,
|
|
1180
|
+
os,
|
|
1181
|
+
// IMAGE IS SHARED STATE (B3): thread the resolved image so the lease stamps it +
|
|
1182
|
+
// conflicts on a live box already running a different image. A SandboxImageConflictError
|
|
1183
|
+
// propagates to the turn activity (an actionable error); a solo image change is handled
|
|
1184
|
+
// by acquireLease recreating the box cold on the new image.
|
|
1185
|
+
...ids.image ? { image: ids.image } : {},
|
|
1186
|
+
leaseTtlMs
|
|
1187
|
+
});
|
|
1188
|
+
if (acquired.role === "fenced") {
|
|
1189
|
+
await release();
|
|
1190
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
1191
|
+
}
|
|
1192
|
+
if (acquired.role === "spawner") {
|
|
1193
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1194
|
+
try {
|
|
1195
|
+
const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1196
|
+
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
1197
|
+
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
1198
|
+
sessionId: ids.sessionId,
|
|
1199
|
+
backendOverride: ids.backend,
|
|
1200
|
+
...ids.environment ? { environment: ids.environment } : {}
|
|
1201
|
+
});
|
|
1202
|
+
await ensureDisplayStack(settings, established);
|
|
1203
|
+
const endpoint = await exposeStreamPort(settings, established);
|
|
1204
|
+
const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
|
|
1205
|
+
const committed = await commitWarmingToWarm(db, {
|
|
1206
|
+
accountId: ids.accountId,
|
|
1207
|
+
workspaceId: ids.workspaceId,
|
|
1208
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1209
|
+
expectedEpoch,
|
|
1210
|
+
instanceId: established.instanceId,
|
|
1211
|
+
dataPlaneUrl: endpoint?.url ?? null,
|
|
1212
|
+
resumeBackendId: established.backendId,
|
|
1213
|
+
resumeState: resumeEnvelope,
|
|
1214
|
+
leaseTtlMs
|
|
1215
|
+
});
|
|
1216
|
+
if (!committed.committed || !committed.lease) {
|
|
1217
|
+
await release();
|
|
1218
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1219
|
+
}
|
|
1220
|
+
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1221
|
+
} catch (error) {
|
|
1222
|
+
if (error instanceof SandboxLeaseSupersededError) {
|
|
1223
|
+
throw error;
|
|
1224
|
+
}
|
|
1225
|
+
await failWarmingToCold(db, {
|
|
1226
|
+
accountId: ids.accountId,
|
|
1227
|
+
workspaceId: ids.workspaceId,
|
|
1228
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1229
|
+
expectedEpoch
|
|
1230
|
+
});
|
|
1231
|
+
await release();
|
|
1232
|
+
throw error;
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1236
|
+
if (acquired.lease.liveness === "warming") {
|
|
1237
|
+
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
1238
|
+
}
|
|
1239
|
+
try {
|
|
1240
|
+
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1241
|
+
const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1242
|
+
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
1243
|
+
sessionId: ids.sessionId,
|
|
1244
|
+
backendOverride: ids.backend,
|
|
1245
|
+
...ids.environment ? { environment: ids.environment } : {}
|
|
1246
|
+
});
|
|
1247
|
+
await ensureDisplayStack(settings, established);
|
|
1248
|
+
return { established, leaseEpoch, release };
|
|
1249
|
+
} catch (error) {
|
|
1250
|
+
await release();
|
|
1251
|
+
throw error;
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
1255
|
+
const { db, settings } = services;
|
|
1256
|
+
const leaseTtlMs = settings.sandboxLeaseTtlMs;
|
|
1257
|
+
let released = false;
|
|
1258
|
+
const release = async () => {
|
|
1259
|
+
if (released) {
|
|
1260
|
+
return;
|
|
1261
|
+
}
|
|
1262
|
+
released = true;
|
|
1263
|
+
await releaseLeaseHolder(db, {
|
|
1264
|
+
accountId: ids.accountId,
|
|
1265
|
+
workspaceId: ids.workspaceId,
|
|
1266
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1267
|
+
kind,
|
|
1268
|
+
holderId,
|
|
1269
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
1270
|
+
});
|
|
1271
|
+
};
|
|
1272
|
+
const acquired = await acquireLease(db, {
|
|
1273
|
+
accountId: ids.accountId,
|
|
1274
|
+
workspaceId: ids.workspaceId,
|
|
1275
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1276
|
+
kind,
|
|
1277
|
+
holderId,
|
|
1278
|
+
subjectId: ids.sessionId,
|
|
1279
|
+
backend: "selfhosted",
|
|
1280
|
+
os: "linux",
|
|
1281
|
+
leaseTtlMs
|
|
1282
|
+
});
|
|
1283
|
+
if (acquired.role === "fenced") {
|
|
1284
|
+
await release();
|
|
1285
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, acquired.lease.leaseEpoch);
|
|
1286
|
+
}
|
|
1287
|
+
if (acquired.role === "spawner") {
|
|
1288
|
+
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1289
|
+
const committed = await commitWarmingToWarm(db, {
|
|
1290
|
+
accountId: ids.accountId,
|
|
1291
|
+
workspaceId: ids.workspaceId,
|
|
1292
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1293
|
+
expectedEpoch,
|
|
1294
|
+
// No provider box id; stamp a clearly non-box marker for diagnostics.
|
|
1295
|
+
instanceId: `selfhosted:${ids.sessionId}`,
|
|
1296
|
+
resumeBackendId: "selfhosted",
|
|
1297
|
+
resumeState: null,
|
|
1298
|
+
leaseTtlMs
|
|
1299
|
+
});
|
|
1300
|
+
if (!committed.committed || !committed.lease) {
|
|
1301
|
+
await release();
|
|
1302
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1303
|
+
}
|
|
1304
|
+
return { leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1305
|
+
}
|
|
1306
|
+
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1307
|
+
}
|
|
1308
|
+
async function waitForWarmOrReacquire(services, ids, kind, holderId) {
|
|
1309
|
+
const { db, settings } = services;
|
|
1310
|
+
const deadline = Date.now() + settings.sandboxLeaseWarmingTtlMs;
|
|
1311
|
+
while (Date.now() < deadline) {
|
|
1312
|
+
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1313
|
+
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1314
|
+
if (!lease) {
|
|
1315
|
+
break;
|
|
1316
|
+
}
|
|
1317
|
+
if (lease.liveness === "warm" || lease.liveness === "draining") {
|
|
1318
|
+
return { liveness: lease.liveness, leaseEpoch: lease.leaseEpoch };
|
|
1319
|
+
}
|
|
1320
|
+
if (lease.liveness === "cold") {
|
|
1321
|
+
break;
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
const reacquired = await acquireLease(db, {
|
|
1325
|
+
accountId: ids.accountId,
|
|
1326
|
+
workspaceId: ids.workspaceId,
|
|
1327
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1328
|
+
kind,
|
|
1329
|
+
holderId,
|
|
1330
|
+
subjectId: ids.sessionId,
|
|
1331
|
+
backend: ids.backend,
|
|
1332
|
+
os: ids.os ?? "linux",
|
|
1333
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
1334
|
+
});
|
|
1335
|
+
if (reacquired.role === "fenced") {
|
|
1336
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
1337
|
+
}
|
|
1338
|
+
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
1339
|
+
}
|
|
1340
|
+
async function ensureDisplayStack(settings, established) {
|
|
1341
|
+
if (!settings.sandboxDesktopEnabled) {
|
|
1342
|
+
return;
|
|
1343
|
+
}
|
|
1344
|
+
if (!desktopCapableBackend(established.backendId)) {
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
try {
|
|
1348
|
+
await ensureDisplayStackOnBox(established.session);
|
|
1349
|
+
} catch (error) {
|
|
1350
|
+
if (error instanceof DisplayStackUnsupportedError || error instanceof DisplayStackError) {
|
|
1351
|
+
console.warn(
|
|
1352
|
+
`[sandbox-resume] ensureDisplayStack degraded to Channel-A-only (turn continues): ${error instanceof Error ? error.message : String(error)}`
|
|
1353
|
+
);
|
|
1354
|
+
return;
|
|
1355
|
+
}
|
|
1356
|
+
throw error;
|
|
1357
|
+
}
|
|
1358
|
+
}
|
|
1359
|
+
async function exposeStreamPort(settings, established) {
|
|
1360
|
+
if (!settings.sandboxDesktopEnabled) {
|
|
1361
|
+
return null;
|
|
1362
|
+
}
|
|
1363
|
+
if (!desktopCapableBackend(established.backendId)) {
|
|
1364
|
+
return null;
|
|
1365
|
+
}
|
|
1366
|
+
const session = established.session;
|
|
1367
|
+
if (typeof session?.resolveExposedPort !== "function") {
|
|
1368
|
+
return null;
|
|
1369
|
+
}
|
|
1370
|
+
try {
|
|
1371
|
+
const endpoint = await session.resolveExposedPort(DESKTOP_STREAM_PORT);
|
|
1372
|
+
const url = buildStreamUrl(endpoint);
|
|
1373
|
+
return { url, expiresAt: new Date(Date.now() + settings.sandboxLeaseTtlMs) };
|
|
1374
|
+
} catch (error) {
|
|
1375
|
+
if (error instanceof StreamPortUnavailableError) {
|
|
1376
|
+
return null;
|
|
1377
|
+
}
|
|
1378
|
+
return null;
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
|
|
1382
|
+
// src/sandbox-routing.ts
|
|
1383
|
+
import { getSandbox, readActiveSandbox } from "@opengeni/db";
|
|
1384
|
+
import {
|
|
1385
|
+
buildSelfhostedBackendSession,
|
|
1386
|
+
makeActiveBackendResolver,
|
|
1387
|
+
NatsControlRpc,
|
|
1388
|
+
RoutingSandboxSession
|
|
1389
|
+
} from "@opengeni/runtime";
|
|
1390
|
+
function relayConfigFromSettings(settings) {
|
|
1391
|
+
const raw = settings.selfhostedRelayUrl?.trim();
|
|
1392
|
+
if (!raw) {
|
|
1393
|
+
return { host: "relay.opengeni.local", port: 443, tls: true };
|
|
1394
|
+
}
|
|
1395
|
+
try {
|
|
1396
|
+
const url = new URL(raw.includes("://") ? raw : `wss://${raw}`);
|
|
1397
|
+
const tls = url.protocol === "wss:" || url.protocol === "https:";
|
|
1398
|
+
const port = url.port ? Number(url.port) : tls ? 443 : 80;
|
|
1399
|
+
return { host: url.hostname, port, tls };
|
|
1400
|
+
} catch {
|
|
1401
|
+
return { host: raw, port: 443, tls: true };
|
|
1402
|
+
}
|
|
1403
|
+
}
|
|
1404
|
+
function controlRpcFactory(bus) {
|
|
1405
|
+
return () => new NatsControlRpc(async () => {
|
|
1406
|
+
if (!bus) {
|
|
1407
|
+
return null;
|
|
1408
|
+
}
|
|
1409
|
+
return bus.getRequestConnection();
|
|
1410
|
+
});
|
|
1411
|
+
}
|
|
1412
|
+
function wrapTurnBoxWithRouting(services, ids, established) {
|
|
1413
|
+
const { db, settings, bus } = services;
|
|
1414
|
+
const resolver = makeActiveBackendResolver({
|
|
1415
|
+
workspaceId: ids.workspaceId,
|
|
1416
|
+
defaultBackend: established.session,
|
|
1417
|
+
defaultKind: established.backendId,
|
|
1418
|
+
getSandbox: async (sandboxId) => {
|
|
1419
|
+
const sandbox = await getSandbox(db, ids.workspaceId, sandboxId);
|
|
1420
|
+
return sandbox ? { id: sandbox.id, kind: sandbox.kind, name: sandbox.name, enrollmentId: sandbox.enrollmentId } : null;
|
|
1421
|
+
},
|
|
1422
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
1423
|
+
relay: relayConfigFromSettings(settings),
|
|
1424
|
+
// The turn's declared environment → a selfhosted swap target's manifest, so the
|
|
1425
|
+
// SDK's per-turn manifest-env delta is empty (no "cannot change manifest
|
|
1426
|
+
// environment variables" throw when the turn pins to a vm). Mirrors the group
|
|
1427
|
+
// box, which is created WITH this same environment (resumeBoxForTurn).
|
|
1428
|
+
...ids.environment !== void 0 ? { environment: ids.environment } : {},
|
|
1429
|
+
// Stage D machine-primary: pin THIS established SelfhostedSession for the machine
|
|
1430
|
+
// pointer so the resolver returns the SAME instance (no two-instance manifest
|
|
1431
|
+
// divergence). `established.session` is the SelfhostedSession the establish branch
|
|
1432
|
+
// bound to (sandboxId, epoch).
|
|
1433
|
+
...ids.pinnedSelfhosted ? {
|
|
1434
|
+
pinnedSelfhosted: {
|
|
1435
|
+
sandboxId: ids.pinnedSelfhosted.sandboxId,
|
|
1436
|
+
epoch: ids.pinnedSelfhosted.epoch,
|
|
1437
|
+
session: established.session
|
|
1438
|
+
}
|
|
1439
|
+
} : {}
|
|
1440
|
+
// A modal swap target in the turn path would need its own lease resume-by-id;
|
|
1441
|
+
// that is a future cross-group-box concern. Until then a modal swap target is
|
|
1442
|
+
// unresolvable (the swap tool validates liveness, so this only triggers if a
|
|
1443
|
+
// session points at a sibling modal box the turn cannot resume here) and the
|
|
1444
|
+
// op surfaces unresolvable — never a silent wrong-box landing.
|
|
1445
|
+
});
|
|
1446
|
+
const proxy = new RoutingSandboxSession({
|
|
1447
|
+
// Seed the DEFAULT backend (the established group box) at construction so
|
|
1448
|
+
// `session.state` is the real backend's state object BEFORE the first op. The
|
|
1449
|
+
// SDK reads `session.state.manifest` at turn START (and writes it back); an
|
|
1450
|
+
// empty `{}` there crashes serializeManifestEnvironment /
|
|
1451
|
+
// validateProvidedSessionManifestUpdate. This is byte-identical to what the
|
|
1452
|
+
// resolver returns for the default pointer (`activeSandboxId === null`).
|
|
1453
|
+
defaultResolved: {
|
|
1454
|
+
session: established.session,
|
|
1455
|
+
sandboxId: null,
|
|
1456
|
+
kind: established.backendId
|
|
1457
|
+
},
|
|
1458
|
+
readPointer: async () => {
|
|
1459
|
+
const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
|
|
1460
|
+
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
1461
|
+
},
|
|
1462
|
+
resolveActiveBackend: resolver
|
|
1463
|
+
});
|
|
1464
|
+
return { ...established, session: proxy };
|
|
1465
|
+
}
|
|
1466
|
+
async function establishSelfhostedTurnSession(services, args) {
|
|
1467
|
+
const { settings, bus } = services;
|
|
1468
|
+
const { client, session } = await buildSelfhostedBackendSession({
|
|
1469
|
+
workspaceId: args.workspaceId,
|
|
1470
|
+
agentId: args.agentId,
|
|
1471
|
+
relay: relayConfigFromSettings(settings),
|
|
1472
|
+
controlRpcFactory: controlRpcFactory(bus),
|
|
1473
|
+
epoch: args.epoch,
|
|
1474
|
+
environment: args.environment,
|
|
1475
|
+
workingDir: args.workingDir
|
|
1476
|
+
});
|
|
1477
|
+
return {
|
|
1478
|
+
client,
|
|
1479
|
+
session,
|
|
1480
|
+
sessionState: { agentId: args.agentId },
|
|
1481
|
+
instanceId: args.agentId,
|
|
1482
|
+
backendId: "selfhosted"
|
|
1483
|
+
};
|
|
1484
|
+
}
|
|
1485
|
+
function routingEnabled(settings) {
|
|
1486
|
+
return settings.sandboxSelfhostedEnabled === true;
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
// src/activities/recording.ts
|
|
1490
|
+
import { deleteRecording, updateRecording } from "@opengeni/db";
|
|
1491
|
+
import {
|
|
1492
|
+
contentTypeForCodec,
|
|
1493
|
+
deleteRecordingArtifacts,
|
|
1494
|
+
readRecordingBytes,
|
|
1495
|
+
RecordingError,
|
|
1496
|
+
recordingStorageKey,
|
|
1497
|
+
startRecording as startRecordingOnBox,
|
|
1498
|
+
stopRecording as stopRecordingOnBox
|
|
1499
|
+
} from "@opengeni/runtime";
|
|
1500
|
+
import { DOWNLOAD_URL_TTL_SECONDS } from "@opengeni/storage";
|
|
1501
|
+
async function beginRecording(args) {
|
|
1502
|
+
const { settings, db } = args;
|
|
1503
|
+
const codec = settings.recordingDefaultCodec;
|
|
1504
|
+
const dimensions = [settings.streamResolutionWidth, settings.streamResolutionHeight];
|
|
1505
|
+
const framerate = settings.recordingFramerate;
|
|
1506
|
+
await import("@opengeni/db").then(({ insertRecording }) => insertRecording(db, {
|
|
1507
|
+
id: args.recordingId,
|
|
1508
|
+
accountId: args.accountId,
|
|
1509
|
+
workspaceId: args.workspaceId,
|
|
1510
|
+
sessionId: args.sessionId,
|
|
1511
|
+
turnId: args.turnId,
|
|
1512
|
+
mode: args.mode,
|
|
1513
|
+
codec,
|
|
1514
|
+
width: dimensions[0],
|
|
1515
|
+
height: dimensions[1],
|
|
1516
|
+
reason: scrubFreeText(args.reason)
|
|
1517
|
+
}));
|
|
1518
|
+
const proc = await startRecordingOnBox(args.session, {
|
|
1519
|
+
recordingId: args.recordingId,
|
|
1520
|
+
codec,
|
|
1521
|
+
framerate,
|
|
1522
|
+
maxSeconds: settings.recordingMaxSeconds,
|
|
1523
|
+
dimensions,
|
|
1524
|
+
...args.runAs ? { runAs: args.runAs } : {}
|
|
1525
|
+
});
|
|
1526
|
+
const active = {
|
|
1527
|
+
recordingId: args.recordingId,
|
|
1528
|
+
turnId: args.turnId,
|
|
1529
|
+
mode: args.mode,
|
|
1530
|
+
proc,
|
|
1531
|
+
dimensions,
|
|
1532
|
+
framerate
|
|
1533
|
+
};
|
|
1534
|
+
const started = {
|
|
1535
|
+
recordingId: args.recordingId,
|
|
1536
|
+
turnId: args.turnId,
|
|
1537
|
+
mode: args.mode,
|
|
1538
|
+
codec,
|
|
1539
|
+
dimensions,
|
|
1540
|
+
framerate,
|
|
1541
|
+
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
1542
|
+
reason: scrubFreeText(args.reason)
|
|
1543
|
+
};
|
|
1544
|
+
return { active, started };
|
|
1545
|
+
}
|
|
1546
|
+
async function finalizeRecording(args) {
|
|
1547
|
+
const { settings, db, objectStorage, active } = args;
|
|
1548
|
+
const codec = settings.recordingDefaultCodec;
|
|
1549
|
+
const fail = async (reason, detail) => {
|
|
1550
|
+
await updateRecording(db, {
|
|
1551
|
+
accountId: args.accountId,
|
|
1552
|
+
workspaceId: args.workspaceId,
|
|
1553
|
+
recordingId: active.recordingId,
|
|
1554
|
+
state: "failed",
|
|
1555
|
+
reason: scrubFreeText(detail)
|
|
1556
|
+
}).catch(() => void 0);
|
|
1557
|
+
return { ok: false, reason, detail: scrubFreeText(detail) };
|
|
1558
|
+
};
|
|
1559
|
+
try {
|
|
1560
|
+
await updateRecording(db, {
|
|
1561
|
+
accountId: args.accountId,
|
|
1562
|
+
workspaceId: args.workspaceId,
|
|
1563
|
+
recordingId: active.recordingId,
|
|
1564
|
+
state: "finalizing"
|
|
1565
|
+
}).catch(() => void 0);
|
|
1566
|
+
await stopRecordingOnBox(args.session, active.proc);
|
|
1567
|
+
const finalized = await readRecordingBytes(args.session, active.proc, settings.recordingMaxBytes);
|
|
1568
|
+
if (!objectStorage) {
|
|
1569
|
+
return await fail("upload-failed", "object storage is not configured");
|
|
1570
|
+
}
|
|
1571
|
+
const key = recordingStorageKey(args.workspaceId, args.sessionId, active.recordingId, codec);
|
|
1572
|
+
try {
|
|
1573
|
+
await objectStorage.putObject({ key, contentType: finalized.contentType, body: finalized.bytes.slice() });
|
|
1574
|
+
} catch (uploadError) {
|
|
1575
|
+
return await fail("upload-failed", uploadError instanceof Error ? uploadError.message : String(uploadError));
|
|
1576
|
+
}
|
|
1577
|
+
await updateRecording(db, {
|
|
1578
|
+
accountId: args.accountId,
|
|
1579
|
+
workspaceId: args.workspaceId,
|
|
1580
|
+
recordingId: active.recordingId,
|
|
1581
|
+
state: "available",
|
|
1582
|
+
storageKey: key,
|
|
1583
|
+
sizeBytes: finalized.sizeBytes,
|
|
1584
|
+
durationSeconds: finalized.durationSeconds
|
|
1585
|
+
});
|
|
1586
|
+
await deleteRecordingArtifacts(args.session, active.proc);
|
|
1587
|
+
const available = {
|
|
1588
|
+
recordingId: active.recordingId,
|
|
1589
|
+
turnId: active.turnId,
|
|
1590
|
+
codec,
|
|
1591
|
+
contentType: contentTypeForCodec(codec),
|
|
1592
|
+
storageKey: key,
|
|
1593
|
+
durationSeconds: finalized.durationSeconds,
|
|
1594
|
+
sizeBytes: finalized.sizeBytes,
|
|
1595
|
+
dimensions: active.dimensions
|
|
1596
|
+
};
|
|
1597
|
+
return { ok: true, available };
|
|
1598
|
+
} catch (error) {
|
|
1599
|
+
const reason = error instanceof RecordingError ? error.reason : "ffmpeg-error";
|
|
1600
|
+
return await fail(reason, error instanceof Error ? error.message : String(error));
|
|
1601
|
+
}
|
|
1602
|
+
}
|
|
1603
|
+
async function discardRecording(args) {
|
|
1604
|
+
const { db, active } = args;
|
|
1605
|
+
try {
|
|
1606
|
+
await stopRecordingOnBox(args.session, active.proc).catch(() => void 0);
|
|
1607
|
+
await deleteRecordingArtifacts(args.session, active.proc).catch(() => void 0);
|
|
1608
|
+
} finally {
|
|
1609
|
+
await deleteRecording(db, {
|
|
1610
|
+
accountId: args.accountId,
|
|
1611
|
+
workspaceId: args.workspaceId,
|
|
1612
|
+
recordingId: active.recordingId
|
|
1613
|
+
}).catch(() => void 0);
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
function scrubFreeText(value) {
|
|
1617
|
+
if (value === null || value === void 0) return null;
|
|
1618
|
+
const trimmed = String(value).slice(0, 2e3);
|
|
1619
|
+
return trimmed.length === 0 ? null : trimmed;
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
// src/activities/agent-turn.ts
|
|
1623
|
+
import { createObjectStorage } from "@opengeni/storage";
|
|
1624
|
+
import { desktopCapableBackend as desktopCapableBackend2, sandboxRunAs } from "@opengeni/runtime";
|
|
1625
|
+
import { CAPABILITY_DESCRIPTORS } from "@opengeni/contracts";
|
|
1626
|
+
import { randomUUID } from "crypto";
|
|
1627
|
+
var PROVIDER_BACKPRESSURE_DELAY_MS = 6e4;
|
|
1628
|
+
function selectCodexCredentialForTurn(args) {
|
|
1629
|
+
const { sessionPinnedCredentialId: pin, activeCredentialId: active, connectedIds } = args;
|
|
1630
|
+
if (pin && connectedIds.has(pin)) {
|
|
1631
|
+
return pin;
|
|
1632
|
+
}
|
|
1633
|
+
if (active && connectedIds.has(active)) {
|
|
1634
|
+
return active;
|
|
1635
|
+
}
|
|
1636
|
+
return null;
|
|
1637
|
+
}
|
|
1638
|
+
var WORKER_SHUTDOWN_RESUME_TEXT = [
|
|
1639
|
+
"[TURN RESUMED AFTER WORKER RESTART] The platform worker running this turn restarted (deploy/rollout) before the turn finished.",
|
|
1640
|
+
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; at most the single in-flight model step after it was lost.",
|
|
1641
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
|
|
1642
|
+
].join("\n");
|
|
1643
|
+
var WORKER_DEATH_RESUME_TEXT = [
|
|
1644
|
+
"[TURN RESUMED AFTER WORKER CRASH] The platform worker running this turn died before the turn finished.",
|
|
1645
|
+
"Your conversation history above, including this turn's earlier work, is preserved up to the last checkpoint; anything in flight after that checkpoint was lost.",
|
|
1646
|
+
"Continue the original task from where it left off. Before repeating any action with side effects, check whether it already happened."
|
|
1647
|
+
].join("\n");
|
|
1648
|
+
function isWorkerShutdownCancellation(error) {
|
|
1649
|
+
return error instanceof CancelledFailure && error.message === "WORKER_SHUTDOWN";
|
|
1650
|
+
}
|
|
1651
|
+
function modelUsageSourceKey(input) {
|
|
1652
|
+
if (input.responseId) {
|
|
1653
|
+
return input.responseId;
|
|
1654
|
+
}
|
|
1655
|
+
return input.dispatchId ? `${input.dispatchId}:${input.positionalKey}` : input.positionalKey;
|
|
1656
|
+
}
|
|
1657
|
+
function historyRowsToAppend(rawHistory, persistedHistoryCount, nextPosition = persistedHistoryCount) {
|
|
1658
|
+
const sanitized = sanitizeHistoryItemsForModel(rawHistory);
|
|
1659
|
+
if (sanitized.length <= persistedHistoryCount) {
|
|
1660
|
+
return { rows: [], nextWatermark: persistedHistoryCount, nextPosition };
|
|
1661
|
+
}
|
|
1662
|
+
const rows = sanitized.slice(persistedHistoryCount).map((item, offset) => ({
|
|
1663
|
+
position: nextPosition + offset,
|
|
1664
|
+
item
|
|
1665
|
+
}));
|
|
1666
|
+
return { rows, nextWatermark: sanitized.length, nextPosition: nextPosition + rows.length };
|
|
1667
|
+
}
|
|
1668
|
+
function reconcileSeedCount(activeSeedRows, modelHistoryFromItems, current) {
|
|
1669
|
+
return sanitizeHistoryItemsForModel(
|
|
1670
|
+
modelHistoryFromItems ? applyCodexHistoryStrip(activeSeedRows, current) : activeSeedRows.map((row) => row.item)
|
|
1671
|
+
).length;
|
|
1672
|
+
}
|
|
1673
|
+
async function resolveActiveSandboxBackend(routingOn, loadPointer, loadSandboxKind) {
|
|
1674
|
+
if (!routingOn) {
|
|
1675
|
+
return void 0;
|
|
1676
|
+
}
|
|
1677
|
+
try {
|
|
1678
|
+
const pointer = await loadPointer();
|
|
1679
|
+
if (!pointer?.activeSandboxId) {
|
|
1680
|
+
return void 0;
|
|
1681
|
+
}
|
|
1682
|
+
const kind = await loadSandboxKind(pointer.activeSandboxId);
|
|
1683
|
+
return kind === "selfhosted" ? "selfhosted" : void 0;
|
|
1684
|
+
} catch (error) {
|
|
1685
|
+
console.error("active sandbox backend resolution failed (turn proceeds on home backend)", error);
|
|
1686
|
+
return void 0;
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts) {
|
|
1690
|
+
const nearPct = settings.codexRotationNearExhaustionPct;
|
|
1691
|
+
const stale = accounts.filter(
|
|
1692
|
+
(a) => a.status === "active" && ((a.primaryUsedPercent ?? 0) >= nearPct || (a.secondaryUsedPercent ?? 0) >= nearPct)
|
|
1693
|
+
);
|
|
1694
|
+
if (stale.length === 0) {
|
|
1695
|
+
return accounts;
|
|
1696
|
+
}
|
|
1697
|
+
await Promise.all(stale.map((a) => fetchCodexUsageForAccount(db, settings, workspaceId, a.id).catch(() => void 0)));
|
|
1698
|
+
return listCodexAccountStatuses(db, workspaceId).catch(() => accounts);
|
|
1699
|
+
}
|
|
1700
|
+
function createRunAgentTurnActivity(services) {
|
|
1701
|
+
return async function runAgentTurn2(input) {
|
|
1702
|
+
const { settings, db, bus, runtime, objectStorage, observability, wakeSessionWorkflow, entitlements, connectionCredentials } = await services();
|
|
1703
|
+
const activityStarted = performance.now();
|
|
1704
|
+
const activitySpan = observability.startSpan("worker.run_agent_segment", {
|
|
1705
|
+
"opengeni.session_id": input.sessionId,
|
|
1706
|
+
"opengeni.workflow_id": input.workflowId,
|
|
1707
|
+
"opengeni.trigger_event_id": input.triggerEventId
|
|
1708
|
+
});
|
|
1709
|
+
let activityStatus = "unknown";
|
|
1710
|
+
let activityError;
|
|
1711
|
+
let turnId;
|
|
1712
|
+
let heartbeatTimer;
|
|
1713
|
+
let resolvedSandbox = null;
|
|
1714
|
+
let setupBoxSession = null;
|
|
1715
|
+
let sandboxHolderId = null;
|
|
1716
|
+
let sandboxGroupId = null;
|
|
1717
|
+
let leaseHeartbeatTimer;
|
|
1718
|
+
let activeRecording = null;
|
|
1719
|
+
let didComputerUse = false;
|
|
1720
|
+
let batcher = null;
|
|
1721
|
+
let preparedTools = null;
|
|
1722
|
+
let publish = null;
|
|
1723
|
+
let turnStartedPublished = false;
|
|
1724
|
+
let stream;
|
|
1725
|
+
let persistedHistoryCount = 0;
|
|
1726
|
+
let historyCountAtTurnStart = 0;
|
|
1727
|
+
let nextHistoryPosition = 0;
|
|
1728
|
+
const reconcileConversationTruth = async () => {
|
|
1729
|
+
if (!stream || !turnId) {
|
|
1730
|
+
return;
|
|
1731
|
+
}
|
|
1732
|
+
try {
|
|
1733
|
+
const rawHistory = stream.state.history;
|
|
1734
|
+
if (Array.isArray(rawHistory)) {
|
|
1735
|
+
const { rows, nextWatermark, nextPosition } = historyRowsToAppend(
|
|
1736
|
+
rawHistory,
|
|
1737
|
+
persistedHistoryCount,
|
|
1738
|
+
nextHistoryPosition
|
|
1739
|
+
);
|
|
1740
|
+
if (rows.length > 0) {
|
|
1741
|
+
await appendSessionHistoryItems(db, {
|
|
1742
|
+
accountId: input.accountId,
|
|
1743
|
+
workspaceId: input.workspaceId,
|
|
1744
|
+
sessionId: input.sessionId,
|
|
1745
|
+
turnId,
|
|
1746
|
+
// Tag each row with the codex account that produced it (null on the
|
|
1747
|
+
// non-codex path). Resolved at line ~504 before any reconcile pass
|
|
1748
|
+
// runs, so this is the turn's effective account. The read path uses
|
|
1749
|
+
// it to strip cross-account reasoning.encrypted_content next turn.
|
|
1750
|
+
producerCodexCredentialId: effectiveCodexCredentialId,
|
|
1751
|
+
items: rows
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
persistedHistoryCount = nextWatermark;
|
|
1755
|
+
nextHistoryPosition = nextPosition;
|
|
1756
|
+
}
|
|
1757
|
+
const envelope = sandboxStateEntryFromRunState(stream.state);
|
|
1758
|
+
if (envelope) {
|
|
1759
|
+
await upsertSandboxSessionEnvelope(db, {
|
|
1760
|
+
accountId: input.accountId,
|
|
1761
|
+
workspaceId: input.workspaceId,
|
|
1762
|
+
sessionId: input.sessionId,
|
|
1763
|
+
envelope
|
|
1764
|
+
});
|
|
1765
|
+
}
|
|
1766
|
+
} catch (persistError) {
|
|
1767
|
+
console.error("session history dual-write failed (run unaffected)", persistError);
|
|
1768
|
+
}
|
|
1769
|
+
};
|
|
1770
|
+
let redact = identityRedactor;
|
|
1771
|
+
let environmentId = "";
|
|
1772
|
+
let effectiveCodexCredentialId = null;
|
|
1773
|
+
let latestCodexUsage = null;
|
|
1774
|
+
let triggerType = null;
|
|
1775
|
+
try {
|
|
1776
|
+
const mcpSettings = await settingsWithEnabledCapabilityMcpServers(db, input.workspaceId, settings);
|
|
1777
|
+
const codexSubscriptionActive = await workspaceCodexSubscriptionActive2(db, mcpSettings, input.workspaceId);
|
|
1778
|
+
const capabilitySettings = await settingsWithCodexCredential(db, input.workspaceId, mcpSettings, codexSubscriptionActive);
|
|
1779
|
+
runtime.configure(capabilitySettings);
|
|
1780
|
+
const session = await requireSession2(db, input.workspaceId, input.sessionId);
|
|
1781
|
+
const trigger = await getSessionEvent3(db, input.workspaceId, input.triggerEventId);
|
|
1782
|
+
if (!trigger) {
|
|
1783
|
+
throw new Error(`Trigger event not found: ${input.triggerEventId}`);
|
|
1784
|
+
}
|
|
1785
|
+
triggerType = trigger.type;
|
|
1786
|
+
let turn = input.turnId ? await getSessionTurn(db, input.workspaceId, input.turnId) : await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
1787
|
+
if (!turn && !input.turnId) {
|
|
1788
|
+
const createdTurnId = await createTurn(db, {
|
|
1789
|
+
workspaceId: input.workspaceId,
|
|
1790
|
+
sessionId: input.sessionId,
|
|
1791
|
+
temporalWorkflowId: input.workflowId,
|
|
1792
|
+
triggerEventId: input.triggerEventId
|
|
1793
|
+
});
|
|
1794
|
+
turn = await getSessionTurn(db, input.workspaceId, createdTurnId);
|
|
1795
|
+
}
|
|
1796
|
+
if (!turn) {
|
|
1797
|
+
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
1798
|
+
}
|
|
1799
|
+
turnId = turn.id;
|
|
1800
|
+
const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
|
|
1801
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1802
|
+
const activityContext = currentActivityContext();
|
|
1803
|
+
const throwIfWorkerShuttingDown = () => {
|
|
1804
|
+
const reason = activityContext?.cancellationSignal.reason;
|
|
1805
|
+
if (isWorkerShutdownCancellation(reason)) {
|
|
1806
|
+
throw reason;
|
|
1807
|
+
}
|
|
1808
|
+
};
|
|
1809
|
+
heartbeatTimer = startActivityHeartbeat(activityContext, {
|
|
1810
|
+
phase: "running",
|
|
1811
|
+
sessionId: input.sessionId,
|
|
1812
|
+
turnId
|
|
1813
|
+
});
|
|
1814
|
+
let producerSeq = 0;
|
|
1815
|
+
const producerId = `${input.workflowId}:${turnId}${activityContext ? `:${activityContext.info.activityId}` : ""}`;
|
|
1816
|
+
const dispatchId = activityContext?.info.activityId ?? null;
|
|
1817
|
+
publish = async (events, immediate = false) => {
|
|
1818
|
+
const inputs = events.map((event) => ({
|
|
1819
|
+
...event,
|
|
1820
|
+
payload: redact(event.payload),
|
|
1821
|
+
turnId,
|
|
1822
|
+
producerId,
|
|
1823
|
+
producerSeq: ++producerSeq
|
|
1824
|
+
}));
|
|
1825
|
+
await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, inputs);
|
|
1826
|
+
activityContext?.heartbeat({ phase: "events_published", sessionId: input.sessionId, turnId, producerSeq });
|
|
1827
|
+
if (immediate) {
|
|
1828
|
+
await Bun.sleep(0);
|
|
1829
|
+
}
|
|
1830
|
+
};
|
|
1831
|
+
activityContext?.heartbeat({ phase: "turn_started", sessionId: input.sessionId, turnId });
|
|
1832
|
+
throwIfWorkerShuttingDown();
|
|
1833
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "running", turnId);
|
|
1834
|
+
await publish([
|
|
1835
|
+
{ type: "session.status.changed", payload: { status: "running" } },
|
|
1836
|
+
{ type: "turn.started", payload: { triggerEventId: input.triggerEventId } }
|
|
1837
|
+
], true);
|
|
1838
|
+
turnStartedPublished = true;
|
|
1839
|
+
if (isCodexTurn) {
|
|
1840
|
+
const [rotation, accounts, sessionCodex] = await Promise.all([
|
|
1841
|
+
getCodexRotationSettings(db, input.workspaceId),
|
|
1842
|
+
listCodexAccountStatuses(db, input.workspaceId),
|
|
1843
|
+
getSessionCodexState(db, input.workspaceId, input.sessionId)
|
|
1844
|
+
]);
|
|
1845
|
+
const connectedIds = new Set(accounts.map((account) => account.id));
|
|
1846
|
+
const sessionPin = sessionCodex?.pinnedCredentialId ?? null;
|
|
1847
|
+
let chosenActive = rotation?.activeCredentialId ?? null;
|
|
1848
|
+
let rotationDecision = null;
|
|
1849
|
+
if (rotation?.rotationEnabled && sessionPin == null) {
|
|
1850
|
+
let rankAccounts = accounts;
|
|
1851
|
+
rotationDecision = chooseRotationActive({
|
|
1852
|
+
rotationStrategy: rotation.rotationStrategy,
|
|
1853
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
1854
|
+
priorCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
1855
|
+
accounts: rankAccounts,
|
|
1856
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
1857
|
+
now: /* @__PURE__ */ new Date(),
|
|
1858
|
+
// P4: the leaving (active) account's cached connector set is the proxy
|
|
1859
|
+
// for "what this session has access to" — prefer a covering target.
|
|
1860
|
+
usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? []
|
|
1861
|
+
});
|
|
1862
|
+
if (rotationDecision.kind === "allCapped") {
|
|
1863
|
+
rankAccounts = await refreshCappedCodexUsageRows(db, settings, input.workspaceId, rankAccounts);
|
|
1864
|
+
rotationDecision = chooseRotationActive({
|
|
1865
|
+
rotationStrategy: rotation.rotationStrategy,
|
|
1866
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
1867
|
+
priorCredentialId: sessionCodex?.lastCredentialId ?? null,
|
|
1868
|
+
accounts: rankAccounts,
|
|
1869
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
1870
|
+
now: /* @__PURE__ */ new Date(),
|
|
1871
|
+
// P4: leaving (active) account's connector set (refreshCappedCodexUsageRows
|
|
1872
|
+
// only touches usage columns, so connectorNamespaces is preserved here).
|
|
1873
|
+
usedConnectors: rankAccounts.find((a) => a.id === rotation.activeCredentialId)?.connectorNamespaces ?? []
|
|
1874
|
+
});
|
|
1875
|
+
}
|
|
1876
|
+
if (rotationDecision.kind === "active") {
|
|
1877
|
+
if (rotationDecision.moved) {
|
|
1878
|
+
await setActiveCodexCredential(db, input.workspaceId, rotationDecision.credentialId);
|
|
1879
|
+
}
|
|
1880
|
+
chosenActive = rotationDecision.credentialId;
|
|
1881
|
+
} else if (rotationDecision.kind === "allCapped" && publish && turnId) {
|
|
1882
|
+
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
1883
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
1884
|
+
const resumeMs = computeIdleDelayMs(rotationDecision.earliestResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS);
|
|
1885
|
+
const failurePayload = codexUsageLimitFailurePayload(
|
|
1886
|
+
{ resetsInSeconds: Math.ceil(resumeMs / 1e3) },
|
|
1887
|
+
"all connected Codex subscriptions are rate-limited",
|
|
1888
|
+
{ allAccounts: true }
|
|
1889
|
+
);
|
|
1890
|
+
await publish([
|
|
1891
|
+
// `rotated:true` (Finding 2): the proactive all-capped wait is the SAME
|
|
1892
|
+
// rotation-wait state as the reactive all-capped path, so it must freeze
|
|
1893
|
+
// autoContinuations identically (evaluateGoalContinuation reads this marker)
|
|
1894
|
+
// — a goal waiting out a long reset must not burn its continuation budget on
|
|
1895
|
+
// the proactive path while the reactive path spares it.
|
|
1896
|
+
{ type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved: false, rotated: true } },
|
|
1897
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
1898
|
+
], true);
|
|
1899
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
1900
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
1901
|
+
activityStatus = "idle";
|
|
1902
|
+
return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
|
|
1903
|
+
}
|
|
1904
|
+
}
|
|
1905
|
+
effectiveCodexCredentialId = selectCodexCredentialForTurn({
|
|
1906
|
+
sessionPinnedCredentialId: sessionPin,
|
|
1907
|
+
// pin still wins, structurally
|
|
1908
|
+
activeCredentialId: chosenActive,
|
|
1909
|
+
// rotation-choice OR today's active
|
|
1910
|
+
connectedIds
|
|
1911
|
+
});
|
|
1912
|
+
if (effectiveCodexCredentialId) {
|
|
1913
|
+
const priorAccountId = sessionCodex?.lastCredentialId ?? null;
|
|
1914
|
+
await recordSessionActiveCodexCredential(db, input.workspaceId, input.sessionId, effectiveCodexCredentialId);
|
|
1915
|
+
if (priorAccountId !== effectiveCodexCredentialId) {
|
|
1916
|
+
const rotated = rotationDecision?.kind === "active" && rotationDecision.moved;
|
|
1917
|
+
const droppedConnectors = rotationDecision?.kind === "active" ? rotationDecision.droppedConnectors : void 0;
|
|
1918
|
+
await publish([{
|
|
1919
|
+
type: "codex.account.switched",
|
|
1920
|
+
payload: {
|
|
1921
|
+
fromAccountId: priorAccountId,
|
|
1922
|
+
toAccountId: effectiveCodexCredentialId,
|
|
1923
|
+
reason: rotated ? "rotation" : "manual",
|
|
1924
|
+
...droppedConnectors && droppedConnectors.length > 0 ? { droppedConnectors } : {}
|
|
1925
|
+
}
|
|
1926
|
+
}]);
|
|
1927
|
+
}
|
|
1928
|
+
}
|
|
1929
|
+
}
|
|
1930
|
+
const packRuntime = await resolveWorkspacePackRuntime(db, input.workspaceId);
|
|
1931
|
+
const workspaceAgentInstructions = await resolveWorkspaceAgentInstructions(db, input.workspaceId);
|
|
1932
|
+
const runSettings = {
|
|
1933
|
+
...settingsWithPackSandboxImage(capabilitySettings, packRuntime.sandboxImage),
|
|
1934
|
+
openaiModel: turn.model,
|
|
1935
|
+
openaiReasoningEffort: turn.reasoningEffort,
|
|
1936
|
+
sandboxBackend: turn.sandboxBackend
|
|
1937
|
+
};
|
|
1938
|
+
const resolvedModel = runtime.resolveTurnModel(capabilitySettings, turn.model);
|
|
1939
|
+
const codexContext = resolvedModel?.provider.kind === "codex-subscription" ? (() => {
|
|
1940
|
+
const resolver = buildCodexTokenResolver(db, runSettings, input.workspaceId, effectiveCodexCredentialId ?? "");
|
|
1941
|
+
return {
|
|
1942
|
+
clientVersion: CODEX_CLIENT_VERSION,
|
|
1943
|
+
getToken: resolver.getToken,
|
|
1944
|
+
refresh: resolver.refresh,
|
|
1945
|
+
resolveModel: buildModelResolver(CODEX_FALLBACK_MODEL_SLUGS2, CODEX_FALLBACK_MODEL_SLUGS2[0]),
|
|
1946
|
+
onUsageHeaders: (snapshot) => {
|
|
1947
|
+
latestCodexUsage = snapshot;
|
|
1948
|
+
}
|
|
1949
|
+
// latest wins; flushed once in finally
|
|
1950
|
+
};
|
|
1951
|
+
})() : null;
|
|
1952
|
+
const withCodex = (fn) => codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
|
|
1953
|
+
const turnResources = mergeResourceRefs(session.resources, turn.resources);
|
|
1954
|
+
const turnTools = withCodexAppsTool(runSettings, withFirstPartyTools(runSettings, mergeToolRefs(session.tools, turn.tools)));
|
|
1955
|
+
const connectionScope = { accountId: input.accountId, workspaceId: input.workspaceId };
|
|
1956
|
+
const workspaceEnvironment = await loadWorkspaceEnvironmentForRunWithCredentials(
|
|
1957
|
+
db,
|
|
1958
|
+
runSettings,
|
|
1959
|
+
connectionScope,
|
|
1960
|
+
session.environmentId,
|
|
1961
|
+
connectionCredentials?.sandboxSecrets
|
|
1962
|
+
);
|
|
1963
|
+
environmentId = workspaceEnvironment?.id ?? "";
|
|
1964
|
+
redact = createSecretRedactor(
|
|
1965
|
+
Object.entries(workspaceEnvironment?.values ?? {}).map(([name, value]) => ({ name, value }))
|
|
1966
|
+
);
|
|
1967
|
+
const routingOn = routingEnabled(settings);
|
|
1968
|
+
const activeSandboxPointer = routingOn ? await readActiveSandbox2(db, input.workspaceId, input.sessionId).catch(() => null) : null;
|
|
1969
|
+
const activeSandboxRecord = routingOn && activeSandboxPointer?.activeSandboxId ? await getSandbox2(db, input.workspaceId, activeSandboxPointer.activeSandboxId).catch(() => null) : null;
|
|
1970
|
+
const activeSandboxBackend = await resolveActiveSandboxBackend(
|
|
1971
|
+
routingOn,
|
|
1972
|
+
async () => activeSandboxPointer,
|
|
1973
|
+
async () => activeSandboxRecord?.kind ?? null
|
|
1974
|
+
);
|
|
1975
|
+
const machinePrimary = activeSandboxBackend === "selfhosted" && Boolean(activeSandboxPointer?.activeSandboxId) && Boolean(activeSandboxRecord?.enrollmentId);
|
|
1976
|
+
const { environment: sandboxEnvironment, gitToken: sandboxGitToken } = await sandboxEnvironmentForRun(
|
|
1977
|
+
runSettings,
|
|
1978
|
+
turnResources,
|
|
1979
|
+
workspaceEnvironment?.values ?? {},
|
|
1980
|
+
{
|
|
1981
|
+
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
1982
|
+
scope: connectionScope,
|
|
1983
|
+
gitCredentials: connectionCredentials?.gitCredentials
|
|
1984
|
+
}
|
|
1985
|
+
);
|
|
1986
|
+
if (settings.sandboxOwnershipEnabled && turn.sandboxBackend !== "none") {
|
|
1987
|
+
sandboxHolderId = dispatchId ?? `turn:${turnId}`;
|
|
1988
|
+
sandboxGroupId = session.sandboxGroupId;
|
|
1989
|
+
const groupBoxBackend = turn.sandboxBackend === "selfhosted" && !machinePrimary ? settings.sandboxBackend : turn.sandboxBackend;
|
|
1990
|
+
if (machinePrimary) {
|
|
1991
|
+
const established = await establishSelfhostedTurnSession(
|
|
1992
|
+
{ db, settings, bus },
|
|
1993
|
+
{
|
|
1994
|
+
workspaceId: input.workspaceId,
|
|
1995
|
+
agentId: activeSandboxRecord.enrollmentId,
|
|
1996
|
+
epoch: activeSandboxPointer.activeEpoch,
|
|
1997
|
+
environment: sandboxEnvironment,
|
|
1998
|
+
workingDir: activeSandboxPointer.workingDir
|
|
1999
|
+
}
|
|
2000
|
+
);
|
|
2001
|
+
const lease = await acquireSelfhostedLeaseForTurn(
|
|
2002
|
+
{ db, settings },
|
|
2003
|
+
{
|
|
2004
|
+
accountId: input.accountId,
|
|
2005
|
+
workspaceId: input.workspaceId,
|
|
2006
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
2007
|
+
sessionId: input.sessionId
|
|
2008
|
+
},
|
|
2009
|
+
"turn",
|
|
2010
|
+
sandboxHolderId
|
|
2011
|
+
);
|
|
2012
|
+
setupBoxSession = established.session;
|
|
2013
|
+
resolvedSandbox = {
|
|
2014
|
+
// Wrap in the SAME routing proxy so a mid-turn swap (to another machine
|
|
2015
|
+
// or back to the group box) still re-routes per op. PIN this established
|
|
2016
|
+
// SelfhostedSession for the machine pointer so the turn-start manifest
|
|
2017
|
+
// write (via the proxy's `state` getter) and the per-op reads hit ONE
|
|
2018
|
+
// instance — no two-instance manifest divergence.
|
|
2019
|
+
established: wrapTurnBoxWithRouting(
|
|
2020
|
+
{ db, settings, bus },
|
|
2021
|
+
{
|
|
2022
|
+
workspaceId: input.workspaceId,
|
|
2023
|
+
sessionId: input.sessionId,
|
|
2024
|
+
environment: sandboxEnvironment,
|
|
2025
|
+
pinnedSelfhosted: {
|
|
2026
|
+
sandboxId: activeSandboxPointer.activeSandboxId,
|
|
2027
|
+
epoch: activeSandboxPointer.activeEpoch
|
|
2028
|
+
}
|
|
2029
|
+
},
|
|
2030
|
+
established
|
|
2031
|
+
),
|
|
2032
|
+
leaseEpoch: lease.leaseEpoch,
|
|
2033
|
+
release: lease.release
|
|
2034
|
+
};
|
|
2035
|
+
} else {
|
|
2036
|
+
resolvedSandbox = await resumeBoxForTurn(
|
|
2037
|
+
{ db, settings },
|
|
2038
|
+
{
|
|
2039
|
+
accountId: input.accountId,
|
|
2040
|
+
workspaceId: input.workspaceId,
|
|
2041
|
+
sandboxGroupId: session.sandboxGroupId,
|
|
2042
|
+
sessionId: input.sessionId,
|
|
2043
|
+
// groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
|
|
2044
|
+
// is not machine-primary resumes a real cloud group box (the
|
|
2045
|
+
// deployment default), never a "selfhosted" box (which would throw
|
|
2046
|
+
// for lack of a bound agentId).
|
|
2047
|
+
backend: groupBoxBackend,
|
|
2048
|
+
os: session.sandboxOs,
|
|
2049
|
+
environment: sandboxEnvironment,
|
|
2050
|
+
// IMAGE IS SHARED STATE (B3, Modal warm-box path only): the container image
|
|
2051
|
+
// this run resolves. The lease stamps it + conflicts on a live shared box
|
|
2052
|
+
// running a DIFFERENT image (solo → recreate on the new image; N-holders →
|
|
2053
|
+
// SandboxImageConflictError surfaced as an actionable turn error). Prefer the
|
|
2054
|
+
// explicit Modal image ref, else the docker image. The selfhosted branch
|
|
2055
|
+
// (establishSelfhostedTurnSession/acquireSelfhostedLeaseForTurn) NEVER passes
|
|
2056
|
+
// an image — B3 lives only on this Modal else-branch.
|
|
2057
|
+
...runSettings.modalImageRef ?? runSettings.dockerImage ? { image: runSettings.modalImageRef ?? runSettings.dockerImage } : {}
|
|
2058
|
+
},
|
|
2059
|
+
"turn",
|
|
2060
|
+
sandboxHolderId
|
|
2061
|
+
);
|
|
2062
|
+
setupBoxSession = resolvedSandbox.established.session;
|
|
2063
|
+
if (routingEnabled(settings)) {
|
|
2064
|
+
resolvedSandbox = {
|
|
2065
|
+
...resolvedSandbox,
|
|
2066
|
+
established: wrapTurnBoxWithRouting(
|
|
2067
|
+
{ db, settings, bus },
|
|
2068
|
+
// Thread the SAME declared environment the group box was created with
|
|
2069
|
+
// (resumeBoxForTurn, above) so a selfhosted swap target's manifest
|
|
2070
|
+
// carries it too — the SDK's per-turn manifest-env delta stays empty
|
|
2071
|
+
// (no "cannot change manifest environment variables" throw).
|
|
2072
|
+
{ workspaceId: input.workspaceId, sessionId: input.sessionId, environment: sandboxEnvironment },
|
|
2073
|
+
resolvedSandbox.established
|
|
2074
|
+
)
|
|
2075
|
+
};
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
const heartbeatEpoch = resolvedSandbox.leaseEpoch;
|
|
2079
|
+
const heartbeatHolderId = sandboxHolderId;
|
|
2080
|
+
const heartbeatGroupId = sandboxGroupId;
|
|
2081
|
+
const warmRate = sandboxWarmRateMicrosPerSecond(settings, activeSandboxBackend ?? groupBoxBackend);
|
|
2082
|
+
leaseHeartbeatTimer = setInterval(() => {
|
|
2083
|
+
void heartbeatLeaseHolder(db, {
|
|
2084
|
+
accountId: input.accountId,
|
|
2085
|
+
workspaceId: input.workspaceId,
|
|
2086
|
+
sandboxGroupId: heartbeatGroupId,
|
|
2087
|
+
kind: "turn",
|
|
2088
|
+
holderId: heartbeatHolderId,
|
|
2089
|
+
leaseTtlMs: settings.sandboxLeaseTtlMs,
|
|
2090
|
+
expectedEpoch: heartbeatEpoch
|
|
2091
|
+
}).catch(() => void 0);
|
|
2092
|
+
void accrueWarmSeconds(db, {
|
|
2093
|
+
accountId: input.accountId,
|
|
2094
|
+
workspaceId: input.workspaceId,
|
|
2095
|
+
sandboxGroupId: heartbeatGroupId,
|
|
2096
|
+
expectedEpoch: heartbeatEpoch,
|
|
2097
|
+
warmRateMicrosPerSecond: warmRate,
|
|
2098
|
+
subjectId: input.sessionId
|
|
2099
|
+
}).catch(() => void 0);
|
|
2100
|
+
}, 1e4);
|
|
2101
|
+
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
2102
|
+
leaseHeartbeatTimer.unref();
|
|
2103
|
+
}
|
|
2104
|
+
}
|
|
2105
|
+
if (resolvedSandbox && settings.sandboxDesktopEnabled && settings.recordingEnabled && desktopCapableBackend2(resolvedSandbox.established.backendId)) {
|
|
2106
|
+
try {
|
|
2107
|
+
const begun = await beginRecording({
|
|
2108
|
+
settings,
|
|
2109
|
+
db,
|
|
2110
|
+
accountId: input.accountId,
|
|
2111
|
+
workspaceId: input.workspaceId,
|
|
2112
|
+
sessionId: input.sessionId,
|
|
2113
|
+
turnId,
|
|
2114
|
+
recordingId: randomUUID(),
|
|
2115
|
+
mode: "on-turn",
|
|
2116
|
+
session: resolvedSandbox.established.session,
|
|
2117
|
+
runAs: sandboxRunAs(settings),
|
|
2118
|
+
reason: null
|
|
2119
|
+
});
|
|
2120
|
+
activeRecording = begun.active;
|
|
2121
|
+
await publish([{ type: "recording.started", payload: begun.started }]);
|
|
2122
|
+
} catch (recordingError) {
|
|
2123
|
+
activeRecording = null;
|
|
2124
|
+
console.error("on-turn recording start failed (turn outcome unaffected)", recordingError);
|
|
2125
|
+
}
|
|
2126
|
+
}
|
|
2127
|
+
const fileResourceDownloads = await sandboxFileDownloadsForRun(runSettings, db, objectStorage, input.workspaceId, turnResources);
|
|
2128
|
+
throwIfWorkerShuttingDown();
|
|
2129
|
+
preparedTools = await withCodex(() => runtime.prepareTools(runSettings, turnTools, {
|
|
2130
|
+
accountId: input.accountId,
|
|
2131
|
+
workspaceId: input.workspaceId,
|
|
2132
|
+
sessionId: input.sessionId,
|
|
2133
|
+
subjectId: "worker:first-party-mcp",
|
|
2134
|
+
subjectLabel: "OpenGeni worker",
|
|
2135
|
+
// Manager-style sessions carry a creation-validated permission set
|
|
2136
|
+
// for their first-party MCP token; null keeps the fixed default.
|
|
2137
|
+
...session.firstPartyMcpPermissions?.length ? { firstPartyPermissions: session.firstPartyMcpPermissions } : {}
|
|
2138
|
+
}));
|
|
2139
|
+
const isGenesisTurn = triggerType === "user.message" && await countSessionHistoryItems(db, input.workspaceId, input.sessionId) === 0;
|
|
2140
|
+
const agent = runtime.buildAgent(runSettings, turnResources, {
|
|
2141
|
+
reasoningEffort: turn.reasoningEffort,
|
|
2142
|
+
genesisTitleHint: isGenesisTurn,
|
|
2143
|
+
sandboxEnvironment,
|
|
2144
|
+
// TOKEN-BROKER (B1): forward the per-turn git token OFF-MANIFEST as the clone
|
|
2145
|
+
// seed. ONLY when the effective backend is NOT selfhosted (the connected
|
|
2146
|
+
// machine uses its own git creds — mirrors the skipGitHubToken gate above)
|
|
2147
|
+
// AND the mint actually produced a token (repo resources present). The runtime
|
|
2148
|
+
// seeds it to the box's token file before the repository-clone runs; it never
|
|
2149
|
+
// touches the box/agent manifest env.
|
|
2150
|
+
...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
2151
|
+
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
2152
|
+
fileResourceDownloads,
|
|
2153
|
+
mcpServers: preparedTools.mcpServers,
|
|
2154
|
+
// LIVE by-reference connector namespaces (fills during this turn's
|
|
2155
|
+
// codex_apps tools/list): the codex tool_search description reads it per
|
|
2156
|
+
// model call so the model sees the account's real connected sources.
|
|
2157
|
+
codexConnectorNamespaces: preparedTools.codexConnectorNamespaces,
|
|
2158
|
+
// Resolved-model routing + gating (legacy defaults when null). The model
|
|
2159
|
+
// is passed as the model *string* (agent.model = runSettings.openaiModel),
|
|
2160
|
+
// NOT a Model instance: an instance only survives the in-process
|
|
2161
|
+
// ("none") run, whereas the SandboxAgent/Modal path drops it and
|
|
2162
|
+
// re-resolves the model *name* through the global MultiProviderModelProvider
|
|
2163
|
+
// configureOpenAI installed — so registry models (Fireworks GLM) route to
|
|
2164
|
+
// their own client instead of 404ing against the built-in Azure/OpenAI
|
|
2165
|
+
// client. The gating still comes from the resolved provider: server-side
|
|
2166
|
+
// store/compaction follow the provider's compaction mode (registry
|
|
2167
|
+
// providers resolve to "client"); encrypted reasoning is only
|
|
2168
|
+
// round-tripped on the Responses wire API; hosted web search is attached
|
|
2169
|
+
// only when the model opts in; the effective context window drives the
|
|
2170
|
+
// compaction threshold.
|
|
2171
|
+
...resolvedModel ? {
|
|
2172
|
+
compactionMode: resolvedModel.provider.compactionMode,
|
|
2173
|
+
hostedWebSearch: resolvedModel.configured.hostedWebSearch,
|
|
2174
|
+
encryptedReasoning: resolvedModel.provider.api === "responses" && runSettings.openaiReasoningEncryptedContent,
|
|
2175
|
+
contextWindowTokens: resolvedModel.configured.contextWindowTokens ?? runSettings.contextWindowTokens,
|
|
2176
|
+
// The ChatGPT/Codex backend rejects the SDK's HOSTED sandbox tools —
|
|
2177
|
+
// the `apply_patch` tool type ("Unsupported tool type: apply_patch")
|
|
2178
|
+
// and structured tool output — which the OpenAIResponsesModel the SDK
|
|
2179
|
+
// binds would otherwise select. Tell buildAgent to emit the function
|
|
2180
|
+
// `apply_patch` + text `view_image` variants the backend accepts. Only
|
|
2181
|
+
// the codex-subscription provider needs this; every other backend
|
|
2182
|
+
// (built-in OpenAI/Azure = real hosted support; registry "chat"
|
|
2183
|
+
// providers = the SDK's own ChatCompletions detection) keeps the SDK
|
|
2184
|
+
// default.
|
|
2185
|
+
structuredToolTransport: resolvedModel.provider.kind !== "codex-subscription"
|
|
2186
|
+
} : {},
|
|
2187
|
+
...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
|
|
2188
|
+
...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
2189
|
+
...workspaceEnvironment ? {
|
|
2190
|
+
workspaceEnvironment: {
|
|
2191
|
+
name: workspaceEnvironment.name,
|
|
2192
|
+
description: workspaceEnvironment.description,
|
|
2193
|
+
variableNames: Object.keys(workspaceEnvironment.values)
|
|
2194
|
+
}
|
|
2195
|
+
} : {}
|
|
2196
|
+
});
|
|
2197
|
+
if (triggerType === "user.message" || triggerType === "goal.continuation") {
|
|
2198
|
+
try {
|
|
2199
|
+
const forced = await consumeSessionCompactionRequest(db, input.workspaceId, input.sessionId);
|
|
2200
|
+
const outcome = await maybeCompactContext(
|
|
2201
|
+
db,
|
|
2202
|
+
runSettings,
|
|
2203
|
+
{ accountId: input.accountId, workspaceId: input.workspaceId, sessionId: input.sessionId, turnId },
|
|
2204
|
+
session.lastInputTokens,
|
|
2205
|
+
// Provider-aware summarizer: when the turn's model resolved to a
|
|
2206
|
+
// registry provider, summarize on THAT provider's client + wire API
|
|
2207
|
+
// (a chat provider can't summarize through OpenAI/Azure). Null
|
|
2208
|
+
// resolution keeps the default built-in Responses summarizer.
|
|
2209
|
+
resolvedModel ? (s, m) => withCodex(() => summarizeForCompaction2(s, m, {
|
|
2210
|
+
client: resolvedModel.client,
|
|
2211
|
+
api: resolvedModel.provider.api,
|
|
2212
|
+
model: resolvedModel.configured.id,
|
|
2213
|
+
maxOutputTokens: s.contextSummaryMaxTokens
|
|
2214
|
+
})) : void 0,
|
|
2215
|
+
forced ? { force: true } : {}
|
|
2216
|
+
);
|
|
2217
|
+
if (outcome.compacted) {
|
|
2218
|
+
const trigger2 = forced ? "operator" : outcome.hardForced ? "hard" : void 0;
|
|
2219
|
+
await publish([{ type: "session.context.compacted", payload: { summaryPosition: outcome.summaryPosition, ...trigger2 ? { trigger: trigger2 } : {} } }]);
|
|
2220
|
+
}
|
|
2221
|
+
} catch (compactError) {
|
|
2222
|
+
console.error("context compaction failed (turn proceeds un-compacted)", compactError);
|
|
2223
|
+
}
|
|
2224
|
+
}
|
|
2225
|
+
const { input: runInput, modelHistoryFromItems } = await turnInput(
|
|
2226
|
+
db,
|
|
2227
|
+
runtime,
|
|
2228
|
+
agent,
|
|
2229
|
+
trigger,
|
|
2230
|
+
runSettings,
|
|
2231
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId }
|
|
2232
|
+
);
|
|
2233
|
+
const activeSeedRows = await getActiveSessionHistoryItems3(db, input.workspaceId, input.sessionId);
|
|
2234
|
+
persistedHistoryCount = reconcileSeedCount(
|
|
2235
|
+
activeSeedRows,
|
|
2236
|
+
modelHistoryFromItems,
|
|
2237
|
+
{ currentCodexCredentialId: effectiveCodexCredentialId }
|
|
2238
|
+
);
|
|
2239
|
+
historyCountAtTurnStart = persistedHistoryCount;
|
|
2240
|
+
nextHistoryPosition = await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId);
|
|
2241
|
+
let responseUsageCount = 0;
|
|
2242
|
+
let lastInputTokensObserved = null;
|
|
2243
|
+
throwIfWorkerShuttingDown();
|
|
2244
|
+
const runStreamOnce = () => runtime.runStream(agent, runInput, runSettings, {
|
|
2245
|
+
sandboxEnvironment,
|
|
2246
|
+
onRuntimeEvent: async (event) => {
|
|
2247
|
+
await publish([{ type: event.type, payload: event.payload }], true);
|
|
2248
|
+
},
|
|
2249
|
+
// P1.2: inject the resumed box NON-OWNED (the SDK never reaps it — the
|
|
2250
|
+
// keystone). Absent when the flag is off -> legacy build-and-discard.
|
|
2251
|
+
...resolvedSandbox ? {
|
|
2252
|
+
ownedSandbox: {
|
|
2253
|
+
client: resolvedSandbox.established.client,
|
|
2254
|
+
session: resolvedSandbox.established.session,
|
|
2255
|
+
sessionState: resolvedSandbox.established.sessionState,
|
|
2256
|
+
// Pin platform setup (hooks + file materialization) to the un-proxied
|
|
2257
|
+
// established box — never through the routing proxy, which would
|
|
2258
|
+
// re-route those execs onto a machine swapped in mid-turn.
|
|
2259
|
+
...setupBoxSession ? { setupSession: setupBoxSession } : {}
|
|
2260
|
+
}
|
|
2261
|
+
} : {}
|
|
2262
|
+
});
|
|
2263
|
+
stream = await withCodex(runStreamOnce);
|
|
2264
|
+
batcher = createRuntimeBatcher(async (events) => {
|
|
2265
|
+
await publish(events);
|
|
2266
|
+
});
|
|
2267
|
+
const iterator = stream.toStream()[Symbol.asyncIterator]();
|
|
2268
|
+
let streamDone = false;
|
|
2269
|
+
try {
|
|
2270
|
+
while (true) {
|
|
2271
|
+
const next = await nextStreamEvent(iterator, activityContext);
|
|
2272
|
+
if (next.done) {
|
|
2273
|
+
streamDone = true;
|
|
2274
|
+
break;
|
|
2275
|
+
}
|
|
2276
|
+
const responseUsage = modelResponseUsageFromSdkEvent(next.value);
|
|
2277
|
+
if (responseUsage) {
|
|
2278
|
+
responseUsageCount += 1;
|
|
2279
|
+
const observed = responseUsage.usage?.inputTokens;
|
|
2280
|
+
if (typeof observed === "number" && observed > 0) {
|
|
2281
|
+
lastInputTokensObserved = observed;
|
|
2282
|
+
}
|
|
2283
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
2284
|
+
accountId: input.accountId,
|
|
2285
|
+
workspaceId: input.workspaceId,
|
|
2286
|
+
sessionId: input.sessionId,
|
|
2287
|
+
turnId,
|
|
2288
|
+
model: turn.model,
|
|
2289
|
+
isCodexTurn,
|
|
2290
|
+
usage: responseUsage.usage,
|
|
2291
|
+
sourceKey: modelUsageSourceKey({
|
|
2292
|
+
responseId: responseUsage.responseId,
|
|
2293
|
+
dispatchId,
|
|
2294
|
+
positionalKey: `response-${responseUsageCount}`
|
|
2295
|
+
})
|
|
2296
|
+
});
|
|
2297
|
+
await reconcileConversationTruth();
|
|
2298
|
+
try {
|
|
2299
|
+
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
2300
|
+
} catch (limitError) {
|
|
2301
|
+
let serializedRunState = null;
|
|
2302
|
+
try {
|
|
2303
|
+
serializedRunState = stream.state.toString();
|
|
2304
|
+
} catch {
|
|
2305
|
+
serializedRunState = null;
|
|
2306
|
+
}
|
|
2307
|
+
throw new BudgetExhaustedError(
|
|
2308
|
+
limitError instanceof Error ? limitError.message : String(limitError),
|
|
2309
|
+
serializedRunState
|
|
2310
|
+
);
|
|
2311
|
+
}
|
|
2312
|
+
}
|
|
2313
|
+
if (activeRecording && !didComputerUse && isComputerCallStreamEvent(next.value)) {
|
|
2314
|
+
didComputerUse = true;
|
|
2315
|
+
}
|
|
2316
|
+
const normalized = normalizeSdkEvent(next.value);
|
|
2317
|
+
for (const event of normalized) {
|
|
2318
|
+
await batcher.push(event);
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
} finally {
|
|
2322
|
+
if (!streamDone) {
|
|
2323
|
+
await iterator.return?.();
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
await batcher.flush();
|
|
2327
|
+
await stream.completed.catch(() => void 0);
|
|
2328
|
+
if (responseUsageCount === 0) {
|
|
2329
|
+
const aggregateInput = stream.state.usage?.inputTokens;
|
|
2330
|
+
if (typeof aggregateInput === "number" && aggregateInput > 0) {
|
|
2331
|
+
lastInputTokensObserved = aggregateInput;
|
|
2332
|
+
}
|
|
2333
|
+
await recordModelUsageAndDebitCredits(settings, db, {
|
|
2334
|
+
accountId: input.accountId,
|
|
2335
|
+
workspaceId: input.workspaceId,
|
|
2336
|
+
sessionId: input.sessionId,
|
|
2337
|
+
turnId,
|
|
2338
|
+
model: turn.model,
|
|
2339
|
+
isCodexTurn,
|
|
2340
|
+
usage: stream.state.usage,
|
|
2341
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2342
|
+
});
|
|
2343
|
+
}
|
|
2344
|
+
if (lastInputTokensObserved !== null) {
|
|
2345
|
+
await setSessionLastInputTokens(db, input.workspaceId, input.sessionId, lastInputTokensObserved).catch((error) => console.error("persist last_input_tokens failed (non-fatal)", error));
|
|
2346
|
+
}
|
|
2347
|
+
if (stream.interruptions.length > 0) {
|
|
2348
|
+
await reconcileConversationTruth();
|
|
2349
|
+
const approvals = runtime.serializeApprovals(stream.interruptions);
|
|
2350
|
+
await saveRunState(db, {
|
|
2351
|
+
accountId: input.accountId,
|
|
2352
|
+
workspaceId: input.workspaceId,
|
|
2353
|
+
sessionId: input.sessionId,
|
|
2354
|
+
turnId,
|
|
2355
|
+
serializedRunState: stream.state.toString(),
|
|
2356
|
+
pendingApprovals: approvals,
|
|
2357
|
+
// Record the account freezing this state so a resume on a DIFFERENT
|
|
2358
|
+
// codex account strips its account-bound reasoning before replay (HOLE C).
|
|
2359
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2360
|
+
});
|
|
2361
|
+
await publish([
|
|
2362
|
+
{ type: "session.requiresAction", payload: { approvals } },
|
|
2363
|
+
{ type: "session.status.changed", payload: { status: "requires_action" } }
|
|
2364
|
+
], true);
|
|
2365
|
+
await finishTurn(db, input.workspaceId, turnId, "requires_action");
|
|
2366
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "requires_action", turnId);
|
|
2367
|
+
activityStatus = "requires_action";
|
|
2368
|
+
return { status: "requires_action" };
|
|
2369
|
+
}
|
|
2370
|
+
const finalOutput = String(stream.finalOutput ?? "");
|
|
2371
|
+
await reconcileConversationTruth();
|
|
2372
|
+
if (settings.sessionHistorySource !== "items") {
|
|
2373
|
+
await saveRunState(db, {
|
|
2374
|
+
accountId: input.accountId,
|
|
2375
|
+
workspaceId: input.workspaceId,
|
|
2376
|
+
sessionId: input.sessionId,
|
|
2377
|
+
turnId,
|
|
2378
|
+
serializedRunState: stream.state.toString(),
|
|
2379
|
+
pendingApprovals: [],
|
|
2380
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2381
|
+
});
|
|
2382
|
+
}
|
|
2383
|
+
await publish([
|
|
2384
|
+
{ type: "agent.message.completed", payload: { text: finalOutput } },
|
|
2385
|
+
{ type: "turn.completed", payload: { output: finalOutput } },
|
|
2386
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2387
|
+
], true);
|
|
2388
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2389
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2390
|
+
await recordUsageEvent2(db, {
|
|
2391
|
+
accountId: input.accountId,
|
|
2392
|
+
workspaceId: input.workspaceId,
|
|
2393
|
+
eventType: "agent_run.completed",
|
|
2394
|
+
quantity: 1,
|
|
2395
|
+
unit: "run",
|
|
2396
|
+
sourceResourceType: "session_turn",
|
|
2397
|
+
sourceResourceId: turnId,
|
|
2398
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`
|
|
2399
|
+
});
|
|
2400
|
+
activityStatus = "idle";
|
|
2401
|
+
return { status: "idle" };
|
|
2402
|
+
} catch (error) {
|
|
2403
|
+
const preemptTurnId = turnId ?? input.turnId;
|
|
2404
|
+
if (error instanceof SandboxLeaseSupersededError2 && preemptTurnId) {
|
|
2405
|
+
try {
|
|
2406
|
+
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
2407
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2408
|
+
activityStatus = "preempted";
|
|
2409
|
+
return { status: "preempted" };
|
|
2410
|
+
} catch (requeueError) {
|
|
2411
|
+
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
2412
|
+
}
|
|
2413
|
+
}
|
|
2414
|
+
if (isWorkerShutdownCancellation(error) && preemptTurnId) {
|
|
2415
|
+
try {
|
|
2416
|
+
await batcher?.flush().catch(() => void 0);
|
|
2417
|
+
await reconcileConversationTruth();
|
|
2418
|
+
const approvalRerun = triggerType === "user.approvalDecision";
|
|
2419
|
+
let resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && persistedHistoryCount > historyCountAtTurnStart;
|
|
2420
|
+
if (settings.sessionHistorySource !== "items" && stream) {
|
|
2421
|
+
try {
|
|
2422
|
+
await saveRunState(db, {
|
|
2423
|
+
accountId: input.accountId,
|
|
2424
|
+
workspaceId: input.workspaceId,
|
|
2425
|
+
sessionId: input.sessionId,
|
|
2426
|
+
turnId: preemptTurnId,
|
|
2427
|
+
serializedRunState: stream.state.toString(),
|
|
2428
|
+
pendingApprovals: runtime.serializeApprovals(stream.interruptions ?? []),
|
|
2429
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2430
|
+
});
|
|
2431
|
+
resumeWithNotice = true;
|
|
2432
|
+
} catch {
|
|
2433
|
+
resumeWithNotice = false;
|
|
2434
|
+
}
|
|
2435
|
+
}
|
|
2436
|
+
const [preemptedEvent] = await appendAndPublishEvents2(db, bus, input.workspaceId, input.sessionId, [
|
|
2437
|
+
{
|
|
2438
|
+
turnId: preemptTurnId,
|
|
2439
|
+
type: "turn.preempted",
|
|
2440
|
+
payload: {
|
|
2441
|
+
triggerEventId: input.triggerEventId,
|
|
2442
|
+
reason: "worker_shutdown",
|
|
2443
|
+
resumeWithNotice,
|
|
2444
|
+
...resumeWithNotice ? { text: WORKER_SHUTDOWN_RESUME_TEXT } : {}
|
|
2445
|
+
}
|
|
2446
|
+
},
|
|
2447
|
+
{
|
|
2448
|
+
turnId: preemptTurnId,
|
|
2449
|
+
type: "session.status.changed",
|
|
2450
|
+
payload: { status: "queued" }
|
|
2451
|
+
}
|
|
2452
|
+
]);
|
|
2453
|
+
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2454
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2455
|
+
activityStatus = "preempted";
|
|
2456
|
+
return { status: "preempted" };
|
|
2457
|
+
} catch (preemptError) {
|
|
2458
|
+
console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
if (error instanceof CancelledFailure) {
|
|
2462
|
+
activityStatus = "cancelled";
|
|
2463
|
+
activityError = error;
|
|
2464
|
+
await batcher?.flush().catch(() => void 0);
|
|
2465
|
+
if (preemptTurnId) {
|
|
2466
|
+
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
|
|
2467
|
+
}
|
|
2468
|
+
throw error;
|
|
2469
|
+
}
|
|
2470
|
+
const maxTurns = maxTurnsExceededRunState(error);
|
|
2471
|
+
if (maxTurns && publish && turnId && turnStartedPublished) {
|
|
2472
|
+
await batcher?.flush().catch(() => void 0);
|
|
2473
|
+
await reconcileConversationTruth();
|
|
2474
|
+
const runStateSaved = Boolean(maxTurns.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2475
|
+
if (maxTurns.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
2476
|
+
await saveRunState(db, {
|
|
2477
|
+
accountId: input.accountId,
|
|
2478
|
+
workspaceId: input.workspaceId,
|
|
2479
|
+
sessionId: input.sessionId,
|
|
2480
|
+
turnId,
|
|
2481
|
+
serializedRunState: maxTurns.serializedRunState,
|
|
2482
|
+
pendingApprovals: [],
|
|
2483
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2484
|
+
});
|
|
2485
|
+
}
|
|
2486
|
+
await publish([
|
|
2487
|
+
{ type: "turn.completed", payload: { output: "", segmentLimit: "max_turns", runStateSaved } },
|
|
2488
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2489
|
+
], true);
|
|
2490
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2491
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2492
|
+
await recordUsageEvent2(db, {
|
|
2493
|
+
accountId: input.accountId,
|
|
2494
|
+
workspaceId: input.workspaceId,
|
|
2495
|
+
eventType: "agent_run.completed",
|
|
2496
|
+
quantity: 1,
|
|
2497
|
+
unit: "run",
|
|
2498
|
+
sourceResourceType: "session_turn",
|
|
2499
|
+
sourceResourceId: turnId,
|
|
2500
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`
|
|
2501
|
+
});
|
|
2502
|
+
activityStatus = "idle";
|
|
2503
|
+
return { status: "idle" };
|
|
2504
|
+
}
|
|
2505
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
2506
|
+
if (usageLimit && publish && turnId && turnStartedPublished) {
|
|
2507
|
+
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
2508
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
2509
|
+
await batcher?.flush().catch(() => void 0);
|
|
2510
|
+
await reconcileConversationTruth();
|
|
2511
|
+
const serializedRunState = agentsErrorRunState(error);
|
|
2512
|
+
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2513
|
+
if (serializedRunState && settings.sessionHistorySource !== "items") {
|
|
2514
|
+
await saveRunState(db, {
|
|
2515
|
+
accountId: input.accountId,
|
|
2516
|
+
workspaceId: input.workspaceId,
|
|
2517
|
+
sessionId: input.sessionId,
|
|
2518
|
+
turnId,
|
|
2519
|
+
serializedRunState,
|
|
2520
|
+
pendingApprovals: [],
|
|
2521
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2522
|
+
});
|
|
2523
|
+
}
|
|
2524
|
+
let rotated = false;
|
|
2525
|
+
let rotationResumeMs = null;
|
|
2526
|
+
let rotationResumeIdleUntilReset = false;
|
|
2527
|
+
let allCappedResetAt = null;
|
|
2528
|
+
if (effectiveCodexCredentialId) {
|
|
2529
|
+
const [rotation, sessionCodex] = await Promise.all([
|
|
2530
|
+
getCodexRotationSettings(db, input.workspaceId).catch(() => null),
|
|
2531
|
+
getSessionCodexState(db, input.workspaceId, input.sessionId).catch(() => null)
|
|
2532
|
+
]);
|
|
2533
|
+
const rotating = Boolean(rotation?.rotationEnabled) && sessionCodex?.pinnedCredentialId == null;
|
|
2534
|
+
if (rotating && rotation) {
|
|
2535
|
+
const accounts = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
|
|
2536
|
+
const serving = accounts.find((a) => a.id === effectiveCodexCredentialId) ?? null;
|
|
2537
|
+
const cachedReset = [serving?.primaryResetAt, serving?.secondaryResetAt].filter((d) => d instanceof Date && d.getTime() > Date.now()).sort((a, b) => a.getTime() - b.getTime())[0] ?? null;
|
|
2538
|
+
const until = usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? new Date(Date.now() + Math.ceil(usageLimit.resetsInSeconds) * 1e3) : cachedReset ?? new Date(Date.now() + CODEX_USAGE_LIMIT_MAX_RESUME_MS);
|
|
2539
|
+
const cooldownPersisted = await setCodexCredentialExhausted(db, input.workspaceId, effectiveCodexCredentialId, until).catch(() => false);
|
|
2540
|
+
const fresh = accounts.map((a) => a.id === effectiveCodexCredentialId ? { ...a, exhaustedUntil: until } : a);
|
|
2541
|
+
const decision = chooseRotationActive({
|
|
2542
|
+
rotationStrategy: rotation.rotationStrategy,
|
|
2543
|
+
activeCredentialId: rotation.activeCredentialId,
|
|
2544
|
+
priorCredentialId: effectiveCodexCredentialId,
|
|
2545
|
+
accounts: fresh,
|
|
2546
|
+
nearExhaustionPct: settings.codexRotationNearExhaustionPct,
|
|
2547
|
+
now: /* @__PURE__ */ new Date(),
|
|
2548
|
+
// P4: the just-capped serving account's connector set is the proxy for
|
|
2549
|
+
// "what this session has access to" — prefer a covering failover target.
|
|
2550
|
+
usedConnectors: serving?.connectorNamespaces ?? []
|
|
2551
|
+
});
|
|
2552
|
+
if (decision.kind === "active") {
|
|
2553
|
+
rotated = true;
|
|
2554
|
+
const priorConsecutiveRotations = await countConsecutiveReactiveRotations(db, input.workspaceId, input.sessionId).catch(() => 0);
|
|
2555
|
+
const resume = computeReactiveRotationResume({
|
|
2556
|
+
cooldownPersisted,
|
|
2557
|
+
priorConsecutiveRotations,
|
|
2558
|
+
connectedAccountCount: accounts.length
|
|
2559
|
+
});
|
|
2560
|
+
rotationResumeMs = resume.continueDelayMs;
|
|
2561
|
+
rotationResumeIdleUntilReset = resume.idleUntilReset;
|
|
2562
|
+
} else if (decision.kind === "allCapped") {
|
|
2563
|
+
rotated = true;
|
|
2564
|
+
allCappedResetAt = decision.earliestResetAt;
|
|
2565
|
+
}
|
|
2566
|
+
}
|
|
2567
|
+
}
|
|
2568
|
+
const failurePayload = allCappedResetAt ? codexUsageLimitFailurePayload(
|
|
2569
|
+
{ resetsInSeconds: Math.ceil(Math.max(0, allCappedResetAt.getTime() - Date.now()) / 1e3) },
|
|
2570
|
+
error instanceof Error ? error.message : String(error),
|
|
2571
|
+
{ allAccounts: true }
|
|
2572
|
+
) : codexUsageLimitFailurePayload(usageLimit, error instanceof Error ? error.message : String(error));
|
|
2573
|
+
await publish([
|
|
2574
|
+
// `rotated:true` ONLY on the reactive rotation path tells evaluateGoalContinuation to
|
|
2575
|
+
// freeze autoContinuations (a rotation walk must not burn the goal's continuation budget).
|
|
2576
|
+
{ type: "turn.failed", payload: { ...failurePayload, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved, ...rotated ? { rotated: true } : {} } },
|
|
2577
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2578
|
+
], true);
|
|
2579
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2580
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2581
|
+
activityStatus = "idle";
|
|
2582
|
+
activityError = error;
|
|
2583
|
+
if (goalActive) {
|
|
2584
|
+
if (rotationResumeMs !== null) {
|
|
2585
|
+
return { status: "idle", continueDelayMs: rotationResumeMs, ...rotationResumeIdleUntilReset ? { idleUntilReset: true } : {} };
|
|
2586
|
+
}
|
|
2587
|
+
const resumeMs = allCappedResetAt ? computeIdleDelayMs(allCappedResetAt, /* @__PURE__ */ new Date(), CODEX_USAGE_LIMIT_MAX_RESUME_MS) : usageLimit.resetsInSeconds !== null && Number.isFinite(usageLimit.resetsInSeconds) && usageLimit.resetsInSeconds > 0 ? Math.min(Math.ceil(usageLimit.resetsInSeconds) * 1e3, CODEX_USAGE_LIMIT_MAX_RESUME_MS) : CODEX_USAGE_LIMIT_MAX_RESUME_MS;
|
|
2588
|
+
return { status: "idle", continueDelayMs: resumeMs, ...allCappedResetAt ? { idleUntilReset: true } : {} };
|
|
2589
|
+
}
|
|
2590
|
+
return { status: "idle" };
|
|
2591
|
+
}
|
|
2592
|
+
if (error instanceof BudgetExhaustedError && publish && turnId && turnStartedPublished) {
|
|
2593
|
+
await batcher?.flush().catch(() => void 0);
|
|
2594
|
+
await reconcileConversationTruth();
|
|
2595
|
+
const runStateSaved = Boolean(error.serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2596
|
+
if (error.serializedRunState && settings.sessionHistorySource !== "items") {
|
|
2597
|
+
await saveRunState(db, {
|
|
2598
|
+
accountId: input.accountId,
|
|
2599
|
+
workspaceId: input.workspaceId,
|
|
2600
|
+
sessionId: input.sessionId,
|
|
2601
|
+
turnId,
|
|
2602
|
+
serializedRunState: error.serializedRunState,
|
|
2603
|
+
pendingApprovals: [],
|
|
2604
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2605
|
+
});
|
|
2606
|
+
}
|
|
2607
|
+
await publish([
|
|
2608
|
+
{ type: "turn.completed", payload: { output: "", segmentLimit: "budget_exhausted", detail: error.message, runStateSaved } },
|
|
2609
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2610
|
+
], true);
|
|
2611
|
+
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
2612
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2613
|
+
await recordUsageEvent2(db, {
|
|
2614
|
+
accountId: input.accountId,
|
|
2615
|
+
workspaceId: input.workspaceId,
|
|
2616
|
+
eventType: "agent_run.completed",
|
|
2617
|
+
quantity: 1,
|
|
2618
|
+
unit: "run",
|
|
2619
|
+
sourceResourceType: "session_turn",
|
|
2620
|
+
sourceResourceId: turnId,
|
|
2621
|
+
idempotencyKey: `usage:agent_run.completed:${turnId}`
|
|
2622
|
+
});
|
|
2623
|
+
activityStatus = "idle";
|
|
2624
|
+
return { status: "idle" };
|
|
2625
|
+
}
|
|
2626
|
+
const failure = agentRunFailurePayload(error);
|
|
2627
|
+
if (failure.retryable && publish && turnId && turnStartedPublished) {
|
|
2628
|
+
const goal = await getSessionGoal3(db, input.workspaceId, input.sessionId).catch(() => null);
|
|
2629
|
+
const goalActive = Boolean(goal && goal.status === "active");
|
|
2630
|
+
await batcher?.flush().catch(() => void 0);
|
|
2631
|
+
await reconcileConversationTruth();
|
|
2632
|
+
const serializedRunState = agentsErrorRunState(error);
|
|
2633
|
+
const runStateSaved = Boolean(serializedRunState) && settings.sessionHistorySource !== "items";
|
|
2634
|
+
if (serializedRunState && settings.sessionHistorySource !== "items") {
|
|
2635
|
+
await saveRunState(db, {
|
|
2636
|
+
accountId: input.accountId,
|
|
2637
|
+
workspaceId: input.workspaceId,
|
|
2638
|
+
sessionId: input.sessionId,
|
|
2639
|
+
turnId,
|
|
2640
|
+
serializedRunState,
|
|
2641
|
+
pendingApprovals: [],
|
|
2642
|
+
frozenCodexCredentialId: effectiveCodexCredentialId
|
|
2643
|
+
});
|
|
2644
|
+
}
|
|
2645
|
+
await publish([
|
|
2646
|
+
{ type: "turn.failed", payload: { ...failure, recovery: goalActive ? "goal_continuation" : "user_message", runStateSaved } },
|
|
2647
|
+
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2648
|
+
], true);
|
|
2649
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2650
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2651
|
+
activityStatus = "idle";
|
|
2652
|
+
activityError = error;
|
|
2653
|
+
return goalActive ? { status: "idle", continueDelayMs: PROVIDER_BACKPRESSURE_DELAY_MS } : { status: "idle" };
|
|
2654
|
+
}
|
|
2655
|
+
activityStatus = "failed";
|
|
2656
|
+
activityError = error;
|
|
2657
|
+
if (!publish || !turnId || !turnStartedPublished) {
|
|
2658
|
+
throw error;
|
|
2659
|
+
}
|
|
2660
|
+
await publish([
|
|
2661
|
+
{ type: "turn.failed", payload: failure },
|
|
2662
|
+
{ type: "session.status.changed", payload: { status: "failed" } }
|
|
2663
|
+
], true);
|
|
2664
|
+
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2665
|
+
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
2666
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
|
|
2667
|
+
return { status: "failed" };
|
|
2668
|
+
} finally {
|
|
2669
|
+
const durationSeconds = (performance.now() - activityStarted) / 1e3;
|
|
2670
|
+
observability.recordWorkerActivity({
|
|
2671
|
+
activity: "runAgentTurn",
|
|
2672
|
+
status: activityStatus,
|
|
2673
|
+
durationSeconds
|
|
2674
|
+
});
|
|
2675
|
+
activitySpan.end({
|
|
2676
|
+
attributes: {
|
|
2677
|
+
"opengeni.turn_id": turnId ?? "",
|
|
2678
|
+
"opengeni.status": activityStatus,
|
|
2679
|
+
"opengeni.environment_id": environmentId,
|
|
2680
|
+
"opengeni.duration_ms": Math.round(durationSeconds * 1e3)
|
|
2681
|
+
},
|
|
2682
|
+
error: activityError
|
|
2683
|
+
});
|
|
2684
|
+
if (effectiveCodexCredentialId) {
|
|
2685
|
+
if (latestCodexUsage) {
|
|
2686
|
+
await recordCodexAccountUsage(db, input.workspaceId, effectiveCodexCredentialId, latestCodexUsage).catch(() => void 0);
|
|
2687
|
+
}
|
|
2688
|
+
const connectorNamespaces = preparedTools?.codexConnectorNamespaces;
|
|
2689
|
+
if (connectorNamespaces && connectorNamespaces.size > 0) {
|
|
2690
|
+
await recordCodexAccountConnectors(db, input.workspaceId, effectiveCodexCredentialId, [...connectorNamespaces]).catch(() => void 0);
|
|
2691
|
+
}
|
|
2692
|
+
}
|
|
2693
|
+
await preparedTools?.close().catch(() => void 0);
|
|
2694
|
+
if (heartbeatTimer) {
|
|
2695
|
+
clearInterval(heartbeatTimer);
|
|
2696
|
+
}
|
|
2697
|
+
if (leaseHeartbeatTimer) {
|
|
2698
|
+
clearInterval(leaseHeartbeatTimer);
|
|
2699
|
+
}
|
|
2700
|
+
if (activeRecording && resolvedSandbox) {
|
|
2701
|
+
try {
|
|
2702
|
+
if (!didComputerUse) {
|
|
2703
|
+
await discardRecording({
|
|
2704
|
+
db,
|
|
2705
|
+
accountId: input.accountId,
|
|
2706
|
+
workspaceId: input.workspaceId,
|
|
2707
|
+
active: activeRecording,
|
|
2708
|
+
session: resolvedSandbox.established.session
|
|
2709
|
+
});
|
|
2710
|
+
} else {
|
|
2711
|
+
const outcome = await finalizeRecording({
|
|
2712
|
+
settings,
|
|
2713
|
+
db,
|
|
2714
|
+
objectStorage,
|
|
2715
|
+
accountId: input.accountId,
|
|
2716
|
+
workspaceId: input.workspaceId,
|
|
2717
|
+
sessionId: input.sessionId,
|
|
2718
|
+
active: activeRecording,
|
|
2719
|
+
session: resolvedSandbox.established.session,
|
|
2720
|
+
runAs: sandboxRunAs(settings)
|
|
2721
|
+
});
|
|
2722
|
+
if (publish) {
|
|
2723
|
+
await publish(outcome.ok ? [{ type: "recording.available", payload: outcome.available }] : [{ type: "recording.failed", payload: { recordingId: activeRecording.recordingId, turnId: activeRecording.turnId, reason: outcome.reason, detail: outcome.detail } }]);
|
|
2724
|
+
}
|
|
2725
|
+
}
|
|
2726
|
+
} catch (finalizeError) {
|
|
2727
|
+
console.error("recording finalize failed (turn outcome unaffected)", finalizeError);
|
|
2728
|
+
} finally {
|
|
2729
|
+
activeRecording = null;
|
|
2730
|
+
}
|
|
2731
|
+
}
|
|
2732
|
+
if (resolvedSandbox) {
|
|
2733
|
+
await resolvedSandbox.release().catch((releaseError) => {
|
|
2734
|
+
console.error("sandbox lease release failed (turn outcome unaffected)", releaseError);
|
|
2735
|
+
});
|
|
2736
|
+
resolvedSandbox = null;
|
|
2737
|
+
}
|
|
2738
|
+
}
|
|
2739
|
+
};
|
|
2740
|
+
}
|
|
2741
|
+
function agentRunFailurePayload(error) {
|
|
2742
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
2743
|
+
const status = typeof error === "object" && error !== null && "status" in error ? Number(error.status) : void 0;
|
|
2744
|
+
const code = typeof error === "object" && error !== null && "code" in error ? String(error.code) : void 0;
|
|
2745
|
+
const usageLimit = classifyCodexUsageLimitError(error);
|
|
2746
|
+
if (usageLimit) {
|
|
2747
|
+
return codexUsageLimitFailurePayload(usageLimit, message);
|
|
2748
|
+
}
|
|
2749
|
+
if (status === 429 || code === "rate_limit_exceeded" || /(?:too many requests|rate.?limit|\b429\b)/i.test(message)) {
|
|
2750
|
+
return {
|
|
2751
|
+
error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
|
|
2752
|
+
code: "provider_rate_limited",
|
|
2753
|
+
retryable: true,
|
|
2754
|
+
...message && message !== "Too Many Requests" ? { detail: message } : {}
|
|
2755
|
+
};
|
|
2756
|
+
}
|
|
2757
|
+
return { error: message };
|
|
2758
|
+
}
|
|
2759
|
+
function humanizeResetWindow(resetsInSeconds) {
|
|
2760
|
+
if (resetsInSeconds === null || !Number.isFinite(resetsInSeconds) || resetsInSeconds <= 0) {
|
|
2761
|
+
return "shortly";
|
|
2762
|
+
}
|
|
2763
|
+
const total = Math.ceil(resetsInSeconds);
|
|
2764
|
+
if (total < 60) {
|
|
2765
|
+
return "in under a minute";
|
|
2766
|
+
}
|
|
2767
|
+
const hours = Math.floor(total / 3600);
|
|
2768
|
+
const minutes = Math.round(total % 3600 / 60);
|
|
2769
|
+
if (hours > 0) {
|
|
2770
|
+
return minutes > 0 ? `in about ${hours}h ${minutes}m` : `in about ${hours}h`;
|
|
2771
|
+
}
|
|
2772
|
+
return `in about ${minutes}m`;
|
|
2773
|
+
}
|
|
2774
|
+
function codexUsageLimitFailurePayload(info, detail, opts) {
|
|
2775
|
+
const error = opts?.allAccounts ? `All connected ChatGPT/Codex subscriptions are rate-limited. Access returns ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for a subscription to reset.` : `Your ChatGPT/Codex subscription usage limit has been reached. Access resets ${humanizeResetWindow(info.resetsInSeconds)}. You can switch this session to a different model in the meantime, or wait for the limit to reset.`;
|
|
2776
|
+
return {
|
|
2777
|
+
error,
|
|
2778
|
+
code: "codex_usage_limit_reached",
|
|
2779
|
+
retryable: false,
|
|
2780
|
+
...detail ? { detail } : {}
|
|
2781
|
+
};
|
|
2782
|
+
}
|
|
2783
|
+
var CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 6e4;
|
|
2784
|
+
function isComputerCallStreamEvent(event) {
|
|
2785
|
+
if (!event || typeof event !== "object") {
|
|
2786
|
+
return false;
|
|
2787
|
+
}
|
|
2788
|
+
if (event.type !== "run_item_stream_event") {
|
|
2789
|
+
return false;
|
|
2790
|
+
}
|
|
2791
|
+
const item = event.item;
|
|
2792
|
+
if (!item || item.type !== "tool_call_item") {
|
|
2793
|
+
return false;
|
|
2794
|
+
}
|
|
2795
|
+
return item.rawItem?.type === "computer_call";
|
|
2796
|
+
}
|
|
2797
|
+
var BudgetExhaustedError = class extends Error {
|
|
2798
|
+
constructor(message, serializedRunState) {
|
|
2799
|
+
super(message);
|
|
2800
|
+
this.serializedRunState = serializedRunState;
|
|
2801
|
+
this.name = "BudgetExhaustedError";
|
|
2802
|
+
}
|
|
2803
|
+
serializedRunState;
|
|
2804
|
+
};
|
|
2805
|
+
async function ensureRunAllowed(settings, db, accountId, workspaceId, isCodexTurn, entitlements) {
|
|
2806
|
+
if (!isCodexTurn && entitlements && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
2807
|
+
const decision = await entitlements.admitRun({
|
|
2808
|
+
accountId,
|
|
2809
|
+
workspaceId,
|
|
2810
|
+
action: "agent_run:create",
|
|
2811
|
+
quantity: 1
|
|
2812
|
+
});
|
|
2813
|
+
if (!decision.allowed) {
|
|
2814
|
+
throw new Error(decision.reason || "insufficient OpenGeni credits");
|
|
2815
|
+
}
|
|
2816
|
+
} else if (!isCodexTurn && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
2817
|
+
const balance = await getBillingBalance2(db, accountId);
|
|
2818
|
+
if (balance.balanceMicros <= 0) {
|
|
2819
|
+
throw new Error("insufficient OpenGeni credits");
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
2823
|
+
const limits = configuredStaticUsageLimits2(settings);
|
|
2824
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
2825
|
+
const used = await sumUsageQuantity2(db, {
|
|
2826
|
+
workspaceId,
|
|
2827
|
+
eventType: "agent_run.created",
|
|
2828
|
+
since: startOfUtcMonth2()
|
|
2829
|
+
});
|
|
2830
|
+
if (used > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
2831
|
+
throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
|
|
2832
|
+
}
|
|
2833
|
+
}
|
|
2834
|
+
if (!isCodexTurn && limits.maxMonthlyTokensPerWorkspace) {
|
|
2835
|
+
const used = await sumUsageQuantity2(db, {
|
|
2836
|
+
workspaceId,
|
|
2837
|
+
eventType: "model.tokens",
|
|
2838
|
+
since: startOfUtcMonth2()
|
|
2839
|
+
});
|
|
2840
|
+
if (used >= limits.maxMonthlyTokensPerWorkspace) {
|
|
2841
|
+
throw new Error(`monthly token limit reached (${limits.maxMonthlyTokensPerWorkspace})`);
|
|
2842
|
+
}
|
|
2843
|
+
}
|
|
2844
|
+
}
|
|
2845
|
+
}
|
|
2846
|
+
async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
2847
|
+
if (!input.usage) {
|
|
2848
|
+
return;
|
|
2849
|
+
}
|
|
2850
|
+
const inputTokens = positiveInt(input.usage.inputTokens);
|
|
2851
|
+
const outputTokens = positiveInt(input.usage.outputTokens);
|
|
2852
|
+
const totalTokens = positiveInt(input.usage.totalTokens) || inputTokens + outputTokens;
|
|
2853
|
+
if (input.isCodexTurn) {
|
|
2854
|
+
await recordUsageEvent2(db, {
|
|
2855
|
+
accountId: input.accountId,
|
|
2856
|
+
workspaceId: input.workspaceId,
|
|
2857
|
+
eventType: "model.cost",
|
|
2858
|
+
quantity: 0,
|
|
2859
|
+
unit: "usd_micros",
|
|
2860
|
+
sourceResourceType: "model_response",
|
|
2861
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2862
|
+
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
|
|
2863
|
+
});
|
|
2864
|
+
return;
|
|
2865
|
+
}
|
|
2866
|
+
if (totalTokens > 0) {
|
|
2867
|
+
await recordUsageEvent2(db, {
|
|
2868
|
+
accountId: input.accountId,
|
|
2869
|
+
workspaceId: input.workspaceId,
|
|
2870
|
+
eventType: "model.tokens",
|
|
2871
|
+
quantity: totalTokens,
|
|
2872
|
+
unit: "tokens",
|
|
2873
|
+
sourceResourceType: "model_response",
|
|
2874
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2875
|
+
idempotencyKey: `usage:model.tokens:${input.turnId}:${input.sourceKey}`
|
|
2876
|
+
});
|
|
2877
|
+
}
|
|
2878
|
+
const shouldDebit = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
2879
|
+
if (!shouldDebit || totalTokens === 0) {
|
|
2880
|
+
return;
|
|
2881
|
+
}
|
|
2882
|
+
if (!configuredModelPricing(settings)[input.model]) {
|
|
2883
|
+
throw new Error(`Missing model pricing for ${input.model}`);
|
|
2884
|
+
}
|
|
2885
|
+
const costMicros = calculateModelUsageCostMicros(settings, input.model, input.usage);
|
|
2886
|
+
await recordUsageEvent2(db, {
|
|
2887
|
+
accountId: input.accountId,
|
|
2888
|
+
workspaceId: input.workspaceId,
|
|
2889
|
+
eventType: "model.cost",
|
|
2890
|
+
quantity: costMicros,
|
|
2891
|
+
unit: "usd_micros",
|
|
2892
|
+
sourceResourceType: "model_response",
|
|
2893
|
+
sourceResourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2894
|
+
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
|
|
2895
|
+
});
|
|
2896
|
+
if (costMicros > 0) {
|
|
2897
|
+
await applyCreditDebitUpToBalance(db, {
|
|
2898
|
+
accountId: input.accountId,
|
|
2899
|
+
workspaceId: input.workspaceId,
|
|
2900
|
+
type: "model_usage_debit",
|
|
2901
|
+
requestedAmountMicros: costMicros,
|
|
2902
|
+
sourceType: "model_response",
|
|
2903
|
+
sourceId: `${input.turnId}:${input.sourceKey}`,
|
|
2904
|
+
idempotencyKey: `credit:model_usage_debit:${input.turnId}:${input.sourceKey}`,
|
|
2905
|
+
metadata: {
|
|
2906
|
+
model: input.model,
|
|
2907
|
+
sessionId: input.sessionId,
|
|
2908
|
+
turnId: input.turnId,
|
|
2909
|
+
sourceKey: input.sourceKey,
|
|
2910
|
+
inputTokens,
|
|
2911
|
+
outputTokens,
|
|
2912
|
+
totalTokens
|
|
2913
|
+
}
|
|
2914
|
+
});
|
|
2915
|
+
}
|
|
2916
|
+
}
|
|
2917
|
+
function positiveInt(value) {
|
|
2918
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
|
2919
|
+
}
|
|
2920
|
+
function startOfUtcMonth2() {
|
|
2921
|
+
const now = /* @__PURE__ */ new Date();
|
|
2922
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
2923
|
+
}
|
|
2924
|
+
async function sandboxFileDownloadsForRun(settings, db, objectStorage, workspaceId, resources) {
|
|
2925
|
+
if (settings.sandboxBackend === "none" || !requiresSignedFileResourceDownloads(settings)) {
|
|
2926
|
+
return [];
|
|
2927
|
+
}
|
|
2928
|
+
const fileResources = resources.filter((resource) => resource.kind === "file");
|
|
2929
|
+
if (fileResources.length === 0) {
|
|
2930
|
+
return [];
|
|
2931
|
+
}
|
|
2932
|
+
if (!objectStorage) {
|
|
2933
|
+
throw new Error(`${settings.objectStorageBackend} file resources require configured object storage`);
|
|
2934
|
+
}
|
|
2935
|
+
const downloadStorage = objectStorageForSandboxDownloads(settings, objectStorage);
|
|
2936
|
+
const downloads = [];
|
|
2937
|
+
for (const resource of fileResources) {
|
|
2938
|
+
const file = await requireFile2(db, workspaceId, resource.fileId);
|
|
2939
|
+
const url = await downloadStorage.createGetUrl({ key: file.objectKey });
|
|
2940
|
+
downloads.push({
|
|
2941
|
+
fileId: file.id,
|
|
2942
|
+
mountPath: resource.mountPath ?? `files/${file.id}`,
|
|
2943
|
+
filename: file.safeFilename,
|
|
2944
|
+
url: url.url,
|
|
2945
|
+
expiresAt: url.expiresAt,
|
|
2946
|
+
sizeBytes: file.sizeBytes
|
|
2947
|
+
});
|
|
2948
|
+
}
|
|
2949
|
+
return downloads;
|
|
2950
|
+
}
|
|
2951
|
+
function requiresSignedFileResourceDownloads(settings) {
|
|
2952
|
+
if (settings.sandboxBackend === "selfhosted") {
|
|
2953
|
+
return true;
|
|
2954
|
+
}
|
|
2955
|
+
const nativeBucketMount = CAPABILITY_DESCRIPTORS[settings.sandboxBackend].nativeBucketMount;
|
|
2956
|
+
return settings.sandboxBackend === "docker" && settings.objectStorageBackend === "s3-compatible" || settings.objectStorageBackend === "aws-s3" || settings.objectStorageBackend === "gcs" || nativeBucketMount && settings.objectStorageBackend === "azure-blob";
|
|
2957
|
+
}
|
|
2958
|
+
function objectStorageForSandboxDownloads(settings, objectStorage) {
|
|
2959
|
+
if (settings.objectStorageBackend !== "s3-compatible" || !settings.objectStorageSandboxEndpoint) {
|
|
2960
|
+
return objectStorage;
|
|
2961
|
+
}
|
|
2962
|
+
return createObjectStorage({
|
|
2963
|
+
...settings,
|
|
2964
|
+
objectStorageEndpoint: settings.objectStorageSandboxEndpoint
|
|
2965
|
+
}) ?? objectStorage;
|
|
2966
|
+
}
|
|
2967
|
+
|
|
2968
|
+
// src/activities/documents.ts
|
|
2969
|
+
import { indexDocumentNow } from "@opengeni/documents";
|
|
2970
|
+
import { configuredStaticUsageLimits as configuredStaticUsageLimits3 } from "@opengeni/config";
|
|
2971
|
+
import {
|
|
2972
|
+
getBillingBalance as getBillingBalance3,
|
|
2973
|
+
recordUsageEvent as recordUsageEvent3,
|
|
2974
|
+
sumUsageQuantity as sumUsageQuantity3,
|
|
2975
|
+
withWorkspaceUsageLock
|
|
2976
|
+
} from "@opengeni/db";
|
|
2977
|
+
function createDocumentActivities(services) {
|
|
2978
|
+
return {
|
|
2979
|
+
indexDocument: async (input) => {
|
|
2980
|
+
const { settings, db, objectStorage, documentServices } = await services();
|
|
2981
|
+
if (!objectStorage) {
|
|
2982
|
+
throw new Error("object storage is not configured");
|
|
2983
|
+
}
|
|
2984
|
+
return await withWorkspaceUsageLock(db, input.workspaceId, async (lockedDb) => {
|
|
2985
|
+
const document = await indexDocumentNow(lockedDb, objectStorage, input.workspaceId, input.documentId, documentServices, {
|
|
2986
|
+
beforeEmbed: async ({ chunkCount }) => {
|
|
2987
|
+
if (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed") {
|
|
2988
|
+
const balance = await getBillingBalance3(lockedDb, input.accountId);
|
|
2989
|
+
if (balance.balanceMicros <= 0) {
|
|
2990
|
+
throw new Error("insufficient OpenGeni credits");
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
if (settings.usageLimitsMode !== "static" && settings.usageLimitsMode !== "managed") {
|
|
2994
|
+
return;
|
|
2995
|
+
}
|
|
2996
|
+
const limit = configuredStaticUsageLimits3(settings).maxDocumentIndexedChunksPerWorkspace;
|
|
2997
|
+
if (!limit) {
|
|
2998
|
+
return;
|
|
2999
|
+
}
|
|
3000
|
+
const used = await sumUsageQuantity3(lockedDb, {
|
|
3001
|
+
workspaceId: input.workspaceId,
|
|
3002
|
+
eventType: "document.indexed",
|
|
3003
|
+
since: startOfUtcMonth3()
|
|
3004
|
+
});
|
|
3005
|
+
if (used + chunkCount > limit) {
|
|
3006
|
+
throw new Error(`monthly document indexing limit reached (${limit} chunks)`);
|
|
3007
|
+
}
|
|
3008
|
+
}
|
|
3009
|
+
});
|
|
3010
|
+
if (document.status === "ready") {
|
|
3011
|
+
await recordUsageEvent3(lockedDb, {
|
|
3012
|
+
accountId: input.accountId,
|
|
3013
|
+
workspaceId: input.workspaceId,
|
|
3014
|
+
eventType: "document.indexed",
|
|
3015
|
+
quantity: document.chunkCount,
|
|
3016
|
+
unit: "chunk",
|
|
3017
|
+
sourceResourceType: "document",
|
|
3018
|
+
sourceResourceId: document.id,
|
|
3019
|
+
idempotencyKey: `document.indexed:${input.workspaceId}:${document.id}:${document.updatedAt}`
|
|
3020
|
+
});
|
|
3021
|
+
}
|
|
3022
|
+
return document;
|
|
3023
|
+
});
|
|
3024
|
+
}
|
|
3025
|
+
};
|
|
3026
|
+
}
|
|
3027
|
+
function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
3028
|
+
return new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), 1));
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
// src/activities/sandbox-lease.ts
|
|
3032
|
+
import {
|
|
3033
|
+
accrueWarmSeconds as accrueWarmSeconds2,
|
|
3034
|
+
confirmDrainCold,
|
|
3035
|
+
forceDrainOverLimitViewerOnlyBoxes,
|
|
3036
|
+
getBillingBalance as getBillingBalance4,
|
|
3037
|
+
listMeterableWarmLeases,
|
|
3038
|
+
persistDrainSnapshot,
|
|
3039
|
+
readLease as readLease2,
|
|
3040
|
+
reapStaleLeaseHoldersGlobal,
|
|
3041
|
+
rlsContextForWorkspace
|
|
3042
|
+
} from "@opengeni/db";
|
|
3043
|
+
import { sandboxWarmRateMicrosPerSecond as sandboxWarmRateMicrosPerSecond2 } from "@opengeni/config";
|
|
3044
|
+
import {
|
|
3045
|
+
createSandboxClientForBackend,
|
|
3046
|
+
deletePriorPersistedSnapshot,
|
|
3047
|
+
deserializeSandboxSessionStateEnvelope,
|
|
3048
|
+
isProviderSandboxNotFoundError
|
|
3049
|
+
} from "@opengeni/runtime";
|
|
3050
|
+
function createSandboxLeaseActivities(services, options = {}) {
|
|
3051
|
+
const terminateBox = options.terminateBox ?? terminateProviderBox;
|
|
3052
|
+
async function reapSandboxLeases2() {
|
|
3053
|
+
const { db, settings, observability } = await services();
|
|
3054
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
3055
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
3056
|
+
}
|
|
3057
|
+
const metered = await accrueWarmTick(db, settings, observability);
|
|
3058
|
+
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3059
|
+
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
3060
|
+
viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
|
|
3061
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
3062
|
+
});
|
|
3063
|
+
let terminated = 0;
|
|
3064
|
+
let skipped = 0;
|
|
3065
|
+
for (const row of drainable) {
|
|
3066
|
+
try {
|
|
3067
|
+
const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
|
|
3068
|
+
if (drainedCold) {
|
|
3069
|
+
terminated += 1;
|
|
3070
|
+
} else {
|
|
3071
|
+
skipped += 1;
|
|
3072
|
+
}
|
|
3073
|
+
} catch (error) {
|
|
3074
|
+
skipped += 1;
|
|
3075
|
+
observability.warn("sandbox reaper: terminate failed for drainable lease", {
|
|
3076
|
+
workspaceId: row.workspaceId,
|
|
3077
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3078
|
+
leaseEpoch: row.leaseEpoch,
|
|
3079
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3080
|
+
});
|
|
3081
|
+
}
|
|
3082
|
+
}
|
|
3083
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0) {
|
|
3084
|
+
observability.info("sandbox reaper swept", {
|
|
3085
|
+
drainable: drainable.length,
|
|
3086
|
+
terminated,
|
|
3087
|
+
skipped,
|
|
3088
|
+
metered: metered.accrued,
|
|
3089
|
+
forceDrained
|
|
3090
|
+
});
|
|
3091
|
+
}
|
|
3092
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
3093
|
+
}
|
|
3094
|
+
return { reapSandboxLeases: reapSandboxLeases2 };
|
|
3095
|
+
}
|
|
3096
|
+
async function accrueWarmTick(db, settings, observability) {
|
|
3097
|
+
const workspaceIds = /* @__PURE__ */ new Set();
|
|
3098
|
+
let accrued = 0;
|
|
3099
|
+
let leases = [];
|
|
3100
|
+
try {
|
|
3101
|
+
leases = await listMeterableWarmLeases(db);
|
|
3102
|
+
} catch (error) {
|
|
3103
|
+
observability.warn("sandbox reaper: warm-lease read failed", {
|
|
3104
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3105
|
+
});
|
|
3106
|
+
return { accrued, workspaceIds };
|
|
3107
|
+
}
|
|
3108
|
+
for (const lease of leases) {
|
|
3109
|
+
workspaceIds.add(lease.workspaceId);
|
|
3110
|
+
try {
|
|
3111
|
+
const rate = sandboxWarmRateMicrosPerSecond2(settings, lease.backend);
|
|
3112
|
+
const result = await accrueWarmSeconds2(db, {
|
|
3113
|
+
accountId: lease.accountId,
|
|
3114
|
+
workspaceId: lease.workspaceId,
|
|
3115
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3116
|
+
expectedEpoch: lease.leaseEpoch,
|
|
3117
|
+
warmRateMicrosPerSecond: rate,
|
|
3118
|
+
subjectId: lease.sandboxGroupId
|
|
3119
|
+
});
|
|
3120
|
+
if (result.accrued) {
|
|
3121
|
+
accrued += 1;
|
|
3122
|
+
}
|
|
3123
|
+
} catch (error) {
|
|
3124
|
+
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3125
|
+
workspaceId: lease.workspaceId,
|
|
3126
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3127
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3128
|
+
});
|
|
3129
|
+
}
|
|
3130
|
+
}
|
|
3131
|
+
return { accrued, workspaceIds };
|
|
3132
|
+
}
|
|
3133
|
+
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3134
|
+
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
3135
|
+
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
3136
|
+
if (!enforceBalance && cap <= 0) {
|
|
3137
|
+
return 0;
|
|
3138
|
+
}
|
|
3139
|
+
let forceDrained = 0;
|
|
3140
|
+
for (const workspaceId of workspaceIds) {
|
|
3141
|
+
try {
|
|
3142
|
+
const { accountId } = await rlsContextForWorkspace(db, workspaceId);
|
|
3143
|
+
const balance = enforceBalance ? await getBillingBalance4(db, accountId) : { balanceMicros: 1 };
|
|
3144
|
+
const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
|
|
3145
|
+
workspaceId,
|
|
3146
|
+
balanceMicros: balance.balanceMicros,
|
|
3147
|
+
enforceBalance,
|
|
3148
|
+
maxWarmSecondsPerWorkspace: cap,
|
|
3149
|
+
idleGraceMs: settings.sandboxIdleGraceMs
|
|
3150
|
+
});
|
|
3151
|
+
if (result.overLimit && result.drained.length > 0) {
|
|
3152
|
+
forceDrained += result.drained.length;
|
|
3153
|
+
observability.info("sandbox reaper: force-drained viewer-only boxes (over limit)", {
|
|
3154
|
+
workspaceId,
|
|
3155
|
+
reason: result.reason,
|
|
3156
|
+
drained: result.drained.length
|
|
3157
|
+
});
|
|
3158
|
+
}
|
|
3159
|
+
} catch (error) {
|
|
3160
|
+
observability.warn("sandbox reaper: force-drain check failed for workspace", {
|
|
3161
|
+
workspaceId,
|
|
3162
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3163
|
+
});
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
return forceDrained;
|
|
3167
|
+
}
|
|
3168
|
+
async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
|
|
3169
|
+
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
3170
|
+
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
3171
|
+
if (!lease) {
|
|
3172
|
+
return false;
|
|
3173
|
+
}
|
|
3174
|
+
if (lease.liveness !== "draining" || lease.refcount !== 0 || lease.leaseEpoch !== row.leaseEpoch) {
|
|
3175
|
+
return false;
|
|
3176
|
+
}
|
|
3177
|
+
const persistArchive = async (archiveBase64) => await persistDrainSnapshot(db, {
|
|
3178
|
+
accountId,
|
|
3179
|
+
workspaceId: row.workspaceId,
|
|
3180
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3181
|
+
expectedEpoch: row.leaseEpoch,
|
|
3182
|
+
workspaceArchive: archiveBase64
|
|
3183
|
+
});
|
|
3184
|
+
const terminated = await terminateBox(settings, lease, observability, persistArchive);
|
|
3185
|
+
if (!terminated) {
|
|
3186
|
+
return false;
|
|
3187
|
+
}
|
|
3188
|
+
const { wentCold } = await confirmDrainCold(db, {
|
|
3189
|
+
accountId,
|
|
3190
|
+
workspaceId: row.workspaceId,
|
|
3191
|
+
sandboxGroupId: row.sandboxGroupId,
|
|
3192
|
+
expectedEpoch: row.leaseEpoch
|
|
3193
|
+
});
|
|
3194
|
+
return wentCold;
|
|
3195
|
+
}
|
|
3196
|
+
async function terminateProviderBox(settings, lease, observability, persistArchive, createClientForBackend = createSandboxClientForBackend) {
|
|
3197
|
+
const backend = lease.resumeBackendId ?? lease.backend;
|
|
3198
|
+
if (!backend || backend === "none") {
|
|
3199
|
+
return true;
|
|
3200
|
+
}
|
|
3201
|
+
if (backend === "selfhosted" || lease.backend === "selfhosted" || lease.resumeBackendId === "selfhosted") {
|
|
3202
|
+
observability.info("sandbox reaper: selfhosted lease drained to cold (NEVER provider-stopped \u2014 it is the user's machine)", {
|
|
3203
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3204
|
+
backend
|
|
3205
|
+
});
|
|
3206
|
+
return true;
|
|
3207
|
+
}
|
|
3208
|
+
if (!lease.resumeState) {
|
|
3209
|
+
return true;
|
|
3210
|
+
}
|
|
3211
|
+
const client = createClientForBackend(backend, settings);
|
|
3212
|
+
if (!client) {
|
|
3213
|
+
return true;
|
|
3214
|
+
}
|
|
3215
|
+
let session;
|
|
3216
|
+
let sessionState;
|
|
3217
|
+
try {
|
|
3218
|
+
if (!client.resume || !client.deserializeSessionState) {
|
|
3219
|
+
return true;
|
|
3220
|
+
}
|
|
3221
|
+
const envelopeSessionState = lease.resumeState.sessionState ?? lease.resumeState;
|
|
3222
|
+
const resumedState = await deserializeSandboxSessionStateEnvelope(client, envelopeSessionState);
|
|
3223
|
+
if (resumedState === void 0) {
|
|
3224
|
+
return true;
|
|
3225
|
+
}
|
|
3226
|
+
session = await client.resume(resumedState);
|
|
3227
|
+
sessionState = resumedState;
|
|
3228
|
+
} catch (error) {
|
|
3229
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3230
|
+
observability.info("sandbox reaper: drainable box already gone (NotFound on resume) \u2014 proceeding to cold", {
|
|
3231
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3232
|
+
backend
|
|
3233
|
+
});
|
|
3234
|
+
return true;
|
|
3235
|
+
}
|
|
3236
|
+
throw error;
|
|
3237
|
+
}
|
|
3238
|
+
let archiveBytes;
|
|
3239
|
+
try {
|
|
3240
|
+
archiveBytes = session?.persistWorkspace ? await session.persistWorkspace() : void 0;
|
|
3241
|
+
} catch (error) {
|
|
3242
|
+
if (isProviderSandboxNotFoundError(client.backendId, error)) {
|
|
3243
|
+
observability.info("sandbox reaper: box gone during persist (NotFound) \u2014 nothing to snapshot, proceeding to cold", {
|
|
3244
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3245
|
+
backend
|
|
3246
|
+
});
|
|
3247
|
+
return true;
|
|
3248
|
+
}
|
|
3249
|
+
observability.warn("sandbox reaper: persistWorkspace failed \u2014 leaving box draining (files NOT lost)", {
|
|
3250
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3251
|
+
backend,
|
|
3252
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3253
|
+
});
|
|
3254
|
+
throw error;
|
|
3255
|
+
}
|
|
3256
|
+
if (archiveBytes && archiveBytes.length > 0) {
|
|
3257
|
+
const archiveBase64 = Buffer.from(archiveBytes).toString("base64");
|
|
3258
|
+
const { wrote, priorArchive } = await persistArchive(archiveBase64);
|
|
3259
|
+
if (!wrote) {
|
|
3260
|
+
observability.info("sandbox reaper: lease re-armed during persist \u2014 leaving box RUNNING (no terminate)", {
|
|
3261
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3262
|
+
backend
|
|
3263
|
+
});
|
|
3264
|
+
return false;
|
|
3265
|
+
}
|
|
3266
|
+
const deleted = await deletePriorPersistedSnapshot(session, priorArchive);
|
|
3267
|
+
if (deleted) {
|
|
3268
|
+
observability.info("sandbox reaper: GC'd superseded workspace snapshot", {
|
|
3269
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3270
|
+
backend,
|
|
3271
|
+
snapshotId: deleted
|
|
3272
|
+
});
|
|
3273
|
+
}
|
|
3274
|
+
} else {
|
|
3275
|
+
const { wrote } = await persistArchive(null);
|
|
3276
|
+
if (!wrote) {
|
|
3277
|
+
observability.info("sandbox reaper: lease re-armed during snapshot window (no-archive path) \u2014 leaving box RUNNING (no terminate)", {
|
|
3278
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3279
|
+
backend
|
|
3280
|
+
});
|
|
3281
|
+
return false;
|
|
3282
|
+
}
|
|
3283
|
+
}
|
|
3284
|
+
try {
|
|
3285
|
+
if (client.delete && sessionState !== void 0) {
|
|
3286
|
+
await client.delete(sessionState);
|
|
3287
|
+
return true;
|
|
3288
|
+
}
|
|
3289
|
+
if (session?.kill) {
|
|
3290
|
+
await session.kill();
|
|
3291
|
+
return true;
|
|
3292
|
+
}
|
|
3293
|
+
if (session?.terminate) {
|
|
3294
|
+
await session.terminate();
|
|
3295
|
+
return true;
|
|
3296
|
+
}
|
|
3297
|
+
if (session?.close && !session.closed) {
|
|
3298
|
+
await session.close();
|
|
3299
|
+
}
|
|
3300
|
+
} catch (error) {
|
|
3301
|
+
observability.warn("sandbox reaper: provider terminate raised (box may already be gone)", {
|
|
3302
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3303
|
+
backend,
|
|
3304
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3305
|
+
});
|
|
3306
|
+
}
|
|
3307
|
+
return true;
|
|
3308
|
+
}
|
|
3309
|
+
|
|
3310
|
+
// src/activities/scheduled-tasks.ts
|
|
3311
|
+
import {
|
|
3312
|
+
appendSessionEventsWithLockedSessionUpdate,
|
|
3313
|
+
createScheduledTaskRun,
|
|
3314
|
+
createSession,
|
|
3315
|
+
createSessionGoal,
|
|
3316
|
+
enqueueSessionTurn as enqueueSessionTurn2,
|
|
3317
|
+
getBillingBalance as getBillingBalance5,
|
|
3318
|
+
getWorkspaceEnvironment,
|
|
3319
|
+
isCodexBilledTurn as isCodexBilledTurn3,
|
|
3320
|
+
recordUsageEvent as recordUsageEvent4,
|
|
3321
|
+
requireScheduledTask,
|
|
3322
|
+
requireSession as requireSession3,
|
|
3323
|
+
setTemporalWorkflowId,
|
|
3324
|
+
sumUsageQuantity as sumUsageQuantity4,
|
|
3325
|
+
updateScheduledTask,
|
|
3326
|
+
updateScheduledTaskRun,
|
|
3327
|
+
upsertSessionGoal
|
|
3328
|
+
} from "@opengeni/db";
|
|
3329
|
+
import { appendAndPublishEvents as appendAndPublishEvents3 } from "@opengeni/events";
|
|
3330
|
+
import { configuredStaticUsageLimits as configuredStaticUsageLimits4 } from "@opengeni/config";
|
|
3331
|
+
function createScheduledTaskActivities(services) {
|
|
3332
|
+
return {
|
|
3333
|
+
dispatchScheduledTaskRun: async (input) => {
|
|
3334
|
+
const { settings, db, bus } = await services();
|
|
3335
|
+
const task = await requireScheduledTask(db, input.workspaceId, input.taskId);
|
|
3336
|
+
const model = task.agentConfig.model ?? settings.openaiModel;
|
|
3337
|
+
const isCodexRun = await isCodexBilledTurn3({ db, settings, workspaceId: task.workspaceId, model });
|
|
3338
|
+
await ensureScheduledRunAllowed(settings, db, task.accountId, task.workspaceId, input.agentRunUsageIdempotencyKey ? 0 : 1, isCodexRun);
|
|
3339
|
+
const run = await createScheduledTaskRun(db, {
|
|
3340
|
+
workspaceId: task.workspaceId,
|
|
3341
|
+
taskId: task.id,
|
|
3342
|
+
triggerType: input.triggerType,
|
|
3343
|
+
scheduledAt: null
|
|
3344
|
+
});
|
|
3345
|
+
await recordUsageEvent4(db, {
|
|
3346
|
+
accountId: task.accountId,
|
|
3347
|
+
workspaceId: task.workspaceId,
|
|
3348
|
+
eventType: "scheduled_task.fired",
|
|
3349
|
+
quantity: 1,
|
|
3350
|
+
unit: "run",
|
|
3351
|
+
sourceResourceType: "scheduled_task_run",
|
|
3352
|
+
sourceResourceId: run.id,
|
|
3353
|
+
idempotencyKey: `usage:scheduled_task.fired:${run.id}`
|
|
3354
|
+
});
|
|
3355
|
+
let result;
|
|
3356
|
+
try {
|
|
3357
|
+
const reasoningEffort = task.agentConfig.reasoningEffort ?? settings.openaiReasoningEffort;
|
|
3358
|
+
const sandboxBackend = task.agentConfig.sandboxBackend ?? settings.sandboxBackend;
|
|
3359
|
+
const goalSpec = task.agentConfig.goal ?? null;
|
|
3360
|
+
const taskTools = withFirstPartyTools(settings, task.agentConfig.tools);
|
|
3361
|
+
if (task.runMode === "new_session_per_run" || !task.reusableSessionId) {
|
|
3362
|
+
const environment = task.environmentId ? await getWorkspaceEnvironment(db, task.workspaceId, task.environmentId) : null;
|
|
3363
|
+
if (task.environmentId && !environment) {
|
|
3364
|
+
throw new Error(`workspace environment not found: ${task.environmentId}`);
|
|
3365
|
+
}
|
|
3366
|
+
const session = await createSession(db, {
|
|
3367
|
+
accountId: task.accountId,
|
|
3368
|
+
workspaceId: task.workspaceId,
|
|
3369
|
+
initialMessage: task.agentConfig.prompt,
|
|
3370
|
+
resources: task.agentConfig.resources,
|
|
3371
|
+
tools: taskTools,
|
|
3372
|
+
metadata: {
|
|
3373
|
+
...task.agentConfig.metadata,
|
|
3374
|
+
model,
|
|
3375
|
+
reasoningEffort,
|
|
3376
|
+
scheduledTaskId: task.id,
|
|
3377
|
+
scheduledTaskRunId: run.id
|
|
3378
|
+
},
|
|
3379
|
+
model,
|
|
3380
|
+
sandboxBackend,
|
|
3381
|
+
environmentId: task.environmentId ?? null
|
|
3382
|
+
});
|
|
3383
|
+
const goal = goalSpec ? await createSessionGoal(db, {
|
|
3384
|
+
accountId: task.accountId,
|
|
3385
|
+
workspaceId: task.workspaceId,
|
|
3386
|
+
sessionId: session.id,
|
|
3387
|
+
text: goalSpec.text,
|
|
3388
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
3389
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
3390
|
+
createdBy: "scheduled_task"
|
|
3391
|
+
}) : null;
|
|
3392
|
+
const workflowId = workflowIdForSession(session.id);
|
|
3393
|
+
await setTemporalWorkflowId(db, task.workspaceId, session.id, workflowId);
|
|
3394
|
+
if (task.runMode === "reusable_session") {
|
|
3395
|
+
await updateScheduledTask(db, task.workspaceId, task.id, { reusableSessionId: session.id });
|
|
3396
|
+
}
|
|
3397
|
+
const events = await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [
|
|
3398
|
+
{
|
|
3399
|
+
type: "session.created",
|
|
3400
|
+
payload: {
|
|
3401
|
+
status: "queued",
|
|
3402
|
+
scheduledTaskId: task.id,
|
|
3403
|
+
scheduledTaskRunId: run.id,
|
|
3404
|
+
// Names/ids only; never values.
|
|
3405
|
+
...environment ? { environmentId: environment.id, environmentName: environment.name } : {}
|
|
3406
|
+
}
|
|
3407
|
+
},
|
|
3408
|
+
...goal ? [{
|
|
3409
|
+
type: "goal.set",
|
|
3410
|
+
payload: {
|
|
3411
|
+
goalId: goal.id,
|
|
3412
|
+
text: goal.text,
|
|
3413
|
+
...goal.successCriteria ? { successCriteria: goal.successCriteria } : {},
|
|
3414
|
+
version: goal.version,
|
|
3415
|
+
actor: "scheduled_task",
|
|
3416
|
+
replaced: false
|
|
3417
|
+
}
|
|
3418
|
+
}] : [],
|
|
3419
|
+
{
|
|
3420
|
+
type: "user.message",
|
|
3421
|
+
payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id)
|
|
3422
|
+
},
|
|
3423
|
+
{ type: "session.status.changed", payload: { status: "queued" } }
|
|
3424
|
+
]);
|
|
3425
|
+
const trigger = events.find((event) => event.type === "user.message");
|
|
3426
|
+
if (!trigger) {
|
|
3427
|
+
throw new Error("failed to append scheduled task trigger event");
|
|
3428
|
+
}
|
|
3429
|
+
const turn = await enqueueSessionTurn2(db, {
|
|
3430
|
+
accountId: task.accountId,
|
|
3431
|
+
workspaceId: task.workspaceId,
|
|
3432
|
+
sessionId: session.id,
|
|
3433
|
+
triggerEventId: trigger.id,
|
|
3434
|
+
temporalWorkflowId: workflowId,
|
|
3435
|
+
source: "scheduled_task",
|
|
3436
|
+
prompt: task.agentConfig.prompt,
|
|
3437
|
+
resources: task.agentConfig.resources,
|
|
3438
|
+
tools: taskTools,
|
|
3439
|
+
model,
|
|
3440
|
+
reasoningEffort,
|
|
3441
|
+
sandboxBackend,
|
|
3442
|
+
metadata: {
|
|
3443
|
+
scheduledTaskId: task.id,
|
|
3444
|
+
scheduledTaskRunId: run.id
|
|
3445
|
+
}
|
|
3446
|
+
});
|
|
3447
|
+
await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [{
|
|
3448
|
+
type: "turn.queued",
|
|
3449
|
+
turnId: turn.id,
|
|
3450
|
+
payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source }
|
|
3451
|
+
}]);
|
|
3452
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
3453
|
+
status: "dispatched",
|
|
3454
|
+
sessionId: session.id,
|
|
3455
|
+
triggerEventId: trigger.id
|
|
3456
|
+
});
|
|
3457
|
+
result = {
|
|
3458
|
+
action: "start",
|
|
3459
|
+
accountId: task.accountId,
|
|
3460
|
+
workspaceId: task.workspaceId,
|
|
3461
|
+
sessionId: session.id,
|
|
3462
|
+
triggerEventId: trigger.id,
|
|
3463
|
+
workflowId
|
|
3464
|
+
};
|
|
3465
|
+
} else {
|
|
3466
|
+
const session = await requireSession3(db, task.workspaceId, task.reusableSessionId);
|
|
3467
|
+
assertReusableSessionRevivable(session.status);
|
|
3468
|
+
if ((session.environmentId ?? null) !== (task.environmentId ?? null)) {
|
|
3469
|
+
throw new Error("scheduled task environment attachment does not match its reusable session");
|
|
3470
|
+
}
|
|
3471
|
+
const reusableGoal = goalSpec ? await upsertSessionGoal(db, {
|
|
3472
|
+
accountId: task.accountId,
|
|
3473
|
+
workspaceId: task.workspaceId,
|
|
3474
|
+
sessionId: session.id,
|
|
3475
|
+
text: goalSpec.text,
|
|
3476
|
+
successCriteria: goalSpec.successCriteria ?? null,
|
|
3477
|
+
maxAutoContinuations: goalSpec.maxAutoContinuations ?? null,
|
|
3478
|
+
createdBy: "scheduled_task"
|
|
3479
|
+
}) : null;
|
|
3480
|
+
const events = await appendSessionEventsWithLockedSessionUpdate(db, task.workspaceId, session.id, (locked) => {
|
|
3481
|
+
assertReusableSessionRevivable(locked.status);
|
|
3482
|
+
return {
|
|
3483
|
+
events: [
|
|
3484
|
+
...reusableGoal ? [{
|
|
3485
|
+
type: "goal.set",
|
|
3486
|
+
payload: {
|
|
3487
|
+
goalId: reusableGoal.goal.id,
|
|
3488
|
+
text: reusableGoal.goal.text,
|
|
3489
|
+
...reusableGoal.goal.successCriteria ? { successCriteria: reusableGoal.goal.successCriteria } : {},
|
|
3490
|
+
version: reusableGoal.goal.version,
|
|
3491
|
+
actor: "scheduled_task",
|
|
3492
|
+
replaced: reusableGoal.replaced
|
|
3493
|
+
}
|
|
3494
|
+
}] : [],
|
|
3495
|
+
{
|
|
3496
|
+
type: "user.message",
|
|
3497
|
+
payload: scheduledUserMessagePayload(task.agentConfig.prompt, task.agentConfig.resources, taskTools, task.id, run.id)
|
|
3498
|
+
}
|
|
3499
|
+
],
|
|
3500
|
+
update: {
|
|
3501
|
+
resources: mergeResourceRefs(locked.resources, task.agentConfig.resources),
|
|
3502
|
+
tools: mergeToolRefs(locked.tools, taskTools)
|
|
3503
|
+
}
|
|
3504
|
+
};
|
|
3505
|
+
});
|
|
3506
|
+
await bus.publish(task.workspaceId, session.id, events);
|
|
3507
|
+
const trigger = events.find((event) => event.type === "user.message");
|
|
3508
|
+
if (!trigger) {
|
|
3509
|
+
throw new Error("failed to append scheduled task trigger event");
|
|
3510
|
+
}
|
|
3511
|
+
const turn = await enqueueSessionTurn2(db, {
|
|
3512
|
+
accountId: task.accountId,
|
|
3513
|
+
workspaceId: task.workspaceId,
|
|
3514
|
+
sessionId: session.id,
|
|
3515
|
+
triggerEventId: trigger.id,
|
|
3516
|
+
temporalWorkflowId: workflowIdForSession(session.id),
|
|
3517
|
+
source: "scheduled_task",
|
|
3518
|
+
prompt: task.agentConfig.prompt,
|
|
3519
|
+
resources: task.agentConfig.resources,
|
|
3520
|
+
tools: taskTools,
|
|
3521
|
+
model,
|
|
3522
|
+
reasoningEffort,
|
|
3523
|
+
sandboxBackend,
|
|
3524
|
+
metadata: {
|
|
3525
|
+
scheduledTaskId: task.id,
|
|
3526
|
+
scheduledTaskRunId: run.id
|
|
3527
|
+
}
|
|
3528
|
+
});
|
|
3529
|
+
await appendAndPublishEvents3(db, bus, task.workspaceId, session.id, [{
|
|
3530
|
+
type: "turn.queued",
|
|
3531
|
+
turnId: turn.id,
|
|
3532
|
+
payload: { turnId: turn.id, triggerEventId: trigger.id, source: turn.source }
|
|
3533
|
+
}]);
|
|
3534
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
3535
|
+
status: "dispatched",
|
|
3536
|
+
sessionId: session.id,
|
|
3537
|
+
triggerEventId: trigger.id
|
|
3538
|
+
});
|
|
3539
|
+
result = {
|
|
3540
|
+
action: "signal",
|
|
3541
|
+
accountId: task.accountId,
|
|
3542
|
+
workspaceId: task.workspaceId,
|
|
3543
|
+
sessionId: session.id,
|
|
3544
|
+
triggerEventId: trigger.id,
|
|
3545
|
+
workflowId: workflowIdForSession(session.id)
|
|
3546
|
+
};
|
|
3547
|
+
}
|
|
3548
|
+
} catch (error) {
|
|
3549
|
+
await updateScheduledTaskRun(db, task.workspaceId, run.id, {
|
|
3550
|
+
status: "failed",
|
|
3551
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3552
|
+
}).catch(() => void 0);
|
|
3553
|
+
throw error;
|
|
3554
|
+
}
|
|
3555
|
+
await recordUsageEvent4(db, {
|
|
3556
|
+
accountId: task.accountId,
|
|
3557
|
+
workspaceId: task.workspaceId,
|
|
3558
|
+
eventType: "agent_run.created",
|
|
3559
|
+
quantity: 1,
|
|
3560
|
+
unit: "run",
|
|
3561
|
+
sourceResourceType: "scheduled_task_run",
|
|
3562
|
+
sourceResourceId: run.id,
|
|
3563
|
+
idempotencyKey: input.agentRunUsageIdempotencyKey ?? `usage:agent_run.created:scheduled:${run.id}`
|
|
3564
|
+
});
|
|
3565
|
+
return result;
|
|
3566
|
+
}
|
|
3567
|
+
};
|
|
3568
|
+
}
|
|
3569
|
+
async function ensureScheduledRunAllowed(settings, db, accountId, workspaceId, requestedAgentRuns, isCodexRun) {
|
|
3570
|
+
if (!isCodexRun && (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")) {
|
|
3571
|
+
const balance = await getBillingBalance5(db, accountId);
|
|
3572
|
+
if (balance.balanceMicros <= 0) {
|
|
3573
|
+
throw new Error("insufficient OpenGeni credits");
|
|
3574
|
+
}
|
|
3575
|
+
}
|
|
3576
|
+
if (settings.usageLimitsMode === "static" || settings.usageLimitsMode === "managed") {
|
|
3577
|
+
const limits = configuredStaticUsageLimits4(settings);
|
|
3578
|
+
if (!isCodexRun && limits.maxMonthlyCostMicrosPerAccount) {
|
|
3579
|
+
const used = await sumUsageQuantity4(db, {
|
|
3580
|
+
accountId,
|
|
3581
|
+
eventType: "model.cost",
|
|
3582
|
+
since: startOfUtcMonth4()
|
|
3583
|
+
});
|
|
3584
|
+
if (used >= limits.maxMonthlyCostMicrosPerAccount) {
|
|
3585
|
+
throw new Error(`monthly model cost limit reached (${limits.maxMonthlyCostMicrosPerAccount} micros)`);
|
|
3586
|
+
}
|
|
3587
|
+
}
|
|
3588
|
+
if (limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
3589
|
+
const used = await sumUsageQuantity4(db, {
|
|
3590
|
+
workspaceId,
|
|
3591
|
+
eventType: "agent_run.created",
|
|
3592
|
+
since: startOfUtcMonth4()
|
|
3593
|
+
});
|
|
3594
|
+
if (used + requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
|
|
3595
|
+
throw new Error(`monthly agent run limit reached (${limits.maxMonthlyAgentRunsPerWorkspace})`);
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
}
|
|
3599
|
+
}
|
|
3600
|
+
function startOfUtcMonth4() {
|
|
3601
|
+
const now = /* @__PURE__ */ new Date();
|
|
3602
|
+
return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
|
|
3603
|
+
}
|
|
3604
|
+
|
|
3605
|
+
// src/activities/session-state.ts
|
|
3606
|
+
import {
|
|
3607
|
+
claimNextQueuedTurn as claimNextQueuedTurnDb2,
|
|
3608
|
+
countTurnSessionHistoryItems,
|
|
3609
|
+
finishTurn as finishTurn2,
|
|
3610
|
+
getSessionEvent as getSessionEvent4,
|
|
3611
|
+
getSessionTurn as getSessionTurn2,
|
|
3612
|
+
incrementTurnWorkerDeathRedispatches,
|
|
3613
|
+
requeuePreemptedTurn as requeuePreemptedTurn2,
|
|
3614
|
+
requireSession as requireSession4,
|
|
3615
|
+
setSessionStatus as setSessionStatus2
|
|
3616
|
+
} from "@opengeni/db";
|
|
3617
|
+
import { appendAndPublishEvents as appendAndPublishEvents4 } from "@opengeni/events";
|
|
3618
|
+
var WORKER_DEATH_MAX_REDISPATCHES = 3;
|
|
3619
|
+
function createSessionStateActivities(services) {
|
|
3620
|
+
async function failSession2(input) {
|
|
3621
|
+
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
3622
|
+
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
3623
|
+
if (session.status === "failed") {
|
|
3624
|
+
return;
|
|
3625
|
+
}
|
|
3626
|
+
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
3627
|
+
const turnId = session.activeTurnId ?? null;
|
|
3628
|
+
await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
|
|
3629
|
+
{
|
|
3630
|
+
type: "turn.failed",
|
|
3631
|
+
turnId,
|
|
3632
|
+
payload: {
|
|
3633
|
+
triggerEventId: input.triggerEventId,
|
|
3634
|
+
trigger: trigger?.payload ?? null,
|
|
3635
|
+
error: input.error ?? "Agent activity failed before it could report a terminal state."
|
|
3636
|
+
}
|
|
3637
|
+
},
|
|
3638
|
+
{
|
|
3639
|
+
type: "session.status.changed",
|
|
3640
|
+
turnId,
|
|
3641
|
+
payload: { status: "failed" }
|
|
3642
|
+
}
|
|
3643
|
+
]);
|
|
3644
|
+
if (turnId) {
|
|
3645
|
+
await finishTurn2(db, input.workspaceId, turnId, "failed");
|
|
3646
|
+
}
|
|
3647
|
+
await setSessionStatus2(db, input.workspaceId, input.sessionId, "failed", null);
|
|
3648
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
|
|
3649
|
+
}
|
|
3650
|
+
async function interruptActiveTurn2(input) {
|
|
3651
|
+
const { db, bus } = await services();
|
|
3652
|
+
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
3653
|
+
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
3654
|
+
if (!isSteerInterrupt(trigger)) {
|
|
3655
|
+
await pauseActiveGoalOnInterrupt(db, bus, input.workspaceId, input.sessionId);
|
|
3656
|
+
}
|
|
3657
|
+
if (!session.activeTurnId) {
|
|
3658
|
+
return;
|
|
3659
|
+
}
|
|
3660
|
+
await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
|
|
3661
|
+
{
|
|
3662
|
+
turnId: session.activeTurnId,
|
|
3663
|
+
type: "turn.cancelled",
|
|
3664
|
+
payload: { triggerEventId: input.triggerEventId, reason: trigger?.payload ?? null }
|
|
3665
|
+
},
|
|
3666
|
+
{
|
|
3667
|
+
turnId: session.activeTurnId,
|
|
3668
|
+
type: "session.status.changed",
|
|
3669
|
+
payload: { status: "queued" }
|
|
3670
|
+
}
|
|
3671
|
+
]);
|
|
3672
|
+
await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
3673
|
+
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
3674
|
+
}
|
|
3675
|
+
async function requeueTurnAfterWorkerDeath2(input) {
|
|
3676
|
+
const { settings, db, bus } = await services();
|
|
3677
|
+
const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
|
|
3678
|
+
if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
|
|
3679
|
+
return { action: "stale" };
|
|
3680
|
+
}
|
|
3681
|
+
const redispatches = await incrementTurnWorkerDeathRedispatches(db, input.workspaceId, input.turnId);
|
|
3682
|
+
if (redispatches > WORKER_DEATH_MAX_REDISPATCHES) {
|
|
3683
|
+
return { action: "exceeded", redispatches: redispatches - 1 };
|
|
3684
|
+
}
|
|
3685
|
+
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
3686
|
+
const approvalRerun = trigger?.type === "user.approvalDecision";
|
|
3687
|
+
const resumeWithNotice = !approvalRerun && settings.sessionHistorySource === "items" && await countTurnSessionHistoryItems(db, input.workspaceId, input.turnId) > 0;
|
|
3688
|
+
const [preemptedEvent] = await appendAndPublishEvents4(db, bus, input.workspaceId, input.sessionId, [
|
|
3689
|
+
{
|
|
3690
|
+
turnId: turn.id,
|
|
3691
|
+
type: "turn.preempted",
|
|
3692
|
+
payload: {
|
|
3693
|
+
triggerEventId: input.triggerEventId,
|
|
3694
|
+
reason: "worker_death",
|
|
3695
|
+
redispatches,
|
|
3696
|
+
resumeWithNotice,
|
|
3697
|
+
...resumeWithNotice ? { text: WORKER_DEATH_RESUME_TEXT } : {}
|
|
3698
|
+
}
|
|
3699
|
+
},
|
|
3700
|
+
{
|
|
3701
|
+
turnId: turn.id,
|
|
3702
|
+
type: "session.status.changed",
|
|
3703
|
+
payload: { status: "queued" }
|
|
3704
|
+
}
|
|
3705
|
+
]);
|
|
3706
|
+
try {
|
|
3707
|
+
await requeuePreemptedTurn2(db, input.workspaceId, turn.id, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
3708
|
+
} catch (requeueError) {
|
|
3709
|
+
const current = await getSessionTurn2(db, input.workspaceId, input.turnId);
|
|
3710
|
+
if (current && current.status !== "running" && current.status !== "requires_action") {
|
|
3711
|
+
return { action: "stale" };
|
|
3712
|
+
}
|
|
3713
|
+
throw requeueError;
|
|
3714
|
+
}
|
|
3715
|
+
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
3716
|
+
return { action: "requeued", redispatches };
|
|
3717
|
+
}
|
|
3718
|
+
async function claimNextQueuedTurn2(input) {
|
|
3719
|
+
const { db } = await services();
|
|
3720
|
+
return await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
3721
|
+
}
|
|
3722
|
+
async function markSessionIdle2(input) {
|
|
3723
|
+
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
3724
|
+
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
3725
|
+
if (session.status === "queued" || session.status === "running") {
|
|
3726
|
+
await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
|
|
3727
|
+
}
|
|
3728
|
+
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
|
|
3729
|
+
}
|
|
3730
|
+
return {
|
|
3731
|
+
failSession: failSession2,
|
|
3732
|
+
interruptActiveTurn: interruptActiveTurn2,
|
|
3733
|
+
requeueTurnAfterWorkerDeath: requeueTurnAfterWorkerDeath2,
|
|
3734
|
+
claimNextQueuedTurn: claimNextQueuedTurn2,
|
|
3735
|
+
markSessionIdle: markSessionIdle2
|
|
3736
|
+
};
|
|
3737
|
+
}
|
|
3738
|
+
|
|
3739
|
+
// src/activities.ts
|
|
3740
|
+
function createActivities(dependencies = {}) {
|
|
3741
|
+
let servicesPromise = null;
|
|
3742
|
+
async function services() {
|
|
3743
|
+
servicesPromise ??= (async () => {
|
|
3744
|
+
const settings = dependencies.settings ?? getSettings();
|
|
3745
|
+
const searchPath = dbSearchPath(settings);
|
|
3746
|
+
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
3747
|
+
...searchPath ? { searchPath } : {},
|
|
3748
|
+
rlsStrategy: settings.rlsStrategy
|
|
3749
|
+
});
|
|
3750
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
3751
|
+
return {
|
|
3752
|
+
settings,
|
|
3753
|
+
db: dependencies.db ?? dbClient.db,
|
|
3754
|
+
// §7 Step G — EventBus binding contract. `bus` is the INJECTED
|
|
3755
|
+
// live-fanout port; the ONE production impl is `createNatsEventBus`,
|
|
3756
|
+
// the default on BOTH processes (this worker edge + the API edge in
|
|
3757
|
+
// `apps/api/src/index.ts`). A host that embeds OpenGeni injects ONE
|
|
3758
|
+
// broker binding (the same `createNatsEventBus(natsUrl)`) here AND on
|
|
3759
|
+
// the mounted API, so the two SEPARATE processes share one broker and
|
|
3760
|
+
// derive the IDENTICAL `sessionSubject` — the only way live fanout
|
|
3761
|
+
// (worker emit → API SSE) works cross-process (SPIKE-1 F5/F6, proven).
|
|
3762
|
+
// NEVER default to an in-memory bus: it fans out intra-process only and
|
|
3763
|
+
// would silently break live SSE. unset → today's NATS default,
|
|
3764
|
+
// byte-for-byte. The bus is live-fanout ONLY — the durable Postgres
|
|
3765
|
+
// `session_events` log is source-of-truth (the API backfills missed
|
|
3766
|
+
// events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
|
|
3767
|
+
bus: dependencies.bus ?? await createNatsEventBus(
|
|
3768
|
+
settings.natsUrl,
|
|
3769
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
|
|
3770
|
+
),
|
|
3771
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime(),
|
|
3772
|
+
objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
|
|
3773
|
+
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
3774
|
+
observability: dependencies.observability ?? createObservability(settings, { component: "worker" }),
|
|
3775
|
+
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
3776
|
+
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
3777
|
+
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
|
3778
|
+
// local ledger exactly as today (mirrors `wakeSessionWorkflow`'s
|
|
3779
|
+
// null-degrades-gracefully shape, not a `createX(settings)` default).
|
|
3780
|
+
entitlements: dependencies.entitlements ?? null,
|
|
3781
|
+
// §7.6 P4a — host connection-credential provider. No constructed
|
|
3782
|
+
// default: standalone owns its own GitHub App + encryption key, so unset
|
|
3783
|
+
// → null → the per-run credential mint self-mints from `settings`
|
|
3784
|
+
// (createGitHubAppInstallationToken + environmentsEncryptionKeyBytes)
|
|
3785
|
+
// exactly as today. Same null-degrades shape as `entitlements`.
|
|
3786
|
+
connectionCredentials: dependencies.connectionCredentials ?? null
|
|
3787
|
+
};
|
|
3788
|
+
})();
|
|
3789
|
+
return servicesPromise;
|
|
3790
|
+
}
|
|
3791
|
+
const runAgentTurn2 = createRunAgentTurnActivity(services);
|
|
3792
|
+
return {
|
|
3793
|
+
runAgentTurn: runAgentTurn2,
|
|
3794
|
+
// Legacy Temporal activity name. In-flight session workflows (which can
|
|
3795
|
+
// legitimately live for days) recorded ScheduleActivityTask events as
|
|
3796
|
+
// "runAgentSegment"; the name must keep resolving until those histories
|
|
3797
|
+
// drain. New workflow code must use runAgentTurn; migrate the session
|
|
3798
|
+
// workflow call site via patched() before removing this alias.
|
|
3799
|
+
runAgentSegment: runAgentTurn2,
|
|
3800
|
+
...createDocumentActivities(services),
|
|
3801
|
+
...createSessionStateActivities(services),
|
|
3802
|
+
...createScheduledTaskActivities(services),
|
|
3803
|
+
...createGoalActivities(services),
|
|
3804
|
+
// P1.3: the SOLE liveness/GC/cost-stop driver. Only reapSandboxLeases — no
|
|
3805
|
+
// *ForViewer activities, no ownerHeartbeat, no resolveOwnerTaskQueue.
|
|
3806
|
+
...createSandboxLeaseActivities(services)
|
|
3807
|
+
};
|
|
3808
|
+
}
|
|
3809
|
+
var defaultActivities = createActivities();
|
|
3810
|
+
var runAgentTurn = defaultActivities.runAgentTurn;
|
|
3811
|
+
var runAgentSegment = defaultActivities.runAgentSegment;
|
|
3812
|
+
var indexDocument = defaultActivities.indexDocument;
|
|
3813
|
+
var failSession = defaultActivities.failSession;
|
|
3814
|
+
var interruptActiveTurn = defaultActivities.interruptActiveTurn;
|
|
3815
|
+
var requeueTurnAfterWorkerDeath = defaultActivities.requeueTurnAfterWorkerDeath;
|
|
3816
|
+
var claimNextQueuedTurn = defaultActivities.claimNextQueuedTurn;
|
|
3817
|
+
var markSessionIdle = defaultActivities.markSessionIdle;
|
|
3818
|
+
var dispatchScheduledTaskRun = defaultActivities.dispatchScheduledTaskRun;
|
|
3819
|
+
var maybeContinueGoal = defaultActivities.maybeContinueGoal;
|
|
3820
|
+
var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
|
|
3821
|
+
var reapSandboxLeases = defaultActivities.reapSandboxLeases;
|
|
3822
|
+
|
|
3823
|
+
// src/index.ts
|
|
3824
|
+
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
3825
|
+
async function createOpenGeniWorker(options = {}) {
|
|
3826
|
+
const settings = options.settings ?? getSettings2();
|
|
3827
|
+
const observability = options.activityDependencies?.observability ?? createObservability2(settings, { component: "worker" });
|
|
3828
|
+
const connection = await retryStartupDependency(
|
|
3829
|
+
"Temporal",
|
|
3830
|
+
() => NativeConnection.connect({ address: settings.temporalHost }),
|
|
3831
|
+
{
|
|
3832
|
+
...startupRetryOptions(settings),
|
|
3833
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
3834
|
+
}
|
|
3835
|
+
);
|
|
3836
|
+
const activities = options.activities ?? createActivities({
|
|
3837
|
+
...options.activityDependencies,
|
|
3838
|
+
settings,
|
|
3839
|
+
observability
|
|
3840
|
+
});
|
|
3841
|
+
const worker = await Worker.create({
|
|
3842
|
+
connection,
|
|
3843
|
+
namespace: settings.temporalNamespace,
|
|
3844
|
+
taskQueue: settings.temporalTaskQueue,
|
|
3845
|
+
workflowsPath: new URL("../src/workflows.ts", import.meta.url).pathname,
|
|
3846
|
+
activities
|
|
3847
|
+
});
|
|
3848
|
+
return { worker, connection };
|
|
3849
|
+
}
|
|
3850
|
+
async function createWorkerWorkflowSignaler(settings) {
|
|
3851
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
3852
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
3853
|
+
return {
|
|
3854
|
+
wakeSessionWorkflow: async ({ accountId, workspaceId, sessionId, workflowId }) => {
|
|
3855
|
+
await temporal.workflow.signalWithStart("sessionWorkflow", {
|
|
3856
|
+
taskQueue: settings.temporalTaskQueue,
|
|
3857
|
+
workflowId,
|
|
3858
|
+
workflowIdReusePolicy: "ALLOW_DUPLICATE",
|
|
3859
|
+
args: [{ accountId, workspaceId, sessionId }],
|
|
3860
|
+
signal: "queueChanged"
|
|
3861
|
+
});
|
|
3862
|
+
},
|
|
3863
|
+
close: async () => {
|
|
3864
|
+
await connection.close();
|
|
3865
|
+
}
|
|
3866
|
+
};
|
|
3867
|
+
}
|
|
3868
|
+
async function registerSandboxReaperSchedule(settings, observability) {
|
|
3869
|
+
if (!settings.sandboxOwnershipEnabled) {
|
|
3870
|
+
return { registered: false, close: async () => {
|
|
3871
|
+
} };
|
|
3872
|
+
}
|
|
3873
|
+
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
3874
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
3875
|
+
try {
|
|
3876
|
+
await temporal.schedule.create({
|
|
3877
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
3878
|
+
spec: {
|
|
3879
|
+
// @every-style interval: fire once per reaper period. The boot invariant
|
|
3880
|
+
// (config) guarantees reaperPeriod < viewerHolderTTL and
|
|
3881
|
+
// reaperPeriod + idleGrace < providerLifetime.
|
|
3882
|
+
intervals: [{ every: settings.sandboxLeaseReaperPeriodMs }]
|
|
3883
|
+
},
|
|
3884
|
+
action: {
|
|
3885
|
+
type: "startWorkflow",
|
|
3886
|
+
workflowType: "sandboxReaperWorkflow",
|
|
3887
|
+
taskQueue: settings.temporalTaskQueue,
|
|
3888
|
+
args: []
|
|
3889
|
+
},
|
|
3890
|
+
policies: {
|
|
3891
|
+
// A slow sweep must never overlap itself; the next fire is skipped.
|
|
3892
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
3893
|
+
catchupWindow: "1m",
|
|
3894
|
+
pauseOnFailure: false
|
|
3895
|
+
}
|
|
3896
|
+
});
|
|
3897
|
+
observability.info("Registered the global sandbox-lease reaper Schedule", {
|
|
3898
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID,
|
|
3899
|
+
reaperPeriodMs: settings.sandboxLeaseReaperPeriodMs
|
|
3900
|
+
});
|
|
3901
|
+
return { registered: true, close: async () => {
|
|
3902
|
+
await connection.close();
|
|
3903
|
+
} };
|
|
3904
|
+
} catch (error) {
|
|
3905
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
3906
|
+
observability.info("Global sandbox-lease reaper Schedule already registered", {
|
|
3907
|
+
scheduleId: SANDBOX_REAPER_SCHEDULE_ID
|
|
3908
|
+
});
|
|
3909
|
+
return { registered: false, close: async () => {
|
|
3910
|
+
await connection.close();
|
|
3911
|
+
} };
|
|
3912
|
+
}
|
|
3913
|
+
await connection.close().catch(() => void 0);
|
|
3914
|
+
throw error;
|
|
3915
|
+
}
|
|
3916
|
+
}
|
|
3917
|
+
async function startWorker() {
|
|
3918
|
+
const settings = getSettings2();
|
|
3919
|
+
const observability = createObservability2(settings, { component: "worker" });
|
|
3920
|
+
const signaler = await retryStartupDependency(
|
|
3921
|
+
"Temporal client",
|
|
3922
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
3923
|
+
{
|
|
3924
|
+
...startupRetryOptions(settings),
|
|
3925
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
3926
|
+
}
|
|
3927
|
+
);
|
|
3928
|
+
const { worker, connection } = await createOpenGeniWorker({
|
|
3929
|
+
settings,
|
|
3930
|
+
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
|
|
3931
|
+
});
|
|
3932
|
+
const reaperSchedule = await retryStartupDependency(
|
|
3933
|
+
"Temporal schedule (sandbox reaper)",
|
|
3934
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
3935
|
+
{
|
|
3936
|
+
...startupRetryOptions(settings),
|
|
3937
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
3938
|
+
}
|
|
3939
|
+
);
|
|
3940
|
+
observability.info("OpenGeni worker listening", {
|
|
3941
|
+
temporalTaskQueue: settings.temporalTaskQueue
|
|
3942
|
+
});
|
|
3943
|
+
try {
|
|
3944
|
+
await worker.run();
|
|
3945
|
+
} finally {
|
|
3946
|
+
await connection.close();
|
|
3947
|
+
await signaler.close();
|
|
3948
|
+
await reaperSchedule.close();
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
if (import.meta.main) {
|
|
3952
|
+
await startWorker();
|
|
3953
|
+
}
|
|
3954
|
+
export {
|
|
3955
|
+
createOpenGeniWorker,
|
|
3956
|
+
createWorkerWorkflowSignaler,
|
|
3957
|
+
registerSandboxReaperSchedule,
|
|
3958
|
+
startWorker
|
|
3959
|
+
};
|
|
3960
|
+
//# sourceMappingURL=index.js.map
|