@opengeni/worker-bundle 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +644 -80
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +32 -4
- package/src/activities/sandbox-lease.ts +130 -3
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +197 -59
package/dist/index.js
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
// src/index.ts
|
|
2
|
-
import { getSettings as getSettings2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
2
|
+
import { dbSearchPath as dbSearchPath2, getSettings as getSettings2, resolveNatsControlPlaneAuth as resolveNatsControlPlaneAuth2, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
|
|
3
|
+
import { createDb as createDb2 } from "@opengeni/db";
|
|
4
|
+
import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
|
|
3
5
|
import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
|
|
4
6
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
5
7
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
@@ -1170,6 +1172,7 @@ import {
|
|
|
1170
1172
|
failWarmingToCold,
|
|
1171
1173
|
getSandboxSessionEnvelope as getSandboxSessionEnvelope2,
|
|
1172
1174
|
readLease,
|
|
1175
|
+
recordWarmingSandboxCreated,
|
|
1173
1176
|
releaseLeaseHolder,
|
|
1174
1177
|
SandboxLeaseSupersededError
|
|
1175
1178
|
} from "@opengeni/db";
|
|
@@ -1181,13 +1184,93 @@ import {
|
|
|
1181
1184
|
DisplayStackError,
|
|
1182
1185
|
DisplayStackUnsupportedError,
|
|
1183
1186
|
buildStreamUrl,
|
|
1184
|
-
StreamPortUnavailableError
|
|
1187
|
+
StreamPortUnavailableError,
|
|
1188
|
+
modalSandboxAttributionEnvironment,
|
|
1189
|
+
tagModalSandbox
|
|
1185
1190
|
} from "@opengeni/runtime";
|
|
1186
1191
|
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
1192
|
+
var SandboxWarmingTimeoutError = class extends Error {
|
|
1193
|
+
constructor(backend, timeoutMs) {
|
|
1194
|
+
super(
|
|
1195
|
+
`Sandbox backend "${backend}" capacity or creation timed out after ${Math.ceil(timeoutMs / 1e3)}s while warming the sandbox lease. Please try again; if this persists, sandbox capacity may be exhausted.`
|
|
1196
|
+
);
|
|
1197
|
+
this.backend = backend;
|
|
1198
|
+
this.timeoutMs = timeoutMs;
|
|
1199
|
+
this.name = "SandboxWarmingTimeoutError";
|
|
1200
|
+
}
|
|
1201
|
+
backend;
|
|
1202
|
+
timeoutMs;
|
|
1203
|
+
code = "sandbox_warming_timeout";
|
|
1204
|
+
};
|
|
1187
1205
|
var WARMING_POLL_INTERVAL_MS = 250;
|
|
1188
1206
|
async function sleep(ms) {
|
|
1189
1207
|
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
1190
1208
|
}
|
|
1209
|
+
async function terminateEstablishedSandbox(established) {
|
|
1210
|
+
if (!established) {
|
|
1211
|
+
return true;
|
|
1212
|
+
}
|
|
1213
|
+
const client = established.client;
|
|
1214
|
+
if (typeof client.delete === "function" && established.sessionState !== void 0) {
|
|
1215
|
+
try {
|
|
1216
|
+
await client.delete(established.sessionState);
|
|
1217
|
+
return true;
|
|
1218
|
+
} catch {
|
|
1219
|
+
return false;
|
|
1220
|
+
}
|
|
1221
|
+
}
|
|
1222
|
+
const session = established.session;
|
|
1223
|
+
try {
|
|
1224
|
+
if (session.terminate) {
|
|
1225
|
+
await session.terminate();
|
|
1226
|
+
return true;
|
|
1227
|
+
} else if (session.kill) {
|
|
1228
|
+
await session.kill();
|
|
1229
|
+
return true;
|
|
1230
|
+
} else if (session.close) {
|
|
1231
|
+
if (!session.closed) {
|
|
1232
|
+
await session.close();
|
|
1233
|
+
}
|
|
1234
|
+
return true;
|
|
1235
|
+
}
|
|
1236
|
+
return false;
|
|
1237
|
+
} catch {
|
|
1238
|
+
return false;
|
|
1239
|
+
}
|
|
1240
|
+
}
|
|
1241
|
+
function asSandboxWarmingError(error, backend, timeoutMs) {
|
|
1242
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1243
|
+
return /sandbox creation timed out|warming timed out|capacity.*timed out/i.test(message) ? new SandboxWarmingTimeoutError(backend, timeoutMs) : error;
|
|
1244
|
+
}
|
|
1245
|
+
function recordSandboxWarmingTimeout(metrics, error) {
|
|
1246
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
1247
|
+
return;
|
|
1248
|
+
}
|
|
1249
|
+
try {
|
|
1250
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
1251
|
+
} catch {
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
function workspaceArchiveFromEnvelope(envelope) {
|
|
1255
|
+
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null ? envelope.sessionState : null;
|
|
1256
|
+
const archive = sessionState?.workspaceArchive;
|
|
1257
|
+
return typeof archive === "string" && archive.length > 0 ? archive : null;
|
|
1258
|
+
}
|
|
1259
|
+
function preserveWorkspaceArchiveOnInterimResumeState(resumeState, archiveSource) {
|
|
1260
|
+
const archive = workspaceArchiveFromEnvelope(archiveSource);
|
|
1261
|
+
if (!archive) {
|
|
1262
|
+
return resumeState;
|
|
1263
|
+
}
|
|
1264
|
+
const existingSessionState = resumeState && typeof resumeState.sessionState === "object" && resumeState.sessionState !== null ? resumeState.sessionState : {};
|
|
1265
|
+
return {
|
|
1266
|
+
...resumeState ?? {},
|
|
1267
|
+
...resumeState?.backendId === void 0 && archiveSource?.backendId !== void 0 ? { backendId: archiveSource.backendId } : {},
|
|
1268
|
+
sessionState: {
|
|
1269
|
+
...existingSessionState,
|
|
1270
|
+
workspaceArchive: archive
|
|
1271
|
+
}
|
|
1272
|
+
};
|
|
1273
|
+
}
|
|
1191
1274
|
async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
1192
1275
|
const { db, settings } = services;
|
|
1193
1276
|
const os = ids.os ?? "linux";
|
|
@@ -1229,14 +1312,51 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1229
1312
|
}
|
|
1230
1313
|
if (acquired.role === "spawner") {
|
|
1231
1314
|
const expectedEpoch = acquired.lease.leaseEpoch;
|
|
1315
|
+
let createdEstablished = null;
|
|
1316
|
+
if (ids.environment && ids.backend === "modal") {
|
|
1317
|
+
Object.assign(ids.environment, modalSandboxAttributionEnvironment({
|
|
1318
|
+
leaseId: acquired.lease.id,
|
|
1319
|
+
workspaceId: ids.workspaceId,
|
|
1320
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1321
|
+
}));
|
|
1322
|
+
}
|
|
1232
1323
|
try {
|
|
1233
1324
|
const envelope = await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1234
1325
|
const spawnEnvelope = acquired.lease.resumeState ?? envelope;
|
|
1235
1326
|
const established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
|
|
1236
1327
|
sessionId: ids.sessionId,
|
|
1237
1328
|
backendOverride: ids.backend,
|
|
1238
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1329
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1330
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
|
|
1331
|
+
onSandboxCreated: async (created) => {
|
|
1332
|
+
createdEstablished = created;
|
|
1333
|
+
const resumeEnvelope2 = preserveWorkspaceArchiveOnInterimResumeState(
|
|
1334
|
+
await serializeEstablishedSandboxEnvelope(created) ?? null,
|
|
1335
|
+
spawnEnvelope
|
|
1336
|
+
);
|
|
1337
|
+
const recorded = await recordWarmingSandboxCreated(db, {
|
|
1338
|
+
accountId: ids.accountId,
|
|
1339
|
+
workspaceId: ids.workspaceId,
|
|
1340
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1341
|
+
expectedEpoch,
|
|
1342
|
+
instanceId: created.instanceId,
|
|
1343
|
+
resumeBackendId: created.backendId,
|
|
1344
|
+
resumeState: resumeEnvelope2,
|
|
1345
|
+
leaseTtlMs
|
|
1346
|
+
});
|
|
1347
|
+
if (!recorded.recorded) {
|
|
1348
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1349
|
+
}
|
|
1350
|
+
if (created.backendId === "modal") {
|
|
1351
|
+
await tagModalSandbox(settings, created.instanceId, {
|
|
1352
|
+
leaseId: acquired.lease.id,
|
|
1353
|
+
workspaceId: ids.workspaceId,
|
|
1354
|
+
sandboxGroupId: ids.sandboxGroupId
|
|
1355
|
+
}).catch(() => void 0);
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1239
1358
|
});
|
|
1359
|
+
createdEstablished = established;
|
|
1240
1360
|
await ensureDisplayStack(settings, established);
|
|
1241
1361
|
const endpoint = await exposeStreamPort(settings, established);
|
|
1242
1362
|
const resumeEnvelope = await serializeEstablishedSandboxEnvelope(established) ?? envelope ?? null;
|
|
@@ -1252,41 +1372,52 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
|
|
|
1252
1372
|
leaseTtlMs
|
|
1253
1373
|
});
|
|
1254
1374
|
if (!committed.committed || !committed.lease) {
|
|
1375
|
+
await terminateEstablishedSandbox(established);
|
|
1255
1376
|
await release();
|
|
1256
1377
|
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
|
|
1257
1378
|
}
|
|
1258
1379
|
return { established, leaseEpoch: committed.lease.leaseEpoch, release };
|
|
1259
1380
|
} catch (error) {
|
|
1260
1381
|
if (error instanceof SandboxLeaseSupersededError) {
|
|
1382
|
+
await terminateEstablishedSandbox(createdEstablished);
|
|
1383
|
+
await release();
|
|
1261
1384
|
throw error;
|
|
1262
1385
|
}
|
|
1263
|
-
await
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1268
|
-
|
|
1386
|
+
const terminated = await terminateEstablishedSandbox(createdEstablished);
|
|
1387
|
+
if (terminated) {
|
|
1388
|
+
await failWarmingToCold(db, {
|
|
1389
|
+
accountId: ids.accountId,
|
|
1390
|
+
workspaceId: ids.workspaceId,
|
|
1391
|
+
sandboxGroupId: ids.sandboxGroupId,
|
|
1392
|
+
expectedEpoch
|
|
1393
|
+
});
|
|
1394
|
+
}
|
|
1269
1395
|
await release();
|
|
1270
|
-
|
|
1396
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1397
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1398
|
+
throw warmingError;
|
|
1271
1399
|
}
|
|
1272
1400
|
}
|
|
1273
|
-
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1274
|
-
if (acquired.lease.liveness === "warming") {
|
|
1275
|
-
leaseEpoch = (await waitForWarmOrReacquire(services, ids, kind, holderId)).leaseEpoch;
|
|
1276
|
-
}
|
|
1277
1401
|
try {
|
|
1402
|
+
let leaseEpoch = acquired.lease.leaseEpoch;
|
|
1403
|
+
if (acquired.lease.liveness === "warming") {
|
|
1404
|
+
leaseEpoch = (await waitForWarm(services, ids)).leaseEpoch;
|
|
1405
|
+
}
|
|
1278
1406
|
const live = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1279
1407
|
const envelope = live?.resumeState ?? await getSandboxSessionEnvelope2(db, ids.workspaceId, ids.sessionId);
|
|
1280
1408
|
const established = await establishSandboxSessionFromEnvelope(settings, envelope, {
|
|
1281
1409
|
sessionId: ids.sessionId,
|
|
1282
1410
|
backendOverride: ids.backend,
|
|
1283
|
-
...ids.environment ? { environment: ids.environment } : {}
|
|
1411
|
+
...ids.environment ? { environment: ids.environment } : {},
|
|
1412
|
+
...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
|
|
1284
1413
|
});
|
|
1285
1414
|
await ensureDisplayStack(settings, established);
|
|
1286
1415
|
return { established, leaseEpoch, release };
|
|
1287
1416
|
} catch (error) {
|
|
1288
1417
|
await release();
|
|
1289
|
-
|
|
1418
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1419
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
1420
|
+
throw warmingError;
|
|
1290
1421
|
}
|
|
1291
1422
|
}
|
|
1292
1423
|
async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
@@ -1343,37 +1474,26 @@ async function acquireSelfhostedLeaseForTurn(services, ids, kind, holderId) {
|
|
|
1343
1474
|
}
|
|
1344
1475
|
return { leaseEpoch: acquired.lease.leaseEpoch, release };
|
|
1345
1476
|
}
|
|
1346
|
-
async function
|
|
1477
|
+
async function waitForWarm(services, ids) {
|
|
1347
1478
|
const { db, settings } = services;
|
|
1348
|
-
const deadline = Date.now() + settings.
|
|
1479
|
+
const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
|
|
1349
1480
|
while (Date.now() < deadline) {
|
|
1350
1481
|
await sleep(WARMING_POLL_INTERVAL_MS);
|
|
1351
1482
|
const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
|
|
1352
1483
|
if (!lease) {
|
|
1353
|
-
|
|
1484
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
|
|
1485
|
+
}
|
|
1486
|
+
if (lease.liveness === "warm") {
|
|
1487
|
+
return { leaseEpoch: lease.leaseEpoch };
|
|
1354
1488
|
}
|
|
1355
|
-
if (lease.liveness === "
|
|
1356
|
-
|
|
1489
|
+
if (lease.liveness === "draining") {
|
|
1490
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1357
1491
|
}
|
|
1358
1492
|
if (lease.liveness === "cold") {
|
|
1359
|
-
|
|
1493
|
+
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, lease.leaseEpoch);
|
|
1360
1494
|
}
|
|
1361
1495
|
}
|
|
1362
|
-
|
|
1363
|
-
accountId: ids.accountId,
|
|
1364
|
-
workspaceId: ids.workspaceId,
|
|
1365
|
-
sandboxGroupId: ids.sandboxGroupId,
|
|
1366
|
-
kind,
|
|
1367
|
-
holderId,
|
|
1368
|
-
subjectId: ids.sessionId,
|
|
1369
|
-
backend: ids.backend,
|
|
1370
|
-
os: ids.os ?? "linux",
|
|
1371
|
-
leaseTtlMs: settings.sandboxLeaseTtlMs
|
|
1372
|
-
});
|
|
1373
|
-
if (reacquired.role === "fenced") {
|
|
1374
|
-
throw new SandboxLeaseSupersededError(ids.sandboxGroupId, reacquired.lease.leaseEpoch);
|
|
1375
|
-
}
|
|
1376
|
-
return { liveness: reacquired.lease.liveness, leaseEpoch: reacquired.lease.leaseEpoch };
|
|
1496
|
+
throw new SandboxWarmingTimeoutError(ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
1377
1497
|
}
|
|
1378
1498
|
async function ensureDisplayStack(settings, established) {
|
|
1379
1499
|
if (!settings.sandboxDesktopEnabled) {
|
|
@@ -1524,6 +1644,201 @@ function routingEnabled(settings) {
|
|
|
1524
1644
|
return settings.sandboxSelfhostedEnabled === true;
|
|
1525
1645
|
}
|
|
1526
1646
|
|
|
1647
|
+
// src/observability-metrics.ts
|
|
1648
|
+
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
1649
|
+
function observabilityEventLogger(observability) {
|
|
1650
|
+
return {
|
|
1651
|
+
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
1652
|
+
warn: (message, attributes) => observability.warn(message, eventAttributes(attributes))
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
function eventAttributes(attributes) {
|
|
1656
|
+
if (!attributes) {
|
|
1657
|
+
return void 0;
|
|
1658
|
+
}
|
|
1659
|
+
const sanitized = {};
|
|
1660
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
1661
|
+
sanitized[key] = eventAttributeValue(value);
|
|
1662
|
+
}
|
|
1663
|
+
return sanitized;
|
|
1664
|
+
}
|
|
1665
|
+
function eventAttributeValue(value) {
|
|
1666
|
+
if (value === null || value === void 0 || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
1667
|
+
return value;
|
|
1668
|
+
}
|
|
1669
|
+
try {
|
|
1670
|
+
return JSON.stringify(value);
|
|
1671
|
+
} catch {
|
|
1672
|
+
return String(value);
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
function runtimeMetricsHooksForObservability(observability) {
|
|
1676
|
+
return {
|
|
1677
|
+
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
1678
|
+
observability.incrementCounter({
|
|
1679
|
+
name: "opengeni_model_calls_total",
|
|
1680
|
+
help: "Total model calls by provider and outcome.",
|
|
1681
|
+
labels: { provider, outcome }
|
|
1682
|
+
});
|
|
1683
|
+
observability.observeHistogram({
|
|
1684
|
+
name: "opengeni_model_call_duration_seconds",
|
|
1685
|
+
help: "Model call duration in seconds by provider.",
|
|
1686
|
+
labels: { provider },
|
|
1687
|
+
value: durationSeconds
|
|
1688
|
+
});
|
|
1689
|
+
},
|
|
1690
|
+
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
1691
|
+
observability.incrementCounter({
|
|
1692
|
+
name: "opengeni_sandbox_creates_total",
|
|
1693
|
+
help: "Total sandbox create attempts by backend and outcome.",
|
|
1694
|
+
labels: { backend, outcome }
|
|
1695
|
+
});
|
|
1696
|
+
observability.observeHistogram({
|
|
1697
|
+
name: "opengeni_sandbox_create_duration_seconds",
|
|
1698
|
+
help: "Sandbox create duration in seconds by backend.",
|
|
1699
|
+
labels: { backend },
|
|
1700
|
+
value: durationSeconds
|
|
1701
|
+
});
|
|
1702
|
+
},
|
|
1703
|
+
onSandboxWarmingTimeout: () => {
|
|
1704
|
+
observability.incrementCounter({
|
|
1705
|
+
name: "opengeni_sandbox_warming_timeouts_total",
|
|
1706
|
+
help: "Total sandbox warming timeouts."
|
|
1707
|
+
});
|
|
1708
|
+
}
|
|
1709
|
+
};
|
|
1710
|
+
}
|
|
1711
|
+
function turnLifecycleMetricsFor(observability) {
|
|
1712
|
+
const existing = turnTrackers.get(observability);
|
|
1713
|
+
if (existing) {
|
|
1714
|
+
return existing;
|
|
1715
|
+
}
|
|
1716
|
+
const tracker = new TurnLifecycleMetrics(observability);
|
|
1717
|
+
turnTrackers.set(observability, tracker);
|
|
1718
|
+
return tracker;
|
|
1719
|
+
}
|
|
1720
|
+
var TurnLifecycleMetrics = class {
|
|
1721
|
+
constructor(observability, options = {}) {
|
|
1722
|
+
this.observability = observability;
|
|
1723
|
+
this.options = options;
|
|
1724
|
+
}
|
|
1725
|
+
observability;
|
|
1726
|
+
options;
|
|
1727
|
+
startedTurns = /* @__PURE__ */ new Map();
|
|
1728
|
+
timer = null;
|
|
1729
|
+
start(turnId) {
|
|
1730
|
+
this.startedTurns.set(turnId, this.now());
|
|
1731
|
+
this.ensureTimer();
|
|
1732
|
+
this.refreshGauges();
|
|
1733
|
+
}
|
|
1734
|
+
finish(turnId, outcome, durationSeconds) {
|
|
1735
|
+
const startedAt = this.startedTurns.get(turnId);
|
|
1736
|
+
if (startedAt !== void 0) {
|
|
1737
|
+
this.startedTurns.delete(turnId);
|
|
1738
|
+
}
|
|
1739
|
+
if (outcome) {
|
|
1740
|
+
const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
|
|
1741
|
+
this.observability.incrementCounter({
|
|
1742
|
+
name: "opengeni_turns_total",
|
|
1743
|
+
help: "Total agent turns by terminal outcome.",
|
|
1744
|
+
labels: { outcome }
|
|
1745
|
+
});
|
|
1746
|
+
this.observability.observeHistogram({
|
|
1747
|
+
name: "opengeni_turn_duration_seconds",
|
|
1748
|
+
help: "Agent turn duration in seconds by terminal outcome.",
|
|
1749
|
+
labels: { outcome },
|
|
1750
|
+
value: observedDuration
|
|
1751
|
+
});
|
|
1752
|
+
}
|
|
1753
|
+
this.refreshGauges();
|
|
1754
|
+
if (this.startedTurns.size === 0) {
|
|
1755
|
+
this.stopTimer();
|
|
1756
|
+
}
|
|
1757
|
+
}
|
|
1758
|
+
refreshGauges() {
|
|
1759
|
+
this.observability.setGauge({
|
|
1760
|
+
name: "opengeni_turns_inflight",
|
|
1761
|
+
help: "Current number of in-flight agent turns in this worker process.",
|
|
1762
|
+
value: this.startedTurns.size
|
|
1763
|
+
});
|
|
1764
|
+
this.observability.setGauge({
|
|
1765
|
+
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
1766
|
+
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
1767
|
+
value: this.oldestInflightAgeSeconds()
|
|
1768
|
+
});
|
|
1769
|
+
}
|
|
1770
|
+
stop() {
|
|
1771
|
+
this.startedTurns.clear();
|
|
1772
|
+
this.refreshGauges();
|
|
1773
|
+
this.stopTimer();
|
|
1774
|
+
}
|
|
1775
|
+
oldestInflightAgeSeconds() {
|
|
1776
|
+
if (this.startedTurns.size === 0) {
|
|
1777
|
+
return 0;
|
|
1778
|
+
}
|
|
1779
|
+
let oldest = Number.POSITIVE_INFINITY;
|
|
1780
|
+
for (const startedAt of this.startedTurns.values()) {
|
|
1781
|
+
oldest = Math.min(oldest, startedAt);
|
|
1782
|
+
}
|
|
1783
|
+
return Math.max(0, (this.now() - oldest) / 1e3);
|
|
1784
|
+
}
|
|
1785
|
+
ensureTimer() {
|
|
1786
|
+
if (this.timer) {
|
|
1787
|
+
return;
|
|
1788
|
+
}
|
|
1789
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15e3);
|
|
1790
|
+
this.timer.unref?.();
|
|
1791
|
+
}
|
|
1792
|
+
stopTimer() {
|
|
1793
|
+
if (!this.timer) {
|
|
1794
|
+
return;
|
|
1795
|
+
}
|
|
1796
|
+
clearInterval(this.timer);
|
|
1797
|
+
this.timer = null;
|
|
1798
|
+
}
|
|
1799
|
+
now() {
|
|
1800
|
+
return this.options.now?.() ?? Date.now();
|
|
1801
|
+
}
|
|
1802
|
+
};
|
|
1803
|
+
function recordTurnsQueuedGauge(observability, value) {
|
|
1804
|
+
observability.setGauge({
|
|
1805
|
+
name: "opengeni_turns_queued",
|
|
1806
|
+
help: "Current number of queued session turns.",
|
|
1807
|
+
value
|
|
1808
|
+
});
|
|
1809
|
+
}
|
|
1810
|
+
function recordSandboxLeaseGauges(observability, counts) {
|
|
1811
|
+
for (const liveness of ["cold", "warming", "warm", "draining"]) {
|
|
1812
|
+
observability.setGauge({
|
|
1813
|
+
name: "opengeni_sandbox_leases",
|
|
1814
|
+
help: "Current sandbox leases by liveness state.",
|
|
1815
|
+
labels: { liveness },
|
|
1816
|
+
value: counts[liveness] ?? 0
|
|
1817
|
+
});
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
function recordSandboxOrphansTerminated(observability, count) {
|
|
1821
|
+
if (count <= 0) {
|
|
1822
|
+
return;
|
|
1823
|
+
}
|
|
1824
|
+
observability.incrementCounter({
|
|
1825
|
+
name: "opengeni_sandbox_orphans_terminated_total",
|
|
1826
|
+
help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
|
|
1827
|
+
amount: count
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1830
|
+
function recordCreditMicros(observability, kind, amountMicros) {
|
|
1831
|
+
if (!observability || amountMicros <= 0) {
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1834
|
+
observability.incrementCounter({
|
|
1835
|
+
name: "opengeni_credit_micros_total",
|
|
1836
|
+
help: "Total credit micros recorded by kind.",
|
|
1837
|
+
labels: { kind },
|
|
1838
|
+
amount: amountMicros
|
|
1839
|
+
});
|
|
1840
|
+
}
|
|
1841
|
+
|
|
1527
1842
|
// src/activities/recording.ts
|
|
1528
1843
|
import { deleteRecording, updateRecording } from "@opengeni/db";
|
|
1529
1844
|
import {
|
|
@@ -1817,6 +2132,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
1817
2132
|
"opengeni.trigger_event_id": input.triggerEventId
|
|
1818
2133
|
});
|
|
1819
2134
|
let activityStatus = "unknown";
|
|
2135
|
+
let turnMetricOutcome = null;
|
|
1820
2136
|
let activityError;
|
|
1821
2137
|
let turnId;
|
|
1822
2138
|
let heartbeatTimer;
|
|
@@ -1919,6 +2235,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
1919
2235
|
throw new Error(`Session turn not found for trigger: ${input.triggerEventId}`);
|
|
1920
2236
|
}
|
|
1921
2237
|
turnId = turn.id;
|
|
2238
|
+
turnLifecycleMetricsFor(observability).start(turnId);
|
|
1922
2239
|
const isCodexTurn = await isCodexBilledTurn2({ db, settings, workspaceId: input.workspaceId, model: turn.model, active: codexSubscriptionActive });
|
|
1923
2240
|
await ensureRunAllowed(settings, db, input.accountId, input.workspaceId, isCodexTurn, entitlements);
|
|
1924
2241
|
const activityContext = currentActivityContext();
|
|
@@ -2019,6 +2336,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2019
2336
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2020
2337
|
], true);
|
|
2021
2338
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
2339
|
+
turnMetricOutcome = "failed";
|
|
2022
2340
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2023
2341
|
activityStatus = "idle";
|
|
2024
2342
|
return goalActive ? { status: "idle", continueDelayMs: resumeMs, idleUntilReset: true } : { status: "idle" };
|
|
@@ -2157,7 +2475,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2157
2475
|
};
|
|
2158
2476
|
} else {
|
|
2159
2477
|
resolvedSandbox = await resumeBoxForTurn(
|
|
2160
|
-
{ db, settings },
|
|
2478
|
+
{ db, settings, sandboxMetrics: runtimeMetricsHooksForObservability(observability) },
|
|
2161
2479
|
{
|
|
2162
2480
|
accountId: input.accountId,
|
|
2163
2481
|
workspaceId: input.workspaceId,
|
|
@@ -2219,7 +2537,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2219
2537
|
expectedEpoch: heartbeatEpoch,
|
|
2220
2538
|
warmRateMicrosPerSecond: warmRate,
|
|
2221
2539
|
subjectId: input.sessionId
|
|
2222
|
-
}).catch(() => void 0);
|
|
2540
|
+
}).then((result) => recordCreditMicros(observability, "usage", result.costMicros)).catch(() => void 0);
|
|
2223
2541
|
}, 1e4);
|
|
2224
2542
|
if ("unref" in leaseHeartbeatTimer && typeof leaseHeartbeatTimer.unref === "function") {
|
|
2225
2543
|
leaseHeartbeatTimer.unref();
|
|
@@ -2322,6 +2640,10 @@ function createRunAgentTurnActivity(services) {
|
|
|
2322
2640
|
} : { computerToolMode: computerToolModeForTurn(null) },
|
|
2323
2641
|
...packRuntime.skills.length > 0 ? { packSkills: packRuntime.skills } : {},
|
|
2324
2642
|
...workspaceAgentInstructions ? { instructionsTemplate: workspaceAgentInstructions } : {},
|
|
2643
|
+
// Per-session persona tier (session > workspace > deployment default).
|
|
2644
|
+
// Composed system-level AFTER the workspace persona so it refines it for
|
|
2645
|
+
// this one session; absent ⇒ byte-identical to today's composition.
|
|
2646
|
+
...session.instructions ? { sessionInstructions: session.instructions } : {},
|
|
2325
2647
|
...workspaceEnvironment ? {
|
|
2326
2648
|
workspaceEnvironment: {
|
|
2327
2649
|
name: workspaceEnvironment.name,
|
|
@@ -2460,7 +2782,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2460
2782
|
responseId: responseUsage.responseId,
|
|
2461
2783
|
dispatchId,
|
|
2462
2784
|
positionalKey: `response-${responseUsageCount}`
|
|
2463
|
-
})
|
|
2785
|
+
}),
|
|
2786
|
+
observability
|
|
2464
2787
|
});
|
|
2465
2788
|
await reconcileConversationTruth();
|
|
2466
2789
|
try {
|
|
@@ -2506,7 +2829,8 @@ function createRunAgentTurnActivity(services) {
|
|
|
2506
2829
|
model: turn.model,
|
|
2507
2830
|
isCodexTurn,
|
|
2508
2831
|
usage: stream.state.usage,
|
|
2509
|
-
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" })
|
|
2832
|
+
sourceKey: modelUsageSourceKey({ responseId: null, dispatchId, positionalKey: "aggregate" }),
|
|
2833
|
+
observability
|
|
2510
2834
|
});
|
|
2511
2835
|
}
|
|
2512
2836
|
if (lastInputTokensObserved !== null) {
|
|
@@ -2554,6 +2878,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2554
2878
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2555
2879
|
], true);
|
|
2556
2880
|
await finishTurn(db, input.workspaceId, activeTurnId, "idle");
|
|
2881
|
+
turnMetricOutcome = "completed";
|
|
2557
2882
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2558
2883
|
await recordUsageEvent2(db, {
|
|
2559
2884
|
accountId: input.accountId,
|
|
@@ -2634,6 +2959,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2634
2959
|
await requeuePreemptedTurn(db, input.workspaceId, activeTurnId, preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2635
2960
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2636
2961
|
activityStatus = "preempted";
|
|
2962
|
+
turnMetricOutcome = "preempted";
|
|
2637
2963
|
observability.info("context compaction recovery succeeded by compacting and auto-continuing", {
|
|
2638
2964
|
sessionId: input.sessionId,
|
|
2639
2965
|
turnId: activeTurnId,
|
|
@@ -2656,6 +2982,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2656
2982
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2657
2983
|
], true);
|
|
2658
2984
|
await finishTurn(db, input.workspaceId, activeTurnId, "failed");
|
|
2985
|
+
turnMetricOutcome = "failed";
|
|
2659
2986
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2660
2987
|
activityStatus = "idle";
|
|
2661
2988
|
activityError = attemptError;
|
|
@@ -2684,6 +3011,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2684
3011
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, input.triggerEventId);
|
|
2685
3012
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2686
3013
|
activityStatus = "preempted";
|
|
3014
|
+
turnMetricOutcome = "preempted";
|
|
2687
3015
|
return { status: "preempted" };
|
|
2688
3016
|
} catch (requeueError) {
|
|
2689
3017
|
console.error("sandbox lease supersession requeue failed; falling back", requeueError);
|
|
@@ -2731,6 +3059,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2731
3059
|
await requeuePreemptedTurn(db, input.workspaceId, preemptTurnId, resumeWithNotice && preemptedEvent ? preemptedEvent.id : input.triggerEventId);
|
|
2732
3060
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null).catch(() => void 0);
|
|
2733
3061
|
activityStatus = "preempted";
|
|
3062
|
+
turnMetricOutcome = "preempted";
|
|
2734
3063
|
return { status: "preempted" };
|
|
2735
3064
|
} catch (preemptError) {
|
|
2736
3065
|
console.error("worker-shutdown preemption failed; falling back to cancellation", preemptError);
|
|
@@ -2742,6 +3071,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2742
3071
|
await flushRuntimeBatcher();
|
|
2743
3072
|
if (preemptTurnId) {
|
|
2744
3073
|
await finishTurn(db, input.workspaceId, preemptTurnId, "cancelled").catch(() => void 0);
|
|
3074
|
+
turnMetricOutcome = "cancelled";
|
|
2745
3075
|
}
|
|
2746
3076
|
throw error;
|
|
2747
3077
|
}
|
|
@@ -2766,6 +3096,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2766
3096
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2767
3097
|
], true);
|
|
2768
3098
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3099
|
+
turnMetricOutcome = "completed";
|
|
2769
3100
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2770
3101
|
await recordUsageEvent2(db, {
|
|
2771
3102
|
accountId: input.accountId,
|
|
@@ -2855,6 +3186,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2855
3186
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2856
3187
|
], true);
|
|
2857
3188
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3189
|
+
turnMetricOutcome = "failed";
|
|
2858
3190
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2859
3191
|
activityStatus = "idle";
|
|
2860
3192
|
activityError = error;
|
|
@@ -2887,6 +3219,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2887
3219
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2888
3220
|
], true);
|
|
2889
3221
|
await finishTurn(db, input.workspaceId, turnId, "idle");
|
|
3222
|
+
turnMetricOutcome = "completed";
|
|
2890
3223
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2891
3224
|
await recordUsageEvent2(db, {
|
|
2892
3225
|
accountId: input.accountId,
|
|
@@ -2925,6 +3258,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2925
3258
|
{ type: "session.status.changed", payload: { status: "idle" } }
|
|
2926
3259
|
], true);
|
|
2927
3260
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3261
|
+
turnMetricOutcome = "failed";
|
|
2928
3262
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
2929
3263
|
activityStatus = "idle";
|
|
2930
3264
|
activityError = error;
|
|
@@ -2940,6 +3274,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
2940
3274
|
{ type: "session.status.changed", payload: { status: "failed" } }
|
|
2941
3275
|
], true);
|
|
2942
3276
|
await finishTurn(db, input.workspaceId, turnId, "failed");
|
|
3277
|
+
turnMetricOutcome = "failed";
|
|
2943
3278
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "failed", null);
|
|
2944
3279
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed", `turn:${turnId}`);
|
|
2945
3280
|
return { status: "failed" };
|
|
@@ -2950,6 +3285,9 @@ function createRunAgentTurnActivity(services) {
|
|
|
2950
3285
|
status: activityStatus,
|
|
2951
3286
|
durationSeconds
|
|
2952
3287
|
});
|
|
3288
|
+
if (turnId && activityStatus !== "unknown") {
|
|
3289
|
+
turnLifecycleMetricsFor(observability).finish(turnId, turnMetricOutcome, durationSeconds);
|
|
3290
|
+
}
|
|
2953
3291
|
activitySpan.end({
|
|
2954
3292
|
attributes: {
|
|
2955
3293
|
"opengeni.turn_id": turnId ?? "",
|
|
@@ -3172,7 +3510,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3172
3510
|
idempotencyKey: `usage:model.cost:${input.turnId}:${input.sourceKey}`
|
|
3173
3511
|
});
|
|
3174
3512
|
if (costMicros > 0) {
|
|
3175
|
-
await applyCreditDebitUpToBalance(db, {
|
|
3513
|
+
const result = await applyCreditDebitUpToBalance(db, {
|
|
3176
3514
|
accountId: input.accountId,
|
|
3177
3515
|
workspaceId: input.workspaceId,
|
|
3178
3516
|
type: "model_usage_debit",
|
|
@@ -3190,6 +3528,7 @@ async function recordModelUsageAndDebitCredits(settings, db, input) {
|
|
|
3190
3528
|
totalTokens
|
|
3191
3529
|
}
|
|
3192
3530
|
});
|
|
3531
|
+
recordCreditMicros(input.observability, "usage", result.debitedMicros);
|
|
3193
3532
|
}
|
|
3194
3533
|
}
|
|
3195
3534
|
function positiveInt(value) {
|
|
@@ -3310,8 +3649,11 @@ function startOfUtcMonth3(date = /* @__PURE__ */ new Date()) {
|
|
|
3310
3649
|
import {
|
|
3311
3650
|
accrueWarmSeconds as accrueWarmSeconds2,
|
|
3312
3651
|
confirmDrainCold,
|
|
3652
|
+
countQueuedTurns,
|
|
3653
|
+
countSandboxLeasesByLiveness,
|
|
3313
3654
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3314
3655
|
getBillingBalance as getBillingBalance4,
|
|
3656
|
+
listLiveModalSandboxLeaseAttributions,
|
|
3315
3657
|
listMeterableWarmLeases,
|
|
3316
3658
|
persistDrainSnapshot,
|
|
3317
3659
|
readLease as readLease2,
|
|
@@ -3323,15 +3665,19 @@ import {
|
|
|
3323
3665
|
createSandboxClientForBackend,
|
|
3324
3666
|
deletePriorPersistedSnapshot,
|
|
3325
3667
|
deserializeSandboxSessionStateEnvelope,
|
|
3326
|
-
isProviderSandboxNotFoundError
|
|
3668
|
+
isProviderSandboxNotFoundError,
|
|
3669
|
+
sweepModalOrphanSandboxes,
|
|
3670
|
+
terminateModalSandboxById
|
|
3327
3671
|
} from "@opengeni/runtime";
|
|
3328
3672
|
function createSandboxLeaseActivities(services, options = {}) {
|
|
3329
3673
|
const terminateBox = options.terminateBox ?? terminateProviderBox;
|
|
3674
|
+
const sweepModalOrphans = options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
|
|
3330
3675
|
async function reapSandboxLeases2() {
|
|
3331
3676
|
const { db, settings, observability } = await services();
|
|
3332
3677
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3333
|
-
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0 };
|
|
3678
|
+
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3334
3679
|
}
|
|
3680
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3335
3681
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3336
3682
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3337
3683
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -3340,6 +3686,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3340
3686
|
});
|
|
3341
3687
|
let terminated = 0;
|
|
3342
3688
|
let skipped = 0;
|
|
3689
|
+
let modalOrphansTerminated = 0;
|
|
3343
3690
|
for (const row of drainable) {
|
|
3344
3691
|
try {
|
|
3345
3692
|
const drainedCold = await terminateDrainableBox(db, settings, row, observability, terminateBox);
|
|
@@ -3358,16 +3705,26 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3358
3705
|
});
|
|
3359
3706
|
}
|
|
3360
3707
|
}
|
|
3361
|
-
|
|
3708
|
+
try {
|
|
3709
|
+
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
3710
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
3711
|
+
} catch (error) {
|
|
3712
|
+
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
3713
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3714
|
+
});
|
|
3715
|
+
}
|
|
3716
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
3717
|
+
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3362
3718
|
observability.info("sandbox reaper swept", {
|
|
3363
3719
|
drainable: drainable.length,
|
|
3364
3720
|
terminated,
|
|
3365
3721
|
skipped,
|
|
3366
3722
|
metered: metered.accrued,
|
|
3367
|
-
forceDrained
|
|
3723
|
+
forceDrained,
|
|
3724
|
+
modalOrphansTerminated
|
|
3368
3725
|
});
|
|
3369
3726
|
}
|
|
3370
|
-
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained };
|
|
3727
|
+
return { examined: drainable.length, terminated, skipped, metered: metered.accrued, forceDrained, modalOrphansTerminated };
|
|
3371
3728
|
}
|
|
3372
3729
|
return { reapSandboxLeases: reapSandboxLeases2 };
|
|
3373
3730
|
}
|
|
@@ -3398,6 +3755,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3398
3755
|
if (result.accrued) {
|
|
3399
3756
|
accrued += 1;
|
|
3400
3757
|
}
|
|
3758
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
3401
3759
|
} catch (error) {
|
|
3402
3760
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
3403
3761
|
workspaceId: lease.workspaceId,
|
|
@@ -3408,6 +3766,22 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3408
3766
|
}
|
|
3409
3767
|
return { accrued, workspaceIds };
|
|
3410
3768
|
}
|
|
3769
|
+
async function refreshQueueAndLeaseGauges(db, observability) {
|
|
3770
|
+
try {
|
|
3771
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
3772
|
+
} catch (error) {
|
|
3773
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
3774
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3775
|
+
});
|
|
3776
|
+
}
|
|
3777
|
+
try {
|
|
3778
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
3779
|
+
} catch (error) {
|
|
3780
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
3781
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3782
|
+
});
|
|
3783
|
+
}
|
|
3784
|
+
}
|
|
3411
3785
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3412
3786
|
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
3413
3787
|
const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
|
|
@@ -3443,6 +3817,37 @@ async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observa
|
|
|
3443
3817
|
}
|
|
3444
3818
|
return forceDrained;
|
|
3445
3819
|
}
|
|
3820
|
+
async function sweepModalOrphansForConfiguredBackend(settings, db, observability) {
|
|
3821
|
+
if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret) {
|
|
3822
|
+
return 0;
|
|
3823
|
+
}
|
|
3824
|
+
let liveLeases;
|
|
3825
|
+
try {
|
|
3826
|
+
liveLeases = await listLiveModalSandboxLeaseAttributions(db);
|
|
3827
|
+
} catch (error) {
|
|
3828
|
+
observability.warn("sandbox reaper: live Modal lease attribution read failed", {
|
|
3829
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3830
|
+
});
|
|
3831
|
+
return 0;
|
|
3832
|
+
}
|
|
3833
|
+
const result = await sweepModalOrphanSandboxes(settings, liveLeases);
|
|
3834
|
+
for (const terminated of result.terminated) {
|
|
3835
|
+
observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
|
|
3836
|
+
sandboxId: terminated.sandboxId,
|
|
3837
|
+
reason: terminated.reason,
|
|
3838
|
+
tags: JSON.stringify(terminated.tags)
|
|
3839
|
+
});
|
|
3840
|
+
}
|
|
3841
|
+
if (result.examined > 0) {
|
|
3842
|
+
observability.info("sandbox reaper: Modal orphan sweep completed", {
|
|
3843
|
+
examined: result.examined,
|
|
3844
|
+
terminated: result.terminated.length,
|
|
3845
|
+
skipped: result.skipped,
|
|
3846
|
+
appName: settings.modalAppName
|
|
3847
|
+
});
|
|
3848
|
+
}
|
|
3849
|
+
return result.terminated.length;
|
|
3850
|
+
}
|
|
3446
3851
|
async function terminateDrainableBox(db, settings, row, observability, terminateBox) {
|
|
3447
3852
|
const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
|
|
3448
3853
|
const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
|
|
@@ -3483,6 +3888,30 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
|
|
|
3483
3888
|
});
|
|
3484
3889
|
return true;
|
|
3485
3890
|
}
|
|
3891
|
+
if (backend === "modal" && !lease.resumeState && lease.instanceId) {
|
|
3892
|
+
const { wrote } = await persistArchive(null);
|
|
3893
|
+
if (!wrote) {
|
|
3894
|
+
observability.info("sandbox reaper: Modal lease re-armed before direct terminate \u2014 leaving sandbox RUNNING", {
|
|
3895
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3896
|
+
backend,
|
|
3897
|
+
instanceId: lease.instanceId
|
|
3898
|
+
});
|
|
3899
|
+
return false;
|
|
3900
|
+
}
|
|
3901
|
+
try {
|
|
3902
|
+
await terminateModalSandboxById(settings, lease.instanceId);
|
|
3903
|
+
} catch (error) {
|
|
3904
|
+
if (!isProviderSandboxNotFoundError("modal", error)) {
|
|
3905
|
+
throw error;
|
|
3906
|
+
}
|
|
3907
|
+
observability.info("sandbox reaper: Modal sandbox already gone during direct terminate", {
|
|
3908
|
+
sandboxGroupId: lease.sandboxGroupId,
|
|
3909
|
+
backend,
|
|
3910
|
+
instanceId: lease.instanceId
|
|
3911
|
+
});
|
|
3912
|
+
}
|
|
3913
|
+
return true;
|
|
3914
|
+
}
|
|
3486
3915
|
if (!lease.resumeState) {
|
|
3487
3916
|
return true;
|
|
3488
3917
|
}
|
|
@@ -3883,6 +4312,7 @@ function startOfUtcMonth4() {
|
|
|
3883
4312
|
// src/activities/session-state.ts
|
|
3884
4313
|
import {
|
|
3885
4314
|
claimNextQueuedTurn as claimNextQueuedTurnDb2,
|
|
4315
|
+
countQueuedTurns as countQueuedTurns2,
|
|
3886
4316
|
countTurnSessionHistoryItems,
|
|
3887
4317
|
finishTurn as finishTurn2,
|
|
3888
4318
|
getSessionEvent as getSessionEvent4,
|
|
@@ -3926,7 +4356,7 @@ function createSessionStateActivities(services) {
|
|
|
3926
4356
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "failed");
|
|
3927
4357
|
}
|
|
3928
4358
|
async function interruptActiveTurn2(input) {
|
|
3929
|
-
const { db, bus } = await services();
|
|
4359
|
+
const { db, bus, observability } = await services();
|
|
3930
4360
|
const session = await requireSession4(db, input.workspaceId, input.sessionId);
|
|
3931
4361
|
const trigger = await getSessionEvent4(db, input.workspaceId, input.triggerEventId);
|
|
3932
4362
|
if (!isSteerInterrupt(trigger)) {
|
|
@@ -3949,9 +4379,10 @@ function createSessionStateActivities(services) {
|
|
|
3949
4379
|
]);
|
|
3950
4380
|
await finishTurn2(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
3951
4381
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4382
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
3952
4383
|
}
|
|
3953
4384
|
async function requeueTurnAfterWorkerDeath2(input) {
|
|
3954
|
-
const { settings, db, bus } = await services();
|
|
4385
|
+
const { settings, db, bus, observability } = await services();
|
|
3955
4386
|
const turn = await getSessionTurn2(db, input.workspaceId, input.turnId);
|
|
3956
4387
|
if (!turn || turn.status !== "running" && turn.status !== "requires_action") {
|
|
3957
4388
|
return { action: "stale" };
|
|
@@ -3991,11 +4422,14 @@ function createSessionStateActivities(services) {
|
|
|
3991
4422
|
throw requeueError;
|
|
3992
4423
|
}
|
|
3993
4424
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "queued", null);
|
|
4425
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
3994
4426
|
return { action: "requeued", redispatches };
|
|
3995
4427
|
}
|
|
3996
4428
|
async function claimNextQueuedTurn2(input) {
|
|
3997
|
-
const { db } = await services();
|
|
3998
|
-
|
|
4429
|
+
const { db, observability } = await services();
|
|
4430
|
+
const turn = await claimNextQueuedTurnDb2(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
4431
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4432
|
+
return turn;
|
|
3999
4433
|
}
|
|
4000
4434
|
async function markSessionIdle2(input) {
|
|
4001
4435
|
const { db, bus, settings, observability, wakeSessionWorkflow } = await services();
|
|
@@ -4003,6 +4437,7 @@ function createSessionStateActivities(services) {
|
|
|
4003
4437
|
if (session.status === "queued" || session.status === "running") {
|
|
4004
4438
|
await setSessionStatus2(db, input.workspaceId, input.sessionId, "idle", null);
|
|
4005
4439
|
}
|
|
4440
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
4006
4441
|
await notifyParentOfChildTerminal({ db, bus, settings, observability, wakeSessionWorkflow }, input.workspaceId, input.sessionId, "idle");
|
|
4007
4442
|
}
|
|
4008
4443
|
return {
|
|
@@ -4013,6 +4448,12 @@ function createSessionStateActivities(services) {
|
|
|
4013
4448
|
markSessionIdle: markSessionIdle2
|
|
4014
4449
|
};
|
|
4015
4450
|
}
|
|
4451
|
+
async function refreshQueuedTurnsGauge(db, observability) {
|
|
4452
|
+
try {
|
|
4453
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns2(db));
|
|
4454
|
+
} catch {
|
|
4455
|
+
}
|
|
4456
|
+
}
|
|
4016
4457
|
|
|
4017
4458
|
// src/activities.ts
|
|
4018
4459
|
function createActivities(dependencies = {}) {
|
|
@@ -4020,6 +4461,7 @@ function createActivities(dependencies = {}) {
|
|
|
4020
4461
|
async function services() {
|
|
4021
4462
|
servicesPromise ??= (async () => {
|
|
4022
4463
|
const settings = dependencies.settings ?? getSettings();
|
|
4464
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
4023
4465
|
const searchPath = dbSearchPath(settings);
|
|
4024
4466
|
const dbClient = dependencies.db ? null : createDb(settings.databaseUrl, {
|
|
4025
4467
|
...searchPath ? { searchPath } : {},
|
|
@@ -4044,12 +4486,15 @@ function createActivities(dependencies = {}) {
|
|
|
4044
4486
|
// events by sequence). See `.agents/skills/opengeni/references/eventbus-binding-contract.md`.
|
|
4045
4487
|
bus: dependencies.bus ?? await createNatsEventBus(
|
|
4046
4488
|
settings.natsUrl,
|
|
4047
|
-
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0
|
|
4489
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4490
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4048
4491
|
),
|
|
4049
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
4492
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
4493
|
+
metrics: runtimeMetricsHooksForObservability(observability)
|
|
4494
|
+
}),
|
|
4050
4495
|
objectStorage: dependencies.objectStorage ?? createObjectStorage2(settings),
|
|
4051
4496
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
4052
|
-
observability
|
|
4497
|
+
observability,
|
|
4053
4498
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
4054
4499
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
4055
4500
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
|
@@ -4098,6 +4543,86 @@ var maybeContinueGoal = defaultActivities.maybeContinueGoal;
|
|
|
4098
4543
|
var pauseGoalForInterrupt = defaultActivities.pauseGoalForInterrupt;
|
|
4099
4544
|
var reapSandboxLeases = defaultActivities.reapSandboxLeases;
|
|
4100
4545
|
|
|
4546
|
+
// src/http.ts
|
|
4547
|
+
import { dbSql } from "@opengeni/db";
|
|
4548
|
+
function startWorkerHttpServer(input) {
|
|
4549
|
+
const { settings, observability, checks } = input;
|
|
4550
|
+
const timeoutMs = input.timeoutMs ?? 2e3;
|
|
4551
|
+
return Bun.serve({
|
|
4552
|
+
hostname: "0.0.0.0",
|
|
4553
|
+
port: settings.workerHttpPort,
|
|
4554
|
+
async fetch(request) {
|
|
4555
|
+
const url = new URL(request.url);
|
|
4556
|
+
if (request.method !== "GET") {
|
|
4557
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
4558
|
+
}
|
|
4559
|
+
if (url.pathname === "/healthz") {
|
|
4560
|
+
return Response.json({
|
|
4561
|
+
service: settings.serviceName,
|
|
4562
|
+
environment: settings.environment,
|
|
4563
|
+
deploymentRevision: settings.deploymentRevision,
|
|
4564
|
+
ok: true
|
|
4565
|
+
});
|
|
4566
|
+
}
|
|
4567
|
+
if (url.pathname === "/metrics") {
|
|
4568
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
4569
|
+
status: 200,
|
|
4570
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" }
|
|
4571
|
+
});
|
|
4572
|
+
}
|
|
4573
|
+
if (url.pathname === "/readyz") {
|
|
4574
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
4575
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
4576
|
+
}
|
|
4577
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
4578
|
+
}
|
|
4579
|
+
});
|
|
4580
|
+
}
|
|
4581
|
+
async function runReadinessChecks(checks, timeoutMs = 2e3) {
|
|
4582
|
+
const entries = await Promise.all(
|
|
4583
|
+
Object.entries(checks).map(async ([name, check]) => {
|
|
4584
|
+
try {
|
|
4585
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
4586
|
+
return [name, { ok: true }];
|
|
4587
|
+
} catch (error) {
|
|
4588
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }];
|
|
4589
|
+
}
|
|
4590
|
+
})
|
|
4591
|
+
);
|
|
4592
|
+
const result = Object.fromEntries(entries);
|
|
4593
|
+
return {
|
|
4594
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
4595
|
+
checks: result
|
|
4596
|
+
};
|
|
4597
|
+
}
|
|
4598
|
+
function dbReadyCheck(db) {
|
|
4599
|
+
return async () => {
|
|
4600
|
+
await db.execute(dbSql`select 1`);
|
|
4601
|
+
};
|
|
4602
|
+
}
|
|
4603
|
+
function natsReadyCheck(bus) {
|
|
4604
|
+
return () => {
|
|
4605
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
4606
|
+
throw new Error("NATS is not connected");
|
|
4607
|
+
}
|
|
4608
|
+
};
|
|
4609
|
+
}
|
|
4610
|
+
async function withTimeout(promise, timeoutMs) {
|
|
4611
|
+
let timer;
|
|
4612
|
+
try {
|
|
4613
|
+
return await Promise.race([
|
|
4614
|
+
promise,
|
|
4615
|
+
new Promise((_, reject) => {
|
|
4616
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
4617
|
+
})
|
|
4618
|
+
]);
|
|
4619
|
+
} finally {
|
|
4620
|
+
if (timer) {
|
|
4621
|
+
clearTimeout(timer);
|
|
4622
|
+
}
|
|
4623
|
+
}
|
|
4624
|
+
}
|
|
4625
|
+
|
|
4101
4626
|
// src/index.ts
|
|
4102
4627
|
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
4103
4628
|
async function createOpenGeniWorker(options = {}) {
|
|
@@ -4138,6 +4663,9 @@ async function createWorkerWorkflowSignaler(settings) {
|
|
|
4138
4663
|
signal: "queueChanged"
|
|
4139
4664
|
});
|
|
4140
4665
|
},
|
|
4666
|
+
check: async () => {
|
|
4667
|
+
await connection.workflowService.getSystemInfo({});
|
|
4668
|
+
},
|
|
4141
4669
|
close: async () => {
|
|
4142
4670
|
await connection.close();
|
|
4143
4671
|
}
|
|
@@ -4195,35 +4723,71 @@ async function registerSandboxReaperSchedule(settings, observability) {
|
|
|
4195
4723
|
async function startWorker() {
|
|
4196
4724
|
const settings = getSettings2();
|
|
4197
4725
|
const observability = createObservability2(settings, { component: "worker" });
|
|
4198
|
-
const
|
|
4199
|
-
|
|
4200
|
-
|
|
4201
|
-
|
|
4202
|
-
|
|
4203
|
-
|
|
4204
|
-
}
|
|
4205
|
-
);
|
|
4206
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
4207
|
-
settings,
|
|
4208
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow }
|
|
4209
|
-
});
|
|
4210
|
-
const reaperSchedule = await retryStartupDependency(
|
|
4211
|
-
"Temporal schedule (sandbox reaper)",
|
|
4212
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
4213
|
-
{
|
|
4214
|
-
...startupRetryOptions(settings),
|
|
4215
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
4216
|
-
}
|
|
4217
|
-
);
|
|
4218
|
-
observability.info("OpenGeni worker listening", {
|
|
4219
|
-
temporalTaskQueue: settings.temporalTaskQueue
|
|
4726
|
+
const retryOptions = startupRetryOptions(settings);
|
|
4727
|
+
const onRetry = (event) => logStartupDependencyRetry(observability, event);
|
|
4728
|
+
const searchPath = dbSearchPath2(settings);
|
|
4729
|
+
const dbClient = createDb2(settings.databaseUrl, {
|
|
4730
|
+
...searchPath ? { searchPath } : {},
|
|
4731
|
+
rlsStrategy: settings.rlsStrategy
|
|
4220
4732
|
});
|
|
4733
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth2(settings);
|
|
4734
|
+
let bus;
|
|
4735
|
+
let signaler;
|
|
4736
|
+
let workerBundle;
|
|
4737
|
+
let reaperSchedule;
|
|
4738
|
+
let httpServer;
|
|
4221
4739
|
try {
|
|
4222
|
-
await
|
|
4740
|
+
bus = await retryStartupDependency(
|
|
4741
|
+
"NATS",
|
|
4742
|
+
() => createNatsEventBus2(
|
|
4743
|
+
settings.natsUrl,
|
|
4744
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : void 0,
|
|
4745
|
+
{ logger: observabilityEventLogger(observability) }
|
|
4746
|
+
),
|
|
4747
|
+
{ ...retryOptions, onRetry }
|
|
4748
|
+
);
|
|
4749
|
+
signaler = await retryStartupDependency(
|
|
4750
|
+
"Temporal client",
|
|
4751
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
4752
|
+
{ ...retryOptions, onRetry }
|
|
4753
|
+
);
|
|
4754
|
+
workerBundle = await createOpenGeniWorker({
|
|
4755
|
+
settings,
|
|
4756
|
+
activityDependencies: {
|
|
4757
|
+
observability,
|
|
4758
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
4759
|
+
db: dbClient.db,
|
|
4760
|
+
bus
|
|
4761
|
+
}
|
|
4762
|
+
});
|
|
4763
|
+
httpServer = startWorkerHttpServer({
|
|
4764
|
+
settings,
|
|
4765
|
+
observability,
|
|
4766
|
+
checks: {
|
|
4767
|
+
db: dbReadyCheck(dbClient.db),
|
|
4768
|
+
nats: natsReadyCheck(bus),
|
|
4769
|
+
temporal: signaler.check
|
|
4770
|
+
}
|
|
4771
|
+
});
|
|
4772
|
+
reaperSchedule = await retryStartupDependency(
|
|
4773
|
+
"Temporal schedule (sandbox reaper)",
|
|
4774
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
4775
|
+
{ ...retryOptions, onRetry }
|
|
4776
|
+
);
|
|
4777
|
+
observability.info("OpenGeni worker listening", {
|
|
4778
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
4779
|
+
httpPort: settings.workerHttpPort
|
|
4780
|
+
});
|
|
4781
|
+
await workerBundle.worker.run();
|
|
4223
4782
|
} finally {
|
|
4224
|
-
|
|
4225
|
-
await
|
|
4226
|
-
|
|
4783
|
+
httpServer?.stop(true);
|
|
4784
|
+
await Promise.allSettled([
|
|
4785
|
+
workerBundle?.connection.close(),
|
|
4786
|
+
signaler?.close(),
|
|
4787
|
+
reaperSchedule?.close(),
|
|
4788
|
+
bus?.close(),
|
|
4789
|
+
dbClient.close()
|
|
4790
|
+
]);
|
|
4227
4791
|
}
|
|
4228
4792
|
}
|
|
4229
4793
|
if (import.meta.main) {
|