@opengeni/worker-bundle 0.13.9 → 0.13.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +80 -68
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +7 -1
- package/dist/sandbox-routing.d.ts +3 -1
- package/package.json +7 -7
- package/src/activities/agent-turn.ts +8 -2
- package/src/activities/sandbox-lease.ts +68 -44
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +38 -26
- package/src/sandbox-routing.ts +5 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type SessionEventType } from "@opengeni/contracts";
|
|
2
2
|
import type { EventLogger } from "@opengeni/events";
|
|
3
3
|
import type { Observability } from "@opengeni/observability";
|
|
4
4
|
import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
|
|
@@ -66,6 +66,10 @@ export declare class TurnLifecycleMetrics {
|
|
|
66
66
|
}
|
|
67
67
|
export declare function recordTurnsQueuedGauge(observability: Observability, value: number): void;
|
|
68
68
|
export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
|
|
69
|
+
export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains"];
|
|
70
|
+
export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
|
|
71
|
+
export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
|
|
72
|
+
export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
|
|
69
73
|
export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
|
|
70
74
|
export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
|
|
71
75
|
export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
|
|
@@ -85,6 +89,8 @@ export declare function recordRetainedProcessInventoryGauges(observability: Obse
|
|
|
85
89
|
activeCount: number;
|
|
86
90
|
terminalOwnerCount: number;
|
|
87
91
|
}>): void;
|
|
92
|
+
export declare const EXPIRED_DRAINING_BACKENDS: readonly [...("blaxel" | "cloudflare" | "daytona" | "docker" | "e2b" | "local" | "modal" | "none" | "runloop" | "selfhosted" | "vercel")[], "unknown"];
|
|
93
|
+
export declare const EXPIRED_DRAINING_AGE_BUCKETS: readonly ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
|
|
88
94
|
export declare function recordExpiredDrainingSandboxLeaseGauges(observability: Observability, counts: Array<{
|
|
89
95
|
backend: string;
|
|
90
96
|
ageBucket: string;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type Settings } from "@opengeni/config";
|
|
2
2
|
import { type Database } from "@opengeni/db";
|
|
3
3
|
import type { EventBus } from "@opengeni/events";
|
|
4
|
-
import { type EstablishedSandboxSession, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
|
|
4
|
+
import { type EstablishedSandboxSession, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
|
|
5
5
|
export type RoutingWiringServices = {
|
|
6
6
|
db: Database;
|
|
7
7
|
settings: Settings;
|
|
@@ -12,6 +12,8 @@ export type RoutingWiringServices = {
|
|
|
12
12
|
/** The per-op observer wired into every selfhosted session this turn builds
|
|
13
13
|
* (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
|
|
14
14
|
onOp?: SelfhostedOpObserver;
|
|
15
|
+
/** Every physical routed provider call, across cloud and selfhosted homes. */
|
|
16
|
+
onSandboxOperation?: RoutingSandboxOperationObserver;
|
|
15
17
|
/** The op-stream durable-resume journal (the Temporal adaptation from
|
|
16
18
|
* op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
|
|
17
19
|
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.13.
|
|
3
|
+
"version": "0.13.11",
|
|
4
4
|
"description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -43,13 +43,13 @@
|
|
|
43
43
|
"@opengeni/codex": "^0.2.9",
|
|
44
44
|
"@opengeni/config": "^0.9.1",
|
|
45
45
|
"@opengeni/contracts": "^0.28.1",
|
|
46
|
-
"@opengeni/core": "^0.16.
|
|
47
|
-
"@opengeni/db": "^0.
|
|
48
|
-
"@opengeni/documents": "^0.2.
|
|
49
|
-
"@opengeni/events": "^0.3.
|
|
46
|
+
"@opengeni/core": "^0.16.3",
|
|
47
|
+
"@opengeni/db": "^0.19.0",
|
|
48
|
+
"@opengeni/documents": "^0.2.64",
|
|
49
|
+
"@opengeni/events": "^0.3.55",
|
|
50
50
|
"@opengeni/github": "^0.4.13",
|
|
51
|
-
"@opengeni/observability": "^0.
|
|
52
|
-
"@opengeni/runtime": "^0.
|
|
51
|
+
"@opengeni/observability": "^0.4.0",
|
|
52
|
+
"@opengeni/runtime": "^0.16.0",
|
|
53
53
|
"@opengeni/storage": "^0.2.50",
|
|
54
54
|
"@temporalio/activity": "^1.17.0",
|
|
55
55
|
"@temporalio/client": "^1.17.0",
|
|
@@ -78,6 +78,7 @@ import {
|
|
|
78
78
|
type SessionTurnRecordingSettlement,
|
|
79
79
|
} from "@opengeni/db";
|
|
80
80
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
81
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
81
82
|
import {
|
|
82
83
|
sandboxStateEntryFromRunState,
|
|
83
84
|
maxTurnsExceededRunState,
|
|
@@ -2297,6 +2298,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
2297
2298
|
const machineOpObserver = makeMachineOpObserver(
|
|
2298
2299
|
runtimeMetricsHooksForObservability(observability),
|
|
2299
2300
|
);
|
|
2301
|
+
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
2300
2302
|
let isCodexTurn = false;
|
|
2301
2303
|
let isExternallyBilledTurn = false;
|
|
2302
2304
|
let executionGeneration = 0;
|
|
@@ -5162,6 +5164,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5162
5164
|
settings,
|
|
5163
5165
|
bus,
|
|
5164
5166
|
onOp: machineOpObserver.observer,
|
|
5167
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5165
5168
|
opJournal,
|
|
5166
5169
|
},
|
|
5167
5170
|
{
|
|
@@ -5190,6 +5193,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5190
5193
|
settings,
|
|
5191
5194
|
bus,
|
|
5192
5195
|
onOp: machineOpObserver.observer,
|
|
5196
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5193
5197
|
onHomeSandboxRebound,
|
|
5194
5198
|
},
|
|
5195
5199
|
{
|
|
@@ -5234,7 +5238,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5234
5238
|
resumeBoxForTurn(
|
|
5235
5239
|
{
|
|
5236
5240
|
db,
|
|
5237
|
-
settings,
|
|
5241
|
+
settings: runSettings,
|
|
5238
5242
|
cancellationSignal: sandboxResumeSignal,
|
|
5239
5243
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
5240
5244
|
onSandboxLost: publishSandboxLost,
|
|
@@ -5297,6 +5301,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5297
5301
|
db,
|
|
5298
5302
|
settings,
|
|
5299
5303
|
bus,
|
|
5304
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5300
5305
|
onHomeSandboxLost: publishSandboxLost,
|
|
5301
5306
|
onHomeSandboxRebound,
|
|
5302
5307
|
},
|
|
@@ -5640,7 +5645,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5640
5645
|
const provisioned = await resumeBoxForTurn(
|
|
5641
5646
|
{
|
|
5642
5647
|
db,
|
|
5643
|
-
settings,
|
|
5648
|
+
settings: runSettings,
|
|
5644
5649
|
cancellationSignal: sandboxResumeSignal,
|
|
5645
5650
|
sandboxMetrics: runtimeMetricsHooksForObservability(observability),
|
|
5646
5651
|
onSandboxLost: publishSandboxLost,
|
|
@@ -5786,6 +5791,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
5786
5791
|
settings,
|
|
5787
5792
|
bus,
|
|
5788
5793
|
onOp: machineOpObserver.observer,
|
|
5794
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
5789
5795
|
onHomeSandboxLost: publishSandboxLost,
|
|
5790
5796
|
onHomeSandboxRebound,
|
|
5791
5797
|
},
|
|
@@ -115,7 +115,10 @@ import {
|
|
|
115
115
|
recordSandboxCheckpointArtifactGauges,
|
|
116
116
|
recordSandboxCheckpointArtifactOutcome,
|
|
117
117
|
recordSandboxDeadlineRotationsRequested,
|
|
118
|
+
recordSandboxInventoryProjectionFailure,
|
|
119
|
+
recordSandboxInventoryProjectionSuccess,
|
|
118
120
|
type RetainedProcessReconciliationOutcome,
|
|
121
|
+
type SandboxInventoryProjectionDomain,
|
|
119
122
|
recordSandboxLeaseGauges,
|
|
120
123
|
recordSandboxOrphansTerminated,
|
|
121
124
|
recordSandboxRotationBacklogGauges,
|
|
@@ -286,9 +289,9 @@ export function createSandboxLeaseActivities(
|
|
|
286
289
|
const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
|
|
287
290
|
/**
|
|
288
291
|
* The one global reaper sweep. Idempotent; concurrency-safe with itself.
|
|
289
|
-
*
|
|
290
|
-
*
|
|
291
|
-
*
|
|
292
|
+
* The global Schedule always runs. A defensive mutation gate here ensures
|
|
293
|
+
* that an ownership-off deployment still refreshes read-only metrics and
|
|
294
|
+
* repairs the system-update outbox but can never terminate a box.
|
|
292
295
|
*/
|
|
293
296
|
async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
|
|
294
297
|
const service = await services();
|
|
@@ -312,6 +315,11 @@ export function createSandboxLeaseActivities(
|
|
|
312
315
|
deleted: expiredSessionListSnapshots,
|
|
313
316
|
});
|
|
314
317
|
}
|
|
318
|
+
// Inventory is useful even while provider ownership is disabled, and the
|
|
319
|
+
// same globally scheduled activity remains its single projection owner.
|
|
320
|
+
// Refresh before the mutation gate so dashboards show truthful zero/history
|
|
321
|
+
// state and absence alerts do not misclassify an intentional rollout flag.
|
|
322
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
315
323
|
if (!settings.sandboxOwnershipEnabled) {
|
|
316
324
|
if (parentUpdates.claimed > 0) {
|
|
317
325
|
observability.info("system-update outbox reconciled", parentUpdates);
|
|
@@ -325,7 +333,6 @@ export function createSandboxLeaseActivities(
|
|
|
325
333
|
modalOrphansTerminated: 0,
|
|
326
334
|
};
|
|
327
335
|
}
|
|
328
|
-
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
329
336
|
|
|
330
337
|
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
331
338
|
db,
|
|
@@ -1218,54 +1225,71 @@ async function refreshQueueLeaseAndCreditGauges(
|
|
|
1218
1225
|
error: error instanceof Error ? error.message : String(error),
|
|
1219
1226
|
});
|
|
1220
1227
|
}
|
|
1221
|
-
|
|
1228
|
+
await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
1222
1229
|
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
1223
|
-
}
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1230
|
+
});
|
|
1231
|
+
await refreshSandboxInventoryGauge(
|
|
1232
|
+
observability,
|
|
1233
|
+
"checkpoint_artifacts",
|
|
1234
|
+
"checkpoint-artifact",
|
|
1235
|
+
async () => {
|
|
1236
|
+
recordSandboxCheckpointArtifactGauges(
|
|
1237
|
+
observability,
|
|
1238
|
+
await countSandboxCheckpointArtifactsByState(db),
|
|
1239
|
+
);
|
|
1240
|
+
},
|
|
1241
|
+
);
|
|
1242
|
+
await refreshSandboxInventoryGauge(
|
|
1243
|
+
observability,
|
|
1244
|
+
"rotation_backlog",
|
|
1245
|
+
"rotation-backlog",
|
|
1246
|
+
async () => {
|
|
1247
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
1248
|
+
},
|
|
1249
|
+
);
|
|
1250
|
+
await refreshSandboxInventoryGauge(
|
|
1251
|
+
observability,
|
|
1252
|
+
"retained_processes",
|
|
1253
|
+
"retained-process",
|
|
1254
|
+
async () => {
|
|
1255
|
+
recordRetainedProcessInventoryGauges(
|
|
1256
|
+
observability,
|
|
1257
|
+
await countActiveRetainedProcessesByOwnerState(db),
|
|
1258
|
+
);
|
|
1259
|
+
},
|
|
1260
|
+
);
|
|
1261
|
+
await refreshSandboxInventoryGauge(
|
|
1262
|
+
observability,
|
|
1263
|
+
"expired_drains",
|
|
1264
|
+
"expired-draining",
|
|
1265
|
+
async () => {
|
|
1266
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
1267
|
+
observability,
|
|
1268
|
+
await countExpiredDrainingSandboxLeases(db),
|
|
1269
|
+
);
|
|
1270
|
+
},
|
|
1271
|
+
);
|
|
1255
1272
|
try {
|
|
1256
|
-
|
|
1257
|
-
observability,
|
|
1258
|
-
await countExpiredDrainingSandboxLeases(db),
|
|
1259
|
-
);
|
|
1273
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
1260
1274
|
} catch (error) {
|
|
1261
|
-
observability.warn("sandbox reaper:
|
|
1275
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
1262
1276
|
error: error instanceof Error ? error.message : String(error),
|
|
1263
1277
|
});
|
|
1264
1278
|
}
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
async function refreshSandboxInventoryGauge(
|
|
1282
|
+
observability: ActivityServices["observability"],
|
|
1283
|
+
domain: SandboxInventoryProjectionDomain,
|
|
1284
|
+
diagnosticName: string,
|
|
1285
|
+
refresh: () => Promise<void>,
|
|
1286
|
+
): Promise<void> {
|
|
1265
1287
|
try {
|
|
1266
|
-
|
|
1288
|
+
await refresh();
|
|
1289
|
+
recordSandboxInventoryProjectionSuccess(observability, domain);
|
|
1267
1290
|
} catch (error) {
|
|
1268
|
-
observability
|
|
1291
|
+
recordSandboxInventoryProjectionFailure(observability, domain);
|
|
1292
|
+
observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
|
|
1269
1293
|
error: error instanceof Error ? error.message : String(error),
|
|
1270
1294
|
});
|
|
1271
1295
|
}
|
package/src/index.ts
CHANGED
|
@@ -351,8 +351,8 @@ export async function createWorkerWorkflowSignaler(
|
|
|
351
351
|
/**
|
|
352
352
|
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
353
353
|
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
354
|
-
* sandbox ownership off the activity performs
|
|
355
|
-
* it never
|
|
354
|
+
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
355
|
+
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
356
356
|
*
|
|
357
357
|
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
358
358
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
|
-
import type
|
|
3
|
+
import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
|
|
4
4
|
import type { EventLogger } from "@opengeni/events";
|
|
5
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
6
6
|
import {
|
|
@@ -19,7 +19,6 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
|
19
19
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
20
20
|
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
21
21
|
const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
|
|
22
|
-
const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
|
|
23
22
|
|
|
24
23
|
// A worker process reaching one trillion tokens in one provider/cache counter is
|
|
25
24
|
// already far outside an ordinary scrape lifetime. Refuse further increments
|
|
@@ -345,6 +344,41 @@ export function recordSandboxLeaseGauges(
|
|
|
345
344
|
}
|
|
346
345
|
}
|
|
347
346
|
|
|
347
|
+
export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
|
|
348
|
+
"leases",
|
|
349
|
+
"checkpoint_artifacts",
|
|
350
|
+
"rotation_backlog",
|
|
351
|
+
"retained_processes",
|
|
352
|
+
"expired_drains",
|
|
353
|
+
] as const;
|
|
354
|
+
|
|
355
|
+
export type SandboxInventoryProjectionDomain =
|
|
356
|
+
(typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
|
|
357
|
+
|
|
358
|
+
export function recordSandboxInventoryProjectionSuccess(
|
|
359
|
+
observability: Observability,
|
|
360
|
+
domain: SandboxInventoryProjectionDomain,
|
|
361
|
+
timestampSeconds = Date.now() / 1_000,
|
|
362
|
+
): void {
|
|
363
|
+
observability.setGauge({
|
|
364
|
+
name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
|
|
365
|
+
help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
|
|
366
|
+
labels: { domain },
|
|
367
|
+
value: timestampSeconds,
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
export function recordSandboxInventoryProjectionFailure(
|
|
372
|
+
observability: Observability,
|
|
373
|
+
domain: SandboxInventoryProjectionDomain,
|
|
374
|
+
): void {
|
|
375
|
+
observability.incrementCounter({
|
|
376
|
+
name: "opengeni_sandbox_inventory_refresh_failures_total",
|
|
377
|
+
help: "Failed sandbox inventory projection refreshes by domain.",
|
|
378
|
+
labels: { domain },
|
|
379
|
+
});
|
|
380
|
+
}
|
|
381
|
+
|
|
348
382
|
export function recordCreditBalanceGauges(
|
|
349
383
|
observability: Observability,
|
|
350
384
|
balances: CreditBalanceGauge[],
|
|
@@ -510,9 +544,7 @@ export function recordRetainedProcessInventoryGauges(
|
|
|
510
544
|
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
511
545
|
}
|
|
512
546
|
|
|
513
|
-
let terminalTotal = 0;
|
|
514
547
|
for (const [ownerState, count] of normalized) {
|
|
515
|
-
terminalTotal += count.terminal;
|
|
516
548
|
observability.setGauge({
|
|
517
549
|
name: "opengeni_retained_processes_active",
|
|
518
550
|
help: "Current active retained provider processes by durable owner state.",
|
|
@@ -526,31 +558,11 @@ export function recordRetainedProcessInventoryGauges(
|
|
|
526
558
|
value: count.terminal,
|
|
527
559
|
});
|
|
528
560
|
}
|
|
529
|
-
const previous = retainedProcessTerminalBacklogTotals.get(observability);
|
|
530
|
-
if (previous !== undefined && terminalTotal > previous) {
|
|
531
|
-
observability.incrementCounter({
|
|
532
|
-
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
533
|
-
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
534
|
-
amount: terminalTotal - previous,
|
|
535
|
-
});
|
|
536
|
-
}
|
|
537
|
-
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
538
561
|
}
|
|
539
562
|
|
|
540
|
-
const EXPIRED_DRAINING_BACKENDS = [
|
|
541
|
-
"none",
|
|
542
|
-
"local",
|
|
543
|
-
"docker",
|
|
544
|
-
"modal",
|
|
545
|
-
"e2b",
|
|
546
|
-
"daytona",
|
|
547
|
-
"blaxel",
|
|
548
|
-
"runloop",
|
|
549
|
-
"selfhosted",
|
|
550
|
-
"unknown",
|
|
551
|
-
] as const;
|
|
563
|
+
export const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"] as const;
|
|
552
564
|
const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
|
|
553
|
-
const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
565
|
+
export const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
|
|
554
566
|
|
|
555
567
|
export function recordExpiredDrainingSandboxLeaseGauges(
|
|
556
568
|
observability: Observability,
|
package/src/sandbox-routing.ts
CHANGED
|
@@ -52,6 +52,7 @@ import {
|
|
|
52
52
|
type RoutableSandbox,
|
|
53
53
|
type ResolvedActiveBackend,
|
|
54
54
|
type RoutingMutationSettlementResult,
|
|
55
|
+
type RoutingSandboxOperationObserver,
|
|
55
56
|
type RoutingRetainedProcess,
|
|
56
57
|
type RoutingRetainedProcessTerminalProof,
|
|
57
58
|
type SelfhostedOpObserver,
|
|
@@ -78,6 +79,8 @@ export type RoutingWiringServices = {
|
|
|
78
79
|
/** The per-op observer wired into every selfhosted session this turn builds
|
|
79
80
|
* (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
|
|
80
81
|
onOp?: SelfhostedOpObserver;
|
|
82
|
+
/** Every physical routed provider call, across cloud and selfhosted homes. */
|
|
83
|
+
onSandboxOperation?: RoutingSandboxOperationObserver;
|
|
81
84
|
/** The op-stream durable-resume journal (the Temporal adaptation from
|
|
82
85
|
* op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
|
|
83
86
|
* the runtime defaults (generation "1", no persistence) — tests / non-turn
|
|
@@ -784,6 +787,7 @@ export function wrapTurnBoxWithRouting(
|
|
|
784
787
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
785
788
|
},
|
|
786
789
|
resolveActiveBackend: resolver,
|
|
790
|
+
...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
|
|
787
791
|
...(beforeMutation ? { beforeMutation } : {}),
|
|
788
792
|
...(afterMutation ? { afterMutation } : {}),
|
|
789
793
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|
|
@@ -927,6 +931,7 @@ export function wrapLazyTurnBoxWithRouting(
|
|
|
927
931
|
}
|
|
928
932
|
return routedResolver(pointer);
|
|
929
933
|
},
|
|
934
|
+
...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
|
|
930
935
|
...(beforeMutation ? { beforeMutation } : {}),
|
|
931
936
|
...(afterMutation ? { afterMutation } : {}),
|
|
932
937
|
...(beforeProcessMutation ? { beforeProcessMutation } : {}),
|