@opengeni/worker-bundle 0.13.10 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import type { SessionEventType } from "@opengeni/contracts";
1
+ import { type SessionEventType } from "@opengeni/contracts";
2
2
  import type { EventLogger } from "@opengeni/events";
3
3
  import type { Observability } from "@opengeni/observability";
4
4
  import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
@@ -66,6 +66,10 @@ export declare class TurnLifecycleMetrics {
66
66
  }
67
67
  export declare function recordTurnsQueuedGauge(observability: Observability, value: number): void;
68
68
  export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
69
+ export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains"];
70
+ export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
71
+ export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
72
+ export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
69
73
  export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
70
74
  export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
71
75
  export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
@@ -85,6 +89,8 @@ export declare function recordRetainedProcessInventoryGauges(observability: Obse
85
89
  activeCount: number;
86
90
  terminalOwnerCount: number;
87
91
  }>): void;
92
+ export declare const EXPIRED_DRAINING_BACKENDS: readonly [...("blaxel" | "cloudflare" | "daytona" | "docker" | "e2b" | "local" | "modal" | "none" | "runloop" | "selfhosted" | "vercel")[], "unknown"];
93
+ export declare const EXPIRED_DRAINING_AGE_BUCKETS: readonly ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
88
94
  export declare function recordExpiredDrainingSandboxLeaseGauges(observability: Observability, counts: Array<{
89
95
  backend: string;
90
96
  ageBucket: string;
@@ -1,7 +1,7 @@
1
1
  import { type Settings } from "@opengeni/config";
2
2
  import { type Database } from "@opengeni/db";
3
3
  import type { EventBus } from "@opengeni/events";
4
- import { type EstablishedSandboxSession, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
4
+ import { type EstablishedSandboxSession, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
5
5
  export type RoutingWiringServices = {
6
6
  db: Database;
7
7
  settings: Settings;
@@ -12,6 +12,8 @@ export type RoutingWiringServices = {
12
12
  /** The per-op observer wired into every selfhosted session this turn builds
13
13
  * (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
14
14
  onOp?: SelfhostedOpObserver;
15
+ /** Every physical routed provider call, across cloud and selfhosted homes. */
16
+ onSandboxOperation?: RoutingSandboxOperationObserver;
15
17
  /** The op-stream durable-resume journal (the Temporal adaptation from
16
18
  * op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
17
19
  * the runtime defaults (generation "1", no persistence) — tests / non-turn
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.13.10",
3
+ "version": "0.14.1",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -41,16 +41,16 @@
41
41
  "dependencies": {
42
42
  "@opengeni/agent-proto": "^0.3.0",
43
43
  "@opengeni/codex": "^0.2.9",
44
- "@opengeni/config": "^0.9.1",
45
- "@opengeni/contracts": "^0.28.1",
46
- "@opengeni/core": "^0.16.2",
47
- "@opengeni/db": "^0.19.0",
48
- "@opengeni/documents": "^0.2.64",
49
- "@opengeni/events": "^0.3.55",
50
- "@opengeni/github": "^0.4.13",
51
- "@opengeni/observability": "^0.3.0",
52
- "@opengeni/runtime": "^0.15.3",
53
- "@opengeni/storage": "^0.2.50",
44
+ "@opengeni/config": "^0.9.3",
45
+ "@opengeni/contracts": "^0.30.0",
46
+ "@opengeni/core": "^0.17.1",
47
+ "@opengeni/db": "^0.21.0",
48
+ "@opengeni/documents": "^0.2.66",
49
+ "@opengeni/events": "^0.3.57",
50
+ "@opengeni/github": "^0.4.15",
51
+ "@opengeni/observability": "^0.4.2",
52
+ "@opengeni/runtime": "^0.16.2",
53
+ "@opengeni/storage": "^0.2.52",
54
54
  "@temporalio/activity": "^1.17.0",
55
55
  "@temporalio/client": "^1.17.0",
56
56
  "@temporalio/worker": "^1.17.0",
@@ -27,6 +27,7 @@ import {
27
27
  CODEX_CREDENTIAL_LEASE_TTL_MS,
28
28
  getCodexRotationSettings,
29
29
  getWorkspaceModelPolicy,
30
+ getWorkspaceGrant,
30
31
  listCodexAccountStatuses,
31
32
  fetchCodexUsageForAccount,
32
33
  getSessionCodexState,
@@ -78,6 +79,7 @@ import {
78
79
  type SessionTurnRecordingSettlement,
79
80
  } from "@opengeni/db";
80
81
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
82
+ import { sandboxOperationMetricObserver } from "@opengeni/observability";
81
83
  import {
82
84
  sandboxStateEntryFromRunState,
83
85
  maxTurnsExceededRunState,
@@ -200,6 +202,8 @@ import { mergeResourceRefs } from "./common";
200
202
  import {
201
203
  assertSessionAllowsProductModel,
202
204
  defaultSessionMcpServerIds,
205
+ directPersonalConnectionSubjectId,
206
+ withFrozenPersonalConnectionDelegations,
203
207
  resolveSessionToolPolicy,
204
208
  } from "@opengeni/core";
205
209
  import { maybeCompactContext, settleFailedContextCompactionLandmark } from "./context-compaction";
@@ -326,7 +330,6 @@ import {
326
330
  type SessionTurn,
327
331
  type ToolAuthNeededPayload,
328
332
  type TurnExecutionPolicyV1,
329
- type TurnInitiator,
330
333
  } from "@opengeni/contracts";
331
334
  import { createHash, randomUUID } from "node:crypto";
332
335
 
@@ -336,11 +339,11 @@ import { createHash, randomUUID } from "node:crypto";
336
339
  // budget against the same window.
337
340
  export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
338
341
 
339
- /** Personal connection authority follows the immutable human turn initiator, never the worker. */
342
+ /** Broad personal lookup is allowed only for a direct human/API command. */
340
343
  export function credentialSubjectIdForTurnInitiator(
341
- initiator: Pick<TurnInitiator, "kind" | "subjectId">,
344
+ turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
342
345
  ): string | undefined {
343
- return initiator.kind === "subject" ? initiator.subjectId : undefined;
346
+ return directPersonalConnectionSubjectId(turn);
344
347
  }
345
348
 
346
349
  /**
@@ -352,7 +355,7 @@ export function credentialSubjectIdForTurnInitiator(
352
355
  export function isDirectHumanTurnInitiation(
353
356
  turn: Pick<SessionTurn, "source" | "initiator" | "initiatorContext">,
354
357
  ): boolean {
355
- if (turn.source !== "user" || turn.initiator.kind !== "subject") {
358
+ if ((turn.source !== "user" && turn.source !== "api") || turn.initiator.kind !== "subject") {
356
359
  return false;
357
360
  }
358
361
  return !["via", "viaTruncated", "provenanceError", "backfill"].some((key) =>
@@ -2297,6 +2300,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2297
2300
  const machineOpObserver = makeMachineOpObserver(
2298
2301
  runtimeMetricsHooksForObservability(observability),
2299
2302
  );
2303
+ const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
2300
2304
  let isCodexTurn = false;
2301
2305
  let isExternallyBilledTurn = false;
2302
2306
  let executionGeneration = 0;
@@ -5162,6 +5166,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5162
5166
  settings,
5163
5167
  bus,
5164
5168
  onOp: machineOpObserver.observer,
5169
+ onSandboxOperation: sandboxOperationObserver,
5165
5170
  opJournal,
5166
5171
  },
5167
5172
  {
@@ -5190,6 +5195,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5190
5195
  settings,
5191
5196
  bus,
5192
5197
  onOp: machineOpObserver.observer,
5198
+ onSandboxOperation: sandboxOperationObserver,
5193
5199
  onHomeSandboxRebound,
5194
5200
  },
5195
5201
  {
@@ -5297,6 +5303,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5297
5303
  db,
5298
5304
  settings,
5299
5305
  bus,
5306
+ onSandboxOperation: sandboxOperationObserver,
5300
5307
  onHomeSandboxLost: publishSandboxLost,
5301
5308
  onHomeSandboxRebound,
5302
5309
  },
@@ -5363,8 +5370,23 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5363
5370
  attemptId: input.attemptId,
5364
5371
  turn,
5365
5372
  });
5373
+ const personalConnectionDelegations = turn.personalConnectionDelegations;
5374
+ const delegatedMembershipChecks = new Map<string, Promise<boolean>>();
5375
+ const delegatedOwnerHasMembership = async (subjectId: string): Promise<boolean> => {
5376
+ const existing = delegatedMembershipChecks.get(subjectId);
5377
+ if (existing) return await existing;
5378
+ const check = getWorkspaceGrant(db, subjectId, input.workspaceId).then(Boolean);
5379
+ delegatedMembershipChecks.set(subjectId, check);
5380
+ return await check;
5381
+ };
5382
+ const resolveFrozenCredential = withFrozenPersonalConnectionDelegations({
5383
+ resolveCredential: rawResolveCredential,
5384
+ settings: runSettings,
5385
+ personalConnectionDelegations,
5386
+ ownerHasWorkspaceMembership: delegatedOwnerHasMembership,
5387
+ });
5366
5388
  const resolveCredential: typeof rawResolveCredential = async (request) => {
5367
- const result = await rawResolveCredential(request);
5389
+ const result = await resolveFrozenCredential(request);
5368
5390
  if (result.status === "ok") {
5369
5391
  registerSecretRedactions(
5370
5392
  headerSecretRedactions("MCP", result.headers, Object.keys(result.headers)),
@@ -5372,7 +5394,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5372
5394
  }
5373
5395
  return result;
5374
5396
  };
5375
- const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn.initiator);
5397
+ const credentialSubjectId = credentialSubjectIdForTurnInitiator(turn);
5376
5398
  preparedTools = await waitForTurnOperation(
5377
5399
  withCodex(() =>
5378
5400
  runtime.prepareTools(runSettings, turnTools, {
@@ -5786,6 +5808,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5786
5808
  settings,
5787
5809
  bus,
5788
5810
  onOp: machineOpObserver.observer,
5811
+ onSandboxOperation: sandboxOperationObserver,
5789
5812
  onHomeSandboxLost: publishSandboxLost,
5790
5813
  onHomeSandboxRebound,
5791
5814
  },
@@ -3,6 +3,7 @@ import type { Session, SessionGoal, SessionSystemUpdatePayload } from "@opengeni
3
3
  import {
4
4
  getSession,
5
5
  getSessionGoal,
6
+ getSessionParentPersonalConnectionDelegations,
6
7
  addSessionSystemUpdateWithSourceMutation,
7
8
  claimPendingSessionSystemUpdateOutbox,
8
9
  claimPendingSessionWorkflowWakes,
@@ -57,6 +58,11 @@ export async function notifyParentOfChildIdle(
57
58
  // guard so a nudge that slips through still cannot revive the goal.
58
59
  const clientEventId = `child-completion:${childSessionId}:${episodeKey}`;
59
60
  const payload = childCompletionPayload(child, goal);
61
+ const personalConnectionDelegations = await getSessionParentPersonalConnectionDelegations(
62
+ svc.db,
63
+ workspaceId,
64
+ childSessionId,
65
+ );
60
66
  const outbox = await getOrCreateSessionSystemUpdateOutbox(svc.db, {
61
67
  accountId: child.accountId,
62
68
  workspaceId,
@@ -69,6 +75,7 @@ export async function notifyParentOfChildIdle(
69
75
  summary: childCompletionSummary(child, goal, "idle"),
70
76
  payload,
71
77
  lineage: { childSessionId: child.id, parentSessionId: child.parentSessionId },
78
+ personalConnectionDelegations,
72
79
  });
73
80
  if (outbox.status === "delivered") {
74
81
  return;
@@ -138,6 +145,7 @@ async function deliverParentSystemUpdateOutbox(
138
145
  summary: outbox.summary,
139
146
  payload: outbox.payload,
140
147
  lineage: outbox.lineage,
148
+ personalConnectionDelegations: outbox.personalConnectionDelegations,
141
149
  },
142
150
  async (tx) => {
143
151
  await markSessionSystemUpdateOutboxDeliveredInTransaction(tx, outbox);
@@ -115,7 +115,10 @@ import {
115
115
  recordSandboxCheckpointArtifactGauges,
116
116
  recordSandboxCheckpointArtifactOutcome,
117
117
  recordSandboxDeadlineRotationsRequested,
118
+ recordSandboxInventoryProjectionFailure,
119
+ recordSandboxInventoryProjectionSuccess,
118
120
  type RetainedProcessReconciliationOutcome,
121
+ type SandboxInventoryProjectionDomain,
119
122
  recordSandboxLeaseGauges,
120
123
  recordSandboxOrphansTerminated,
121
124
  recordSandboxRotationBacklogGauges,
@@ -286,9 +289,9 @@ export function createSandboxLeaseActivities(
286
289
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
287
290
  /**
288
291
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
289
- * Gated by the caller (the Schedule is only registered when
290
- * sandboxOwnershipEnabled); a defensive no-op here too so a manual trigger
291
- * with the flag off can never terminate a box.
292
+ * The global Schedule always runs. A defensive mutation gate here ensures
293
+ * that an ownership-off deployment still refreshes read-only metrics and
294
+ * repairs the system-update outbox but can never terminate a box.
292
295
  */
293
296
  async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
294
297
  const service = await services();
@@ -312,6 +315,11 @@ export function createSandboxLeaseActivities(
312
315
  deleted: expiredSessionListSnapshots,
313
316
  });
314
317
  }
318
+ // Inventory is useful even while provider ownership is disabled, and the
319
+ // same globally scheduled activity remains its single projection owner.
320
+ // Refresh before the mutation gate so dashboards show truthful zero/history
321
+ // state and absence alerts do not misclassify an intentional rollout flag.
322
+ await refreshQueueLeaseAndCreditGauges(db, observability);
315
323
  if (!settings.sandboxOwnershipEnabled) {
316
324
  if (parentUpdates.claimed > 0) {
317
325
  observability.info("system-update outbox reconciled", parentUpdates);
@@ -325,7 +333,6 @@ export function createSandboxLeaseActivities(
325
333
  modalOrphansTerminated: 0,
326
334
  };
327
335
  }
328
- await refreshQueueLeaseAndCreditGauges(db, observability);
329
336
 
330
337
  const rotationsRequested = await requestDueSandboxRotationsGlobal(
331
338
  db,
@@ -1218,54 +1225,71 @@ async function refreshQueueLeaseAndCreditGauges(
1218
1225
  error: error instanceof Error ? error.message : String(error),
1219
1226
  });
1220
1227
  }
1221
- try {
1228
+ await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
1222
1229
  recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
1223
- } catch (error) {
1224
- observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
1225
- error: error instanceof Error ? error.message : String(error),
1226
- });
1227
- }
1228
- try {
1229
- recordSandboxCheckpointArtifactGauges(
1230
- observability,
1231
- await countSandboxCheckpointArtifactsByState(db),
1232
- );
1233
- } catch (error) {
1234
- observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
1235
- error: error instanceof Error ? error.message : String(error),
1236
- });
1237
- }
1238
- try {
1239
- recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1240
- } catch (error) {
1241
- observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
1242
- error: error instanceof Error ? error.message : String(error),
1243
- });
1244
- }
1245
- try {
1246
- recordRetainedProcessInventoryGauges(
1247
- observability,
1248
- await countActiveRetainedProcessesByOwnerState(db),
1249
- );
1250
- } catch (error) {
1251
- observability.warn("sandbox reaper: retained-process gauge refresh failed", {
1252
- error: error instanceof Error ? error.message : String(error),
1253
- });
1254
- }
1230
+ });
1231
+ await refreshSandboxInventoryGauge(
1232
+ observability,
1233
+ "checkpoint_artifacts",
1234
+ "checkpoint-artifact",
1235
+ async () => {
1236
+ recordSandboxCheckpointArtifactGauges(
1237
+ observability,
1238
+ await countSandboxCheckpointArtifactsByState(db),
1239
+ );
1240
+ },
1241
+ );
1242
+ await refreshSandboxInventoryGauge(
1243
+ observability,
1244
+ "rotation_backlog",
1245
+ "rotation-backlog",
1246
+ async () => {
1247
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1248
+ },
1249
+ );
1250
+ await refreshSandboxInventoryGauge(
1251
+ observability,
1252
+ "retained_processes",
1253
+ "retained-process",
1254
+ async () => {
1255
+ recordRetainedProcessInventoryGauges(
1256
+ observability,
1257
+ await countActiveRetainedProcessesByOwnerState(db),
1258
+ );
1259
+ },
1260
+ );
1261
+ await refreshSandboxInventoryGauge(
1262
+ observability,
1263
+ "expired_drains",
1264
+ "expired-draining",
1265
+ async () => {
1266
+ recordExpiredDrainingSandboxLeaseGauges(
1267
+ observability,
1268
+ await countExpiredDrainingSandboxLeases(db),
1269
+ );
1270
+ },
1271
+ );
1255
1272
  try {
1256
- recordExpiredDrainingSandboxLeaseGauges(
1257
- observability,
1258
- await countExpiredDrainingSandboxLeases(db),
1259
- );
1273
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1260
1274
  } catch (error) {
1261
- observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
1275
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1262
1276
  error: error instanceof Error ? error.message : String(error),
1263
1277
  });
1264
1278
  }
1279
+ }
1280
+
1281
+ async function refreshSandboxInventoryGauge(
1282
+ observability: ActivityServices["observability"],
1283
+ domain: SandboxInventoryProjectionDomain,
1284
+ diagnosticName: string,
1285
+ refresh: () => Promise<void>,
1286
+ ): Promise<void> {
1265
1287
  try {
1266
- recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1288
+ await refresh();
1289
+ recordSandboxInventoryProjectionSuccess(observability, domain);
1267
1290
  } catch (error) {
1268
- observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1291
+ recordSandboxInventoryProjectionFailure(observability, domain);
1292
+ observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
1269
1293
  error: error instanceof Error ? error.message : String(error),
1270
1294
  });
1271
1295
  }
@@ -17,6 +17,7 @@ import {
17
17
  enqueueSessionWorkflowWakeIfRunnable,
18
18
  getBillingBalance,
19
19
  getScheduledTask,
20
+ getScheduledTaskPersonalConnectionDelegations,
20
21
  getRig,
21
22
  getVariableSet,
22
23
  isCodexBilledTurn,
@@ -63,6 +64,11 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
63
64
  if (!task) {
64
65
  return { action: "deleted" };
65
66
  }
67
+ const taskPersonalConnectionDelegations = await getScheduledTaskPersonalConnectionDelegations(
68
+ db,
69
+ task.workspaceId,
70
+ task.id,
71
+ );
66
72
  // A scheduled bot selection was authorized when the task was written,
67
73
  // but connection status and tenant/role binding are mutable. Revalidate
68
74
  // before any session/model cost and never fall back to a personal Slack
@@ -237,6 +243,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
237
243
  variableSetId: task.variableSetId ?? null,
238
244
  rigId: frozenRigId,
239
245
  rigVersionId: frozenRigVersionId,
246
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
240
247
  maxNestedAgentDepthOverride: task.agentConfig.maxNestedAgentDepth ?? null,
241
248
  // The durable agent config was privilege-checked when the task
242
249
  // was created/updated. Preserve that explicit policy if a broader
@@ -328,6 +335,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
328
335
  run.id,
329
336
  ),
330
337
  lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
338
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
331
339
  },
332
340
  async (tx, wakeEventId) => {
333
341
  if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
@@ -444,6 +452,7 @@ export function createScheduledTaskActivities(services: () => Promise<ActivitySe
444
452
  run.id,
445
453
  ),
446
454
  lineage: { scheduledTaskId: task.id, scheduledTaskRunId: run.id },
455
+ personalConnectionDelegations: taskPersonalConnectionDelegations,
447
456
  },
448
457
  async (tx, wakeEventId) => {
449
458
  if (!wakeEventId) throw new Error("Scheduled delivery has no wake event");
package/src/index.ts CHANGED
@@ -351,8 +351,8 @@ export async function createWorkerWorkflowSignaler(
351
351
  /**
352
352
  * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
353
353
  * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
354
- * sandbox ownership off the activity performs only bounded DB outbox repair;
355
- * it never reads/terminates sandbox leases.
354
+ * sandbox ownership off the activity performs bounded DB outbox repair and
355
+ * read-only observability projections; it never mutates or terminates sandbox leases.
356
356
  *
357
357
  * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
358
358
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
@@ -1,6 +1,6 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3
- import type { SessionEventType } from "@opengeni/contracts";
3
+ import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
4
  import type { EventLogger } from "@opengeni/events";
5
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
6
  import {
@@ -19,7 +19,6 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
19
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
20
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
21
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
- const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
23
22
 
24
23
  // A worker process reaching one trillion tokens in one provider/cache counter is
25
24
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -345,6 +344,41 @@ export function recordSandboxLeaseGauges(
345
344
  }
346
345
  }
347
346
 
347
+ export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
348
+ "leases",
349
+ "checkpoint_artifacts",
350
+ "rotation_backlog",
351
+ "retained_processes",
352
+ "expired_drains",
353
+ ] as const;
354
+
355
+ export type SandboxInventoryProjectionDomain =
356
+ (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
357
+
358
+ export function recordSandboxInventoryProjectionSuccess(
359
+ observability: Observability,
360
+ domain: SandboxInventoryProjectionDomain,
361
+ timestampSeconds = Date.now() / 1_000,
362
+ ): void {
363
+ observability.setGauge({
364
+ name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
365
+ help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
366
+ labels: { domain },
367
+ value: timestampSeconds,
368
+ });
369
+ }
370
+
371
+ export function recordSandboxInventoryProjectionFailure(
372
+ observability: Observability,
373
+ domain: SandboxInventoryProjectionDomain,
374
+ ): void {
375
+ observability.incrementCounter({
376
+ name: "opengeni_sandbox_inventory_refresh_failures_total",
377
+ help: "Failed sandbox inventory projection refreshes by domain.",
378
+ labels: { domain },
379
+ });
380
+ }
381
+
348
382
  export function recordCreditBalanceGauges(
349
383
  observability: Observability,
350
384
  balances: CreditBalanceGauge[],
@@ -510,9 +544,7 @@ export function recordRetainedProcessInventoryGauges(
510
544
  current.terminal += Math.max(0, count.terminalOwnerCount);
511
545
  }
512
546
 
513
- let terminalTotal = 0;
514
547
  for (const [ownerState, count] of normalized) {
515
- terminalTotal += count.terminal;
516
548
  observability.setGauge({
517
549
  name: "opengeni_retained_processes_active",
518
550
  help: "Current active retained provider processes by durable owner state.",
@@ -526,31 +558,11 @@ export function recordRetainedProcessInventoryGauges(
526
558
  value: count.terminal,
527
559
  });
528
560
  }
529
- const previous = retainedProcessTerminalBacklogTotals.get(observability);
530
- if (previous !== undefined && terminalTotal > previous) {
531
- observability.incrementCounter({
532
- name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
533
- help: "Positive growth in terminal-owner retained-process backlog between app samples.",
534
- amount: terminalTotal - previous,
535
- });
536
- }
537
- retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
538
561
  }
539
562
 
540
- const EXPIRED_DRAINING_BACKENDS = [
541
- "none",
542
- "local",
543
- "docker",
544
- "modal",
545
- "e2b",
546
- "daytona",
547
- "blaxel",
548
- "runloop",
549
- "selfhosted",
550
- "unknown",
551
- ] as const;
563
+ export const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"] as const;
552
564
  const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
553
- const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
565
+ export const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
554
566
 
555
567
  export function recordExpiredDrainingSandboxLeaseGauges(
556
568
  observability: Observability,
@@ -52,6 +52,7 @@ import {
52
52
  type RoutableSandbox,
53
53
  type ResolvedActiveBackend,
54
54
  type RoutingMutationSettlementResult,
55
+ type RoutingSandboxOperationObserver,
55
56
  type RoutingRetainedProcess,
56
57
  type RoutingRetainedProcessTerminalProof,
57
58
  type SelfhostedOpObserver,
@@ -78,6 +79,8 @@ export type RoutingWiringServices = {
78
79
  /** The per-op observer wired into every selfhosted session this turn builds
79
80
  * (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
80
81
  onOp?: SelfhostedOpObserver;
82
+ /** Every physical routed provider call, across cloud and selfhosted homes. */
83
+ onSandboxOperation?: RoutingSandboxOperationObserver;
81
84
  /** The op-stream durable-resume journal (the Temporal adaptation from
82
85
  * op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
83
86
  * the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -784,6 +787,7 @@ export function wrapTurnBoxWithRouting(
784
787
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
785
788
  },
786
789
  resolveActiveBackend: resolver,
790
+ ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
787
791
  ...(beforeMutation ? { beforeMutation } : {}),
788
792
  ...(afterMutation ? { afterMutation } : {}),
789
793
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
@@ -927,6 +931,7 @@ export function wrapLazyTurnBoxWithRouting(
927
931
  }
928
932
  return routedResolver(pointer);
929
933
  },
934
+ ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
930
935
  ...(beforeMutation ? { beforeMutation } : {}),
931
936
  ...(afterMutation ? { afterMutation } : {}),
932
937
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),