@opengeni/worker-bundle 0.13.9 → 0.13.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import type { SessionEventType } from "@opengeni/contracts";
1
+ import { type SessionEventType } from "@opengeni/contracts";
2
2
  import type { EventLogger } from "@opengeni/events";
3
3
  import type { Observability } from "@opengeni/observability";
4
4
  import { type RuntimeMetricsHooks, type SelfhostedOpObservation, type SelfhostedOpObserver } from "@opengeni/runtime";
@@ -66,6 +66,10 @@ export declare class TurnLifecycleMetrics {
66
66
  }
67
67
  export declare function recordTurnsQueuedGauge(observability: Observability, value: number): void;
68
68
  export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
69
+ export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "rotation_backlog", "retained_processes", "expired_drains"];
70
+ export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
71
+ export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
72
+ export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
69
73
  export declare function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void;
70
74
  export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
71
75
  export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
@@ -85,6 +89,8 @@ export declare function recordRetainedProcessInventoryGauges(observability: Obse
85
89
  activeCount: number;
86
90
  terminalOwnerCount: number;
87
91
  }>): void;
92
+ export declare const EXPIRED_DRAINING_BACKENDS: readonly [...("blaxel" | "cloudflare" | "daytona" | "docker" | "e2b" | "local" | "modal" | "none" | "runloop" | "selfhosted" | "vercel")[], "unknown"];
93
+ export declare const EXPIRED_DRAINING_AGE_BUCKETS: readonly ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
88
94
  export declare function recordExpiredDrainingSandboxLeaseGauges(observability: Observability, counts: Array<{
89
95
  backend: string;
90
96
  ageBucket: string;
@@ -1,7 +1,7 @@
1
1
  import { type Settings } from "@opengeni/config";
2
2
  import { type Database } from "@opengeni/db";
3
3
  import type { EventBus } from "@opengeni/events";
4
- import { type EstablishedSandboxSession, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
4
+ import { type EstablishedSandboxSession, type RoutingSandboxOperationObserver, type SelfhostedOpObserver, type SelfhostedRelayConfig, type OpStreamJournal } from "@opengeni/runtime";
5
5
  export type RoutingWiringServices = {
6
6
  db: Database;
7
7
  settings: Settings;
@@ -12,6 +12,8 @@ export type RoutingWiringServices = {
12
12
  /** The per-op observer wired into every selfhosted session this turn builds
13
13
  * (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
14
14
  onOp?: SelfhostedOpObserver;
15
+ /** Every physical routed provider call, across cloud and selfhosted homes. */
16
+ onSandboxOperation?: RoutingSandboxOperationObserver;
15
17
  /** The op-stream durable-resume journal (the Temporal adaptation from
16
18
  * op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
17
19
  * the runtime defaults (generation "1", no persistence) — tests / non-turn
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opengeni/worker-bundle",
3
- "version": "0.13.9",
3
+ "version": "0.13.11",
4
4
  "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -43,13 +43,13 @@
43
43
  "@opengeni/codex": "^0.2.9",
44
44
  "@opengeni/config": "^0.9.1",
45
45
  "@opengeni/contracts": "^0.28.1",
46
- "@opengeni/core": "^0.16.1",
47
- "@opengeni/db": "^0.18.1",
48
- "@opengeni/documents": "^0.2.63",
49
- "@opengeni/events": "^0.3.54",
46
+ "@opengeni/core": "^0.16.3",
47
+ "@opengeni/db": "^0.19.0",
48
+ "@opengeni/documents": "^0.2.64",
49
+ "@opengeni/events": "^0.3.55",
50
50
  "@opengeni/github": "^0.4.13",
51
- "@opengeni/observability": "^0.3.0",
52
- "@opengeni/runtime": "^0.15.3",
51
+ "@opengeni/observability": "^0.4.0",
52
+ "@opengeni/runtime": "^0.16.0",
53
53
  "@opengeni/storage": "^0.2.50",
54
54
  "@temporalio/activity": "^1.17.0",
55
55
  "@temporalio/client": "^1.17.0",
@@ -78,6 +78,7 @@ import {
78
78
  type SessionTurnRecordingSettlement,
79
79
  } from "@opengeni/db";
80
80
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
81
+ import { sandboxOperationMetricObserver } from "@opengeni/observability";
81
82
  import {
82
83
  sandboxStateEntryFromRunState,
83
84
  maxTurnsExceededRunState,
@@ -2297,6 +2298,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
2297
2298
  const machineOpObserver = makeMachineOpObserver(
2298
2299
  runtimeMetricsHooksForObservability(observability),
2299
2300
  );
2301
+ const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
2300
2302
  let isCodexTurn = false;
2301
2303
  let isExternallyBilledTurn = false;
2302
2304
  let executionGeneration = 0;
@@ -5162,6 +5164,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5162
5164
  settings,
5163
5165
  bus,
5164
5166
  onOp: machineOpObserver.observer,
5167
+ onSandboxOperation: sandboxOperationObserver,
5165
5168
  opJournal,
5166
5169
  },
5167
5170
  {
@@ -5190,6 +5193,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5190
5193
  settings,
5191
5194
  bus,
5192
5195
  onOp: machineOpObserver.observer,
5196
+ onSandboxOperation: sandboxOperationObserver,
5193
5197
  onHomeSandboxRebound,
5194
5198
  },
5195
5199
  {
@@ -5234,7 +5238,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5234
5238
  resumeBoxForTurn(
5235
5239
  {
5236
5240
  db,
5237
- settings,
5241
+ settings: runSettings,
5238
5242
  cancellationSignal: sandboxResumeSignal,
5239
5243
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
5240
5244
  onSandboxLost: publishSandboxLost,
@@ -5297,6 +5301,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5297
5301
  db,
5298
5302
  settings,
5299
5303
  bus,
5304
+ onSandboxOperation: sandboxOperationObserver,
5300
5305
  onHomeSandboxLost: publishSandboxLost,
5301
5306
  onHomeSandboxRebound,
5302
5307
  },
@@ -5640,7 +5645,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5640
5645
  const provisioned = await resumeBoxForTurn(
5641
5646
  {
5642
5647
  db,
5643
- settings,
5648
+ settings: runSettings,
5644
5649
  cancellationSignal: sandboxResumeSignal,
5645
5650
  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
5646
5651
  onSandboxLost: publishSandboxLost,
@@ -5786,6 +5791,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
5786
5791
  settings,
5787
5792
  bus,
5788
5793
  onOp: machineOpObserver.observer,
5794
+ onSandboxOperation: sandboxOperationObserver,
5789
5795
  onHomeSandboxLost: publishSandboxLost,
5790
5796
  onHomeSandboxRebound,
5791
5797
  },
@@ -115,7 +115,10 @@ import {
115
115
  recordSandboxCheckpointArtifactGauges,
116
116
  recordSandboxCheckpointArtifactOutcome,
117
117
  recordSandboxDeadlineRotationsRequested,
118
+ recordSandboxInventoryProjectionFailure,
119
+ recordSandboxInventoryProjectionSuccess,
118
120
  type RetainedProcessReconciliationOutcome,
121
+ type SandboxInventoryProjectionDomain,
119
122
  recordSandboxLeaseGauges,
120
123
  recordSandboxOrphansTerminated,
121
124
  recordSandboxRotationBacklogGauges,
@@ -286,9 +289,9 @@ export function createSandboxLeaseActivities(
286
289
  const probeDrainableProvider = options.probeDrainableProvider ?? probeDrainableProviderReadiness;
287
290
  /**
288
291
  * The one global reaper sweep. Idempotent; concurrency-safe with itself.
289
- * Gated by the caller (the Schedule is only registered when
290
- * sandboxOwnershipEnabled); a defensive no-op here too so a manual trigger
291
- * with the flag off can never terminate a box.
292
+ * The global Schedule always runs. A defensive mutation gate here ensures
293
+ * that an ownership-off deployment still refreshes read-only metrics and
294
+ * repairs the system-update outbox but can never terminate a box.
292
295
  */
293
296
  async function reapSandboxLeases(): Promise<ReapSandboxLeasesResult> {
294
297
  const service = await services();
@@ -312,6 +315,11 @@ export function createSandboxLeaseActivities(
312
315
  deleted: expiredSessionListSnapshots,
313
316
  });
314
317
  }
318
+ // Inventory is useful even while provider ownership is disabled, and the
319
+ // same globally scheduled activity remains its single projection owner.
320
+ // Refresh before the mutation gate so dashboards show truthful zero/history
321
+ // state and absence alerts do not misclassify an intentional rollout flag.
322
+ await refreshQueueLeaseAndCreditGauges(db, observability);
315
323
  if (!settings.sandboxOwnershipEnabled) {
316
324
  if (parentUpdates.claimed > 0) {
317
325
  observability.info("system-update outbox reconciled", parentUpdates);
@@ -325,7 +333,6 @@ export function createSandboxLeaseActivities(
325
333
  modalOrphansTerminated: 0,
326
334
  };
327
335
  }
328
- await refreshQueueLeaseAndCreditGauges(db, observability);
329
336
 
330
337
  const rotationsRequested = await requestDueSandboxRotationsGlobal(
331
338
  db,
@@ -1218,54 +1225,71 @@ async function refreshQueueLeaseAndCreditGauges(
1218
1225
  error: error instanceof Error ? error.message : String(error),
1219
1226
  });
1220
1227
  }
1221
- try {
1228
+ await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
1222
1229
  recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
1223
- } catch (error) {
1224
- observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
1225
- error: error instanceof Error ? error.message : String(error),
1226
- });
1227
- }
1228
- try {
1229
- recordSandboxCheckpointArtifactGauges(
1230
- observability,
1231
- await countSandboxCheckpointArtifactsByState(db),
1232
- );
1233
- } catch (error) {
1234
- observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
1235
- error: error instanceof Error ? error.message : String(error),
1236
- });
1237
- }
1238
- try {
1239
- recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1240
- } catch (error) {
1241
- observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
1242
- error: error instanceof Error ? error.message : String(error),
1243
- });
1244
- }
1245
- try {
1246
- recordRetainedProcessInventoryGauges(
1247
- observability,
1248
- await countActiveRetainedProcessesByOwnerState(db),
1249
- );
1250
- } catch (error) {
1251
- observability.warn("sandbox reaper: retained-process gauge refresh failed", {
1252
- error: error instanceof Error ? error.message : String(error),
1253
- });
1254
- }
1230
+ });
1231
+ await refreshSandboxInventoryGauge(
1232
+ observability,
1233
+ "checkpoint_artifacts",
1234
+ "checkpoint-artifact",
1235
+ async () => {
1236
+ recordSandboxCheckpointArtifactGauges(
1237
+ observability,
1238
+ await countSandboxCheckpointArtifactsByState(db),
1239
+ );
1240
+ },
1241
+ );
1242
+ await refreshSandboxInventoryGauge(
1243
+ observability,
1244
+ "rotation_backlog",
1245
+ "rotation-backlog",
1246
+ async () => {
1247
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
1248
+ },
1249
+ );
1250
+ await refreshSandboxInventoryGauge(
1251
+ observability,
1252
+ "retained_processes",
1253
+ "retained-process",
1254
+ async () => {
1255
+ recordRetainedProcessInventoryGauges(
1256
+ observability,
1257
+ await countActiveRetainedProcessesByOwnerState(db),
1258
+ );
1259
+ },
1260
+ );
1261
+ await refreshSandboxInventoryGauge(
1262
+ observability,
1263
+ "expired_drains",
1264
+ "expired-draining",
1265
+ async () => {
1266
+ recordExpiredDrainingSandboxLeaseGauges(
1267
+ observability,
1268
+ await countExpiredDrainingSandboxLeases(db),
1269
+ );
1270
+ },
1271
+ );
1255
1272
  try {
1256
- recordExpiredDrainingSandboxLeaseGauges(
1257
- observability,
1258
- await countExpiredDrainingSandboxLeases(db),
1259
- );
1273
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1260
1274
  } catch (error) {
1261
- observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
1275
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1262
1276
  error: error instanceof Error ? error.message : String(error),
1263
1277
  });
1264
1278
  }
1279
+ }
1280
+
1281
+ async function refreshSandboxInventoryGauge(
1282
+ observability: ActivityServices["observability"],
1283
+ domain: SandboxInventoryProjectionDomain,
1284
+ diagnosticName: string,
1285
+ refresh: () => Promise<void>,
1286
+ ): Promise<void> {
1265
1287
  try {
1266
- recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
1288
+ await refresh();
1289
+ recordSandboxInventoryProjectionSuccess(observability, domain);
1267
1290
  } catch (error) {
1268
- observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
1291
+ recordSandboxInventoryProjectionFailure(observability, domain);
1292
+ observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
1269
1293
  error: error instanceof Error ? error.message : String(error),
1270
1294
  });
1271
1295
  }
package/src/index.ts CHANGED
@@ -351,8 +351,8 @@ export async function createWorkerWorkflowSignaler(
351
351
  /**
352
352
  * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
353
353
  * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
354
- * sandbox ownership off the activity performs only bounded DB outbox repair;
355
- * it never reads/terminates sandbox leases.
354
+ * sandbox ownership off the activity performs bounded DB outbox repair and
355
+ * read-only observability projections; it never mutates or terminates sandbox leases.
356
356
  *
357
357
  * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
358
358
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
@@ -1,6 +1,6 @@
1
1
  import { createHash } from "node:crypto";
2
2
  import { errorCodeToJSON } from "@opengeni/agent-proto";
3
- import type { SessionEventType } from "@opengeni/contracts";
3
+ import { SandboxBackend, type SessionEventType } from "@opengeni/contracts";
4
4
  import type { EventLogger } from "@opengeni/events";
5
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
6
6
  import {
@@ -19,7 +19,6 @@ export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
19
19
  const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
20
20
  const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
21
21
  const modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();
22
- const retainedProcessTerminalBacklogTotals = new WeakMap<Observability, number>();
23
22
 
24
23
  // A worker process reaching one trillion tokens in one provider/cache counter is
25
24
  // already far outside an ordinary scrape lifetime. Refuse further increments
@@ -345,6 +344,41 @@ export function recordSandboxLeaseGauges(
345
344
  }
346
345
  }
347
346
 
347
+ export const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [
348
+ "leases",
349
+ "checkpoint_artifacts",
350
+ "rotation_backlog",
351
+ "retained_processes",
352
+ "expired_drains",
353
+ ] as const;
354
+
355
+ export type SandboxInventoryProjectionDomain =
356
+ (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
357
+
358
+ export function recordSandboxInventoryProjectionSuccess(
359
+ observability: Observability,
360
+ domain: SandboxInventoryProjectionDomain,
361
+ timestampSeconds = Date.now() / 1_000,
362
+ ): void {
363
+ observability.setGauge({
364
+ name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
365
+ help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
366
+ labels: { domain },
367
+ value: timestampSeconds,
368
+ });
369
+ }
370
+
371
+ export function recordSandboxInventoryProjectionFailure(
372
+ observability: Observability,
373
+ domain: SandboxInventoryProjectionDomain,
374
+ ): void {
375
+ observability.incrementCounter({
376
+ name: "opengeni_sandbox_inventory_refresh_failures_total",
377
+ help: "Failed sandbox inventory projection refreshes by domain.",
378
+ labels: { domain },
379
+ });
380
+ }
381
+
348
382
  export function recordCreditBalanceGauges(
349
383
  observability: Observability,
350
384
  balances: CreditBalanceGauge[],
@@ -510,9 +544,7 @@ export function recordRetainedProcessInventoryGauges(
510
544
  current.terminal += Math.max(0, count.terminalOwnerCount);
511
545
  }
512
546
 
513
- let terminalTotal = 0;
514
547
  for (const [ownerState, count] of normalized) {
515
- terminalTotal += count.terminal;
516
548
  observability.setGauge({
517
549
  name: "opengeni_retained_processes_active",
518
550
  help: "Current active retained provider processes by durable owner state.",
@@ -526,31 +558,11 @@ export function recordRetainedProcessInventoryGauges(
526
558
  value: count.terminal,
527
559
  });
528
560
  }
529
- const previous = retainedProcessTerminalBacklogTotals.get(observability);
530
- if (previous !== undefined && terminalTotal > previous) {
531
- observability.incrementCounter({
532
- name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
533
- help: "Positive growth in terminal-owner retained-process backlog between app samples.",
534
- amount: terminalTotal - previous,
535
- });
536
- }
537
- retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
538
561
  }
539
562
 
540
- const EXPIRED_DRAINING_BACKENDS = [
541
- "none",
542
- "local",
543
- "docker",
544
- "modal",
545
- "e2b",
546
- "daytona",
547
- "blaxel",
548
- "runloop",
549
- "selfhosted",
550
- "unknown",
551
- ] as const;
563
+ export const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"] as const;
552
564
  const EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);
553
- const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
565
+ export const EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"] as const;
554
566
 
555
567
  export function recordExpiredDrainingSandboxLeaseGauges(
556
568
  observability: Observability,
@@ -52,6 +52,7 @@ import {
52
52
  type RoutableSandbox,
53
53
  type ResolvedActiveBackend,
54
54
  type RoutingMutationSettlementResult,
55
+ type RoutingSandboxOperationObserver,
55
56
  type RoutingRetainedProcess,
56
57
  type RoutingRetainedProcessTerminalProof,
57
58
  type SelfhostedOpObserver,
@@ -78,6 +79,8 @@ export type RoutingWiringServices = {
78
79
  /** The per-op observer wired into every selfhosted session this turn builds
79
80
  * (out-of-band telemetry — op metrics + machine.* events). Absent ⇒ no-op. */
80
81
  onOp?: SelfhostedOpObserver;
82
+ /** Every physical routed provider call, across cloud and selfhosted homes. */
83
+ onSandboxOperation?: RoutingSandboxOperationObserver;
81
84
  /** The op-stream durable-resume journal (the Temporal adaptation from
82
85
  * op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
83
86
  * the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -784,6 +787,7 @@ export function wrapTurnBoxWithRouting(
784
787
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
785
788
  },
786
789
  resolveActiveBackend: resolver,
790
+ ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
787
791
  ...(beforeMutation ? { beforeMutation } : {}),
788
792
  ...(afterMutation ? { afterMutation } : {}),
789
793
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
@@ -927,6 +931,7 @@ export function wrapLazyTurnBoxWithRouting(
927
931
  }
928
932
  return routedResolver(pointer);
929
933
  },
934
+ ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
930
935
  ...(beforeMutation ? { beforeMutation } : {}),
931
936
  ...(afterMutation ? { afterMutation } : {}),
932
937
  ...(beforeProcessMutation ? { beforeProcessMutation } : {}),