@opengeni/worker-bundle 0.13.10 → 0.13.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.d.ts CHANGED
@@ -62,8 +62,8 @@ export declare function createWorkerWorkflowSignaler(settings: Settings, db: Dat
62
62
  /**
63
63
  * Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
64
64
  * driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
65
- * sandbox ownership off the activity performs only bounded DB outbox repair;
66
- * it never reads/terminates sandbox leases.
65
+ * sandbox ownership off the activity performs bounded DB outbox repair and
66
+ * read-only observability projections; it never mutates or terminates sandbox leases.
67
67
  *
68
68
  * The Schedule fires sandboxReaperWorkflow on the worker's global task queue
69
69
  * every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
package/dist/index.js CHANGED
@@ -115,6 +115,7 @@ import {
115
115
  markSessionAttemptQuiesced
116
116
  } from "@opengeni/db";
117
117
  import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
118
+ import { sandboxOperationMetricObserver } from "@opengeni/observability";
118
119
  import {
119
120
  sandboxStateEntryFromRunState,
120
121
  maxTurnsExceededRunState,
@@ -4474,6 +4475,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
4474
4475
  return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
4475
4476
  },
4476
4477
  resolveActiveBackend: resolver,
4478
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
4477
4479
  ...beforeMutation ? { beforeMutation } : {},
4478
4480
  ...afterMutation ? { afterMutation } : {},
4479
4481
  ...beforeProcessMutation ? { beforeProcessMutation } : {},
@@ -4581,6 +4583,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
4581
4583
  }
4582
4584
  return routedResolver(pointer);
4583
4585
  },
4586
+ ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
4584
4587
  ...beforeMutation ? { beforeMutation } : {},
4585
4588
  ...afterMutation ? { afterMutation } : {},
4586
4589
  ...beforeProcessMutation ? { beforeProcessMutation } : {},
@@ -4690,6 +4693,7 @@ function makeTurnOpJournal(context, details) {
4690
4693
  // src/observability-metrics.ts
4691
4694
  import { createHash as createHash2 } from "crypto";
4692
4695
  import { errorCodeToJSON } from "@opengeni/agent-proto";
4696
+ import { SandboxBackend } from "@opengeni/contracts";
4693
4697
  import {
4694
4698
  SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
4695
4699
  modelUsageTokenCountOrNull
@@ -4697,7 +4701,6 @@ import {
4697
4701
  var turnTrackers = /* @__PURE__ */ new WeakMap();
4698
4702
  var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
4699
4703
  var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
4700
- var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
4701
4704
  var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
4702
4705
  function observabilityEventLogger(observability) {
4703
4706
  return {
@@ -4949,6 +4952,21 @@ function recordSandboxLeaseGauges(observability, counts) {
4949
4952
  });
4950
4953
  }
4951
4954
  }
4955
+ function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
4956
+ observability.setGauge({
4957
+ name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
4958
+ help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
4959
+ labels: { domain },
4960
+ value: timestampSeconds
4961
+ });
4962
+ }
4963
+ function recordSandboxInventoryProjectionFailure(observability, domain) {
4964
+ observability.incrementCounter({
4965
+ name: "opengeni_sandbox_inventory_refresh_failures_total",
4966
+ help: "Failed sandbox inventory projection refreshes by domain.",
4967
+ labels: { domain }
4968
+ });
4969
+ }
4952
4970
  function recordCreditBalanceGauges(observability, balances) {
4953
4971
  const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
4954
4972
  const current = /* @__PURE__ */ new Set();
@@ -5062,9 +5080,7 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
5062
5080
  current.active += Math.max(0, count.activeCount);
5063
5081
  current.terminal += Math.max(0, count.terminalOwnerCount);
5064
5082
  }
5065
- let terminalTotal = 0;
5066
5083
  for (const [ownerState, count] of normalized) {
5067
- terminalTotal += count.terminal;
5068
5084
  observability.setGauge({
5069
5085
  name: "opengeni_retained_processes_active",
5070
5086
  help: "Current active retained provider processes by durable owner state.",
@@ -5078,28 +5094,8 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
5078
5094
  value: count.terminal
5079
5095
  });
5080
5096
  }
5081
- const previous = retainedProcessTerminalBacklogTotals.get(observability);
5082
- if (previous !== void 0 && terminalTotal > previous) {
5083
- observability.incrementCounter({
5084
- name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
5085
- help: "Positive growth in terminal-owner retained-process backlog between app samples.",
5086
- amount: terminalTotal - previous
5087
- });
5088
- }
5089
- retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
5090
- }
5091
- var EXPIRED_DRAINING_BACKENDS = [
5092
- "none",
5093
- "local",
5094
- "docker",
5095
- "modal",
5096
- "e2b",
5097
- "daytona",
5098
- "blaxel",
5099
- "runloop",
5100
- "selfhosted",
5101
- "unknown"
5102
- ];
5097
+ }
5098
+ var EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"];
5103
5099
  var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
5104
5100
  var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
5105
5101
  function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
@@ -7291,6 +7287,7 @@ function createRunAgentTurnActivity(services) {
7291
7287
  const machineOpObserver = makeMachineOpObserver(
7292
7288
  runtimeMetricsHooksForObservability(observability)
7293
7289
  );
7290
+ const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
7294
7291
  let isCodexTurn = false;
7295
7292
  let isExternallyBilledTurn = false;
7296
7293
  let executionGeneration = 0;
@@ -9459,6 +9456,7 @@ function createRunAgentTurnActivity(services) {
9459
9456
  settings,
9460
9457
  bus,
9461
9458
  onOp: machineOpObserver.observer,
9459
+ onSandboxOperation: sandboxOperationObserver,
9462
9460
  opJournal
9463
9461
  },
9464
9462
  {
@@ -9484,6 +9482,7 @@ function createRunAgentTurnActivity(services) {
9484
9482
  settings,
9485
9483
  bus,
9486
9484
  onOp: machineOpObserver.observer,
9485
+ onSandboxOperation: sandboxOperationObserver,
9487
9486
  onHomeSandboxRebound
9488
9487
  },
9489
9488
  {
@@ -9572,6 +9571,7 @@ function createRunAgentTurnActivity(services) {
9572
9571
  db,
9573
9572
  settings,
9574
9573
  bus,
9574
+ onSandboxOperation: sandboxOperationObserver,
9575
9575
  onHomeSandboxLost: publishSandboxLost,
9576
9576
  onHomeSandboxRebound
9577
9577
  },
@@ -9970,6 +9970,7 @@ function createRunAgentTurnActivity(services) {
9970
9970
  settings,
9971
9971
  bus,
9972
9972
  onOp: machineOpObserver.observer,
9973
+ onSandboxOperation: sandboxOperationObserver,
9973
9974
  onHomeSandboxLost: publishSandboxLost,
9974
9975
  onHomeSandboxRebound
9975
9976
  },
@@ -12724,6 +12725,7 @@ function createSandboxLeaseActivities(services, options = {}) {
12724
12725
  deleted: expiredSessionListSnapshots
12725
12726
  });
12726
12727
  }
12728
+ await refreshQueueLeaseAndCreditGauges(db, observability);
12727
12729
  if (!settings.sandboxOwnershipEnabled) {
12728
12730
  if (parentUpdates.claimed > 0) {
12729
12731
  observability.info("system-update outbox reconciled", parentUpdates);
@@ -12737,7 +12739,6 @@ function createSandboxLeaseActivities(services, options = {}) {
12737
12739
  modalOrphansTerminated: 0
12738
12740
  };
12739
12741
  }
12740
- await refreshQueueLeaseAndCreditGauges(db, observability);
12741
12742
  const rotationsRequested = await requestDueSandboxRotationsGlobal(
12742
12743
  db,
12743
12744
  settings.sandboxRotationLeadMs,
@@ -13453,54 +13454,65 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
13453
13454
  error: error instanceof Error ? error.message : String(error)
13454
13455
  });
13455
13456
  }
13456
- try {
13457
+ await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
13457
13458
  recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
13458
- } catch (error) {
13459
- observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
13460
- error: error instanceof Error ? error.message : String(error)
13461
- });
13462
- }
13463
- try {
13464
- recordSandboxCheckpointArtifactGauges(
13465
- observability,
13466
- await countSandboxCheckpointArtifactsByState(db)
13467
- );
13468
- } catch (error) {
13469
- observability.warn("sandbox reaper: checkpoint-artifact gauge refresh failed", {
13470
- error: error instanceof Error ? error.message : String(error)
13471
- });
13472
- }
13473
- try {
13474
- recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
13475
- } catch (error) {
13476
- observability.warn("sandbox reaper: rotation-backlog gauge refresh failed", {
13477
- error: error instanceof Error ? error.message : String(error)
13478
- });
13479
- }
13480
- try {
13481
- recordRetainedProcessInventoryGauges(
13482
- observability,
13483
- await countActiveRetainedProcessesByOwnerState(db)
13484
- );
13485
- } catch (error) {
13486
- observability.warn("sandbox reaper: retained-process gauge refresh failed", {
13487
- error: error instanceof Error ? error.message : String(error)
13488
- });
13489
- }
13459
+ });
13460
+ await refreshSandboxInventoryGauge(
13461
+ observability,
13462
+ "checkpoint_artifacts",
13463
+ "checkpoint-artifact",
13464
+ async () => {
13465
+ recordSandboxCheckpointArtifactGauges(
13466
+ observability,
13467
+ await countSandboxCheckpointArtifactsByState(db)
13468
+ );
13469
+ }
13470
+ );
13471
+ await refreshSandboxInventoryGauge(
13472
+ observability,
13473
+ "rotation_backlog",
13474
+ "rotation-backlog",
13475
+ async () => {
13476
+ recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
13477
+ }
13478
+ );
13479
+ await refreshSandboxInventoryGauge(
13480
+ observability,
13481
+ "retained_processes",
13482
+ "retained-process",
13483
+ async () => {
13484
+ recordRetainedProcessInventoryGauges(
13485
+ observability,
13486
+ await countActiveRetainedProcessesByOwnerState(db)
13487
+ );
13488
+ }
13489
+ );
13490
+ await refreshSandboxInventoryGauge(
13491
+ observability,
13492
+ "expired_drains",
13493
+ "expired-draining",
13494
+ async () => {
13495
+ recordExpiredDrainingSandboxLeaseGauges(
13496
+ observability,
13497
+ await countExpiredDrainingSandboxLeases(db)
13498
+ );
13499
+ }
13500
+ );
13490
13501
  try {
13491
- recordExpiredDrainingSandboxLeaseGauges(
13492
- observability,
13493
- await countExpiredDrainingSandboxLeases(db)
13494
- );
13502
+ recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
13495
13503
  } catch (error) {
13496
- observability.warn("sandbox reaper: expired-draining gauge refresh failed", {
13504
+ observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
13497
13505
  error: error instanceof Error ? error.message : String(error)
13498
13506
  });
13499
13507
  }
13508
+ }
13509
+ async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
13500
13510
  try {
13501
- recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
13511
+ await refresh();
13512
+ recordSandboxInventoryProjectionSuccess(observability, domain);
13502
13513
  } catch (error) {
13503
- observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
13514
+ recordSandboxInventoryProjectionFailure(observability, domain);
13515
+ observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
13504
13516
  error: error instanceof Error ? error.message : String(error)
13505
13517
  });
13506
13518
  }