@opengeni/worker-bundle 0.13.10 → 0.13.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +2 -2
- package/dist/index.js +78 -66
- package/dist/index.js.map +1 -1
- package/dist/observability-metrics.d.ts +7 -1
- package/dist/sandbox-routing.d.ts +3 -1
- package/package.json +4 -4
- package/src/activities/agent-turn.ts +6 -0
- package/src/activities/sandbox-lease.ts +68 -44
- package/src/index.ts +2 -2
- package/src/observability-metrics.ts +38 -26
- package/src/sandbox-routing.ts +5 -0
package/dist/index.d.ts
CHANGED
|
@@ -62,8 +62,8 @@ export declare function createWorkerWorkflowSignaler(settings: Settings, db: Dat
|
|
|
62
62
|
/**
|
|
63
63
|
* Register the ONE global reaper Temporal Schedule (the sole liveness/GC/cost-stop
|
|
64
64
|
* driver — P1.3 / OD-3) and durable system-update outbox repair cadence. With
|
|
65
|
-
* sandbox ownership off the activity performs
|
|
66
|
-
* it never
|
|
65
|
+
* sandbox ownership off the activity performs bounded DB outbox repair and
|
|
66
|
+
* read-only observability projections; it never mutates or terminates sandbox leases.
|
|
67
67
|
*
|
|
68
68
|
* The Schedule fires sandboxReaperWorkflow on the worker's global task queue
|
|
69
69
|
* every settings.sandboxLeaseReaperPeriodMs (the SAME cadence the boot invariant
|
package/dist/index.js
CHANGED
|
@@ -115,6 +115,7 @@ import {
|
|
|
115
115
|
markSessionAttemptQuiesced
|
|
116
116
|
} from "@opengeni/db";
|
|
117
117
|
import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@opengeni/events";
|
|
118
|
+
import { sandboxOperationMetricObserver } from "@opengeni/observability";
|
|
118
119
|
import {
|
|
119
120
|
sandboxStateEntryFromRunState,
|
|
120
121
|
maxTurnsExceededRunState,
|
|
@@ -4474,6 +4475,7 @@ function wrapTurnBoxWithRouting(services, ids, established) {
|
|
|
4474
4475
|
return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
|
|
4475
4476
|
},
|
|
4476
4477
|
resolveActiveBackend: resolver,
|
|
4478
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
4477
4479
|
...beforeMutation ? { beforeMutation } : {},
|
|
4478
4480
|
...afterMutation ? { afterMutation } : {},
|
|
4479
4481
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
@@ -4581,6 +4583,7 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
|
|
|
4581
4583
|
}
|
|
4582
4584
|
return routedResolver(pointer);
|
|
4583
4585
|
},
|
|
4586
|
+
...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
|
|
4584
4587
|
...beforeMutation ? { beforeMutation } : {},
|
|
4585
4588
|
...afterMutation ? { afterMutation } : {},
|
|
4586
4589
|
...beforeProcessMutation ? { beforeProcessMutation } : {},
|
|
@@ -4690,6 +4693,7 @@ function makeTurnOpJournal(context, details) {
|
|
|
4690
4693
|
// src/observability-metrics.ts
|
|
4691
4694
|
import { createHash as createHash2 } from "crypto";
|
|
4692
4695
|
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
4696
|
+
import { SandboxBackend } from "@opengeni/contracts";
|
|
4693
4697
|
import {
|
|
4694
4698
|
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
4695
4699
|
modelUsageTokenCountOrNull
|
|
@@ -4697,7 +4701,6 @@ import {
|
|
|
4697
4701
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
4698
4702
|
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
4699
4703
|
var modelCacheCounterTotals = /* @__PURE__ */ new WeakMap();
|
|
4700
|
-
var retainedProcessTerminalBacklogTotals = /* @__PURE__ */ new WeakMap();
|
|
4701
4704
|
var MAX_MODEL_CACHE_COUNTER_TOTAL = 1e12;
|
|
4702
4705
|
function observabilityEventLogger(observability) {
|
|
4703
4706
|
return {
|
|
@@ -4949,6 +4952,21 @@ function recordSandboxLeaseGauges(observability, counts) {
|
|
|
4949
4952
|
});
|
|
4950
4953
|
}
|
|
4951
4954
|
}
|
|
4955
|
+
function recordSandboxInventoryProjectionSuccess(observability, domain, timestampSeconds = Date.now() / 1e3) {
|
|
4956
|
+
observability.setGauge({
|
|
4957
|
+
name: "opengeni_sandbox_inventory_refresh_timestamp_seconds",
|
|
4958
|
+
help: "Unix timestamp of the last complete successful sandbox inventory projection by domain.",
|
|
4959
|
+
labels: { domain },
|
|
4960
|
+
value: timestampSeconds
|
|
4961
|
+
});
|
|
4962
|
+
}
|
|
4963
|
+
function recordSandboxInventoryProjectionFailure(observability, domain) {
|
|
4964
|
+
observability.incrementCounter({
|
|
4965
|
+
name: "opengeni_sandbox_inventory_refresh_failures_total",
|
|
4966
|
+
help: "Failed sandbox inventory projection refreshes by domain.",
|
|
4967
|
+
labels: { domain }
|
|
4968
|
+
});
|
|
4969
|
+
}
|
|
4952
4970
|
function recordCreditBalanceGauges(observability, balances) {
|
|
4953
4971
|
const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
|
|
4954
4972
|
const current = /* @__PURE__ */ new Set();
|
|
@@ -5062,9 +5080,7 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
|
|
|
5062
5080
|
current.active += Math.max(0, count.activeCount);
|
|
5063
5081
|
current.terminal += Math.max(0, count.terminalOwnerCount);
|
|
5064
5082
|
}
|
|
5065
|
-
let terminalTotal = 0;
|
|
5066
5083
|
for (const [ownerState, count] of normalized) {
|
|
5067
|
-
terminalTotal += count.terminal;
|
|
5068
5084
|
observability.setGauge({
|
|
5069
5085
|
name: "opengeni_retained_processes_active",
|
|
5070
5086
|
help: "Current active retained provider processes by durable owner state.",
|
|
@@ -5078,28 +5094,8 @@ function recordRetainedProcessInventoryGauges(observability, counts) {
|
|
|
5078
5094
|
value: count.terminal
|
|
5079
5095
|
});
|
|
5080
5096
|
}
|
|
5081
|
-
|
|
5082
|
-
|
|
5083
|
-
observability.incrementCounter({
|
|
5084
|
-
name: "opengeni_retained_process_terminal_owner_backlog_growth_total",
|
|
5085
|
-
help: "Positive growth in terminal-owner retained-process backlog between app samples.",
|
|
5086
|
-
amount: terminalTotal - previous
|
|
5087
|
-
});
|
|
5088
|
-
}
|
|
5089
|
-
retainedProcessTerminalBacklogTotals.set(observability, terminalTotal);
|
|
5090
|
-
}
|
|
5091
|
-
var EXPIRED_DRAINING_BACKENDS = [
|
|
5092
|
-
"none",
|
|
5093
|
-
"local",
|
|
5094
|
-
"docker",
|
|
5095
|
-
"modal",
|
|
5096
|
-
"e2b",
|
|
5097
|
-
"daytona",
|
|
5098
|
-
"blaxel",
|
|
5099
|
-
"runloop",
|
|
5100
|
-
"selfhosted",
|
|
5101
|
-
"unknown"
|
|
5102
|
-
];
|
|
5097
|
+
}
|
|
5098
|
+
var EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, "unknown"];
|
|
5103
5099
|
var EXPIRED_DRAINING_BACKEND_SET = new Set(EXPIRED_DRAINING_BACKENDS);
|
|
5104
5100
|
var EXPIRED_DRAINING_AGE_BUCKETS = ["lt_5m", "5m_1h", "1h_1d", "gte_1d"];
|
|
5105
5101
|
function recordExpiredDrainingSandboxLeaseGauges(observability, counts) {
|
|
@@ -7291,6 +7287,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
7291
7287
|
const machineOpObserver = makeMachineOpObserver(
|
|
7292
7288
|
runtimeMetricsHooksForObservability(observability)
|
|
7293
7289
|
);
|
|
7290
|
+
const sandboxOperationObserver = sandboxOperationMetricObserver(observability);
|
|
7294
7291
|
let isCodexTurn = false;
|
|
7295
7292
|
let isExternallyBilledTurn = false;
|
|
7296
7293
|
let executionGeneration = 0;
|
|
@@ -9459,6 +9456,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9459
9456
|
settings,
|
|
9460
9457
|
bus,
|
|
9461
9458
|
onOp: machineOpObserver.observer,
|
|
9459
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9462
9460
|
opJournal
|
|
9463
9461
|
},
|
|
9464
9462
|
{
|
|
@@ -9484,6 +9482,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9484
9482
|
settings,
|
|
9485
9483
|
bus,
|
|
9486
9484
|
onOp: machineOpObserver.observer,
|
|
9485
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9487
9486
|
onHomeSandboxRebound
|
|
9488
9487
|
},
|
|
9489
9488
|
{
|
|
@@ -9572,6 +9571,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9572
9571
|
db,
|
|
9573
9572
|
settings,
|
|
9574
9573
|
bus,
|
|
9574
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9575
9575
|
onHomeSandboxLost: publishSandboxLost,
|
|
9576
9576
|
onHomeSandboxRebound
|
|
9577
9577
|
},
|
|
@@ -9970,6 +9970,7 @@ function createRunAgentTurnActivity(services) {
|
|
|
9970
9970
|
settings,
|
|
9971
9971
|
bus,
|
|
9972
9972
|
onOp: machineOpObserver.observer,
|
|
9973
|
+
onSandboxOperation: sandboxOperationObserver,
|
|
9973
9974
|
onHomeSandboxLost: publishSandboxLost,
|
|
9974
9975
|
onHomeSandboxRebound
|
|
9975
9976
|
},
|
|
@@ -12724,6 +12725,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
12724
12725
|
deleted: expiredSessionListSnapshots
|
|
12725
12726
|
});
|
|
12726
12727
|
}
|
|
12728
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
12727
12729
|
if (!settings.sandboxOwnershipEnabled) {
|
|
12728
12730
|
if (parentUpdates.claimed > 0) {
|
|
12729
12731
|
observability.info("system-update outbox reconciled", parentUpdates);
|
|
@@ -12737,7 +12739,6 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
12737
12739
|
modalOrphansTerminated: 0
|
|
12738
12740
|
};
|
|
12739
12741
|
}
|
|
12740
|
-
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
12741
12742
|
const rotationsRequested = await requestDueSandboxRotationsGlobal(
|
|
12742
12743
|
db,
|
|
12743
12744
|
settings.sandboxRotationLeadMs,
|
|
@@ -13453,54 +13454,65 @@ async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
|
13453
13454
|
error: error instanceof Error ? error.message : String(error)
|
|
13454
13455
|
});
|
|
13455
13456
|
}
|
|
13456
|
-
|
|
13457
|
+
await refreshSandboxInventoryGauge(observability, "leases", "sandbox-lease", async () => {
|
|
13457
13458
|
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
13458
|
-
}
|
|
13459
|
-
|
|
13460
|
-
|
|
13461
|
-
|
|
13462
|
-
|
|
13463
|
-
|
|
13464
|
-
|
|
13465
|
-
|
|
13466
|
-
|
|
13467
|
-
|
|
13468
|
-
|
|
13469
|
-
|
|
13470
|
-
|
|
13471
|
-
|
|
13472
|
-
|
|
13473
|
-
|
|
13474
|
-
|
|
13475
|
-
|
|
13476
|
-
|
|
13477
|
-
|
|
13478
|
-
|
|
13479
|
-
|
|
13480
|
-
|
|
13481
|
-
|
|
13482
|
-
|
|
13483
|
-
|
|
13484
|
-
|
|
13485
|
-
|
|
13486
|
-
|
|
13487
|
-
|
|
13488
|
-
|
|
13489
|
-
|
|
13459
|
+
});
|
|
13460
|
+
await refreshSandboxInventoryGauge(
|
|
13461
|
+
observability,
|
|
13462
|
+
"checkpoint_artifacts",
|
|
13463
|
+
"checkpoint-artifact",
|
|
13464
|
+
async () => {
|
|
13465
|
+
recordSandboxCheckpointArtifactGauges(
|
|
13466
|
+
observability,
|
|
13467
|
+
await countSandboxCheckpointArtifactsByState(db)
|
|
13468
|
+
);
|
|
13469
|
+
}
|
|
13470
|
+
);
|
|
13471
|
+
await refreshSandboxInventoryGauge(
|
|
13472
|
+
observability,
|
|
13473
|
+
"rotation_backlog",
|
|
13474
|
+
"rotation-backlog",
|
|
13475
|
+
async () => {
|
|
13476
|
+
recordSandboxRotationBacklogGauges(observability, await readSandboxRotationBacklog(db));
|
|
13477
|
+
}
|
|
13478
|
+
);
|
|
13479
|
+
await refreshSandboxInventoryGauge(
|
|
13480
|
+
observability,
|
|
13481
|
+
"retained_processes",
|
|
13482
|
+
"retained-process",
|
|
13483
|
+
async () => {
|
|
13484
|
+
recordRetainedProcessInventoryGauges(
|
|
13485
|
+
observability,
|
|
13486
|
+
await countActiveRetainedProcessesByOwnerState(db)
|
|
13487
|
+
);
|
|
13488
|
+
}
|
|
13489
|
+
);
|
|
13490
|
+
await refreshSandboxInventoryGauge(
|
|
13491
|
+
observability,
|
|
13492
|
+
"expired_drains",
|
|
13493
|
+
"expired-draining",
|
|
13494
|
+
async () => {
|
|
13495
|
+
recordExpiredDrainingSandboxLeaseGauges(
|
|
13496
|
+
observability,
|
|
13497
|
+
await countExpiredDrainingSandboxLeases(db)
|
|
13498
|
+
);
|
|
13499
|
+
}
|
|
13500
|
+
);
|
|
13490
13501
|
try {
|
|
13491
|
-
|
|
13492
|
-
observability,
|
|
13493
|
-
await countExpiredDrainingSandboxLeases(db)
|
|
13494
|
-
);
|
|
13502
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
13495
13503
|
} catch (error) {
|
|
13496
|
-
observability.warn("sandbox reaper:
|
|
13504
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
13497
13505
|
error: error instanceof Error ? error.message : String(error)
|
|
13498
13506
|
});
|
|
13499
13507
|
}
|
|
13508
|
+
}
|
|
13509
|
+
async function refreshSandboxInventoryGauge(observability, domain, diagnosticName, refresh) {
|
|
13500
13510
|
try {
|
|
13501
|
-
|
|
13511
|
+
await refresh();
|
|
13512
|
+
recordSandboxInventoryProjectionSuccess(observability, domain);
|
|
13502
13513
|
} catch (error) {
|
|
13503
|
-
observability
|
|
13514
|
+
recordSandboxInventoryProjectionFailure(observability, domain);
|
|
13515
|
+
observability.warn(`sandbox reaper: ${diagnosticName} gauge refresh failed`, {
|
|
13504
13516
|
error: error instanceof Error ? error.message : String(error)
|
|
13505
13517
|
});
|
|
13506
13518
|
}
|