@opengeni/worker-bundle 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +54 -6
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +8 -2
- package/src/activities/environment.ts +8 -2
- package/src/activities/sandbox-lease.ts +12 -3
- package/src/index.ts +13 -0
- package/src/observability-metrics.ts +27 -0
package/dist/index.js
CHANGED
|
@@ -5,6 +5,7 @@ import { createNatsEventBus as createNatsEventBus2 } from "@opengeni/events";
|
|
|
5
5
|
import { createObservability as createObservability2, logStartupDependencyRetry } from "@opengeni/observability";
|
|
6
6
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
7
7
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
8
|
+
import { ensureModalRegistryImage } from "@opengeni/runtime";
|
|
8
9
|
|
|
9
10
|
// src/activities.ts
|
|
10
11
|
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
|
|
@@ -520,7 +521,7 @@ async function sandboxEnvironmentForRun(settings, resources, workspaceEnvironmen
|
|
|
520
521
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
521
522
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
522
523
|
let toolspaceToken;
|
|
523
|
-
if (settings.toolspaceEnabled &&
|
|
524
|
+
if (settings.toolspaceEnabled && settings.delegationSecret && options.scope && options.sessionId && options.runId) {
|
|
524
525
|
toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
525
526
|
accountId: options.scope.accountId,
|
|
526
527
|
workspaceId: options.scope.workspaceId,
|
|
@@ -1710,6 +1711,7 @@ function routingEnabled(settings) {
|
|
|
1710
1711
|
|
|
1711
1712
|
// src/observability-metrics.ts
|
|
1712
1713
|
var turnTrackers = /* @__PURE__ */ new WeakMap();
|
|
1714
|
+
var creditBalanceGaugeAccounts = /* @__PURE__ */ new WeakMap();
|
|
1713
1715
|
function observabilityEventLogger(observability) {
|
|
1714
1716
|
return {
|
|
1715
1717
|
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
@@ -1881,6 +1883,30 @@ function recordSandboxLeaseGauges(observability, counts) {
|
|
|
1881
1883
|
});
|
|
1882
1884
|
}
|
|
1883
1885
|
}
|
|
1886
|
+
function recordCreditBalanceGauges(observability, balances) {
|
|
1887
|
+
const previous = creditBalanceGaugeAccounts.get(observability) ?? /* @__PURE__ */ new Set();
|
|
1888
|
+
const current = /* @__PURE__ */ new Set();
|
|
1889
|
+
for (const balance of balances) {
|
|
1890
|
+
current.add(balance.accountId);
|
|
1891
|
+
observability.setGauge({
|
|
1892
|
+
name: "opengeni_credit_balance_micros",
|
|
1893
|
+
help: "Current credit balance in micros by account.",
|
|
1894
|
+
labels: { account_id: balance.accountId },
|
|
1895
|
+
value: balance.balanceMicros
|
|
1896
|
+
});
|
|
1897
|
+
}
|
|
1898
|
+
for (const accountId of previous) {
|
|
1899
|
+
if (!current.has(accountId)) {
|
|
1900
|
+
observability.setGauge({
|
|
1901
|
+
name: "opengeni_credit_balance_micros",
|
|
1902
|
+
help: "Current credit balance in micros by account.",
|
|
1903
|
+
labels: { account_id: accountId },
|
|
1904
|
+
value: 0
|
|
1905
|
+
});
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
creditBalanceGaugeAccounts.set(observability, current);
|
|
1909
|
+
}
|
|
1884
1910
|
function recordSandboxOrphansTerminated(observability, count) {
|
|
1885
1911
|
if (count <= 0) {
|
|
1886
1912
|
return;
|
|
@@ -2492,7 +2518,6 @@ function createRunAgentTurnActivity(services) {
|
|
|
2492
2518
|
workspaceEnvironment?.values ?? {},
|
|
2493
2519
|
{
|
|
2494
2520
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
2495
|
-
skipToolspaceToken: activeSandboxBackend === "selfhosted",
|
|
2496
2521
|
scope: connectionScope,
|
|
2497
2522
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
2498
2523
|
sessionId: input.sessionId,
|
|
@@ -2677,7 +2702,14 @@ function createRunAgentTurnActivity(services) {
|
|
|
2677
2702
|
// seeds it to the box's token file before the repository-clone runs; it never
|
|
2678
2703
|
// touches the box/agent manifest env.
|
|
2679
2704
|
...activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {},
|
|
2680
|
-
|
|
2705
|
+
// Toolspace is delivered on EVERY backend including selfhosted. The git-
|
|
2706
|
+
// token skip does NOT transfer: that token is inert on a connected
|
|
2707
|
+
// machine (it uses its own git creds), but the toolspace token is the
|
|
2708
|
+
// machine's ONLY path to programmatic tool calling and grants no more
|
|
2709
|
+
// than toolspace:call for its own session (own-session-bound, turn TTL,
|
|
2710
|
+
// budgeted, approval-tools excluded). The runtime seeds it to the box's
|
|
2711
|
+
// token file over the same exec channel, off-manifest, on every backend.
|
|
2712
|
+
...sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {},
|
|
2681
2713
|
...activeSandboxBackend ? { activeSandboxBackend } : {},
|
|
2682
2714
|
fileResourceDownloads,
|
|
2683
2715
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -3771,6 +3803,7 @@ import {
|
|
|
3771
3803
|
countSandboxLeasesByLiveness,
|
|
3772
3804
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
3773
3805
|
getBillingBalance as getBillingBalance4,
|
|
3806
|
+
listCreditBalancesByAccount,
|
|
3774
3807
|
listLiveModalSandboxLeaseAttributions,
|
|
3775
3808
|
listMeterableWarmLeases,
|
|
3776
3809
|
persistDrainSnapshot,
|
|
@@ -3795,7 +3828,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3795
3828
|
if (!settings.sandboxOwnershipEnabled) {
|
|
3796
3829
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
3797
3830
|
}
|
|
3798
|
-
await
|
|
3831
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
3799
3832
|
const metered = await accrueWarmTick(db, settings, observability);
|
|
3800
3833
|
const forceDrained = await forceDrainOverLimitWorkspaces(db, settings, metered.workspaceIds, observability);
|
|
3801
3834
|
const drainable = await reapStaleLeaseHoldersGlobal(db, {
|
|
@@ -3831,7 +3864,7 @@ function createSandboxLeaseActivities(services, options = {}) {
|
|
|
3831
3864
|
error: error instanceof Error ? error.message : String(error)
|
|
3832
3865
|
});
|
|
3833
3866
|
}
|
|
3834
|
-
await
|
|
3867
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
3835
3868
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
3836
3869
|
observability.info("sandbox reaper swept", {
|
|
3837
3870
|
drainable: drainable.length,
|
|
@@ -3884,7 +3917,7 @@ async function accrueWarmTick(db, settings, observability) {
|
|
|
3884
3917
|
}
|
|
3885
3918
|
return { accrued, workspaceIds };
|
|
3886
3919
|
}
|
|
3887
|
-
async function
|
|
3920
|
+
async function refreshQueueLeaseAndCreditGauges(db, observability) {
|
|
3888
3921
|
try {
|
|
3889
3922
|
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
3890
3923
|
} catch (error) {
|
|
@@ -3899,6 +3932,13 @@ async function refreshQueueAndLeaseGauges(db, observability) {
|
|
|
3899
3932
|
error: error instanceof Error ? error.message : String(error)
|
|
3900
3933
|
});
|
|
3901
3934
|
}
|
|
3935
|
+
try {
|
|
3936
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
3937
|
+
} catch (error) {
|
|
3938
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
3939
|
+
error: error instanceof Error ? error.message : String(error)
|
|
3940
|
+
});
|
|
3941
|
+
}
|
|
3902
3942
|
}
|
|
3903
3943
|
async function forceDrainOverLimitWorkspaces(db, settings, workspaceIds, observability) {
|
|
3904
3944
|
const enforceBalance = settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
|
|
@@ -4746,6 +4786,14 @@ var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
|
4746
4786
|
async function createOpenGeniWorker(options = {}) {
|
|
4747
4787
|
const settings = options.settings ?? getSettings2();
|
|
4748
4788
|
const observability = options.activityDependencies?.observability ?? createObservability2(settings, { component: "worker" });
|
|
4789
|
+
await retryStartupDependency(
|
|
4790
|
+
"Modal private-registry image",
|
|
4791
|
+
() => ensureModalRegistryImage(settings),
|
|
4792
|
+
{
|
|
4793
|
+
...startupRetryOptions(settings),
|
|
4794
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event)
|
|
4795
|
+
}
|
|
4796
|
+
);
|
|
4749
4797
|
const connection = await retryStartupDependency(
|
|
4750
4798
|
"Temporal",
|
|
4751
4799
|
() => NativeConnection.connect({ address: settings.temporalHost }),
|