@opengeni/worker-bundle 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +54 -6
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +8 -2
- package/src/activities/environment.ts +8 -2
- package/src/activities/sandbox-lease.ts +12 -3
- package/src/index.ts +13 -0
- package/src/observability-metrics.ts +27 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@opengeni/worker-bundle",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.7.0",
|
|
4
4
|
"description": "OpenGeni worker entry: runOpenGeniWorker/createOpenGeniWorker(deps) + the un-bundled Temporal workflow source tree. An engine-distribution surface — its runtime closure includes engine-internal packages and the workflow bundle ships as on-disk TS source.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -39,15 +39,15 @@
|
|
|
39
39
|
},
|
|
40
40
|
"dependencies": {
|
|
41
41
|
"@opengeni/codex": "^0.2.1",
|
|
42
|
-
"@opengeni/config": "^0.
|
|
42
|
+
"@opengeni/config": "^0.4.0",
|
|
43
43
|
"@opengeni/contracts": "^0.9.0",
|
|
44
|
-
"@opengeni/core": "^0.4.
|
|
45
|
-
"@opengeni/db": "^0.6.
|
|
46
|
-
"@opengeni/documents": "^0.2.
|
|
47
|
-
"@opengeni/events": "^0.2.
|
|
44
|
+
"@opengeni/core": "^0.4.5",
|
|
45
|
+
"@opengeni/db": "^0.6.1",
|
|
46
|
+
"@opengeni/documents": "^0.2.8",
|
|
47
|
+
"@opengeni/events": "^0.2.8",
|
|
48
48
|
"@opengeni/observability": "^0.2.1",
|
|
49
|
-
"@opengeni/runtime": "^0.
|
|
50
|
-
"@opengeni/storage": "^0.2.
|
|
49
|
+
"@opengeni/runtime": "^0.6.0",
|
|
50
|
+
"@opengeni/storage": "^0.2.8",
|
|
51
51
|
"@temporalio/activity": "^1.17.0",
|
|
52
52
|
"@temporalio/client": "^1.17.0",
|
|
53
53
|
"@temporalio/worker": "^1.17.0",
|
|
@@ -1027,7 +1027,6 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1027
1027
|
workspaceEnvironment?.values ?? {},
|
|
1028
1028
|
{
|
|
1029
1029
|
skipGitHubToken: activeSandboxBackend === "selfhosted",
|
|
1030
|
-
skipToolspaceToken: activeSandboxBackend === "selfhosted",
|
|
1031
1030
|
scope: connectionScope,
|
|
1032
1031
|
gitCredentials: connectionCredentials?.gitCredentials,
|
|
1033
1032
|
sessionId: input.sessionId,
|
|
@@ -1312,7 +1311,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
|
|
|
1312
1311
|
// seeds it to the box's token file before the repository-clone runs; it never
|
|
1313
1312
|
// touches the box/agent manifest env.
|
|
1314
1313
|
...(activeSandboxBackend !== "selfhosted" && sandboxGitToken ? { gitTokenSeed: sandboxGitToken } : {}),
|
|
1315
|
-
|
|
1314
|
+
// Toolspace is delivered on EVERY backend including selfhosted. The git-
|
|
1315
|
+
// token skip does NOT transfer: that token is inert on a connected
|
|
1316
|
+
// machine (it uses its own git creds), but the toolspace token is the
|
|
1317
|
+
// machine's ONLY path to programmatic tool calling and grants no more
|
|
1318
|
+
// than toolspace:call for its own session (own-session-bound, turn TTL,
|
|
1319
|
+
// budgeted, approval-tools excluded). The runtime seeds it to the box's
|
|
1320
|
+
// token file over the same exec channel, off-manifest, on every backend.
|
|
1321
|
+
...(sandboxToolspaceToken ? { toolspaceTokenSeed: sandboxToolspaceToken } : {}),
|
|
1316
1322
|
...(activeSandboxBackend ? { activeSandboxBackend } : {}),
|
|
1317
1323
|
fileResourceDownloads,
|
|
1318
1324
|
mcpServers: preparedTools.mcpServers,
|
|
@@ -108,7 +108,6 @@ export async function sandboxEnvironmentForRun(
|
|
|
108
108
|
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
109
109
|
sessionId?: string;
|
|
110
110
|
runId?: string;
|
|
111
|
-
skipToolspaceToken?: boolean;
|
|
112
111
|
} = {},
|
|
113
112
|
): Promise<{ environment: Record<string, string>; gitToken?: string; toolspaceToken?: string }> {
|
|
114
113
|
// Precedence: deployment allowlist < git identity < workspace environment
|
|
@@ -129,10 +128,17 @@ export async function sandboxEnvironmentForRun(
|
|
|
129
128
|
// rotates. The agent can refresh the token mid-turn via the `github_token` MCP tool.
|
|
130
129
|
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
131
130
|
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
131
|
+
// TOOLSPACE (selfhosted parity): the toolspace token is minted for EVERY
|
|
132
|
+
// backend, including a connected machine. Unlike the platform GitHub token
|
|
133
|
+
// (inert on selfhosted → skipped above), the toolspace token is the machine's
|
|
134
|
+
// only path to programmatic tool calling, and it grants no more than the
|
|
135
|
+
// machine owner's own authority (toolspace:call, own-session-bound, turn TTL,
|
|
136
|
+
// budgeted, approval-tools excluded). Delivery mirrors the docker path: the
|
|
137
|
+
// caller threads it OFF-MANIFEST as the seed the runtime writes to
|
|
138
|
+
// $OPENGENI_TOOLSPACE_TOKEN_FILE over the box's exec channel.
|
|
132
139
|
let toolspaceToken: string | undefined;
|
|
133
140
|
if (
|
|
134
141
|
settings.toolspaceEnabled
|
|
135
|
-
&& !options.skipToolspaceToken
|
|
136
142
|
&& settings.delegationSecret
|
|
137
143
|
&& options.scope
|
|
138
144
|
&& options.sessionId
|
|
@@ -37,6 +37,7 @@ import {
|
|
|
37
37
|
countSandboxLeasesByLiveness,
|
|
38
38
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
39
39
|
getBillingBalance,
|
|
40
|
+
listCreditBalancesByAccount,
|
|
40
41
|
listLiveModalSandboxLeaseAttributions,
|
|
41
42
|
listMeterableWarmLeases,
|
|
42
43
|
persistDrainSnapshot,
|
|
@@ -63,6 +64,7 @@ import {
|
|
|
63
64
|
} from "@opengeni/runtime";
|
|
64
65
|
import type { ActivityServices } from "./types";
|
|
65
66
|
import {
|
|
67
|
+
recordCreditBalanceGauges,
|
|
66
68
|
recordCreditMicros,
|
|
67
69
|
recordSandboxLeaseGauges,
|
|
68
70
|
recordSandboxOrphansTerminated,
|
|
@@ -174,7 +176,7 @@ export function createSandboxLeaseActivities(
|
|
|
174
176
|
if (!settings.sandboxOwnershipEnabled) {
|
|
175
177
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
176
178
|
}
|
|
177
|
-
await
|
|
179
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
178
180
|
|
|
179
181
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
180
182
|
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
@@ -229,7 +231,7 @@ export function createSandboxLeaseActivities(
|
|
|
229
231
|
});
|
|
230
232
|
}
|
|
231
233
|
|
|
232
|
-
await
|
|
234
|
+
await refreshQueueLeaseAndCreditGauges(db, observability);
|
|
233
235
|
|
|
234
236
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
235
237
|
observability.info("sandbox reaper swept", {
|
|
@@ -299,7 +301,7 @@ async function accrueWarmTick(
|
|
|
299
301
|
return { accrued, workspaceIds };
|
|
300
302
|
}
|
|
301
303
|
|
|
302
|
-
async function
|
|
304
|
+
async function refreshQueueLeaseAndCreditGauges(
|
|
303
305
|
db: ActivityServices["db"],
|
|
304
306
|
observability: ActivityServices["observability"],
|
|
305
307
|
): Promise<void> {
|
|
@@ -317,6 +319,13 @@ async function refreshQueueAndLeaseGauges(
|
|
|
317
319
|
error: error instanceof Error ? error.message : String(error),
|
|
318
320
|
});
|
|
319
321
|
}
|
|
322
|
+
try {
|
|
323
|
+
recordCreditBalanceGauges(observability, await listCreditBalancesByAccount(db));
|
|
324
|
+
} catch (error) {
|
|
325
|
+
observability.warn("sandbox reaper: credit-balance gauge refresh failed", {
|
|
326
|
+
error: error instanceof Error ? error.message : String(error),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
320
329
|
}
|
|
321
330
|
|
|
322
331
|
/**
|
package/src/index.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { createNatsEventBus } from "@opengeni/events";
|
|
|
4
4
|
import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
|
|
5
5
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
6
6
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
7
|
+
import { ensureModalRegistryImage } from "@opengeni/runtime";
|
|
7
8
|
import { createActivities, type ActivityDependencies } from "./activities";
|
|
8
9
|
import type { WakeSessionWorkflowSignal } from "./activities/types";
|
|
9
10
|
import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
|
|
@@ -33,6 +34,18 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
33
34
|
}> {
|
|
34
35
|
const settings = options.settings ?? getSettings();
|
|
35
36
|
const observability = options.activityDependencies?.observability ?? createObservability(settings, { component: "worker" });
|
|
37
|
+
// Pre-resolve a PRIVATE-registry sandbox image before any turn creates a box.
|
|
38
|
+
// No-op unless OPENGENI_MODAL_IMAGE_REGISTRY_SECRET + OPENGENI_MODAL_IMAGE_REF are
|
|
39
|
+
// both set (so non-modal / public-image deployments are byte-unchanged and never
|
|
40
|
+
// load the modal SDK here). Memoized in the provider, so this runs once per process.
|
|
41
|
+
await retryStartupDependency(
|
|
42
|
+
"Modal private-registry image",
|
|
43
|
+
() => ensureModalRegistryImage(settings),
|
|
44
|
+
{
|
|
45
|
+
...startupRetryOptions(settings),
|
|
46
|
+
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
47
|
+
},
|
|
48
|
+
);
|
|
36
49
|
const connection = await retryStartupDependency(
|
|
37
50
|
"Temporal",
|
|
38
51
|
() => NativeConnection.connect({ address: settings.temporalHost }),
|
|
@@ -5,8 +5,10 @@ import type { RuntimeMetricsHooks } from "@opengeni/runtime";
|
|
|
5
5
|
export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
|
|
6
6
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
7
7
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
8
|
+
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
8
9
|
|
|
9
10
|
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
11
|
+
const creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();
|
|
10
12
|
|
|
11
13
|
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
12
14
|
return {
|
|
@@ -194,6 +196,31 @@ export function recordSandboxLeaseGauges(observability: Observability, counts: P
|
|
|
194
196
|
}
|
|
195
197
|
}
|
|
196
198
|
|
|
199
|
+
export function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void {
|
|
200
|
+
const previous = creditBalanceGaugeAccounts.get(observability) ?? new Set<string>();
|
|
201
|
+
const current = new Set<string>();
|
|
202
|
+
for (const balance of balances) {
|
|
203
|
+
current.add(balance.accountId);
|
|
204
|
+
observability.setGauge({
|
|
205
|
+
name: "opengeni_credit_balance_micros",
|
|
206
|
+
help: "Current credit balance in micros by account.",
|
|
207
|
+
labels: { account_id: balance.accountId },
|
|
208
|
+
value: balance.balanceMicros,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
for (const accountId of previous) {
|
|
212
|
+
if (!current.has(accountId)) {
|
|
213
|
+
observability.setGauge({
|
|
214
|
+
name: "opengeni_credit_balance_micros",
|
|
215
|
+
help: "Current credit balance in micros by account.",
|
|
216
|
+
labels: { account_id: accountId },
|
|
217
|
+
value: 0,
|
|
218
|
+
});
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
creditBalanceGaugeAccounts.set(observability, current);
|
|
222
|
+
}
|
|
223
|
+
|
|
197
224
|
export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
|
|
198
225
|
if (count <= 0) {
|
|
199
226
|
return;
|