@opengeni/worker-bundle 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +593 -86
- package/dist/index.js.map +1 -1
- package/package.json +11 -15
- package/src/activities/agent-turn.ts +95 -12
- package/src/activities/capabilities.ts +2 -30
- package/src/activities/context-compaction.ts +96 -10
- package/src/activities/environment.ts +30 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities/streaming.ts +1 -0
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
applyGitAuthPointerEnvironment,
|
|
3
|
+
firstPartyMcpWorkspaceUrl,
|
|
3
4
|
stableSandboxEnvironmentForRun,
|
|
4
5
|
type Settings,
|
|
5
6
|
} from "@opengeni/config";
|
|
6
7
|
import {
|
|
8
|
+
signDelegatedAccessToken,
|
|
7
9
|
type ConnectionCredentialsPort,
|
|
8
10
|
type GitCredentials,
|
|
9
11
|
type ResourceRef,
|
|
@@ -104,8 +106,11 @@ export async function sandboxEnvironmentForRun(
|
|
|
104
106
|
skipGitHubToken?: boolean;
|
|
105
107
|
scope?: ConnectionScope;
|
|
106
108
|
gitCredentials?: ConnectionCredentialsPort["gitCredentials"];
|
|
109
|
+
sessionId?: string;
|
|
110
|
+
runId?: string;
|
|
111
|
+
skipToolspaceToken?: boolean;
|
|
107
112
|
} = {},
|
|
108
|
-
): Promise<{ environment: Record<string, string>; gitToken?: string }> {
|
|
113
|
+
): Promise<{ environment: Record<string, string>; gitToken?: string; toolspaceToken?: string }> {
|
|
109
114
|
// Precedence: deployment allowlist < git identity < workspace environment
|
|
110
115
|
// < backend-aware HOME (the STABLE base, shared with the API-direct attach
|
|
111
116
|
// paths via stableSandboxEnvironmentForRun) < platform run-scoped GitHub auth
|
|
@@ -122,7 +127,28 @@ export async function sandboxEnvironmentForRun(
|
|
|
122
127
|
// OPENGENI_GIT_TOKEN_FILE), so the token VALUE never rides the manifest and the
|
|
123
128
|
// SDK's per-turn provided-session env delta stays empty even though the token
|
|
124
129
|
// rotates. The agent can refresh the token mid-turn via the `github_token` MCP tool.
|
|
125
|
-
const
|
|
130
|
+
const stableOptions = options.scope ? { workspaceId: options.scope.workspaceId } : {};
|
|
131
|
+
const environment = stableSandboxEnvironmentForRun(settings, workspaceEnvironment, stableOptions);
|
|
132
|
+
let toolspaceToken: string | undefined;
|
|
133
|
+
if (
|
|
134
|
+
settings.toolspaceEnabled
|
|
135
|
+
&& !options.skipToolspaceToken
|
|
136
|
+
&& settings.delegationSecret
|
|
137
|
+
&& options.scope
|
|
138
|
+
&& options.sessionId
|
|
139
|
+
&& options.runId
|
|
140
|
+
) {
|
|
141
|
+
toolspaceToken = await signDelegatedAccessToken(settings.delegationSecret, {
|
|
142
|
+
accountId: options.scope.accountId,
|
|
143
|
+
workspaceId: options.scope.workspaceId,
|
|
144
|
+
subjectId: `sandbox:${options.runId}`,
|
|
145
|
+
subjectLabel: "sandbox toolspace",
|
|
146
|
+
permissions: ["toolspace:call"],
|
|
147
|
+
sessionId: options.sessionId,
|
|
148
|
+
exp: Math.floor(Date.now() / 1000) + 60 * 60,
|
|
149
|
+
});
|
|
150
|
+
environment.OPENGENI_TOOLSPACE_URL ??= firstPartyMcpWorkspaceUrl(settings, options.scope.workspaceId);
|
|
151
|
+
}
|
|
126
152
|
const selection = githubRepositorySelection(resources);
|
|
127
153
|
// NO-TOKEN SKIP (Stage D, change B): when the turn's EFFECTIVE compute backend is
|
|
128
154
|
// a connected machine (selfhosted), the platform GitHub App installation token is
|
|
@@ -134,7 +160,7 @@ export async function sandboxEnvironmentForRun(
|
|
|
134
160
|
// (validateNoEnvironmentDelta). The API-direct viewer attach path already drops the
|
|
135
161
|
// token under this exact contract — proof a box runs fine without it.
|
|
136
162
|
if (!selection || options.skipGitHubToken) {
|
|
137
|
-
return { environment };
|
|
163
|
+
return { environment, ...(toolspaceToken ? { toolspaceToken } : {}) };
|
|
138
164
|
}
|
|
139
165
|
// Run-scoped sandbox preparation for GitHub App repository resources. When a
|
|
140
166
|
// host binds `gitCredentials`, the HOST mints the installation token (BYO
|
|
@@ -173,7 +199,7 @@ export async function sandboxEnvironmentForRun(
|
|
|
173
199
|
// cold-creates the box for a repo-attached session — an attach-warmed box
|
|
174
200
|
// missing these keys kills the next repo turn on the SDK's manifest-env guard.
|
|
175
201
|
applyGitAuthPointerEnvironment(environment, identity);
|
|
176
|
-
return { environment, gitToken: token };
|
|
202
|
+
return { environment, gitToken: token, ...(toolspaceToken ? { toolspaceToken } : {}) };
|
|
177
203
|
}
|
|
178
204
|
|
|
179
205
|
function githubRepositorySelection(resources: ResourceRef[]): { installationId: number; repositoryIds: number[] } | null {
|
|
@@ -33,6 +33,8 @@
|
|
|
33
33
|
import {
|
|
34
34
|
accrueWarmSeconds,
|
|
35
35
|
confirmDrainCold,
|
|
36
|
+
countQueuedTurns,
|
|
37
|
+
countSandboxLeasesByLiveness,
|
|
36
38
|
forceDrainOverLimitViewerOnlyBoxes,
|
|
37
39
|
getBillingBalance,
|
|
38
40
|
listLiveModalSandboxLeaseAttributions,
|
|
@@ -60,6 +62,12 @@ import {
|
|
|
60
62
|
terminateModalSandboxById,
|
|
61
63
|
} from "@opengeni/runtime";
|
|
62
64
|
import type { ActivityServices } from "./types";
|
|
65
|
+
import {
|
|
66
|
+
recordCreditMicros,
|
|
67
|
+
recordSandboxLeaseGauges,
|
|
68
|
+
recordSandboxOrphansTerminated,
|
|
69
|
+
recordTurnsQueuedGauge,
|
|
70
|
+
} from "../observability-metrics";
|
|
63
71
|
|
|
64
72
|
export type ReapSandboxLeasesResult = {
|
|
65
73
|
/** Stale viewer holders + warming-death rows the sweep touched is folded into
|
|
@@ -166,6 +174,7 @@ export function createSandboxLeaseActivities(
|
|
|
166
174
|
if (!settings.sandboxOwnershipEnabled) {
|
|
167
175
|
return { examined: 0, terminated: 0, skipped: 0, metered: 0, forceDrained: 0, modalOrphansTerminated: 0 };
|
|
168
176
|
}
|
|
177
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
169
178
|
|
|
170
179
|
// (0) Warm-meter tick (P2.1) — accrue warm-seconds for every WARM viewer-only
|
|
171
180
|
// box (turn-held boxes meter on the turn heartbeat, so the list fn excludes
|
|
@@ -213,12 +222,15 @@ export function createSandboxLeaseActivities(
|
|
|
213
222
|
|
|
214
223
|
try {
|
|
215
224
|
modalOrphansTerminated = await sweepModalOrphans(settings, db, observability);
|
|
225
|
+
recordSandboxOrphansTerminated(observability, modalOrphansTerminated);
|
|
216
226
|
} catch (error) {
|
|
217
227
|
observability.warn("sandbox reaper: Modal orphan sweep failed", {
|
|
218
228
|
error: error instanceof Error ? error.message : String(error),
|
|
219
229
|
});
|
|
220
230
|
}
|
|
221
231
|
|
|
232
|
+
await refreshQueueAndLeaseGauges(db, observability);
|
|
233
|
+
|
|
222
234
|
if (drainable.length > 0 || metered.accrued > 0 || forceDrained > 0 || modalOrphansTerminated > 0) {
|
|
223
235
|
observability.info("sandbox reaper swept", {
|
|
224
236
|
drainable: drainable.length,
|
|
@@ -275,6 +287,7 @@ async function accrueWarmTick(
|
|
|
275
287
|
if (result.accrued) {
|
|
276
288
|
accrued += 1;
|
|
277
289
|
}
|
|
290
|
+
recordCreditMicros(observability, "usage", result.costMicros);
|
|
278
291
|
} catch (error) {
|
|
279
292
|
observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
|
|
280
293
|
workspaceId: lease.workspaceId,
|
|
@@ -286,6 +299,26 @@ async function accrueWarmTick(
|
|
|
286
299
|
return { accrued, workspaceIds };
|
|
287
300
|
}
|
|
288
301
|
|
|
302
|
+
async function refreshQueueAndLeaseGauges(
|
|
303
|
+
db: ActivityServices["db"],
|
|
304
|
+
observability: ActivityServices["observability"],
|
|
305
|
+
): Promise<void> {
|
|
306
|
+
try {
|
|
307
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
308
|
+
} catch (error) {
|
|
309
|
+
observability.warn("sandbox reaper: queued-turn gauge refresh failed", {
|
|
310
|
+
error: error instanceof Error ? error.message : String(error),
|
|
311
|
+
});
|
|
312
|
+
}
|
|
313
|
+
try {
|
|
314
|
+
recordSandboxLeaseGauges(observability, await countSandboxLeasesByLiveness(db));
|
|
315
|
+
} catch (error) {
|
|
316
|
+
observability.warn("sandbox reaper: sandbox-lease gauge refresh failed", {
|
|
317
|
+
error: error instanceof Error ? error.message : String(error),
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
289
322
|
/**
|
|
290
323
|
* The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
|
|
291
324
|
* live warm box, under the usage lock: if it is at 0 balance (when a billing /
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
claimNextQueuedTurn as claimNextQueuedTurnDb,
|
|
3
|
+
countQueuedTurns,
|
|
3
4
|
countTurnSessionHistoryItems,
|
|
4
5
|
finishTurn,
|
|
5
6
|
getSessionEvent,
|
|
@@ -13,6 +14,7 @@ import { appendAndPublishEvents } from "@opengeni/events";
|
|
|
13
14
|
import { WORKER_DEATH_RESUME_TEXT } from "./agent-turn";
|
|
14
15
|
import { isSteerInterrupt, pauseActiveGoalOnInterrupt } from "./goals";
|
|
15
16
|
import { notifyParentOfChildTerminal } from "./parent-wake";
|
|
17
|
+
import { recordTurnsQueuedGauge } from "../observability-metrics";
|
|
16
18
|
import type {
|
|
17
19
|
ActivityServices,
|
|
18
20
|
ClaimNextQueuedTurnInput,
|
|
@@ -67,7 +69,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
async function interruptActiveTurn(input: RunAgentTurnInput): Promise<void> {
|
|
70
|
-
const { db, bus } = await services();
|
|
72
|
+
const { db, bus, observability } = await services();
|
|
71
73
|
const session = await requireSession(db, input.workspaceId, input.sessionId);
|
|
72
74
|
const trigger = await getSessionEvent(db, input.workspaceId, input.triggerEventId);
|
|
73
75
|
// Pause an active goal before the early return below: an interrupt can
|
|
@@ -96,6 +98,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
96
98
|
]);
|
|
97
99
|
await finishTurn(db, input.workspaceId, session.activeTurnId, "cancelled");
|
|
98
100
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
101
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
99
102
|
}
|
|
100
103
|
|
|
101
104
|
/**
|
|
@@ -116,7 +119,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
116
119
|
* WORKER_DEATH_MAX_REDISPATCHES, persisted on the turn row.
|
|
117
120
|
*/
|
|
118
121
|
async function requeueTurnAfterWorkerDeath(input: RequeueTurnAfterWorkerDeathInput): Promise<RequeueTurnAfterWorkerDeathResult> {
|
|
119
|
-
const { settings, db, bus } = await services();
|
|
122
|
+
const { settings, db, bus, observability } = await services();
|
|
120
123
|
const turn = await getSessionTurn(db, input.workspaceId, input.turnId);
|
|
121
124
|
if (!turn || (turn.status !== "running" && turn.status !== "requires_action")) {
|
|
122
125
|
// The timed-out attempt was a zombie that actually settled the turn
|
|
@@ -169,12 +172,15 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
169
172
|
throw requeueError;
|
|
170
173
|
}
|
|
171
174
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "queued", null);
|
|
175
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
172
176
|
return { action: "requeued", redispatches };
|
|
173
177
|
}
|
|
174
178
|
|
|
175
179
|
async function claimNextQueuedTurn(input: ClaimNextQueuedTurnInput) {
|
|
176
|
-
const { db } = await services();
|
|
177
|
-
|
|
180
|
+
const { db, observability } = await services();
|
|
181
|
+
const turn = await claimNextQueuedTurnDb(db, input.workspaceId, input.sessionId, input.workflowId);
|
|
182
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
183
|
+
return turn;
|
|
178
184
|
}
|
|
179
185
|
|
|
180
186
|
async function markSessionIdle(input: MarkSessionIdleInput): Promise<void> {
|
|
@@ -183,6 +189,7 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
183
189
|
if (session.status === "queued" || session.status === "running") {
|
|
184
190
|
await setSessionStatus(db, input.workspaceId, input.sessionId, "idle", null);
|
|
185
191
|
}
|
|
192
|
+
await refreshQueuedTurnsGauge(db, observability);
|
|
186
193
|
// The workflow reaches markSessionIdle exactly when it has decided to stop
|
|
187
194
|
// for now (no queued turn, no goal continuation): the terminal-for-now
|
|
188
195
|
// point for a spawned worker, whatever the cause (goal completed, agent or
|
|
@@ -199,3 +206,14 @@ export function createSessionStateActivities(services: () => Promise<ActivitySer
|
|
|
199
206
|
markSessionIdle,
|
|
200
207
|
};
|
|
201
208
|
}
|
|
209
|
+
|
|
210
|
+
async function refreshQueuedTurnsGauge(
|
|
211
|
+
db: ActivityServices["db"],
|
|
212
|
+
observability: ActivityServices["observability"],
|
|
213
|
+
): Promise<void> {
|
|
214
|
+
try {
|
|
215
|
+
recordTurnsQueuedGauge(observability, await countQueuedTurns(db));
|
|
216
|
+
} catch {
|
|
217
|
+
// Best-effort telemetry; session state transitions remain authoritative.
|
|
218
|
+
}
|
|
219
|
+
}
|
package/src/activities.ts
CHANGED
|
@@ -12,6 +12,7 @@ import { createSandboxLeaseActivities } from "./activities/sandbox-lease";
|
|
|
12
12
|
import { createScheduledTaskActivities } from "./activities/scheduled-tasks";
|
|
13
13
|
import { createSessionStateActivities } from "./activities/session-state";
|
|
14
14
|
import type { ActivityDependencies, ActivityServices } from "./activities/types";
|
|
15
|
+
import { observabilityEventLogger, runtimeMetricsHooksForObservability } from "./observability-metrics";
|
|
15
16
|
|
|
16
17
|
export type {
|
|
17
18
|
ActivityDependencies,
|
|
@@ -33,6 +34,7 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
33
34
|
async function services(): Promise<ActivityServices> {
|
|
34
35
|
servicesPromise ??= (async () => {
|
|
35
36
|
const settings = dependencies.settings ?? getSettings();
|
|
37
|
+
const observability = dependencies.observability ?? createObservability(settings, { component: "worker" });
|
|
36
38
|
// Step I: when not injected, build the standalone handle — searchPath
|
|
37
39
|
// undefined for standalone (public), scoped to the dedicated schema +
|
|
38
40
|
// host RLS strategy when embedded config is set. An embedded host injects
|
|
@@ -67,11 +69,14 @@ export function createActivities(dependencies: ActivityDependencies = {}) {
|
|
|
67
69
|
(await createNatsEventBus(
|
|
68
70
|
settings.natsUrl,
|
|
69
71
|
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
72
|
+
{ logger: observabilityEventLogger(observability) },
|
|
70
73
|
)),
|
|
71
|
-
runtime: dependencies.runtime ?? createProductionAgentRuntime(
|
|
74
|
+
runtime: dependencies.runtime ?? createProductionAgentRuntime({
|
|
75
|
+
metrics: runtimeMetricsHooksForObservability(observability),
|
|
76
|
+
}),
|
|
72
77
|
objectStorage: dependencies.objectStorage ?? createObjectStorage(settings),
|
|
73
78
|
documentServices: dependencies.documentServices ?? createDocumentServices(settings),
|
|
74
|
-
observability
|
|
79
|
+
observability,
|
|
75
80
|
wakeSessionWorkflow: dependencies.wakeSessionWorkflow ?? null,
|
|
76
81
|
// §7.5 P3 — host-entitlements port. No constructed default: standalone
|
|
77
82
|
// has no host meter, so unset → null → `ensureRunAllowed` reads the
|
package/src/http.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import type { Settings } from "@opengeni/config";
|
|
2
|
+
import { dbSql, type Database } from "@opengeni/db";
|
|
3
|
+
import type { EventBus } from "@opengeni/events";
|
|
4
|
+
import type { Observability } from "@opengeni/observability";
|
|
5
|
+
|
|
6
|
+
export type ReadinessCheckName = "db" | "nats" | "temporal";
|
|
7
|
+
export type ReadinessChecks = Record<ReadinessCheckName, () => Promise<void> | void>;
|
|
8
|
+
|
|
9
|
+
export type ReadinessResult = {
|
|
10
|
+
ok: boolean;
|
|
11
|
+
checks: Record<ReadinessCheckName, { ok: boolean; error?: string }>;
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export function startWorkerHttpServer(input: {
|
|
15
|
+
settings: Settings;
|
|
16
|
+
observability: Observability;
|
|
17
|
+
checks: ReadinessChecks;
|
|
18
|
+
timeoutMs?: number;
|
|
19
|
+
}): ReturnType<typeof Bun.serve> {
|
|
20
|
+
const { settings, observability, checks } = input;
|
|
21
|
+
const timeoutMs = input.timeoutMs ?? 2_000;
|
|
22
|
+
return Bun.serve({
|
|
23
|
+
hostname: "0.0.0.0",
|
|
24
|
+
port: settings.workerHttpPort,
|
|
25
|
+
async fetch(request) {
|
|
26
|
+
const url = new URL(request.url);
|
|
27
|
+
if (request.method !== "GET") {
|
|
28
|
+
return Response.json({ error: "method_not_allowed" }, { status: 405 });
|
|
29
|
+
}
|
|
30
|
+
if (url.pathname === "/healthz") {
|
|
31
|
+
return Response.json({
|
|
32
|
+
service: settings.serviceName,
|
|
33
|
+
environment: settings.environment,
|
|
34
|
+
deploymentRevision: settings.deploymentRevision,
|
|
35
|
+
ok: true,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
if (url.pathname === "/metrics") {
|
|
39
|
+
return new Response(await observability.prometheusMetrics(), {
|
|
40
|
+
status: 200,
|
|
41
|
+
headers: { "content-type": "text/plain; version=0.0.4; charset=utf-8" },
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
if (url.pathname === "/readyz") {
|
|
45
|
+
const result = await runReadinessChecks(checks, timeoutMs);
|
|
46
|
+
return Response.json(result, { status: result.ok ? 200 : 503 });
|
|
47
|
+
}
|
|
48
|
+
return Response.json({ error: "not_found" }, { status: 404 });
|
|
49
|
+
},
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export async function runReadinessChecks(checks: ReadinessChecks, timeoutMs = 2_000): Promise<ReadinessResult> {
|
|
54
|
+
const entries = await Promise.all(
|
|
55
|
+
(Object.entries(checks) as Array<[ReadinessCheckName, () => Promise<void> | void]>)
|
|
56
|
+
.map(async ([name, check]) => {
|
|
57
|
+
try {
|
|
58
|
+
await withTimeout(Promise.resolve().then(check), timeoutMs);
|
|
59
|
+
return [name, { ok: true }] as const;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
return [name, { ok: false, error: error instanceof Error ? error.message : String(error) }] as const;
|
|
62
|
+
}
|
|
63
|
+
}),
|
|
64
|
+
);
|
|
65
|
+
const result = Object.fromEntries(entries) as ReadinessResult["checks"];
|
|
66
|
+
return {
|
|
67
|
+
ok: Object.values(result).every((check) => check.ok),
|
|
68
|
+
checks: result,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function dbReadyCheck(db: Database): () => Promise<void> {
|
|
73
|
+
return async () => {
|
|
74
|
+
await db.execute(dbSql`select 1`);
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function natsReadyCheck(bus: EventBus): () => void {
|
|
79
|
+
return () => {
|
|
80
|
+
if (bus.isConnected && !bus.isConnected()) {
|
|
81
|
+
throw new Error("NATS is not connected");
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
async function withTimeout<T>(promise: Promise<T>, timeoutMs: number): Promise<T> {
|
|
87
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
88
|
+
try {
|
|
89
|
+
return await Promise.race([
|
|
90
|
+
promise,
|
|
91
|
+
new Promise<never>((_, reject) => {
|
|
92
|
+
timer = setTimeout(() => reject(new Error(`readiness check timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
93
|
+
}),
|
|
94
|
+
]);
|
|
95
|
+
} finally {
|
|
96
|
+
if (timer) {
|
|
97
|
+
clearTimeout(timer);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
1
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
2
|
+
import { createDb } from "@opengeni/db";
|
|
3
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
2
4
|
import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
|
|
3
5
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
4
6
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
5
7
|
import { createActivities, type ActivityDependencies } from "./activities";
|
|
6
8
|
import type { WakeSessionWorkflowSignal } from "./activities/types";
|
|
9
|
+
import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
|
|
10
|
+
import { observabilityEventLogger } from "./observability-metrics";
|
|
7
11
|
|
|
8
12
|
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
9
13
|
// create() is idempotent across every worker in the pool: the first worker to
|
|
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
57
61
|
// idled and let its run finish, so a plain signal would not start one).
|
|
58
62
|
// Separate from the worker's NativeConnection: the @temporalio/client
|
|
59
63
|
// Connection is what exposes workflow.signalWithStart.
|
|
60
|
-
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
64
|
+
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
65
|
+
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
66
|
+
check: () => Promise<void>;
|
|
67
|
+
close: () => Promise<void>;
|
|
68
|
+
}> {
|
|
61
69
|
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
62
70
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
63
71
|
return {
|
|
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
|
|
|
70
78
|
signal: "queueChanged",
|
|
71
79
|
});
|
|
72
80
|
},
|
|
81
|
+
check: async () => {
|
|
82
|
+
await connection.workflowService.getSystemInfo({});
|
|
83
|
+
},
|
|
73
84
|
close: async () => {
|
|
74
85
|
await connection.close();
|
|
75
86
|
},
|
|
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
|
|
|
148
159
|
export async function startWorker() {
|
|
149
160
|
const settings = getSettings();
|
|
150
161
|
const observability = createObservability(settings, { component: "worker" });
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
},
|
|
158
|
-
);
|
|
159
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
160
|
-
settings,
|
|
161
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
|
|
162
|
-
});
|
|
163
|
-
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
164
|
-
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
165
|
-
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
166
|
-
// per-session worker factory.
|
|
167
|
-
const reaperSchedule = await retryStartupDependency(
|
|
168
|
-
"Temporal schedule (sandbox reaper)",
|
|
169
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
170
|
-
{
|
|
171
|
-
...startupRetryOptions(settings),
|
|
172
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
173
|
-
},
|
|
174
|
-
);
|
|
175
|
-
observability.info("OpenGeni worker listening", {
|
|
176
|
-
temporalTaskQueue: settings.temporalTaskQueue,
|
|
162
|
+
const retryOptions = startupRetryOptions(settings);
|
|
163
|
+
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
|
|
164
|
+
const searchPath = dbSearchPath(settings);
|
|
165
|
+
const dbClient = createDb(settings.databaseUrl, {
|
|
166
|
+
...(searchPath ? { searchPath } : {}),
|
|
167
|
+
rlsStrategy: settings.rlsStrategy,
|
|
177
168
|
});
|
|
169
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
170
|
+
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
171
|
+
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
172
|
+
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
173
|
+
let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
|
|
174
|
+
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
178
175
|
try {
|
|
179
|
-
await
|
|
176
|
+
bus = await retryStartupDependency(
|
|
177
|
+
"NATS",
|
|
178
|
+
() =>
|
|
179
|
+
createNatsEventBus(
|
|
180
|
+
settings.natsUrl,
|
|
181
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
182
|
+
{ logger: observabilityEventLogger(observability) },
|
|
183
|
+
),
|
|
184
|
+
{ ...retryOptions, onRetry },
|
|
185
|
+
);
|
|
186
|
+
signaler = await retryStartupDependency(
|
|
187
|
+
"Temporal client",
|
|
188
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
189
|
+
{ ...retryOptions, onRetry },
|
|
190
|
+
);
|
|
191
|
+
workerBundle = await createOpenGeniWorker({
|
|
192
|
+
settings,
|
|
193
|
+
activityDependencies: {
|
|
194
|
+
observability,
|
|
195
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
196
|
+
db: dbClient.db,
|
|
197
|
+
bus,
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
httpServer = startWorkerHttpServer({
|
|
201
|
+
settings,
|
|
202
|
+
observability,
|
|
203
|
+
checks: {
|
|
204
|
+
db: dbReadyCheck(dbClient.db),
|
|
205
|
+
nats: natsReadyCheck(bus),
|
|
206
|
+
temporal: signaler.check,
|
|
207
|
+
},
|
|
208
|
+
});
|
|
209
|
+
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
210
|
+
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
211
|
+
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
212
|
+
// per-session worker factory.
|
|
213
|
+
reaperSchedule = await retryStartupDependency(
|
|
214
|
+
"Temporal schedule (sandbox reaper)",
|
|
215
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
216
|
+
{ ...retryOptions, onRetry },
|
|
217
|
+
);
|
|
218
|
+
observability.info("OpenGeni worker listening", {
|
|
219
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
220
|
+
httpPort: settings.workerHttpPort,
|
|
221
|
+
});
|
|
222
|
+
await workerBundle.worker.run();
|
|
180
223
|
} finally {
|
|
181
|
-
|
|
182
|
-
await
|
|
183
|
-
|
|
224
|
+
httpServer?.stop(true);
|
|
225
|
+
await Promise.allSettled([
|
|
226
|
+
workerBundle?.connection.close(),
|
|
227
|
+
signaler?.close(),
|
|
228
|
+
reaperSchedule?.close(),
|
|
229
|
+
bus?.close(),
|
|
230
|
+
dbClient.close(),
|
|
231
|
+
]);
|
|
184
232
|
}
|
|
185
233
|
}
|
|
186
234
|
|