@opengeni/worker-bundle 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +3 -0
- package/dist/index.js +644 -80
- package/dist/index.js.map +1 -1
- package/package.json +8 -8
- package/src/activities/agent-turn.ts +32 -4
- package/src/activities/sandbox-lease.ts +130 -3
- package/src/activities/session-state.ts +22 -4
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +197 -59
package/src/index.ts
CHANGED
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
1
|
+
import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
|
|
2
|
+
import { createDb } from "@opengeni/db";
|
|
3
|
+
import { createNatsEventBus } from "@opengeni/events";
|
|
2
4
|
import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
|
|
3
5
|
import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
|
|
4
6
|
import { NativeConnection, Worker } from "@temporalio/worker";
|
|
5
7
|
import { createActivities, type ActivityDependencies } from "./activities";
|
|
6
8
|
import type { WakeSessionWorkflowSignal } from "./activities/types";
|
|
9
|
+
import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
|
|
10
|
+
import { observabilityEventLogger } from "./observability-metrics";
|
|
7
11
|
|
|
8
12
|
// The deterministic id of the ONE global reaper Schedule. A single id means
|
|
9
13
|
// create() is idempotent across every worker in the pool: the first worker to
|
|
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
|
|
|
57
61
|
// idled and let its run finish, so a plain signal would not start one).
|
|
58
62
|
// Separate from the worker's NativeConnection: the @temporalio/client
|
|
59
63
|
// Connection is what exposes workflow.signalWithStart.
|
|
60
|
-
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
64
|
+
export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
|
|
65
|
+
wakeSessionWorkflow: WakeSessionWorkflowSignal;
|
|
66
|
+
check: () => Promise<void>;
|
|
67
|
+
close: () => Promise<void>;
|
|
68
|
+
}> {
|
|
61
69
|
const connection = await Connection.connect({ address: settings.temporalHost });
|
|
62
70
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
63
71
|
return {
|
|
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
|
|
|
70
78
|
signal: "queueChanged",
|
|
71
79
|
});
|
|
72
80
|
},
|
|
81
|
+
check: async () => {
|
|
82
|
+
await connection.workflowService.getSystemInfo({});
|
|
83
|
+
},
|
|
73
84
|
close: async () => {
|
|
74
85
|
await connection.close();
|
|
75
86
|
},
|
|
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
|
|
|
148
159
|
export async function startWorker() {
|
|
149
160
|
const settings = getSettings();
|
|
150
161
|
const observability = createObservability(settings, { component: "worker" });
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
},
|
|
158
|
-
);
|
|
159
|
-
const { worker, connection } = await createOpenGeniWorker({
|
|
160
|
-
settings,
|
|
161
|
-
activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
|
|
162
|
-
});
|
|
163
|
-
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
164
|
-
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
165
|
-
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
166
|
-
// per-session worker factory.
|
|
167
|
-
const reaperSchedule = await retryStartupDependency(
|
|
168
|
-
"Temporal schedule (sandbox reaper)",
|
|
169
|
-
() => registerSandboxReaperSchedule(settings, observability),
|
|
170
|
-
{
|
|
171
|
-
...startupRetryOptions(settings),
|
|
172
|
-
onRetry: (event) => logStartupDependencyRetry(observability, event),
|
|
173
|
-
},
|
|
174
|
-
);
|
|
175
|
-
observability.info("OpenGeni worker listening", {
|
|
176
|
-
temporalTaskQueue: settings.temporalTaskQueue,
|
|
162
|
+
const retryOptions = startupRetryOptions(settings);
|
|
163
|
+
const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
|
|
164
|
+
const searchPath = dbSearchPath(settings);
|
|
165
|
+
const dbClient = createDb(settings.databaseUrl, {
|
|
166
|
+
...(searchPath ? { searchPath } : {}),
|
|
167
|
+
rlsStrategy: settings.rlsStrategy,
|
|
177
168
|
});
|
|
169
|
+
const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
|
|
170
|
+
let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
|
|
171
|
+
let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
|
|
172
|
+
let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
|
|
173
|
+
let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
|
|
174
|
+
let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
|
|
178
175
|
try {
|
|
179
|
-
await
|
|
176
|
+
bus = await retryStartupDependency(
|
|
177
|
+
"NATS",
|
|
178
|
+
() =>
|
|
179
|
+
createNatsEventBus(
|
|
180
|
+
settings.natsUrl,
|
|
181
|
+
controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
|
|
182
|
+
{ logger: observabilityEventLogger(observability) },
|
|
183
|
+
),
|
|
184
|
+
{ ...retryOptions, onRetry },
|
|
185
|
+
);
|
|
186
|
+
signaler = await retryStartupDependency(
|
|
187
|
+
"Temporal client",
|
|
188
|
+
() => createWorkerWorkflowSignaler(settings),
|
|
189
|
+
{ ...retryOptions, onRetry },
|
|
190
|
+
);
|
|
191
|
+
workerBundle = await createOpenGeniWorker({
|
|
192
|
+
settings,
|
|
193
|
+
activityDependencies: {
|
|
194
|
+
observability,
|
|
195
|
+
wakeSessionWorkflow: signaler.wakeSessionWorkflow,
|
|
196
|
+
db: dbClient.db,
|
|
197
|
+
bus,
|
|
198
|
+
},
|
|
199
|
+
});
|
|
200
|
+
httpServer = startWorkerHttpServer({
|
|
201
|
+
settings,
|
|
202
|
+
observability,
|
|
203
|
+
checks: {
|
|
204
|
+
db: dbReadyCheck(dbClient.db),
|
|
205
|
+
nats: natsReadyCheck(bus),
|
|
206
|
+
temporal: signaler.check,
|
|
207
|
+
},
|
|
208
|
+
});
|
|
209
|
+
// Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
|
|
210
|
+
// is false). Idempotent across the pool: only the first worker creates it. The
|
|
211
|
+
// static global-queue Worker.create above is UNCHANGED — there is no
|
|
212
|
+
// per-session worker factory.
|
|
213
|
+
reaperSchedule = await retryStartupDependency(
|
|
214
|
+
"Temporal schedule (sandbox reaper)",
|
|
215
|
+
() => registerSandboxReaperSchedule(settings, observability),
|
|
216
|
+
{ ...retryOptions, onRetry },
|
|
217
|
+
);
|
|
218
|
+
observability.info("OpenGeni worker listening", {
|
|
219
|
+
temporalTaskQueue: settings.temporalTaskQueue,
|
|
220
|
+
httpPort: settings.workerHttpPort,
|
|
221
|
+
});
|
|
222
|
+
await workerBundle.worker.run();
|
|
180
223
|
} finally {
|
|
181
|
-
|
|
182
|
-
await
|
|
183
|
-
|
|
224
|
+
httpServer?.stop(true);
|
|
225
|
+
await Promise.allSettled([
|
|
226
|
+
workerBundle?.connection.close(),
|
|
227
|
+
signaler?.close(),
|
|
228
|
+
reaperSchedule?.close(),
|
|
229
|
+
bus?.close(),
|
|
230
|
+
dbClient.close(),
|
|
231
|
+
]);
|
|
184
232
|
}
|
|
185
233
|
}
|
|
186
234
|
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type { EventLogger } from "@opengeni/events";
|
|
2
|
+
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
3
|
+
import type { RuntimeMetricsHooks } from "@opengeni/runtime";
|
|
4
|
+
|
|
5
|
+
export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
|
|
6
|
+
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
7
|
+
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
8
|
+
|
|
9
|
+
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
10
|
+
|
|
11
|
+
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
12
|
+
return {
|
|
13
|
+
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
14
|
+
warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {
|
|
19
|
+
if (!attributes) {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
const sanitized: Attributes = {};
|
|
23
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
24
|
+
sanitized[key] = eventAttributeValue(value);
|
|
25
|
+
}
|
|
26
|
+
return sanitized;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function eventAttributeValue(value: unknown): AttributeValue {
|
|
30
|
+
if (value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
return JSON.stringify(value);
|
|
35
|
+
} catch {
|
|
36
|
+
return String(value);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks {
|
|
41
|
+
return {
|
|
42
|
+
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
43
|
+
observability.incrementCounter({
|
|
44
|
+
name: "opengeni_model_calls_total",
|
|
45
|
+
help: "Total model calls by provider and outcome.",
|
|
46
|
+
labels: { provider, outcome },
|
|
47
|
+
});
|
|
48
|
+
observability.observeHistogram({
|
|
49
|
+
name: "opengeni_model_call_duration_seconds",
|
|
50
|
+
help: "Model call duration in seconds by provider.",
|
|
51
|
+
labels: { provider },
|
|
52
|
+
value: durationSeconds,
|
|
53
|
+
});
|
|
54
|
+
},
|
|
55
|
+
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
56
|
+
observability.incrementCounter({
|
|
57
|
+
name: "opengeni_sandbox_creates_total",
|
|
58
|
+
help: "Total sandbox create attempts by backend and outcome.",
|
|
59
|
+
labels: { backend, outcome },
|
|
60
|
+
});
|
|
61
|
+
observability.observeHistogram({
|
|
62
|
+
name: "opengeni_sandbox_create_duration_seconds",
|
|
63
|
+
help: "Sandbox create duration in seconds by backend.",
|
|
64
|
+
labels: { backend },
|
|
65
|
+
value: durationSeconds,
|
|
66
|
+
});
|
|
67
|
+
},
|
|
68
|
+
onSandboxWarmingTimeout: () => {
|
|
69
|
+
observability.incrementCounter({
|
|
70
|
+
name: "opengeni_sandbox_warming_timeouts_total",
|
|
71
|
+
help: "Total sandbox warming timeouts.",
|
|
72
|
+
});
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {
|
|
78
|
+
const existing = turnTrackers.get(observability);
|
|
79
|
+
if (existing) {
|
|
80
|
+
return existing;
|
|
81
|
+
}
|
|
82
|
+
const tracker = new TurnLifecycleMetrics(observability);
|
|
83
|
+
turnTrackers.set(observability, tracker);
|
|
84
|
+
return tracker;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export class TurnLifecycleMetrics {
|
|
88
|
+
private readonly startedTurns = new Map<string, number>();
|
|
89
|
+
private timer: ReturnType<typeof setInterval> | null = null;
|
|
90
|
+
|
|
91
|
+
constructor(
|
|
92
|
+
private readonly observability: Observability,
|
|
93
|
+
private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},
|
|
94
|
+
) {}
|
|
95
|
+
|
|
96
|
+
start(turnId: string): void {
|
|
97
|
+
this.startedTurns.set(turnId, this.now());
|
|
98
|
+
this.ensureTimer();
|
|
99
|
+
this.refreshGauges();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
|
|
103
|
+
const startedAt = this.startedTurns.get(turnId);
|
|
104
|
+
if (startedAt !== undefined) {
|
|
105
|
+
this.startedTurns.delete(turnId);
|
|
106
|
+
}
|
|
107
|
+
if (outcome) {
|
|
108
|
+
const observedDuration = durationSeconds ?? (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
|
|
109
|
+
this.observability.incrementCounter({
|
|
110
|
+
name: "opengeni_turns_total",
|
|
111
|
+
help: "Total agent turns by terminal outcome.",
|
|
112
|
+
labels: { outcome },
|
|
113
|
+
});
|
|
114
|
+
this.observability.observeHistogram({
|
|
115
|
+
name: "opengeni_turn_duration_seconds",
|
|
116
|
+
help: "Agent turn duration in seconds by terminal outcome.",
|
|
117
|
+
labels: { outcome },
|
|
118
|
+
value: observedDuration,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
this.refreshGauges();
|
|
122
|
+
if (this.startedTurns.size === 0) {
|
|
123
|
+
this.stopTimer();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
refreshGauges(): void {
|
|
128
|
+
this.observability.setGauge({
|
|
129
|
+
name: "opengeni_turns_inflight",
|
|
130
|
+
help: "Current number of in-flight agent turns in this worker process.",
|
|
131
|
+
value: this.startedTurns.size,
|
|
132
|
+
});
|
|
133
|
+
this.observability.setGauge({
|
|
134
|
+
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
135
|
+
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
136
|
+
value: this.oldestInflightAgeSeconds(),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
stop(): void {
|
|
141
|
+
this.startedTurns.clear();
|
|
142
|
+
this.refreshGauges();
|
|
143
|
+
this.stopTimer();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
private oldestInflightAgeSeconds(): number {
|
|
147
|
+
if (this.startedTurns.size === 0) {
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
let oldest = Number.POSITIVE_INFINITY;
|
|
151
|
+
for (const startedAt of this.startedTurns.values()) {
|
|
152
|
+
oldest = Math.min(oldest, startedAt);
|
|
153
|
+
}
|
|
154
|
+
return Math.max(0, (this.now() - oldest) / 1000);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
private ensureTimer(): void {
|
|
158
|
+
if (this.timer) {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
|
|
162
|
+
this.timer.unref?.();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private stopTimer(): void {
|
|
166
|
+
if (!this.timer) {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
clearInterval(this.timer);
|
|
170
|
+
this.timer = null;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private now(): number {
|
|
174
|
+
return this.options.now?.() ?? Date.now();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
|
|
179
|
+
observability.setGauge({
|
|
180
|
+
name: "opengeni_turns_queued",
|
|
181
|
+
help: "Current number of queued session turns.",
|
|
182
|
+
value,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void {
|
|
187
|
+
for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
|
|
188
|
+
observability.setGauge({
|
|
189
|
+
name: "opengeni_sandbox_leases",
|
|
190
|
+
help: "Current sandbox leases by liveness state.",
|
|
191
|
+
labels: { liveness },
|
|
192
|
+
value: counts[liveness] ?? 0,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
|
|
198
|
+
if (count <= 0) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
observability.incrementCounter({
|
|
202
|
+
name: "opengeni_sandbox_orphans_terminated_total",
|
|
203
|
+
help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
|
|
204
|
+
amount: count,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void {
|
|
209
|
+
if (!observability || amountMicros <= 0) {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
observability.incrementCounter({
|
|
213
|
+
name: "opengeni_credit_micros_total",
|
|
214
|
+
help: "Total credit micros recorded by kind.",
|
|
215
|
+
labels: { kind },
|
|
216
|
+
amount: amountMicros,
|
|
217
|
+
});
|
|
218
|
+
}
|