@opengeni/worker-bundle 0.3.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts CHANGED
@@ -1,9 +1,13 @@
1
- import { getSettings, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
1
+ import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions, type Settings } from "@opengeni/config";
2
+ import { createDb } from "@opengeni/db";
3
+ import { createNatsEventBus } from "@opengeni/events";
2
4
  import { createObservability, logStartupDependencyRetry, type Observability } from "@opengeni/observability";
3
5
  import { Connection, ScheduleAlreadyRunning, ScheduleOverlapPolicy, Client as TemporalClient } from "@temporalio/client";
4
6
  import { NativeConnection, Worker } from "@temporalio/worker";
5
7
  import { createActivities, type ActivityDependencies } from "./activities";
6
8
  import type { WakeSessionWorkflowSignal } from "./activities/types";
9
+ import { dbReadyCheck, natsReadyCheck, startWorkerHttpServer } from "./http";
10
+ import { observabilityEventLogger } from "./observability-metrics";
7
11
 
8
12
  // The deterministic id of the ONE global reaper Schedule. A single id means
9
13
  // create() is idempotent across every worker in the pool: the first worker to
@@ -57,7 +61,11 @@ export async function createOpenGeniWorker(options: WorkerOptions = {}): Promise
57
61
  // idled and let its run finish, so a plain signal would not start one).
58
62
  // Separate from the worker's NativeConnection: the @temporalio/client
59
63
  // Connection is what exposes workflow.signalWithStart.
60
- export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{ wakeSessionWorkflow: WakeSessionWorkflowSignal; close: () => Promise<void> }> {
64
+ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<{
65
+ wakeSessionWorkflow: WakeSessionWorkflowSignal;
66
+ check: () => Promise<void>;
67
+ close: () => Promise<void>;
68
+ }> {
61
69
  const connection = await Connection.connect({ address: settings.temporalHost });
62
70
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
63
71
  return {
@@ -70,6 +78,9 @@ export async function createWorkerWorkflowSignaler(settings: Settings): Promise<
70
78
  signal: "queueChanged",
71
79
  });
72
80
  },
81
+ check: async () => {
82
+ await connection.workflowService.getSystemInfo({});
83
+ },
73
84
  close: async () => {
74
85
  await connection.close();
75
86
  },
@@ -148,39 +159,76 @@ export async function registerSandboxReaperSchedule(
148
159
  export async function startWorker() {
149
160
  const settings = getSettings();
150
161
  const observability = createObservability(settings, { component: "worker" });
151
- const signaler = await retryStartupDependency(
152
- "Temporal client",
153
- () => createWorkerWorkflowSignaler(settings),
154
- {
155
- ...startupRetryOptions(settings),
156
- onRetry: (event) => logStartupDependencyRetry(observability, event),
157
- },
158
- );
159
- const { worker, connection } = await createOpenGeniWorker({
160
- settings,
161
- activityDependencies: { observability, wakeSessionWorkflow: signaler.wakeSessionWorkflow },
162
- });
163
- // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
164
- // is false). Idempotent across the pool: only the first worker creates it. The
165
- // static global-queue Worker.create above is UNCHANGED — there is no
166
- // per-session worker factory.
167
- const reaperSchedule = await retryStartupDependency(
168
- "Temporal schedule (sandbox reaper)",
169
- () => registerSandboxReaperSchedule(settings, observability),
170
- {
171
- ...startupRetryOptions(settings),
172
- onRetry: (event) => logStartupDependencyRetry(observability, event),
173
- },
174
- );
175
- observability.info("OpenGeni worker listening", {
176
- temporalTaskQueue: settings.temporalTaskQueue,
162
+ const retryOptions = startupRetryOptions(settings);
163
+ const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) => logStartupDependencyRetry(observability, event);
164
+ const searchPath = dbSearchPath(settings);
165
+ const dbClient = createDb(settings.databaseUrl, {
166
+ ...(searchPath ? { searchPath } : {}),
167
+ rlsStrategy: settings.rlsStrategy,
177
168
  });
169
+ const controlPlaneAuth = resolveNatsControlPlaneAuth(settings);
170
+ let bus: Awaited<ReturnType<typeof createNatsEventBus>> | undefined;
171
+ let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
172
+ let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
173
+ let reaperSchedule: Awaited<ReturnType<typeof registerSandboxReaperSchedule>> | undefined;
174
+ let httpServer: ReturnType<typeof startWorkerHttpServer> | undefined;
178
175
  try {
179
- await worker.run();
176
+ bus = await retryStartupDependency(
177
+ "NATS",
178
+ () =>
179
+ createNatsEventBus(
180
+ settings.natsUrl,
181
+ controlPlaneAuth ? { user: controlPlaneAuth.user, pass: controlPlaneAuth.password } : undefined,
182
+ { logger: observabilityEventLogger(observability) },
183
+ ),
184
+ { ...retryOptions, onRetry },
185
+ );
186
+ signaler = await retryStartupDependency(
187
+ "Temporal client",
188
+ () => createWorkerWorkflowSignaler(settings),
189
+ { ...retryOptions, onRetry },
190
+ );
191
+ workerBundle = await createOpenGeniWorker({
192
+ settings,
193
+ activityDependencies: {
194
+ observability,
195
+ wakeSessionWorkflow: signaler.wakeSessionWorkflow,
196
+ db: dbClient.db,
197
+ bus,
198
+ },
199
+ });
200
+ httpServer = startWorkerHttpServer({
201
+ settings,
202
+ observability,
203
+ checks: {
204
+ db: dbReadyCheck(dbClient.db),
205
+ nats: natsReadyCheck(bus),
206
+ temporal: signaler.check,
207
+ },
208
+ });
209
+ // Register the ONE global reaper Schedule (no-op when sandboxOwnershipEnabled
210
+ // is false). Idempotent across the pool: only the first worker creates it. The
211
+ // static global-queue Worker.create above is UNCHANGED — there is no
212
+ // per-session worker factory.
213
+ reaperSchedule = await retryStartupDependency(
214
+ "Temporal schedule (sandbox reaper)",
215
+ () => registerSandboxReaperSchedule(settings, observability),
216
+ { ...retryOptions, onRetry },
217
+ );
218
+ observability.info("OpenGeni worker listening", {
219
+ temporalTaskQueue: settings.temporalTaskQueue,
220
+ httpPort: settings.workerHttpPort,
221
+ });
222
+ await workerBundle.worker.run();
180
223
  } finally {
181
- await connection.close();
182
- await signaler.close();
183
- await reaperSchedule.close();
224
+ httpServer?.stop(true);
225
+ await Promise.allSettled([
226
+ workerBundle?.connection.close(),
227
+ signaler?.close(),
228
+ reaperSchedule?.close(),
229
+ bus?.close(),
230
+ dbClient.close(),
231
+ ]);
184
232
  }
185
233
  }
186
234
 
@@ -0,0 +1,218 @@
1
+ import type { EventLogger } from "@opengeni/events";
2
+ import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
3
+ import type { RuntimeMetricsHooks } from "@opengeni/runtime";
4
+
5
+ export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
6
+ export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
7
+ export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
8
+
9
+ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
10
+
11
+ export function observabilityEventLogger(observability: Observability): EventLogger {
12
+ return {
13
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
14
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),
15
+ };
16
+ }
17
+
18
+ function eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {
19
+ if (!attributes) {
20
+ return undefined;
21
+ }
22
+ const sanitized: Attributes = {};
23
+ for (const [key, value] of Object.entries(attributes)) {
24
+ sanitized[key] = eventAttributeValue(value);
25
+ }
26
+ return sanitized;
27
+ }
28
+
29
+ function eventAttributeValue(value: unknown): AttributeValue {
30
+ if (value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
31
+ return value;
32
+ }
33
+ try {
34
+ return JSON.stringify(value);
35
+ } catch {
36
+ return String(value);
37
+ }
38
+ }
39
+
40
+ export function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks {
41
+ return {
42
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
43
+ observability.incrementCounter({
44
+ name: "opengeni_model_calls_total",
45
+ help: "Total model calls by provider and outcome.",
46
+ labels: { provider, outcome },
47
+ });
48
+ observability.observeHistogram({
49
+ name: "opengeni_model_call_duration_seconds",
50
+ help: "Model call duration in seconds by provider.",
51
+ labels: { provider },
52
+ value: durationSeconds,
53
+ });
54
+ },
55
+ onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
56
+ observability.incrementCounter({
57
+ name: "opengeni_sandbox_creates_total",
58
+ help: "Total sandbox create attempts by backend and outcome.",
59
+ labels: { backend, outcome },
60
+ });
61
+ observability.observeHistogram({
62
+ name: "opengeni_sandbox_create_duration_seconds",
63
+ help: "Sandbox create duration in seconds by backend.",
64
+ labels: { backend },
65
+ value: durationSeconds,
66
+ });
67
+ },
68
+ onSandboxWarmingTimeout: () => {
69
+ observability.incrementCounter({
70
+ name: "opengeni_sandbox_warming_timeouts_total",
71
+ help: "Total sandbox warming timeouts.",
72
+ });
73
+ },
74
+ };
75
+ }
76
+
77
+ export function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {
78
+ const existing = turnTrackers.get(observability);
79
+ if (existing) {
80
+ return existing;
81
+ }
82
+ const tracker = new TurnLifecycleMetrics(observability);
83
+ turnTrackers.set(observability, tracker);
84
+ return tracker;
85
+ }
86
+
87
+ export class TurnLifecycleMetrics {
88
+ private readonly startedTurns = new Map<string, number>();
89
+ private timer: ReturnType<typeof setInterval> | null = null;
90
+
91
+ constructor(
92
+ private readonly observability: Observability,
93
+ private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},
94
+ ) {}
95
+
96
+ start(turnId: string): void {
97
+ this.startedTurns.set(turnId, this.now());
98
+ this.ensureTimer();
99
+ this.refreshGauges();
100
+ }
101
+
102
+ finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
103
+ const startedAt = this.startedTurns.get(turnId);
104
+ if (startedAt !== undefined) {
105
+ this.startedTurns.delete(turnId);
106
+ }
107
+ if (outcome) {
108
+ const observedDuration = durationSeconds ?? (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
109
+ this.observability.incrementCounter({
110
+ name: "opengeni_turns_total",
111
+ help: "Total agent turns by terminal outcome.",
112
+ labels: { outcome },
113
+ });
114
+ this.observability.observeHistogram({
115
+ name: "opengeni_turn_duration_seconds",
116
+ help: "Agent turn duration in seconds by terminal outcome.",
117
+ labels: { outcome },
118
+ value: observedDuration,
119
+ });
120
+ }
121
+ this.refreshGauges();
122
+ if (this.startedTurns.size === 0) {
123
+ this.stopTimer();
124
+ }
125
+ }
126
+
127
+ refreshGauges(): void {
128
+ this.observability.setGauge({
129
+ name: "opengeni_turns_inflight",
130
+ help: "Current number of in-flight agent turns in this worker process.",
131
+ value: this.startedTurns.size,
132
+ });
133
+ this.observability.setGauge({
134
+ name: "opengeni_turn_oldest_inflight_age_seconds",
135
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
136
+ value: this.oldestInflightAgeSeconds(),
137
+ });
138
+ }
139
+
140
+ stop(): void {
141
+ this.startedTurns.clear();
142
+ this.refreshGauges();
143
+ this.stopTimer();
144
+ }
145
+
146
+ private oldestInflightAgeSeconds(): number {
147
+ if (this.startedTurns.size === 0) {
148
+ return 0;
149
+ }
150
+ let oldest = Number.POSITIVE_INFINITY;
151
+ for (const startedAt of this.startedTurns.values()) {
152
+ oldest = Math.min(oldest, startedAt);
153
+ }
154
+ return Math.max(0, (this.now() - oldest) / 1000);
155
+ }
156
+
157
+ private ensureTimer(): void {
158
+ if (this.timer) {
159
+ return;
160
+ }
161
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
162
+ this.timer.unref?.();
163
+ }
164
+
165
+ private stopTimer(): void {
166
+ if (!this.timer) {
167
+ return;
168
+ }
169
+ clearInterval(this.timer);
170
+ this.timer = null;
171
+ }
172
+
173
+ private now(): number {
174
+ return this.options.now?.() ?? Date.now();
175
+ }
176
+ }
177
+
178
+ export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
179
+ observability.setGauge({
180
+ name: "opengeni_turns_queued",
181
+ help: "Current number of queued session turns.",
182
+ value,
183
+ });
184
+ }
185
+
186
+ export function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void {
187
+ for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
188
+ observability.setGauge({
189
+ name: "opengeni_sandbox_leases",
190
+ help: "Current sandbox leases by liveness state.",
191
+ labels: { liveness },
192
+ value: counts[liveness] ?? 0,
193
+ });
194
+ }
195
+ }
196
+
197
+ export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
198
+ if (count <= 0) {
199
+ return;
200
+ }
201
+ observability.incrementCounter({
202
+ name: "opengeni_sandbox_orphans_terminated_total",
203
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
204
+ amount: count,
205
+ });
206
+ }
207
+
208
+ export function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void {
209
+ if (!observability || amountMicros <= 0) {
210
+ return;
211
+ }
212
+ observability.incrementCounter({
213
+ name: "opengeni_credit_micros_total",
214
+ help: "Total credit micros recorded by kind.",
215
+ labels: { kind },
216
+ amount: amountMicros,
217
+ });
218
+ }