@opengeni/worker-bundle 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,218 @@
1
+ import type { EventLogger } from "@opengeni/events";
2
+ import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
3
+ import type { RuntimeMetricsHooks } from "@opengeni/runtime";
4
+
5
+ export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
6
+ export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
7
+ export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
8
+
9
+ const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
10
+
11
+ export function observabilityEventLogger(observability: Observability): EventLogger {
12
+ return {
13
+ debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
14
+ warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),
15
+ };
16
+ }
17
+
18
+ function eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {
19
+ if (!attributes) {
20
+ return undefined;
21
+ }
22
+ const sanitized: Attributes = {};
23
+ for (const [key, value] of Object.entries(attributes)) {
24
+ sanitized[key] = eventAttributeValue(value);
25
+ }
26
+ return sanitized;
27
+ }
28
+
29
+ function eventAttributeValue(value: unknown): AttributeValue {
30
+ if (value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
31
+ return value;
32
+ }
33
+ try {
34
+ return JSON.stringify(value);
35
+ } catch {
36
+ return String(value);
37
+ }
38
+ }
39
+
40
+ export function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks {
41
+ return {
42
+ onModelCall: ({ provider, outcome, durationSeconds }) => {
43
+ observability.incrementCounter({
44
+ name: "opengeni_model_calls_total",
45
+ help: "Total model calls by provider and outcome.",
46
+ labels: { provider, outcome },
47
+ });
48
+ observability.observeHistogram({
49
+ name: "opengeni_model_call_duration_seconds",
50
+ help: "Model call duration in seconds by provider.",
51
+ labels: { provider },
52
+ value: durationSeconds,
53
+ });
54
+ },
55
+ onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
56
+ observability.incrementCounter({
57
+ name: "opengeni_sandbox_creates_total",
58
+ help: "Total sandbox create attempts by backend and outcome.",
59
+ labels: { backend, outcome },
60
+ });
61
+ observability.observeHistogram({
62
+ name: "opengeni_sandbox_create_duration_seconds",
63
+ help: "Sandbox create duration in seconds by backend.",
64
+ labels: { backend },
65
+ value: durationSeconds,
66
+ });
67
+ },
68
+ onSandboxWarmingTimeout: () => {
69
+ observability.incrementCounter({
70
+ name: "opengeni_sandbox_warming_timeouts_total",
71
+ help: "Total sandbox warming timeouts.",
72
+ });
73
+ },
74
+ };
75
+ }
76
+
77
+ export function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {
78
+ const existing = turnTrackers.get(observability);
79
+ if (existing) {
80
+ return existing;
81
+ }
82
+ const tracker = new TurnLifecycleMetrics(observability);
83
+ turnTrackers.set(observability, tracker);
84
+ return tracker;
85
+ }
86
+
87
+ export class TurnLifecycleMetrics {
88
+ private readonly startedTurns = new Map<string, number>();
89
+ private timer: ReturnType<typeof setInterval> | null = null;
90
+
91
+ constructor(
92
+ private readonly observability: Observability,
93
+ private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},
94
+ ) {}
95
+
96
+ start(turnId: string): void {
97
+ this.startedTurns.set(turnId, this.now());
98
+ this.ensureTimer();
99
+ this.refreshGauges();
100
+ }
101
+
102
+ finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
103
+ const startedAt = this.startedTurns.get(turnId);
104
+ if (startedAt !== undefined) {
105
+ this.startedTurns.delete(turnId);
106
+ }
107
+ if (outcome) {
108
+ const observedDuration = durationSeconds ?? (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
109
+ this.observability.incrementCounter({
110
+ name: "opengeni_turns_total",
111
+ help: "Total agent turns by terminal outcome.",
112
+ labels: { outcome },
113
+ });
114
+ this.observability.observeHistogram({
115
+ name: "opengeni_turn_duration_seconds",
116
+ help: "Agent turn duration in seconds by terminal outcome.",
117
+ labels: { outcome },
118
+ value: observedDuration,
119
+ });
120
+ }
121
+ this.refreshGauges();
122
+ if (this.startedTurns.size === 0) {
123
+ this.stopTimer();
124
+ }
125
+ }
126
+
127
+ refreshGauges(): void {
128
+ this.observability.setGauge({
129
+ name: "opengeni_turns_inflight",
130
+ help: "Current number of in-flight agent turns in this worker process.",
131
+ value: this.startedTurns.size,
132
+ });
133
+ this.observability.setGauge({
134
+ name: "opengeni_turn_oldest_inflight_age_seconds",
135
+ help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
136
+ value: this.oldestInflightAgeSeconds(),
137
+ });
138
+ }
139
+
140
+ stop(): void {
141
+ this.startedTurns.clear();
142
+ this.refreshGauges();
143
+ this.stopTimer();
144
+ }
145
+
146
+ private oldestInflightAgeSeconds(): number {
147
+ if (this.startedTurns.size === 0) {
148
+ return 0;
149
+ }
150
+ let oldest = Number.POSITIVE_INFINITY;
151
+ for (const startedAt of this.startedTurns.values()) {
152
+ oldest = Math.min(oldest, startedAt);
153
+ }
154
+ return Math.max(0, (this.now() - oldest) / 1000);
155
+ }
156
+
157
+ private ensureTimer(): void {
158
+ if (this.timer) {
159
+ return;
160
+ }
161
+ this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
162
+ this.timer.unref?.();
163
+ }
164
+
165
+ private stopTimer(): void {
166
+ if (!this.timer) {
167
+ return;
168
+ }
169
+ clearInterval(this.timer);
170
+ this.timer = null;
171
+ }
172
+
173
+ private now(): number {
174
+ return this.options.now?.() ?? Date.now();
175
+ }
176
+ }
177
+
178
+ export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
179
+ observability.setGauge({
180
+ name: "opengeni_turns_queued",
181
+ help: "Current number of queued session turns.",
182
+ value,
183
+ });
184
+ }
185
+
186
+ export function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void {
187
+ for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
188
+ observability.setGauge({
189
+ name: "opengeni_sandbox_leases",
190
+ help: "Current sandbox leases by liveness state.",
191
+ labels: { liveness },
192
+ value: counts[liveness] ?? 0,
193
+ });
194
+ }
195
+ }
196
+
197
+ export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
198
+ if (count <= 0) {
199
+ return;
200
+ }
201
+ observability.incrementCounter({
202
+ name: "opengeni_sandbox_orphans_terminated_total",
203
+ help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
204
+ amount: count,
205
+ });
206
+ }
207
+
208
+ export function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void {
209
+ if (!observability || amountMicros <= 0) {
210
+ return;
211
+ }
212
+ observability.incrementCounter({
213
+ name: "opengeni_credit_micros_total",
214
+ help: "Total credit micros recorded by kind.",
215
+ labels: { kind },
216
+ amount: amountMicros,
217
+ });
218
+ }
@@ -47,6 +47,7 @@ import {
47
47
  tagModalSandbox,
48
48
  type EstablishedSandboxSession,
49
49
  type ExposedPortEndpoint,
50
+ type RuntimeMetricsHooks,
50
51
  } from "@opengeni/runtime";
51
52
  import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
52
53
 
@@ -60,6 +61,7 @@ export type ResumeHolderKind = LeaseHolderKind;
60
61
  export type SandboxResumeServices = {
61
62
  db: Database;
62
63
  settings: Settings;
64
+ sandboxMetrics?: RuntimeMetricsHooks;
63
65
  };
64
66
 
65
67
  export type ResumeBoxIds = {
@@ -173,6 +175,17 @@ function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: numbe
173
175
  : error;
174
176
  }
175
177
 
178
+ function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, error: unknown): void {
179
+ if (!(error instanceof SandboxWarmingTimeoutError)) {
180
+ return;
181
+ }
182
+ try {
183
+ metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
184
+ } catch {
185
+ // Metrics emission must never affect sandbox recovery or error propagation.
186
+ }
187
+ }
188
+
176
189
  function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
177
190
  const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
178
191
  ? envelope.sessionState as Record<string, unknown>
@@ -293,6 +306,7 @@ export async function resumeBoxForTurn(
293
306
  sessionId: ids.sessionId,
294
307
  backendOverride: ids.backend as never,
295
308
  ...(ids.environment ? { environment: ids.environment } : {}),
309
+ ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
296
310
  onSandboxCreated: async (created) => {
297
311
  createdEstablished = created;
298
312
  const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
@@ -374,7 +388,9 @@ export async function resumeBoxForTurn(
374
388
  });
375
389
  }
376
390
  await release();
377
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
391
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
392
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
393
+ throw warmingError;
378
394
  }
379
395
  }
380
396
 
@@ -400,6 +416,7 @@ export async function resumeBoxForTurn(
400
416
  sessionId: ids.sessionId,
401
417
  backendOverride: ids.backend as never,
402
418
  ...(ids.environment ? { environment: ids.environment } : {}),
419
+ ...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
403
420
  });
404
421
  // Re-ensure the desktop display stack on the ATTACHED/REARMED path too — NOT
405
422
  // just the spawner path. A turn attaching to a warm box whose :0 was never
@@ -414,7 +431,9 @@ export async function resumeBoxForTurn(
414
431
  return { established, leaseEpoch, release };
415
432
  } catch (error) {
416
433
  await release();
417
- throw asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
434
+ const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
435
+ recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
436
+ throw warmingError;
418
437
  }
419
438
  }
420
439