@opengeni/worker-bundle 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +190 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +593 -86
- package/dist/index.js.map +1 -1
- package/package.json +11 -15
- package/src/activities/agent-turn.ts +95 -12
- package/src/activities/capabilities.ts +2 -30
- package/src/activities/context-compaction.ts +96 -10
- package/src/activities/environment.ts +30 -4
- package/src/activities/sandbox-lease.ts +33 -0
- package/src/activities/session-state.ts +22 -4
- package/src/activities/streaming.ts +1 -0
- package/src/activities.ts +7 -2
- package/src/http.ts +100 -0
- package/src/index.ts +80 -32
- package/src/observability-metrics.ts +218 -0
- package/src/sandbox-resume.ts +21 -2
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import type { EventLogger } from "@opengeni/events";
|
|
2
|
+
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
3
|
+
import type { RuntimeMetricsHooks } from "@opengeni/runtime";
|
|
4
|
+
|
|
5
|
+
export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
|
|
6
|
+
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
7
|
+
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
8
|
+
|
|
9
|
+
const turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();
|
|
10
|
+
|
|
11
|
+
export function observabilityEventLogger(observability: Observability): EventLogger {
|
|
12
|
+
return {
|
|
13
|
+
debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),
|
|
14
|
+
warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),
|
|
15
|
+
};
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {
|
|
19
|
+
if (!attributes) {
|
|
20
|
+
return undefined;
|
|
21
|
+
}
|
|
22
|
+
const sanitized: Attributes = {};
|
|
23
|
+
for (const [key, value] of Object.entries(attributes)) {
|
|
24
|
+
sanitized[key] = eventAttributeValue(value);
|
|
25
|
+
}
|
|
26
|
+
return sanitized;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function eventAttributeValue(value: unknown): AttributeValue {
|
|
30
|
+
if (value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
31
|
+
return value;
|
|
32
|
+
}
|
|
33
|
+
try {
|
|
34
|
+
return JSON.stringify(value);
|
|
35
|
+
} catch {
|
|
36
|
+
return String(value);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks {
|
|
41
|
+
return {
|
|
42
|
+
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
43
|
+
observability.incrementCounter({
|
|
44
|
+
name: "opengeni_model_calls_total",
|
|
45
|
+
help: "Total model calls by provider and outcome.",
|
|
46
|
+
labels: { provider, outcome },
|
|
47
|
+
});
|
|
48
|
+
observability.observeHistogram({
|
|
49
|
+
name: "opengeni_model_call_duration_seconds",
|
|
50
|
+
help: "Model call duration in seconds by provider.",
|
|
51
|
+
labels: { provider },
|
|
52
|
+
value: durationSeconds,
|
|
53
|
+
});
|
|
54
|
+
},
|
|
55
|
+
onSandboxCreate: ({ backend, outcome, durationSeconds }) => {
|
|
56
|
+
observability.incrementCounter({
|
|
57
|
+
name: "opengeni_sandbox_creates_total",
|
|
58
|
+
help: "Total sandbox create attempts by backend and outcome.",
|
|
59
|
+
labels: { backend, outcome },
|
|
60
|
+
});
|
|
61
|
+
observability.observeHistogram({
|
|
62
|
+
name: "opengeni_sandbox_create_duration_seconds",
|
|
63
|
+
help: "Sandbox create duration in seconds by backend.",
|
|
64
|
+
labels: { backend },
|
|
65
|
+
value: durationSeconds,
|
|
66
|
+
});
|
|
67
|
+
},
|
|
68
|
+
onSandboxWarmingTimeout: () => {
|
|
69
|
+
observability.incrementCounter({
|
|
70
|
+
name: "opengeni_sandbox_warming_timeouts_total",
|
|
71
|
+
help: "Total sandbox warming timeouts.",
|
|
72
|
+
});
|
|
73
|
+
},
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
export function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {
|
|
78
|
+
const existing = turnTrackers.get(observability);
|
|
79
|
+
if (existing) {
|
|
80
|
+
return existing;
|
|
81
|
+
}
|
|
82
|
+
const tracker = new TurnLifecycleMetrics(observability);
|
|
83
|
+
turnTrackers.set(observability, tracker);
|
|
84
|
+
return tracker;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export class TurnLifecycleMetrics {
|
|
88
|
+
private readonly startedTurns = new Map<string, number>();
|
|
89
|
+
private timer: ReturnType<typeof setInterval> | null = null;
|
|
90
|
+
|
|
91
|
+
constructor(
|
|
92
|
+
private readonly observability: Observability,
|
|
93
|
+
private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},
|
|
94
|
+
) {}
|
|
95
|
+
|
|
96
|
+
start(turnId: string): void {
|
|
97
|
+
this.startedTurns.set(turnId, this.now());
|
|
98
|
+
this.ensureTimer();
|
|
99
|
+
this.refreshGauges();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {
|
|
103
|
+
const startedAt = this.startedTurns.get(turnId);
|
|
104
|
+
if (startedAt !== undefined) {
|
|
105
|
+
this.startedTurns.delete(turnId);
|
|
106
|
+
}
|
|
107
|
+
if (outcome) {
|
|
108
|
+
const observedDuration = durationSeconds ?? (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
|
|
109
|
+
this.observability.incrementCounter({
|
|
110
|
+
name: "opengeni_turns_total",
|
|
111
|
+
help: "Total agent turns by terminal outcome.",
|
|
112
|
+
labels: { outcome },
|
|
113
|
+
});
|
|
114
|
+
this.observability.observeHistogram({
|
|
115
|
+
name: "opengeni_turn_duration_seconds",
|
|
116
|
+
help: "Agent turn duration in seconds by terminal outcome.",
|
|
117
|
+
labels: { outcome },
|
|
118
|
+
value: observedDuration,
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
this.refreshGauges();
|
|
122
|
+
if (this.startedTurns.size === 0) {
|
|
123
|
+
this.stopTimer();
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
refreshGauges(): void {
|
|
128
|
+
this.observability.setGauge({
|
|
129
|
+
name: "opengeni_turns_inflight",
|
|
130
|
+
help: "Current number of in-flight agent turns in this worker process.",
|
|
131
|
+
value: this.startedTurns.size,
|
|
132
|
+
});
|
|
133
|
+
this.observability.setGauge({
|
|
134
|
+
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
135
|
+
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
136
|
+
value: this.oldestInflightAgeSeconds(),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
stop(): void {
|
|
141
|
+
this.startedTurns.clear();
|
|
142
|
+
this.refreshGauges();
|
|
143
|
+
this.stopTimer();
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
private oldestInflightAgeSeconds(): number {
|
|
147
|
+
if (this.startedTurns.size === 0) {
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
let oldest = Number.POSITIVE_INFINITY;
|
|
151
|
+
for (const startedAt of this.startedTurns.values()) {
|
|
152
|
+
oldest = Math.min(oldest, startedAt);
|
|
153
|
+
}
|
|
154
|
+
return Math.max(0, (this.now() - oldest) / 1000);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
private ensureTimer(): void {
|
|
158
|
+
if (this.timer) {
|
|
159
|
+
return;
|
|
160
|
+
}
|
|
161
|
+
this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);
|
|
162
|
+
this.timer.unref?.();
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private stopTimer(): void {
|
|
166
|
+
if (!this.timer) {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
clearInterval(this.timer);
|
|
170
|
+
this.timer = null;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
private now(): number {
|
|
174
|
+
return this.options.now?.() ?? Date.now();
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
export function recordTurnsQueuedGauge(observability: Observability, value: number): void {
|
|
179
|
+
observability.setGauge({
|
|
180
|
+
name: "opengeni_turns_queued",
|
|
181
|
+
help: "Current number of queued session turns.",
|
|
182
|
+
value,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void {
|
|
187
|
+
for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
|
|
188
|
+
observability.setGauge({
|
|
189
|
+
name: "opengeni_sandbox_leases",
|
|
190
|
+
help: "Current sandbox leases by liveness state.",
|
|
191
|
+
labels: { liveness },
|
|
192
|
+
value: counts[liveness] ?? 0,
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export function recordSandboxOrphansTerminated(observability: Observability, count: number): void {
|
|
198
|
+
if (count <= 0) {
|
|
199
|
+
return;
|
|
200
|
+
}
|
|
201
|
+
observability.incrementCounter({
|
|
202
|
+
name: "opengeni_sandbox_orphans_terminated_total",
|
|
203
|
+
help: "Total provider-side orphan sandboxes terminated by defensive sweeps.",
|
|
204
|
+
amount: count,
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
export function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void {
|
|
209
|
+
if (!observability || amountMicros <= 0) {
|
|
210
|
+
return;
|
|
211
|
+
}
|
|
212
|
+
observability.incrementCounter({
|
|
213
|
+
name: "opengeni_credit_micros_total",
|
|
214
|
+
help: "Total credit micros recorded by kind.",
|
|
215
|
+
labels: { kind },
|
|
216
|
+
amount: amountMicros,
|
|
217
|
+
});
|
|
218
|
+
}
|
package/src/sandbox-resume.ts
CHANGED
|
@@ -47,6 +47,7 @@ import {
|
|
|
47
47
|
tagModalSandbox,
|
|
48
48
|
type EstablishedSandboxSession,
|
|
49
49
|
type ExposedPortEndpoint,
|
|
50
|
+
type RuntimeMetricsHooks,
|
|
50
51
|
} from "@opengeni/runtime";
|
|
51
52
|
import { DESKTOP_STREAM_PORT } from "@opengeni/contracts";
|
|
52
53
|
|
|
@@ -60,6 +61,7 @@ export type ResumeHolderKind = LeaseHolderKind;
|
|
|
60
61
|
export type SandboxResumeServices = {
|
|
61
62
|
db: Database;
|
|
62
63
|
settings: Settings;
|
|
64
|
+
sandboxMetrics?: RuntimeMetricsHooks;
|
|
63
65
|
};
|
|
64
66
|
|
|
65
67
|
export type ResumeBoxIds = {
|
|
@@ -173,6 +175,17 @@ function asSandboxWarmingError(error: unknown, backend: string, timeoutMs: numbe
|
|
|
173
175
|
: error;
|
|
174
176
|
}
|
|
175
177
|
|
|
178
|
+
function recordSandboxWarmingTimeout(metrics: RuntimeMetricsHooks | undefined, error: unknown): void {
|
|
179
|
+
if (!(error instanceof SandboxWarmingTimeoutError)) {
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
try {
|
|
183
|
+
metrics?.onSandboxWarmingTimeout?.({ backend: error.backend });
|
|
184
|
+
} catch {
|
|
185
|
+
// Metrics emission must never affect sandbox recovery or error propagation.
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
176
189
|
function workspaceArchiveFromEnvelope(envelope: Record<string, unknown> | null | undefined): string | null {
|
|
177
190
|
const sessionState = envelope && typeof envelope.sessionState === "object" && envelope.sessionState !== null
|
|
178
191
|
? envelope.sessionState as Record<string, unknown>
|
|
@@ -293,6 +306,7 @@ export async function resumeBoxForTurn(
|
|
|
293
306
|
sessionId: ids.sessionId,
|
|
294
307
|
backendOverride: ids.backend as never,
|
|
295
308
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
309
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
296
310
|
onSandboxCreated: async (created) => {
|
|
297
311
|
createdEstablished = created;
|
|
298
312
|
const resumeEnvelope = preserveWorkspaceArchiveOnInterimResumeState(
|
|
@@ -374,7 +388,9 @@ export async function resumeBoxForTurn(
|
|
|
374
388
|
});
|
|
375
389
|
}
|
|
376
390
|
await release();
|
|
377
|
-
|
|
391
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
392
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
393
|
+
throw warmingError;
|
|
378
394
|
}
|
|
379
395
|
}
|
|
380
396
|
|
|
@@ -400,6 +416,7 @@ export async function resumeBoxForTurn(
|
|
|
400
416
|
sessionId: ids.sessionId,
|
|
401
417
|
backendOverride: ids.backend as never,
|
|
402
418
|
...(ids.environment ? { environment: ids.environment } : {}),
|
|
419
|
+
...(services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}),
|
|
403
420
|
});
|
|
404
421
|
// Re-ensure the desktop display stack on the ATTACHED/REARMED path too — NOT
|
|
405
422
|
// just the spawner path. A turn attaching to a warm box whose :0 was never
|
|
@@ -414,7 +431,9 @@ export async function resumeBoxForTurn(
|
|
|
414
431
|
return { established, leaseEpoch, release };
|
|
415
432
|
} catch (error) {
|
|
416
433
|
await release();
|
|
417
|
-
|
|
434
|
+
const warmingError = asSandboxWarmingError(error, ids.backend, settings.sandboxWarmingTimeoutMs);
|
|
435
|
+
recordSandboxWarmingTimeout(services.sandboxMetrics, warmingError);
|
|
436
|
+
throw warmingError;
|
|
418
437
|
}
|
|
419
438
|
}
|
|
420
439
|
|