@opengeni/worker-bundle 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +182 -67
- package/dist/index.js +7127 -1654
- package/dist/index.js.map +1 -1
- package/package.json +19 -19
- package/src/activities/agent-turn.ts +4405 -1089
- package/src/activities/capabilities.ts +43 -8
- package/src/activities/codex-capacity.ts +223 -0
- package/src/activities/codex-rotation.ts +550 -54
- package/src/activities/common.ts +7 -1
- package/src/activities/context-compaction.ts +152 -118
- package/src/activities/documents.ts +32 -27
- package/src/activities/environment.ts +318 -75
- package/src/activities/file-upload-reaper.ts +79 -0
- package/src/activities/git-credential-renewal.ts +183 -0
- package/src/activities/goals.ts +142 -123
- package/src/activities/packs.ts +55 -5
- package/src/activities/parent-wake.ts +228 -41
- package/src/activities/recording.ts +154 -122
- package/src/activities/redaction.ts +3 -1
- package/src/activities/rig-verification.ts +441 -0
- package/src/activities/run-input.ts +110 -122
- package/src/activities/sandbox-lease.ts +186 -51
- package/src/activities/scheduled-tasks.ts +287 -171
- package/src/activities/session-state.ts +168 -157
- package/src/activities/streaming.ts +143 -19
- package/src/activities/turn-attempt-fenced.ts +14 -0
- package/src/activities/types.ts +102 -32
- package/src/activities/workflow-wake.ts +44 -0
- package/src/activities/workspace-capture.ts +964 -0
- package/src/activities.ts +75 -35
- package/src/concurrency.ts +11 -0
- package/src/http.ts +16 -6
- package/src/index.ts +322 -49
- package/src/observability-metrics.ts +441 -8
- package/src/op-journal.ts +46 -0
- package/src/sandbox-resume.ts +431 -201
- package/src/sandbox-routing.ts +209 -22
- package/src/workflow-wake-contract.ts +5 -0
- package/src/workflows/activities.ts +58 -7
- package/src/workflows/file-upload-reaper.ts +12 -0
- package/src/workflows/rig-verification.ts +28 -0
- package/src/workflows/scheduled-tasks.ts +12 -26
- package/src/workflows/session.ts +330 -173
- package/src/workflows/workflow-wake.ts +12 -0
- package/src/workflows.ts +9 -7
|
@@ -1,8 +1,16 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { errorCodeToJSON } from "@opengeni/agent-proto";
|
|
3
|
+
import type { SessionEventType } from "@opengeni/contracts";
|
|
1
4
|
import type { EventLogger } from "@opengeni/events";
|
|
2
5
|
import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
|
|
3
|
-
import
|
|
6
|
+
import {
|
|
7
|
+
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
|
|
8
|
+
type RuntimeMetricsHooks,
|
|
9
|
+
type SelfhostedOpObservation,
|
|
10
|
+
type SelfhostedOpObserver,
|
|
11
|
+
} from "@opengeni/runtime";
|
|
4
12
|
|
|
5
|
-
export type TurnOutcome = "completed" | "failed" | "cancelled" | "
|
|
13
|
+
export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
|
|
6
14
|
export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
|
|
7
15
|
export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
|
|
8
16
|
export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
|
|
@@ -29,7 +37,13 @@ function eventAttributes(attributes: Record<string, unknown> | undefined): Attri
|
|
|
29
37
|
}
|
|
30
38
|
|
|
31
39
|
function eventAttributeValue(value: unknown): AttributeValue {
|
|
32
|
-
if (
|
|
40
|
+
if (
|
|
41
|
+
value === null ||
|
|
42
|
+
value === undefined ||
|
|
43
|
+
typeof value === "string" ||
|
|
44
|
+
typeof value === "number" ||
|
|
45
|
+
typeof value === "boolean"
|
|
46
|
+
) {
|
|
33
47
|
return value;
|
|
34
48
|
}
|
|
35
49
|
try {
|
|
@@ -39,7 +53,9 @@ function eventAttributeValue(value: unknown): AttributeValue {
|
|
|
39
53
|
}
|
|
40
54
|
}
|
|
41
55
|
|
|
42
|
-
export function runtimeMetricsHooksForObservability(
|
|
56
|
+
export function runtimeMetricsHooksForObservability(
|
|
57
|
+
observability: Observability,
|
|
58
|
+
): RuntimeMetricsHooks {
|
|
43
59
|
return {
|
|
44
60
|
onModelCall: ({ provider, outcome, durationSeconds }) => {
|
|
45
61
|
observability.incrementCounter({
|
|
@@ -73,6 +89,125 @@ export function runtimeMetricsHooksForObservability(observability: Observability
|
|
|
73
89
|
help: "Total sandbox warming timeouts.",
|
|
74
90
|
});
|
|
75
91
|
},
|
|
92
|
+
onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
|
|
93
|
+
observability.incrementCounter({
|
|
94
|
+
name: "opengeni_machine_op_total",
|
|
95
|
+
help: "Total Connected Machine control ops by op, outcome, and typed fault code.",
|
|
96
|
+
labels: { backend, op, outcome, code: code ?? "" },
|
|
97
|
+
});
|
|
98
|
+
observability.observeHistogram({
|
|
99
|
+
name: "opengeni_machine_op_duration_seconds",
|
|
100
|
+
help: "Connected Machine control-op duration in seconds by op.",
|
|
101
|
+
labels: { backend, op },
|
|
102
|
+
value: durationSeconds,
|
|
103
|
+
});
|
|
104
|
+
// The healed-fault leading indicator: an op that only succeeded after a retry
|
|
105
|
+
// (a blip/backpressure the transport absorbed). The doctrine: healed faults are
|
|
106
|
+
// the leading indicator of the next unhealed one, so they are always recorded.
|
|
107
|
+
if (healed) {
|
|
108
|
+
observability.incrementCounter({
|
|
109
|
+
name: "opengeni_machine_op_healed_total",
|
|
110
|
+
help: "Connected Machine ops that succeeded only after ≥1 in-call retry.",
|
|
111
|
+
labels: { backend, op },
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
// The payload-wall indicator (bytes known only on a PAYLOAD_TOO_LARGE fault today).
|
|
115
|
+
if (replyBytes !== undefined) {
|
|
116
|
+
observability.observeHistogram({
|
|
117
|
+
name: "opengeni_machine_op_reply_bytes",
|
|
118
|
+
help: "Connected Machine control-op reply size in bytes (payload-wall indicator).",
|
|
119
|
+
labels: { backend, op },
|
|
120
|
+
value: replyBytes,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Adapt the runtime's transport-agnostic `SelfhostedOpObserver` to the metrics
|
|
129
|
+
* hooks: map a completed-op observation onto `onSandboxOp`, converting the wire
|
|
130
|
+
* `ErrorCode` to its stable enum-name string (a bounded metric label) and the
|
|
131
|
+
* duration to seconds. Wired into the selfhosted session build so every Connected
|
|
132
|
+
* Machine control op meters.
|
|
133
|
+
*/
|
|
134
|
+
export function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): SelfhostedOpObserver {
|
|
135
|
+
return (o) => {
|
|
136
|
+
hooks.onSandboxOp?.({
|
|
137
|
+
backend: "selfhosted",
|
|
138
|
+
op: o.op,
|
|
139
|
+
outcome: o.outcome,
|
|
140
|
+
healed: o.healed,
|
|
141
|
+
retries: o.retries,
|
|
142
|
+
durationSeconds: o.durationMs / 1000,
|
|
143
|
+
...(o.code !== undefined ? { code: errorCodeToJSON(o.code) } : {}),
|
|
144
|
+
...(o.replyBytes !== undefined ? { replyBytes: o.replyBytes } : {}),
|
|
145
|
+
});
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** A session-scoped `machine.op.*` event mapped from a completed-op observation. */
|
|
150
|
+
export type MachineOpSessionEvent = {
|
|
151
|
+
type: "machine.op.failed" | "machine.op.recovered";
|
|
152
|
+
payload: { op: string; faultClass: string; attempts: number; machineId?: string };
|
|
153
|
+
};
|
|
154
|
+
|
|
155
|
+
/** Map an observation to a `machine.op.*` session event, or null if it is not
|
|
156
|
+
* eventable. `machine.op.failed` fires ONLY for infrastructure fault classes (a
|
|
157
|
+
* semantic miss the model asked about is an outcome, not an infra fault);
|
|
158
|
+
* `machine.op.recovered` fires for a healed op (success after ≥1 retry). */
|
|
159
|
+
export function machineOpSessionEventFor(o: SelfhostedOpObservation): MachineOpSessionEvent | null {
|
|
160
|
+
if (
|
|
161
|
+
o.outcome === "failed" &&
|
|
162
|
+
o.faultClass &&
|
|
163
|
+
SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES.has(o.faultClass)
|
|
164
|
+
) {
|
|
165
|
+
return {
|
|
166
|
+
type: "machine.op.failed",
|
|
167
|
+
payload: {
|
|
168
|
+
op: o.op,
|
|
169
|
+
faultClass: o.faultClass,
|
|
170
|
+
attempts: o.retries,
|
|
171
|
+
...(o.machineId ? { machineId: o.machineId } : {}),
|
|
172
|
+
},
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
if (o.outcome === "ok" && o.healed) {
|
|
176
|
+
return {
|
|
177
|
+
type: "machine.op.recovered",
|
|
178
|
+
payload: {
|
|
179
|
+
op: o.op,
|
|
180
|
+
faultClass: o.faultClass ?? "unknown",
|
|
181
|
+
attempts: o.retries,
|
|
182
|
+
...(o.machineId ? { machineId: o.machineId } : {}),
|
|
183
|
+
},
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
return null;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* The Connected Machine op observer wired into a turn: it meters EVERY op (the
|
|
191
|
+
* metrics sink) and BUFFERS the eventable ops (infra failures + healed recoveries)
|
|
192
|
+
* as `machine.op.*` session events. The observer is SYNC (fire-and-forget), so the
|
|
193
|
+
* turn drains the buffer to durable session events at a known checkpoint (turn end),
|
|
194
|
+
* awaited — never an unawaited DB write inside the Temporal activity.
|
|
195
|
+
*/
|
|
196
|
+
export function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {
|
|
197
|
+
observer: SelfhostedOpObserver;
|
|
198
|
+
drainEvents(): MachineOpSessionEvent[];
|
|
199
|
+
} {
|
|
200
|
+
const meter = selfhostedOpObserverForMetrics(hooks);
|
|
201
|
+
const buffered: MachineOpSessionEvent[] = [];
|
|
202
|
+
return {
|
|
203
|
+
observer: (o) => {
|
|
204
|
+
meter(o);
|
|
205
|
+
const event = machineOpSessionEventFor(o);
|
|
206
|
+
if (event) {
|
|
207
|
+
buffered.push(event);
|
|
208
|
+
}
|
|
209
|
+
},
|
|
210
|
+
drainEvents: () => buffered.splice(0, buffered.length),
|
|
76
211
|
};
|
|
77
212
|
}
|
|
78
213
|
|
|
@@ -107,7 +242,9 @@ export class TurnLifecycleMetrics {
|
|
|
107
242
|
this.startedTurns.delete(turnId);
|
|
108
243
|
}
|
|
109
244
|
if (outcome) {
|
|
110
|
-
const observedDuration =
|
|
245
|
+
const observedDuration =
|
|
246
|
+
durationSeconds ??
|
|
247
|
+
(startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
|
|
111
248
|
this.observability.incrementCounter({
|
|
112
249
|
name: "opengeni_turns_total",
|
|
113
250
|
help: "Total agent turns by terminal outcome.",
|
|
@@ -185,7 +322,10 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
|
|
|
185
322
|
});
|
|
186
323
|
}
|
|
187
324
|
|
|
188
|
-
export function recordSandboxLeaseGauges(
|
|
325
|
+
export function recordSandboxLeaseGauges(
|
|
326
|
+
observability: Observability,
|
|
327
|
+
counts: Partial<Record<SandboxLeaseLiveness, number>>,
|
|
328
|
+
): void {
|
|
189
329
|
for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
|
|
190
330
|
observability.setGauge({
|
|
191
331
|
name: "opengeni_sandbox_leases",
|
|
@@ -196,7 +336,10 @@ export function recordSandboxLeaseGauges(observability: Observability, counts: P
|
|
|
196
336
|
}
|
|
197
337
|
}
|
|
198
338
|
|
|
199
|
-
export function recordCreditBalanceGauges(
|
|
339
|
+
export function recordCreditBalanceGauges(
|
|
340
|
+
observability: Observability,
|
|
341
|
+
balances: CreditBalanceGauge[],
|
|
342
|
+
): void {
|
|
200
343
|
const previous = creditBalanceGaugeAccounts.get(observability) ?? new Set<string>();
|
|
201
344
|
const current = new Set<string>();
|
|
202
345
|
for (const balance of balances) {
|
|
@@ -232,7 +375,11 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
|
|
|
232
375
|
});
|
|
233
376
|
}
|
|
234
377
|
|
|
235
|
-
export function recordCreditMicros(
|
|
378
|
+
export function recordCreditMicros(
|
|
379
|
+
observability: Observability | undefined,
|
|
380
|
+
kind: CreditMicrosKind,
|
|
381
|
+
amountMicros: number,
|
|
382
|
+
): void {
|
|
236
383
|
if (!observability || amountMicros <= 0) {
|
|
237
384
|
return;
|
|
238
385
|
}
|
|
@@ -243,3 +390,289 @@ export function recordCreditMicros(observability: Observability | undefined, kin
|
|
|
243
390
|
amount: amountMicros,
|
|
244
391
|
});
|
|
245
392
|
}
|
|
393
|
+
|
|
394
|
+
// ── Streaming SLIs ────────────────────────────────────────────────────────────
|
|
395
|
+
// Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
|
|
396
|
+
// not a vibe. Split across the three attributable stages so an operator can tell
|
|
397
|
+
// WHERE the latency lives: the model (TTFT + inter-delta gaps), our durable write
|
|
398
|
+
// path (append latency), or delivery (publish latency + batcher flush shape). All
|
|
399
|
+
// labels are bounded (provider from the model registry, a two-value delta class) —
|
|
400
|
+
// never a session id or a raw user-supplied model string.
|
|
401
|
+
|
|
402
|
+
export type StreamDeltaClass = "message" | "reasoning";
|
|
403
|
+
|
|
404
|
+
/** Content-delta classes only. A `null` return means "not a content delta" — the
|
|
405
|
+
* event that re-arms the TTFT anchor and closes an inter-delta run. */
|
|
406
|
+
function contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {
|
|
407
|
+
if (type === "agent.message.delta") {
|
|
408
|
+
return "message";
|
|
409
|
+
}
|
|
410
|
+
if (type === "agent.reasoning.delta") {
|
|
411
|
+
return "reasoning";
|
|
412
|
+
}
|
|
413
|
+
return null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few
|
|
417
|
+
// seconds), so they get their own SHORT buckets — the default duration buckets
|
|
418
|
+
// (which run to 3600s) would collapse every real streaming value into one bucket.
|
|
419
|
+
const STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];
|
|
420
|
+
const STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
|
|
421
|
+
|
|
422
|
+
/**
|
|
423
|
+
* Per-turn stream-timing tracker fed every normalized runtime event in push order.
|
|
424
|
+
* It emits two model-responsiveness SLIs from the worker's seat on the stream:
|
|
425
|
+
*
|
|
426
|
+
* - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
|
|
427
|
+
* first streamed content delta. The anchor starts at construction (≈ runStream
|
|
428
|
+
* start, so the first observation is "how long until text appears") and re-arms
|
|
429
|
+
* on every non-content event (a tool call, a completed message, a usage frame),
|
|
430
|
+
* so a post-tool response measures the model's restart latency, NOT our own
|
|
431
|
+
* tool-execution time.
|
|
432
|
+
* - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
|
|
433
|
+
* consecutive content deltas of the SAME class. The run resets on any
|
|
434
|
+
* non-content event so a gap never spans a tool call or a model boundary — it
|
|
435
|
+
* measures only the choppiness of a live token stream.
|
|
436
|
+
*
|
|
437
|
+
* Purely observational and clock-injectable; it never touches the events it sees.
|
|
438
|
+
*/
|
|
439
|
+
export class StreamTimingMetrics {
|
|
440
|
+
private readonly now: () => number;
|
|
441
|
+
private ttftAnchor: number;
|
|
442
|
+
private ttftArmed = true;
|
|
443
|
+
private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();
|
|
444
|
+
|
|
445
|
+
constructor(
|
|
446
|
+
private readonly observability: Observability,
|
|
447
|
+
private readonly options: { provider: string; now?: () => number },
|
|
448
|
+
) {
|
|
449
|
+
this.now = options.now ?? (() => performance.now());
|
|
450
|
+
this.ttftAnchor = this.now();
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
onEvent(type: SessionEventType): void {
|
|
454
|
+
const deltaClass = contentDeltaClass(type);
|
|
455
|
+
if (deltaClass === null) {
|
|
456
|
+
// A non-content event: the model paused emitting. Re-arm TTFT so the next
|
|
457
|
+
// content delta measures (re)start latency, and close every inter-delta run
|
|
458
|
+
// so no gap spans a tool call / model boundary.
|
|
459
|
+
this.ttftAnchor = this.now();
|
|
460
|
+
this.ttftArmed = true;
|
|
461
|
+
this.lastDeltaAt.clear();
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
const at = this.now();
|
|
465
|
+
if (this.ttftArmed) {
|
|
466
|
+
this.observability.observeHistogram({
|
|
467
|
+
name: "opengeni_stream_ttft_seconds",
|
|
468
|
+
help: "Seconds from a model (re)start to its first streamed content delta.",
|
|
469
|
+
buckets: STREAM_TTFT_BUCKETS,
|
|
470
|
+
labels: { provider: this.options.provider },
|
|
471
|
+
value: Math.max(0, (at - this.ttftAnchor) / 1000),
|
|
472
|
+
});
|
|
473
|
+
this.ttftArmed = false;
|
|
474
|
+
}
|
|
475
|
+
const last = this.lastDeltaAt.get(deltaClass);
|
|
476
|
+
if (last !== undefined) {
|
|
477
|
+
this.observability.observeHistogram({
|
|
478
|
+
name: "opengeni_stream_inter_delta_gap_seconds",
|
|
479
|
+
help: "Seconds between consecutive streamed content deltas of the same class.",
|
|
480
|
+
buckets: STREAM_INTER_DELTA_BUCKETS,
|
|
481
|
+
labels: { provider: this.options.provider, class: deltaClass },
|
|
482
|
+
value: Math.max(0, (at - last) / 1000),
|
|
483
|
+
});
|
|
484
|
+
}
|
|
485
|
+
this.lastDeltaAt.set(deltaClass, at);
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
// Batch shapes: sizes are small integers; durations are the append+publish round
|
|
490
|
+
// trip the flush performs (sub-ms to a couple seconds under contention).
|
|
491
|
+
const STREAM_BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500];
|
|
492
|
+
const STREAM_IO_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5];
|
|
493
|
+
|
|
494
|
+
/** Flush shape of the streaming batcher: how many events coalesced into one flush
|
|
495
|
+
* (the coalescing win) and how long that flush took (append + publish). */
|
|
496
|
+
export function recordBatchFlush(
|
|
497
|
+
observability: Observability,
|
|
498
|
+
input: { events: number; durationSeconds: number },
|
|
499
|
+
): void {
|
|
500
|
+
observability.observeHistogram({
|
|
501
|
+
name: "opengeni_stream_batch_flush_events",
|
|
502
|
+
help: "Session events coalesced into one streaming batcher flush.",
|
|
503
|
+
buckets: STREAM_BATCH_SIZE_BUCKETS,
|
|
504
|
+
value: input.events,
|
|
505
|
+
});
|
|
506
|
+
observability.observeHistogram({
|
|
507
|
+
name: "opengeni_stream_batch_flush_duration_seconds",
|
|
508
|
+
help: "Duration in seconds of one streaming batcher flush (append + publish).",
|
|
509
|
+
buckets: STREAM_IO_BUCKETS,
|
|
510
|
+
value: input.durationSeconds,
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/** Latency of the durable `appendSessionEvents` DB write — the write path. A p99
|
|
515
|
+
* climb here is our Postgres, not the model or NATS. */
|
|
516
|
+
export function recordSessionEventAppendLatency(
|
|
517
|
+
observability: Observability,
|
|
518
|
+
input: { durationSeconds: number },
|
|
519
|
+
): void {
|
|
520
|
+
observability.observeHistogram({
|
|
521
|
+
name: "opengeni_session_event_append_seconds",
|
|
522
|
+
help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
|
|
523
|
+
buckets: STREAM_IO_BUCKETS,
|
|
524
|
+
value: input.durationSeconds,
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
|
|
529
|
+
* here (with append healthy) is delivery, not the write path. */
|
|
530
|
+
export function recordSessionEventPublishLatency(
|
|
531
|
+
observability: Observability,
|
|
532
|
+
input: { durationSeconds: number },
|
|
533
|
+
): void {
|
|
534
|
+
observability.observeHistogram({
|
|
535
|
+
name: "opengeni_session_event_publish_seconds",
|
|
536
|
+
help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
|
|
537
|
+
buckets: STREAM_IO_BUCKETS,
|
|
538
|
+
value: input.durationSeconds,
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
// Context tokens per response span a wide range; buckets track the pressure toward
|
|
543
|
+
// a model's window so "sessions are running hot but never compacting" is queryable.
|
|
544
|
+
const MODEL_INPUT_TOKENS_BUCKETS = [
|
|
545
|
+
1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
|
|
546
|
+
];
|
|
547
|
+
|
|
548
|
+
/** Observed input (context) tokens per model response — the context-pressure
|
|
549
|
+
* signal. Paired with `opengeni_context_compactions_total`, it makes the
|
|
550
|
+
* "compaction never firing while contexts run hot" failure mode expressible. */
|
|
551
|
+
export function recordModelInputTokens(
|
|
552
|
+
observability: Observability,
|
|
553
|
+
provider: string,
|
|
554
|
+
inputTokens: number,
|
|
555
|
+
): void {
|
|
556
|
+
if (!(inputTokens > 0)) {
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
observability.observeHistogram({
|
|
560
|
+
name: "opengeni_model_input_tokens",
|
|
561
|
+
help: "Observed input (context) tokens per model response, by provider.",
|
|
562
|
+
buckets: MODEL_INPUT_TOKENS_BUCKETS,
|
|
563
|
+
labels: { provider },
|
|
564
|
+
value: inputTokens,
|
|
565
|
+
});
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
/** A context compaction actually fired, by trigger (operator | overflow | proactive
|
|
569
|
+
* | auto). The rate of this — against the input-tokens histogram above — is how an
|
|
570
|
+
* operator sees compaction working (or silently not). */
|
|
571
|
+
export function recordContextCompaction(observability: Observability, trigger: string): void {
|
|
572
|
+
observability.incrementCounter({
|
|
573
|
+
name: "opengeni_context_compactions_total",
|
|
574
|
+
help: "Total context compactions performed, by trigger.",
|
|
575
|
+
labels: { trigger },
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
|
|
579
|
+
// ── Prompt-cache efficiency ─────────────────────────────────────────────────
|
|
580
|
+
// Per model-call prompt-cache signal, provider-labelled only (bounded
|
|
581
|
+
// cardinality — never a session id or account). `cached_tokens` is the slice of
|
|
582
|
+
// the prompt the provider served from its prompt cache; the ratio cached/prompt
|
|
583
|
+
// is the efficiency of that call. The account-switch hypothesis (a codex account
|
|
584
|
+
// rotation cold-starts the provider's per-account prompt cache) is tested from
|
|
585
|
+
// the per-call STRUCTURED LOG below — the account id is unbounded, so it is
|
|
586
|
+
// hashed into a log field and NEVER a Prometheus label.
|
|
587
|
+
|
|
588
|
+
// The hit ratio lives in [0, 1]; bucket tighter around the alerting threshold so
|
|
589
|
+
// a p50 near 40% resolves. The default duration buckets (to 3600) would collapse
|
|
590
|
+
// every ratio into the first bucket.
|
|
591
|
+
const MODEL_CACHE_HIT_RATIO_BUCKETS = [
|
|
592
|
+
0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1,
|
|
593
|
+
];
|
|
594
|
+
|
|
595
|
+
/**
|
|
596
|
+
* Prompt-cache efficiency of one model response, by provider. Reads from the SAME
|
|
597
|
+
* usage frame that feeds input-token accounting, so the two are always consistent:
|
|
598
|
+
* - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
|
|
599
|
+
* provider served from cache. Advanced only when >0, so a provider that never
|
|
600
|
+
* reports cached tokens contributes nothing rather than phantom zero-increments.
|
|
601
|
+
* - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
|
|
602
|
+
* observed whenever prompt tokens are known (>0). A call whose provider does NOT
|
|
603
|
+
* report cached_tokens records a real 0 here — "we saw a call and the cache did
|
|
604
|
+
* nothing" is exactly the signal the alert watches, so it must not be swallowed.
|
|
605
|
+
* Absent/zero/non-finite `cachedTokens` (providers that don't report it) is safe: no
|
|
606
|
+
* counter increment, ratio 0. A call with no prompt tokens has no ratio (skipped).
|
|
607
|
+
*/
|
|
608
|
+
export function recordModelCacheTokens(
|
|
609
|
+
observability: Observability,
|
|
610
|
+
provider: string,
|
|
611
|
+
input: { cachedTokens: number | null | undefined; promptTokens: number | null | undefined },
|
|
612
|
+
): void {
|
|
613
|
+
const cached = nonNegativeTokenCount(input.cachedTokens);
|
|
614
|
+
const prompt = nonNegativeTokenCount(input.promptTokens);
|
|
615
|
+
if (cached > 0) {
|
|
616
|
+
observability.incrementCounter({
|
|
617
|
+
name: "opengeni_model_cached_tokens_total",
|
|
618
|
+
help: "Total prompt tokens served from the provider's prompt cache, by provider.",
|
|
619
|
+
labels: { provider },
|
|
620
|
+
amount: cached,
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
if (prompt > 0) {
|
|
624
|
+
observability.observeHistogram({
|
|
625
|
+
name: "opengeni_model_cache_hit_ratio",
|
|
626
|
+
help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
|
|
627
|
+
buckets: MODEL_CACHE_HIT_RATIO_BUCKETS,
|
|
628
|
+
labels: { provider },
|
|
629
|
+
// Clamp to [0, 1]: a provider that (rarely) reports cached >= prompt must
|
|
630
|
+
// not skew the histogram past 1.0.
|
|
631
|
+
value: Math.min(1, cached / prompt),
|
|
632
|
+
});
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
function nonNegativeTokenCount(value: number | null | undefined): number {
|
|
637
|
+
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/**
|
|
641
|
+
* An opaque, stable, non-reversible tag for a codex credential (account) — the
|
|
642
|
+
* per-call log dimension the account-switch hypothesis correlates against. It is
|
|
643
|
+
* a hash of the NON-SECRET credential ROW id (never a token/bearer), truncated so
|
|
644
|
+
* it is short in logs while still distinguishing a handful of accounts without
|
|
645
|
+
* collision. A null/absent credential (non-codex turn, no active account) tags as
|
|
646
|
+
* "none" so the field is always present and never leaks an id verbatim.
|
|
647
|
+
*/
|
|
648
|
+
export function stableAccountHash(credentialId: string | null | undefined): string {
|
|
649
|
+
if (!credentialId) {
|
|
650
|
+
return "none";
|
|
651
|
+
}
|
|
652
|
+
return createHash("sha256").update(credentialId).digest("hex").slice(0, 12);
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
/**
|
|
656
|
+
* The per-call account dimensions for the usage log: the opaque serving-account
|
|
657
|
+
* tag and whether that account CHANGED versus the session's previous call. Within
|
|
658
|
+
* one turn the serving credential is fixed, so a switch can only surface on the
|
|
659
|
+
* turn's FIRST call (compared against the session's durably-recorded prior
|
|
660
|
+
* credential); later calls in the same turn report `false`. A switch is reported
|
|
661
|
+
* only when there was a KNOWN prior account that differs — a session's very first
|
|
662
|
+
* call (no prior) is a cold start, not a switch.
|
|
663
|
+
*/
|
|
664
|
+
export function modelCallAccountContext(input: {
|
|
665
|
+
servingCredentialId: string | null;
|
|
666
|
+
priorSessionCredentialId: string | null;
|
|
667
|
+
isFirstCallOfTurn: boolean;
|
|
668
|
+
}): { servingAccountHash: string; accountChangedFromPrevCall: boolean } {
|
|
669
|
+
const accountChangedFromPrevCall =
|
|
670
|
+
input.isFirstCallOfTurn &&
|
|
671
|
+
input.servingCredentialId !== null &&
|
|
672
|
+
input.priorSessionCredentialId !== null &&
|
|
673
|
+
input.priorSessionCredentialId !== input.servingCredentialId;
|
|
674
|
+
return {
|
|
675
|
+
servingAccountHash: stableAccountHash(input.servingCredentialId),
|
|
676
|
+
accountChangedFromPrevCall,
|
|
677
|
+
};
|
|
678
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// The Temporal adaptation of the op-stream durable-resume journal
|
|
2
|
+
// (`OpStreamJournal`, @opengeni/runtime): the runtime leaf stays
|
|
3
|
+
// Temporal-free; this is where its two seams bind to the activity.
|
|
4
|
+
//
|
|
5
|
+
// * `attachGeneration` — the consumer generation the runner fences zombie
|
|
6
|
+
// consumers by. `Info.attempt` CANNOT serve here: `runAgentTurn` runs with
|
|
7
|
+
// `maximumAttempts: 1`, so worker death is healed by the WORKFLOW
|
|
8
|
+
// redispatching a NEW activity whose attempt is 1 again. The activity's
|
|
9
|
+
// `currentAttemptScheduledTimestampMs` is Temporal-server-assigned and
|
|
10
|
+
// strictly larger for a later dispatch (death detection is bounded below by
|
|
11
|
+
// the 2-minute heartbeat timeout, far above ms resolution), so it is a
|
|
12
|
+
// monotonic generation with zero workflow plumbing. Intra-activity
|
|
13
|
+
// re-attaches (blip/gap heals) reuse it — the runner's equal-generation
|
|
14
|
+
// replay path.
|
|
15
|
+
//
|
|
16
|
+
// * `persistSettled` — the durable-before-wire-ack ordering hook: fold the
|
|
17
|
+
// settled op's frontier into the activity's shared heartbeat details and
|
|
18
|
+
// flush a heartbeat BEFORE the client publishes the final ack. The shared
|
|
19
|
+
// `details` object is the SAME one every other heartbeat site spreads, so a
|
|
20
|
+
// later phase heartbeat re-carries the roster instead of clobbering it.
|
|
21
|
+
|
|
22
|
+
import type { Context } from "@temporalio/activity";
|
|
23
|
+
import type { OpStreamJournal } from "@opengeni/runtime";
|
|
24
|
+
|
|
25
|
+
/** The activity's shared heartbeat-details object: every heartbeat call site
|
|
26
|
+
* spreads THIS one object (plus its own phase), so fields like the op-ack
|
|
27
|
+
* roster survive across sites instead of being lost to last-write-wins. */
|
|
28
|
+
export interface TurnHeartbeatDetails extends Record<string, unknown> {
|
|
29
|
+
/** Settled-op roster: durable op id → its exit seq (the final-ack frontier). */
|
|
30
|
+
opAcks: Record<string, string>;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function makeTurnOpJournal(
|
|
34
|
+
context: Context | null,
|
|
35
|
+
details: TurnHeartbeatDetails,
|
|
36
|
+
): OpStreamJournal {
|
|
37
|
+
return {
|
|
38
|
+
attachGeneration: () => String(context?.info.currentAttemptScheduledTimestampMs ?? 1),
|
|
39
|
+
persistSettled: (opId, exitSeq) => {
|
|
40
|
+
details.opAcks[opId] = exitSeq;
|
|
41
|
+
// Flush immediately (heartbeats are also sent on the 10s timer; this one
|
|
42
|
+
// exists so the persist precedes the wire final ack, not just eventually).
|
|
43
|
+
context?.heartbeat({ ...details, phase: "op_settled", at: new Date().toISOString() });
|
|
44
|
+
},
|
|
45
|
+
};
|
|
46
|
+
}
|