@opengeni/worker-bundle 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7113 -1652
  3. package/dist/index.js.map +1 -1
  4. package/package.json +18 -18
  5. package/src/activities/agent-turn.ts +4337 -1052
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -1,8 +1,16 @@
1
+ import { createHash } from "node:crypto";
2
+ import { errorCodeToJSON } from "@opengeni/agent-proto";
3
+ import type { SessionEventType } from "@opengeni/contracts";
1
4
  import type { EventLogger } from "@opengeni/events";
2
5
  import type { Attributes, AttributeValue, Observability } from "@opengeni/observability";
3
- import type { RuntimeMetricsHooks } from "@opengeni/runtime";
6
+ import {
7
+ SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,
8
+ type RuntimeMetricsHooks,
9
+ type SelfhostedOpObservation,
10
+ type SelfhostedOpObserver,
11
+ } from "@opengeni/runtime";
4
12
 
5
- export type TurnOutcome = "completed" | "failed" | "cancelled" | "preempted";
13
+ export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
6
14
  export type CreditMicrosKind = "usage" | "grant" | "topup" | "refund";
7
15
  export type SandboxLeaseLiveness = "cold" | "warming" | "warm" | "draining";
8
16
  export type CreditBalanceGauge = { accountId: string; balanceMicros: number };
@@ -29,7 +37,13 @@ function eventAttributes(attributes: Record<string, unknown> | undefined): Attri
29
37
  }
30
38
 
31
39
  function eventAttributeValue(value: unknown): AttributeValue {
32
- if (value === null || value === undefined || typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
40
+ if (
41
+ value === null ||
42
+ value === undefined ||
43
+ typeof value === "string" ||
44
+ typeof value === "number" ||
45
+ typeof value === "boolean"
46
+ ) {
33
47
  return value;
34
48
  }
35
49
  try {
@@ -39,7 +53,9 @@ function eventAttributeValue(value: unknown): AttributeValue {
39
53
  }
40
54
  }
41
55
 
42
- export function runtimeMetricsHooksForObservability(observability: Observability): RuntimeMetricsHooks {
56
+ export function runtimeMetricsHooksForObservability(
57
+ observability: Observability,
58
+ ): RuntimeMetricsHooks {
43
59
  return {
44
60
  onModelCall: ({ provider, outcome, durationSeconds }) => {
45
61
  observability.incrementCounter({
@@ -73,6 +89,125 @@ export function runtimeMetricsHooksForObservability(observability: Observability
73
89
  help: "Total sandbox warming timeouts.",
74
90
  });
75
91
  },
92
+ onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {
93
+ observability.incrementCounter({
94
+ name: "opengeni_machine_op_total",
95
+ help: "Total Connected Machine control ops by op, outcome, and typed fault code.",
96
+ labels: { backend, op, outcome, code: code ?? "" },
97
+ });
98
+ observability.observeHistogram({
99
+ name: "opengeni_machine_op_duration_seconds",
100
+ help: "Connected Machine control-op duration in seconds by op.",
101
+ labels: { backend, op },
102
+ value: durationSeconds,
103
+ });
104
+ // The healed-fault leading indicator: an op that only succeeded after a retry
105
+ // (a blip/backpressure the transport absorbed). The doctrine: healed faults are
106
+ // the leading indicator of the next unhealed one, so they are always recorded.
107
+ if (healed) {
108
+ observability.incrementCounter({
109
+ name: "opengeni_machine_op_healed_total",
110
+ help: "Connected Machine ops that succeeded only after ≥1 in-call retry.",
111
+ labels: { backend, op },
112
+ });
113
+ }
114
+ // The payload-wall indicator (bytes known only on a PAYLOAD_TOO_LARGE fault today).
115
+ if (replyBytes !== undefined) {
116
+ observability.observeHistogram({
117
+ name: "opengeni_machine_op_reply_bytes",
118
+ help: "Connected Machine control-op reply size in bytes (payload-wall indicator).",
119
+ labels: { backend, op },
120
+ value: replyBytes,
121
+ });
122
+ }
123
+ },
124
+ };
125
+ }
126
+
127
+ /**
128
+ * Adapt the runtime's transport-agnostic `SelfhostedOpObserver` to the metrics
129
+ * hooks: map a completed-op observation onto `onSandboxOp`, converting the wire
130
+ * `ErrorCode` to its stable enum-name string (a bounded metric label) and the
131
+ * duration to seconds. Wired into the selfhosted session build so every Connected
132
+ * Machine control op meters.
133
+ */
134
+ export function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): SelfhostedOpObserver {
135
+ return (o) => {
136
+ hooks.onSandboxOp?.({
137
+ backend: "selfhosted",
138
+ op: o.op,
139
+ outcome: o.outcome,
140
+ healed: o.healed,
141
+ retries: o.retries,
142
+ durationSeconds: o.durationMs / 1000,
143
+ ...(o.code !== undefined ? { code: errorCodeToJSON(o.code) } : {}),
144
+ ...(o.replyBytes !== undefined ? { replyBytes: o.replyBytes } : {}),
145
+ });
146
+ };
147
+ }
148
+
149
+ /** A session-scoped `machine.op.*` event mapped from a completed-op observation. */
150
+ export type MachineOpSessionEvent = {
151
+ type: "machine.op.failed" | "machine.op.recovered";
152
+ payload: { op: string; faultClass: string; attempts: number; machineId?: string };
153
+ };
154
+
155
+ /** Map an observation to a `machine.op.*` session event, or null if it is not
156
+ * eventable. `machine.op.failed` fires ONLY for infrastructure fault classes (a
157
+ * semantic miss the model asked about is an outcome, not an infra fault);
158
+ * `machine.op.recovered` fires for a healed op (success after ≥1 retry). */
159
+ export function machineOpSessionEventFor(o: SelfhostedOpObservation): MachineOpSessionEvent | null {
160
+ if (
161
+ o.outcome === "failed" &&
162
+ o.faultClass &&
163
+ SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES.has(o.faultClass)
164
+ ) {
165
+ return {
166
+ type: "machine.op.failed",
167
+ payload: {
168
+ op: o.op,
169
+ faultClass: o.faultClass,
170
+ attempts: o.retries,
171
+ ...(o.machineId ? { machineId: o.machineId } : {}),
172
+ },
173
+ };
174
+ }
175
+ if (o.outcome === "ok" && o.healed) {
176
+ return {
177
+ type: "machine.op.recovered",
178
+ payload: {
179
+ op: o.op,
180
+ faultClass: o.faultClass ?? "unknown",
181
+ attempts: o.retries,
182
+ ...(o.machineId ? { machineId: o.machineId } : {}),
183
+ },
184
+ };
185
+ }
186
+ return null;
187
+ }
188
+
189
+ /**
190
+ * The Connected Machine op observer wired into a turn: it meters EVERY op (the
191
+ * metrics sink) and BUFFERS the eventable ops (infra failures + healed recoveries)
192
+ * as `machine.op.*` session events. The observer is SYNC (fire-and-forget), so the
193
+ * turn drains the buffer to durable session events at a known checkpoint (turn end),
194
+ * awaited — never an unawaited DB write inside the Temporal activity.
195
+ */
196
+ export function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {
197
+ observer: SelfhostedOpObserver;
198
+ drainEvents(): MachineOpSessionEvent[];
199
+ } {
200
+ const meter = selfhostedOpObserverForMetrics(hooks);
201
+ const buffered: MachineOpSessionEvent[] = [];
202
+ return {
203
+ observer: (o) => {
204
+ meter(o);
205
+ const event = machineOpSessionEventFor(o);
206
+ if (event) {
207
+ buffered.push(event);
208
+ }
209
+ },
210
+ drainEvents: () => buffered.splice(0, buffered.length),
76
211
  };
77
212
  }
78
213
 
@@ -107,7 +242,9 @@ export class TurnLifecycleMetrics {
107
242
  this.startedTurns.delete(turnId);
108
243
  }
109
244
  if (outcome) {
110
- const observedDuration = durationSeconds ?? (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
245
+ const observedDuration =
246
+ durationSeconds ??
247
+ (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));
111
248
  this.observability.incrementCounter({
112
249
  name: "opengeni_turns_total",
113
250
  help: "Total agent turns by terminal outcome.",
@@ -185,7 +322,10 @@ export function recordTurnsQueuedGauge(observability: Observability, value: numb
185
322
  });
186
323
  }
187
324
 
188
- export function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void {
325
+ export function recordSandboxLeaseGauges(
326
+ observability: Observability,
327
+ counts: Partial<Record<SandboxLeaseLiveness, number>>,
328
+ ): void {
189
329
  for (const liveness of ["cold", "warming", "warm", "draining"] as const) {
190
330
  observability.setGauge({
191
331
  name: "opengeni_sandbox_leases",
@@ -196,7 +336,10 @@ export function recordSandboxLeaseGauges(observability: Observability, counts: P
196
336
  }
197
337
  }
198
338
 
199
- export function recordCreditBalanceGauges(observability: Observability, balances: CreditBalanceGauge[]): void {
339
+ export function recordCreditBalanceGauges(
340
+ observability: Observability,
341
+ balances: CreditBalanceGauge[],
342
+ ): void {
200
343
  const previous = creditBalanceGaugeAccounts.get(observability) ?? new Set<string>();
201
344
  const current = new Set<string>();
202
345
  for (const balance of balances) {
@@ -232,7 +375,11 @@ export function recordSandboxOrphansTerminated(observability: Observability, cou
232
375
  });
233
376
  }
234
377
 
235
- export function recordCreditMicros(observability: Observability | undefined, kind: CreditMicrosKind, amountMicros: number): void {
378
+ export function recordCreditMicros(
379
+ observability: Observability | undefined,
380
+ kind: CreditMicrosKind,
381
+ amountMicros: number,
382
+ ): void {
236
383
  if (!observability || amountMicros <= 0) {
237
384
  return;
238
385
  }
@@ -243,3 +390,289 @@ export function recordCreditMicros(observability: Observability | undefined, kin
243
390
  amount: amountMicros,
244
391
  });
245
392
  }
393
+
394
+ // ── Streaming SLIs ────────────────────────────────────────────────────────────
395
+ // Instruments the token-streaming pipeline so "streaming is sluggish" is a number,
396
+ // not a vibe. Split across the three attributable stages so an operator can tell
397
+ // WHERE the latency lives: the model (TTFT + inter-delta gaps), our durable write
398
+ // path (append latency), or delivery (publish latency + batcher flush shape). All
399
+ // labels are bounded (provider from the model registry, a two-value delta class) —
400
+ // never a session id or a raw user-supplied model string.
401
+
402
+ export type StreamDeltaClass = "message" | "reasoning";
403
+
404
+ /** Content-delta classes only. A `null` return means "not a content delta" — the
405
+ * event that re-arms the TTFT anchor and closes an inter-delta run. */
406
+ function contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {
407
+ if (type === "agent.message.delta") {
408
+ return "message";
409
+ }
410
+ if (type === "agent.reasoning.delta") {
411
+ return "reasoning";
412
+ }
413
+ return null;
414
+ }
415
+
416
+ // TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few
417
+ // seconds), so they get their own SHORT buckets — the default duration buckets
418
+ // (which run to 3600s) would collapse every real streaming value into one bucket.
419
+ const STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];
420
+ const STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];
421
+
422
+ /**
423
+ * Per-turn stream-timing tracker fed every normalized runtime event in push order.
424
+ * It emits two model-responsiveness SLIs from the worker's seat on the stream:
425
+ *
426
+ * - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
427
+ * first streamed content delta. The anchor starts at construction (≈ runStream
428
+ * start, so the first observation is "how long until text appears") and re-arms
429
+ * on every non-content event (a tool call, a completed message, a usage frame),
430
+ * so a post-tool response measures the model's restart latency, NOT our own
431
+ * tool-execution time.
432
+ * - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
433
+ * consecutive content deltas of the SAME class. The run resets on any
434
+ * non-content event so a gap never spans a tool call or a model boundary — it
435
+ * measures only the choppiness of a live token stream.
436
+ *
437
+ * Purely observational and clock-injectable; it never touches the events it sees.
438
+ */
439
+ export class StreamTimingMetrics {
440
+ private readonly now: () => number;
441
+ private ttftAnchor: number;
442
+ private ttftArmed = true;
443
+ private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();
444
+
445
+ constructor(
446
+ private readonly observability: Observability,
447
+ private readonly options: { provider: string; now?: () => number },
448
+ ) {
449
+ this.now = options.now ?? (() => performance.now());
450
+ this.ttftAnchor = this.now();
451
+ }
452
+
453
+ onEvent(type: SessionEventType): void {
454
+ const deltaClass = contentDeltaClass(type);
455
+ if (deltaClass === null) {
456
+ // A non-content event: the model paused emitting. Re-arm TTFT so the next
457
+ // content delta measures (re)start latency, and close every inter-delta run
458
+ // so no gap spans a tool call / model boundary.
459
+ this.ttftAnchor = this.now();
460
+ this.ttftArmed = true;
461
+ this.lastDeltaAt.clear();
462
+ return;
463
+ }
464
+ const at = this.now();
465
+ if (this.ttftArmed) {
466
+ this.observability.observeHistogram({
467
+ name: "opengeni_stream_ttft_seconds",
468
+ help: "Seconds from a model (re)start to its first streamed content delta.",
469
+ buckets: STREAM_TTFT_BUCKETS,
470
+ labels: { provider: this.options.provider },
471
+ value: Math.max(0, (at - this.ttftAnchor) / 1000),
472
+ });
473
+ this.ttftArmed = false;
474
+ }
475
+ const last = this.lastDeltaAt.get(deltaClass);
476
+ if (last !== undefined) {
477
+ this.observability.observeHistogram({
478
+ name: "opengeni_stream_inter_delta_gap_seconds",
479
+ help: "Seconds between consecutive streamed content deltas of the same class.",
480
+ buckets: STREAM_INTER_DELTA_BUCKETS,
481
+ labels: { provider: this.options.provider, class: deltaClass },
482
+ value: Math.max(0, (at - last) / 1000),
483
+ });
484
+ }
485
+ this.lastDeltaAt.set(deltaClass, at);
486
+ }
487
+ }
488
+
489
+ // Batch shapes: sizes are small integers; durations are the append+publish round
490
+ // trip the flush performs (sub-ms to a couple seconds under contention).
491
+ const STREAM_BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500];
492
+ const STREAM_IO_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5];
493
+
494
+ /** Flush shape of the streaming batcher: how many events coalesced into one flush
495
+ * (the coalescing win) and how long that flush took (append + publish). */
496
+ export function recordBatchFlush(
497
+ observability: Observability,
498
+ input: { events: number; durationSeconds: number },
499
+ ): void {
500
+ observability.observeHistogram({
501
+ name: "opengeni_stream_batch_flush_events",
502
+ help: "Session events coalesced into one streaming batcher flush.",
503
+ buckets: STREAM_BATCH_SIZE_BUCKETS,
504
+ value: input.events,
505
+ });
506
+ observability.observeHistogram({
507
+ name: "opengeni_stream_batch_flush_duration_seconds",
508
+ help: "Duration in seconds of one streaming batcher flush (append + publish).",
509
+ buckets: STREAM_IO_BUCKETS,
510
+ value: input.durationSeconds,
511
+ });
512
+ }
513
+
514
+ /** Latency of the durable `appendSessionEvents` DB write — the write path. A p99
515
+ * climb here is our Postgres, not the model or NATS. */
516
+ export function recordSessionEventAppendLatency(
517
+ observability: Observability,
518
+ input: { durationSeconds: number },
519
+ ): void {
520
+ observability.observeHistogram({
521
+ name: "opengeni_session_event_append_seconds",
522
+ help: "Duration in seconds of an appendSessionEvents DB write (the durable write path).",
523
+ buckets: STREAM_IO_BUCKETS,
524
+ value: input.durationSeconds,
525
+ });
526
+ }
527
+
528
+ /** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb
529
+ * here (with append healthy) is delivery, not the write path. */
530
+ export function recordSessionEventPublishLatency(
531
+ observability: Observability,
532
+ input: { durationSeconds: number },
533
+ ): void {
534
+ observability.observeHistogram({
535
+ name: "opengeni_session_event_publish_seconds",
536
+ help: "Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).",
537
+ buckets: STREAM_IO_BUCKETS,
538
+ value: input.durationSeconds,
539
+ });
540
+ }
541
+
542
+ // Context tokens per response span a wide range; buckets track the pressure toward
543
+ // a model's window so "sessions are running hot but never compacting" is queryable.
544
+ const MODEL_INPUT_TOKENS_BUCKETS = [
545
+ 1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,
546
+ ];
547
+
548
+ /** Observed input (context) tokens per model response — the context-pressure
549
+ * signal. Paired with `opengeni_context_compactions_total`, it makes the
550
+ * "compaction never firing while contexts run hot" failure mode expressible. */
551
+ export function recordModelInputTokens(
552
+ observability: Observability,
553
+ provider: string,
554
+ inputTokens: number,
555
+ ): void {
556
+ if (!(inputTokens > 0)) {
557
+ return;
558
+ }
559
+ observability.observeHistogram({
560
+ name: "opengeni_model_input_tokens",
561
+ help: "Observed input (context) tokens per model response, by provider.",
562
+ buckets: MODEL_INPUT_TOKENS_BUCKETS,
563
+ labels: { provider },
564
+ value: inputTokens,
565
+ });
566
+ }
567
+
568
+ /** A context compaction actually fired, by trigger (operator | overflow | proactive
569
+ * | auto). The rate of this — against the input-tokens histogram above — is how an
570
+ * operator sees compaction working (or silently not). */
571
+ export function recordContextCompaction(observability: Observability, trigger: string): void {
572
+ observability.incrementCounter({
573
+ name: "opengeni_context_compactions_total",
574
+ help: "Total context compactions performed, by trigger.",
575
+ labels: { trigger },
576
+ });
577
+ }
578
+
579
+ // ── Prompt-cache efficiency ─────────────────────────────────────────────────
580
+ // Per model-call prompt-cache signal, provider-labelled only (bounded
581
+ // cardinality — never a session id or account). `cached_tokens` is the slice of
582
+ // the prompt the provider served from its prompt cache; the ratio cached/prompt
583
+ // is the efficiency of that call. The account-switch hypothesis (a codex account
584
+ // rotation cold-starts the provider's per-account prompt cache) is tested from
585
+ // the per-call STRUCTURED LOG below — the account id is unbounded, so it is
586
+ // hashed into a log field and NEVER a Prometheus label.
587
+
588
+ // The hit ratio lives in [0, 1]; bucket tighter around the alerting threshold so
589
+ // a p50 near 40% resolves. The default duration buckets (to 3600) would collapse
590
+ // every ratio into the first bucket.
591
+ const MODEL_CACHE_HIT_RATIO_BUCKETS = [
592
+ 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1,
593
+ ];
594
+
595
+ /**
596
+ * Prompt-cache efficiency of one model response, by provider. Reads from the SAME
597
+ * usage frame that feeds input-token accounting, so the two are always consistent:
598
+ * - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the
599
+ * provider served from cache. Advanced only when >0, so a provider that never
600
+ * reports cached tokens contributes nothing rather than phantom zero-increments.
601
+ * - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,
602
+ * observed whenever prompt tokens are known (>0). A call whose provider does NOT
603
+ * report cached_tokens records a real 0 here — "we saw a call and the cache did
604
+ * nothing" is exactly the signal the alert watches, so it must not be swallowed.
605
+ * Absent/zero/non-finite `cachedTokens` (providers that don't report it) is safe: no
606
+ * counter increment, ratio 0. A call with no prompt tokens has no ratio (skipped).
607
+ */
608
+ export function recordModelCacheTokens(
609
+ observability: Observability,
610
+ provider: string,
611
+ input: { cachedTokens: number | null | undefined; promptTokens: number | null | undefined },
612
+ ): void {
613
+ const cached = nonNegativeTokenCount(input.cachedTokens);
614
+ const prompt = nonNegativeTokenCount(input.promptTokens);
615
+ if (cached > 0) {
616
+ observability.incrementCounter({
617
+ name: "opengeni_model_cached_tokens_total",
618
+ help: "Total prompt tokens served from the provider's prompt cache, by provider.",
619
+ labels: { provider },
620
+ amount: cached,
621
+ });
622
+ }
623
+ if (prompt > 0) {
624
+ observability.observeHistogram({
625
+ name: "opengeni_model_cache_hit_ratio",
626
+ help: "Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.",
627
+ buckets: MODEL_CACHE_HIT_RATIO_BUCKETS,
628
+ labels: { provider },
629
+ // Clamp to [0, 1]: a provider that (rarely) reports cached >= prompt must
630
+ // not skew the histogram past 1.0.
631
+ value: Math.min(1, cached / prompt),
632
+ });
633
+ }
634
+ }
635
+
636
+ function nonNegativeTokenCount(value: number | null | undefined): number {
637
+ return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : 0;
638
+ }
639
+
640
+ /**
641
+ * An opaque, stable, non-reversible tag for a codex credential (account) — the
642
+ * per-call log dimension the account-switch hypothesis correlates against. It is
643
+ * a hash of the NON-SECRET credential ROW id (never a token/bearer), truncated so
644
+ * it is short in logs while still distinguishing a handful of accounts without
645
+ * collision. A null/absent credential (non-codex turn, no active account) tags as
646
+ * "none" so the field is always present and never leaks an id verbatim.
647
+ */
648
+ export function stableAccountHash(credentialId: string | null | undefined): string {
649
+ if (!credentialId) {
650
+ return "none";
651
+ }
652
+ return createHash("sha256").update(credentialId).digest("hex").slice(0, 12);
653
+ }
654
+
655
+ /**
656
+ * The per-call account dimensions for the usage log: the opaque serving-account
657
+ * tag and whether that account CHANGED versus the session's previous call. Within
658
+ * one turn the serving credential is fixed, so a switch can only surface on the
659
+ * turn's FIRST call (compared against the session's durably-recorded prior
660
+ * credential); later calls in the same turn report `false`. A switch is reported
661
+ * only when there was a KNOWN prior account that differs — a session's very first
662
+ * call (no prior) is a cold start, not a switch.
663
+ */
664
+ export function modelCallAccountContext(input: {
665
+ servingCredentialId: string | null;
666
+ priorSessionCredentialId: string | null;
667
+ isFirstCallOfTurn: boolean;
668
+ }): { servingAccountHash: string; accountChangedFromPrevCall: boolean } {
669
+ const accountChangedFromPrevCall =
670
+ input.isFirstCallOfTurn &&
671
+ input.servingCredentialId !== null &&
672
+ input.priorSessionCredentialId !== null &&
673
+ input.priorSessionCredentialId !== input.servingCredentialId;
674
+ return {
675
+ servingAccountHash: stableAccountHash(input.servingCredentialId),
676
+ accountChangedFromPrevCall,
677
+ };
678
+ }
@@ -0,0 +1,46 @@
1
+ // The Temporal adaptation of the op-stream durable-resume journal
2
+ // (`OpStreamJournal`, @opengeni/runtime): the runtime leaf stays
3
+ // Temporal-free; this is where its two seams bind to the activity.
4
+ //
5
+ // * `attachGeneration` — the consumer generation the runner fences zombie
6
+ // consumers by. `Info.attempt` CANNOT serve here: `runAgentTurn` runs with
7
+ // `maximumAttempts: 1`, so worker death is healed by the WORKFLOW
8
+ // redispatching a NEW activity whose attempt is 1 again. The activity's
9
+ // `currentAttemptScheduledTimestampMs` is Temporal-server-assigned and
10
+ // strictly larger for a later dispatch (death detection is bounded below by
11
+ // the 2-minute heartbeat timeout, far above ms resolution), so it is a
12
+ // monotonic generation with zero workflow plumbing. Intra-activity
13
+ // re-attaches (blip/gap heals) reuse it — the runner's equal-generation
14
+ // replay path.
15
+ //
16
+ // * `persistSettled` — the durable-before-wire-ack ordering hook: fold the
17
+ // settled op's frontier into the activity's shared heartbeat details and
18
+ // flush a heartbeat BEFORE the client publishes the final ack. The shared
19
+ // `details` object is the SAME one every other heartbeat site spreads, so a
20
+ // later phase heartbeat re-carries the roster instead of clobbering it.
21
+
22
+ import type { Context } from "@temporalio/activity";
23
+ import type { OpStreamJournal } from "@opengeni/runtime";
24
+
25
+ /** The activity's shared heartbeat-details object: every heartbeat call site
26
+ * spreads THIS one object (plus its own phase), so fields like the op-ack
27
+ * roster survive across sites instead of being lost to last-write-wins. */
28
+ export interface TurnHeartbeatDetails extends Record<string, unknown> {
29
+ /** Settled-op roster: durable op id → its exit seq (the final-ack frontier). */
30
+ opAcks: Record<string, string>;
31
+ }
32
+
33
+ export function makeTurnOpJournal(
34
+ context: Context | null,
35
+ details: TurnHeartbeatDetails,
36
+ ): OpStreamJournal {
37
+ return {
38
+ attachGeneration: () => String(context?.info.currentAttemptScheduledTimestampMs ?? 1),
39
+ persistSettled: (opId, exitSeq) => {
40
+ details.opAcks[opId] = exitSeq;
41
+ // Flush immediately (heartbeats are also sent on the 10s timer; this one
42
+ // exists so the persist precedes the wire final ack, not just eventually).
43
+ context?.heartbeat({ ...details, phase: "op_settled", at: new Date().toISOString() });
44
+ },
45
+ };
46
+ }