@opengeni/worker-bundle 0.17.2 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/dist/activities/agent-run-admission.d.ts +12 -0
  2. package/dist/activities/agent-turn.d.ts +25 -0
  3. package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
  4. package/dist/activities/capabilities.d.ts +6 -0
  5. package/dist/activities/codex-capacity.d.ts +1 -0
  6. package/dist/activities/environment.d.ts +7 -1
  7. package/dist/activities/packs.d.ts +8 -10
  8. package/dist/activities/session-state.d.ts +4 -2
  9. package/dist/activities/site-auth-maintenance.d.ts +27 -0
  10. package/dist/activities/types.d.ts +16 -0
  11. package/dist/activities/video-generation-admission.d.ts +9 -4
  12. package/dist/activities/video-generation-credential.d.ts +17 -0
  13. package/dist/activities/video-reference-staging.d.ts +7 -1
  14. package/dist/activities/xai-auth.d.ts +23 -0
  15. package/dist/activities/xai-image-generation.d.ts +26 -0
  16. package/dist/activities/xai-video-generation.d.ts +18 -0
  17. package/dist/{activities-control-44KBWLSM.js → activities-control-XURE4VKN.js} +387 -51
  18. package/dist/activities-control-XURE4VKN.js.map +1 -0
  19. package/dist/activities-control.d.ts +8 -0
  20. package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-6EBXGGLS.js} +5312 -3951
  21. package/dist/activities-turn-6EBXGGLS.js.map +1 -0
  22. package/dist/activities.d.ts +9 -1
  23. package/dist/{chunk-KKCRGAGZ.js → chunk-5GRJIP5H.js} +99 -98
  24. package/dist/chunk-5GRJIP5H.js.map +1 -0
  25. package/dist/{chunk-E3RI3I33.js → chunk-TYNPCG4U.js} +88 -10
  26. package/dist/chunk-TYNPCG4U.js.map +1 -0
  27. package/dist/index.d.ts +13 -0
  28. package/dist/index.js +49 -2
  29. package/dist/index.js.map +1 -1
  30. package/dist/model-checkpoint-memory-collector.d.ts +1 -0
  31. package/dist/model-context-contributions.d.ts +40 -0
  32. package/dist/observability-metrics.d.ts +20 -1
  33. package/dist/sandbox-routing.d.ts +4 -0
  34. package/dist/workflow-bundle.js +59 -15
  35. package/dist/workflows/activities.d.ts +1 -1
  36. package/dist/workflows/site-auth-maintenance.d.ts +2 -0
  37. package/dist/workflows.d.ts +1 -0
  38. package/package.json +16 -15
  39. package/src/activities/agent-run-admission.ts +76 -0
  40. package/src/activities/agent-turn.ts +1178 -142
  41. package/src/activities/browser-state-artifact-reaper.ts +120 -0
  42. package/src/activities/capabilities.ts +10 -0
  43. package/src/activities/codex-capacity.ts +48 -1
  44. package/src/activities/environment.ts +14 -8
  45. package/src/activities/goals.ts +2 -1
  46. package/src/activities/packs.ts +45 -122
  47. package/src/activities/parent-wake.ts +13 -1
  48. package/src/activities/scheduled-tasks.ts +30 -73
  49. package/src/activities/session-state.ts +18 -0
  50. package/src/activities/site-auth-maintenance.ts +269 -0
  51. package/src/activities/types.ts +17 -0
  52. package/src/activities/video-generation-admission.ts +37 -4
  53. package/src/activities/video-generation-credential.ts +84 -6
  54. package/src/activities/video-generation-reconciliation.ts +212 -19
  55. package/src/activities/video-media-validation.ts +8 -2
  56. package/src/activities/video-reference-staging.ts +121 -34
  57. package/src/activities/xai-auth.ts +122 -0
  58. package/src/activities/xai-image-generation.ts +86 -0
  59. package/src/activities/xai-video-generation.ts +69 -0
  60. package/src/activities-control.ts +4 -0
  61. package/src/activities.ts +5 -0
  62. package/src/index.ts +51 -0
  63. package/src/model-checkpoint-memory-collector.ts +8 -1
  64. package/src/model-context-contributions.ts +159 -0
  65. package/src/observability-metrics.ts +98 -9
  66. package/src/sandbox-routing.ts +13 -0
  67. package/src/workflows/activities.ts +1 -0
  68. package/src/workflows/file-upload-reaper.ts +8 -2
  69. package/src/workflows/session.ts +20 -6
  70. package/src/workflows/site-auth-maintenance.ts +14 -0
  71. package/src/workflows.ts +1 -0
  72. package/dist/activities-control-44KBWLSM.js.map +0 -1
  73. package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
  74. package/dist/chunk-E3RI3I33.js.map +0 -1
  75. package/dist/chunk-KKCRGAGZ.js.map +0 -1
@@ -168,17 +168,23 @@ var TurnLifecycleMetrics = class {
168
168
  this.observability = observability;
169
169
  this.options = options;
170
170
  }
171
- startedTurns = /* @__PURE__ */ new Map();
171
+ turns = /* @__PURE__ */ new Map();
172
172
  timer = null;
173
173
  start(turnId) {
174
- this.startedTurns.set(turnId, this.now());
174
+ const now = this.now();
175
+ this.turns.set(turnId, { startedAt: now, lastProgressAt: now });
175
176
  this.ensureTimer();
176
177
  this.refreshGauges();
177
178
  }
179
+ progress(turnId) {
180
+ const turn = this.turns.get(turnId);
181
+ if (!turn) return;
182
+ turn.lastProgressAt = this.now();
183
+ }
178
184
  finish(turnId, outcome, durationSeconds) {
179
- const startedAt = this.startedTurns.get(turnId);
185
+ const startedAt = this.turns.get(turnId)?.startedAt;
180
186
  if (startedAt !== void 0) {
181
- this.startedTurns.delete(turnId);
187
+ this.turns.delete(turnId);
182
188
  }
183
189
  if (outcome) {
184
190
  const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
@@ -195,7 +201,7 @@ var TurnLifecycleMetrics = class {
195
201
  });
196
202
  }
197
203
  this.refreshGauges();
198
- if (this.startedTurns.size === 0) {
204
+ if (this.turns.size === 0) {
199
205
  this.stopTimer();
200
206
  }
201
207
  }
@@ -203,29 +209,42 @@ var TurnLifecycleMetrics = class {
203
209
  this.observability.setGauge({
204
210
  name: "opengeni_turns_inflight",
205
211
  help: "Current number of in-flight agent turns in this worker process.",
206
- value: this.startedTurns.size
212
+ value: this.turns.size
207
213
  });
208
214
  this.observability.setGauge({
209
215
  name: "opengeni_turn_oldest_inflight_age_seconds",
210
216
  help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
211
217
  value: this.oldestInflightAgeSeconds()
212
218
  });
219
+ this.observability.setGauge({
220
+ name: "opengeni_turn_oldest_no_progress_age_seconds",
221
+ help: "Seconds since durable progress for the least recently progressing in-flight turn.",
222
+ value: this.oldestNoProgressAgeSeconds()
223
+ });
213
224
  }
214
225
  stop() {
215
- this.startedTurns.clear();
226
+ this.turns.clear();
216
227
  this.refreshGauges();
217
228
  this.stopTimer();
218
229
  }
219
230
  oldestInflightAgeSeconds() {
220
- if (this.startedTurns.size === 0) {
231
+ if (this.turns.size === 0) {
221
232
  return 0;
222
233
  }
223
234
  let oldest = Number.POSITIVE_INFINITY;
224
- for (const startedAt of this.startedTurns.values()) {
235
+ for (const { startedAt } of this.turns.values()) {
225
236
  oldest = Math.min(oldest, startedAt);
226
237
  }
227
238
  return Math.max(0, (this.now() - oldest) / 1e3);
228
239
  }
240
+ oldestNoProgressAgeSeconds() {
241
+ if (this.turns.size === 0) return 0;
242
+ let leastRecentProgress = Number.POSITIVE_INFINITY;
243
+ for (const { lastProgressAt } of this.turns.values()) {
244
+ leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);
245
+ }
246
+ return Math.max(0, (this.now() - leastRecentProgress) / 1e3);
247
+ }
229
248
  ensureTimer() {
230
249
  if (this.timer) {
231
250
  return;
@@ -567,6 +586,38 @@ function recordCreditMicros(observability, kind, amountMicros) {
567
586
  amount: amountMicros
568
587
  });
569
588
  }
589
+ var MODEL_REQUEST_PHASE_BUCKETS = [
590
+ 5e-3,
591
+ 0.01,
592
+ 0.025,
593
+ 0.05,
594
+ 0.1,
595
+ 0.2,
596
+ 0.35,
597
+ 0.5,
598
+ 1,
599
+ 2,
600
+ 5,
601
+ 10,
602
+ 30,
603
+ 60,
604
+ 300
605
+ ];
606
+ function recordModelRequestPhase(observability, input) {
607
+ const outcome = input.outcome ?? "";
608
+ observability.incrementCounter({
609
+ name: "opengeni_model_request_phases_total",
610
+ help: "Provider model-request lifecycle phases by bounded phase and outcome.",
611
+ labels: { provider: input.provider, phase: input.phase, outcome }
612
+ });
613
+ observability.observeHistogram({
614
+ name: "opengeni_model_request_phase_duration_seconds",
615
+ help: "Monotonic seconds from provider dispatch to a model-request lifecycle phase.",
616
+ buckets: MODEL_REQUEST_PHASE_BUCKETS,
617
+ labels: { provider: input.provider, phase: input.phase },
618
+ value: Math.max(0, input.durationSeconds)
619
+ });
620
+ }
570
621
  function contentDeltaClass(type) {
571
622
  if (type === "agent.message.delta") {
572
623
  return "message";
@@ -686,6 +737,31 @@ function recordContextCompaction(observability, trigger) {
686
737
  labels: { trigger }
687
738
  });
688
739
  }
740
+ var MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2048, 8192];
741
+ function recordCompanyBrainContributions(observability, receipt) {
742
+ for (const contribution of receipt.contributions) {
743
+ const labels = {
744
+ category: contribution.category,
745
+ source: contribution.source,
746
+ inclusion_reason: contribution.inclusionReason,
747
+ authority_scope: contribution.authorityScope,
748
+ session_role: receipt.sessionRole,
749
+ memory_prompt_mode: receipt.memoryPromptMode
750
+ };
751
+ observability.incrementCounter({
752
+ name: "opengeni_model_context_contributions_total",
753
+ help: "Company Brain contributions exposed to a model, by bounded reason and authority.",
754
+ labels
755
+ });
756
+ observability.observeHistogram({
757
+ name: "opengeni_model_context_contribution_estimated_tokens",
758
+ help: "Estimated tokens per model-visible Company Brain contribution.",
759
+ buckets: MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS,
760
+ labels,
761
+ value: contribution.estimatedTokens
762
+ });
763
+ }
764
+ }
689
765
  var MODEL_CACHE_HIT_RATIO_BUCKETS = [
690
766
  0.05,
691
767
  0.1,
@@ -816,13 +892,15 @@ export {
816
892
  recordExpiredDrainingSandboxLeaseGauges,
817
893
  recordRetainedProcessReconciliation,
818
894
  recordCreditMicros,
895
+ recordModelRequestPhase,
819
896
  StreamTimingMetrics,
820
897
  recordBatchFlush,
821
898
  recordSessionEventAppendLatency,
822
899
  recordSessionEventPublishLatency,
823
900
  recordModelInputTokens,
824
901
  recordContextCompaction,
902
+ recordCompanyBrainContributions,
825
903
  recordModelCacheTokens,
826
904
  modelCallAccountContext
827
905
  };
828
- //# sourceMappingURL=chunk-E3RI3I33.js.map
906
+ //# sourceMappingURL=chunk-TYNPCG4U.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/observability-metrics.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { errorCodeToJSON } from \"@opengeni/agent-proto\";\nimport { SandboxBackend, type SessionEventType } from \"@opengeni/contracts\";\nimport type { EventLogger } from \"@opengeni/events\";\nimport type { Attributes, AttributeValue, Observability } from \"@opengeni/observability\";\nimport type { CompanyBrainContributionReceipt } from \"./model-context-contributions\";\nimport {\n SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,\n modelUsageTokenCountOrNull,\n type RuntimeMetricsHooks,\n type SelfhostedOpObservation,\n type SelfhostedOpObserver,\n} from \"@opengeni/runtime\";\n\nexport type TurnOutcome = \"completed\" | \"failed\" | \"cancelled\" | \"recovering\";\nexport type CreditMicrosKind = \"usage\" | \"grant\" | \"topup\" | \"refund\";\nexport type SandboxLeaseLiveness = \"cold\" | \"warming\" | \"warm\" | \"draining\";\nexport type CreditBalanceGauge = { accountId: string; balanceMicros: number };\nexport type TurnTaskQueueStats = {\n /** Temporal activity tasks ready for a turn worker to accept. */\n eligibleBacklog: number;\n oldestBacklogAgeSeconds: number;\n tasksAddRate: number;\n tasksDispatchRate: number;\n};\n\nexport type TemporalTurnTaskQueueStats = {\n approximateBacklogCount?: unknown;\n approximateBacklogAge?: {\n seconds?: unknown;\n nanos?: unknown;\n } | null;\n tasksAddRate?: unknown;\n tasksDispatchRate?: unknown;\n} | null;\n\nconst turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();\nconst creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();\nconst modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();\n\n// A worker process reaching one trillion tokens in one provider/cache counter is\n// already far outside an ordinary scrape lifetime. Refuse further increments\n// before floating-point precision can degrade; a restart resets both the\n// process-local Prometheus registry and this guard together.\nconst MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;\n\nexport function observabilityEventLogger(observability: Observability): EventLogger {\n return {\n debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),\n warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),\n };\n}\n\nfunction eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {\n if (!attributes) {\n return undefined;\n }\n const projected: Attributes = {};\n for (const [key, value] of Object.entries(attributes)) {\n projected[key] = eventAttributeValue(value);\n }\n return projected;\n}\n\nfunction eventAttributeValue(value: unknown): AttributeValue {\n if (\n value === null ||\n value === undefined ||\n typeof value === \"string\" ||\n typeof value === \"number\" ||\n typeof value === \"boolean\"\n ) {\n return value;\n }\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nexport function runtimeMetricsHooksForObservability(\n observability: Observability,\n): RuntimeMetricsHooks {\n return {\n onModelCall: ({ provider, outcome, durationSeconds }) => {\n observability.incrementCounter({\n name: \"opengeni_model_calls_total\",\n help: \"Total model calls by provider and outcome.\",\n labels: { provider, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_model_call_duration_seconds\",\n help: \"Model call duration in seconds by provider.\",\n labels: { provider },\n value: durationSeconds,\n });\n },\n onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {\n observability.incrementCounter({\n name: \"opengeni_sandbox_creates_total\",\n help: \"Total sandbox create attempts by backend and outcome.\",\n labels: { backend, image_source: imageSource, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_sandbox_create_duration_seconds\",\n help: \"Sandbox create duration in seconds by backend.\",\n labels: { backend, image_source: imageSource },\n value: durationSeconds,\n });\n },\n onSandboxWarmingTimeout: ({ backend, stage }) => {\n observability.incrementCounter({\n name: \"opengeni_sandbox_warming_timeouts_total\",\n help: \"Total sandbox warming timeouts.\",\n labels: { backend, stage },\n });\n },\n onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {\n observability.incrementCounter({\n name: \"opengeni_machine_op_total\",\n help: \"Total Connected Machine control ops by op, outcome, and typed fault code.\",\n labels: { backend, op, outcome, code: code ?? \"\" },\n });\n observability.observeHistogram({\n name: \"opengeni_machine_op_duration_seconds\",\n help: \"Connected Machine control-op duration in seconds by op.\",\n labels: { backend, op },\n value: durationSeconds,\n });\n // The healed-fault leading indicator: an op that only succeeded after a retry\n // (a blip/backpressure the transport absorbed). The doctrine: healed faults are\n // the leading indicator of the next unhealed one, so they are always recorded.\n if (healed) {\n observability.incrementCounter({\n name: \"opengeni_machine_op_healed_total\",\n help: \"Connected Machine ops that succeeded only after ≥1 in-call retry.\",\n labels: { backend, op },\n });\n }\n // The payload-wall indicator (bytes known only on a PAYLOAD_TOO_LARGE fault today).\n if (replyBytes !== undefined) {\n observability.observeHistogram({\n name: \"opengeni_machine_op_reply_bytes\",\n help: \"Connected Machine control-op reply size in bytes (payload-wall indicator).\",\n labels: { backend, op },\n value: replyBytes,\n });\n }\n },\n };\n}\n\n/**\n * Adapt the runtime's transport-agnostic `SelfhostedOpObserver` to the metrics\n * hooks: map a completed-op observation onto `onSandboxOp`, converting the wire\n * `ErrorCode` to its stable enum-name string (a bounded metric label) and the\n * duration to seconds. Wired into the selfhosted session build so every Connected\n * Machine control op meters.\n */\nexport function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): SelfhostedOpObserver {\n return (o) => {\n hooks.onSandboxOp?.({\n backend: \"selfhosted\",\n op: o.op,\n outcome: o.outcome,\n healed: o.healed,\n retries: o.retries,\n durationSeconds: o.durationMs / 1000,\n ...(o.code !== undefined ? { code: errorCodeToJSON(o.code) } : {}),\n ...(o.replyBytes !== undefined ? { replyBytes: o.replyBytes } : {}),\n });\n };\n}\n\n/** A session-scoped `machine.op.*` event mapped from a completed-op observation. */\nexport type MachineOpSessionEvent = {\n type: \"machine.op.failed\" | \"machine.op.recovered\";\n payload: { op: string; faultClass: string; attempts: number; machineId?: string };\n};\n\n/** Map an observation to a `machine.op.*` session event, or null if it is not\n * eventable. `machine.op.failed` fires ONLY for infrastructure fault classes (a\n * semantic miss the model asked about is an outcome, not an infra fault);\n * `machine.op.recovered` fires for a healed op (success after ≥1 retry). */\nexport function machineOpSessionEventFor(o: SelfhostedOpObservation): MachineOpSessionEvent | null {\n if (\n o.outcome === \"failed\" &&\n o.faultClass &&\n SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES.has(o.faultClass)\n ) {\n return {\n type: \"machine.op.failed\",\n payload: {\n op: o.op,\n faultClass: o.faultClass,\n attempts: o.retries,\n ...(o.machineId ? { machineId: o.machineId } : {}),\n },\n };\n }\n if (o.outcome === \"ok\" && o.healed) {\n return {\n type: \"machine.op.recovered\",\n payload: {\n op: o.op,\n faultClass: o.faultClass ?? \"unknown\",\n attempts: o.retries,\n ...(o.machineId ? { machineId: o.machineId } : {}),\n },\n };\n }\n return null;\n}\n\n/**\n * The Connected Machine op observer wired into a turn: it meters EVERY op (the\n * metrics sink) and BUFFERS the eventable ops (infra failures + healed recoveries)\n * as `machine.op.*` session events. The observer is SYNC (fire-and-forget), so the\n * turn drains the buffer to durable session events at a known checkpoint (turn end),\n * awaited — never an unawaited DB write inside the Temporal activity.\n */\nexport function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {\n observer: SelfhostedOpObserver;\n drainEvents(): MachineOpSessionEvent[];\n} {\n const meter = selfhostedOpObserverForMetrics(hooks);\n const buffered: MachineOpSessionEvent[] = [];\n return {\n observer: (o) => {\n meter(o);\n const event = machineOpSessionEventFor(o);\n if (event) {\n buffered.push(event);\n }\n },\n drainEvents: () => buffered.splice(0, buffered.length),\n };\n}\n\nexport function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {\n const existing = turnTrackers.get(observability);\n if (existing) {\n return existing;\n }\n const tracker = new TurnLifecycleMetrics(observability);\n turnTrackers.set(observability, tracker);\n return tracker;\n}\n\nexport class TurnLifecycleMetrics {\n private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();\n private timer: ReturnType<typeof setInterval> | null = null;\n\n constructor(\n private readonly observability: Observability,\n private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},\n ) {}\n\n start(turnId: string): void {\n const now = this.now();\n this.turns.set(turnId, { startedAt: now, lastProgressAt: now });\n this.ensureTimer();\n this.refreshGauges();\n }\n\n progress(turnId: string): void {\n const turn = this.turns.get(turnId);\n if (!turn) return;\n turn.lastProgressAt = this.now();\n }\n\n finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {\n const startedAt = this.turns.get(turnId)?.startedAt;\n if (startedAt !== undefined) {\n this.turns.delete(turnId);\n }\n if (outcome) {\n const observedDuration =\n durationSeconds ??\n (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));\n this.observability.incrementCounter({\n name: \"opengeni_turns_total\",\n help: \"Total agent turns by terminal outcome.\",\n labels: { outcome },\n });\n this.observability.observeHistogram({\n name: \"opengeni_turn_duration_seconds\",\n help: \"Agent turn duration in seconds by terminal outcome.\",\n labels: { outcome },\n value: observedDuration,\n });\n }\n this.refreshGauges();\n if (this.turns.size === 0) {\n this.stopTimer();\n }\n }\n\n refreshGauges(): void {\n this.observability.setGauge({\n name: \"opengeni_turns_inflight\",\n help: \"Current number of in-flight agent turns in this worker process.\",\n value: this.turns.size,\n });\n this.observability.setGauge({\n name: \"opengeni_turn_oldest_inflight_age_seconds\",\n help: \"Age in seconds of the oldest in-flight agent turn in this worker process.\",\n value: this.oldestInflightAgeSeconds(),\n });\n this.observability.setGauge({\n name: \"opengeni_turn_oldest_no_progress_age_seconds\",\n help: \"Seconds since durable progress for the least recently progressing in-flight turn.\",\n value: this.oldestNoProgressAgeSeconds(),\n });\n }\n\n stop(): void {\n this.turns.clear();\n this.refreshGauges();\n this.stopTimer();\n }\n\n private oldestInflightAgeSeconds(): number {\n if (this.turns.size === 0) {\n return 0;\n }\n let oldest = Number.POSITIVE_INFINITY;\n for (const { startedAt } of this.turns.values()) {\n oldest = Math.min(oldest, startedAt);\n }\n return Math.max(0, (this.now() - oldest) / 1000);\n }\n\n private oldestNoProgressAgeSeconds(): number {\n if (this.turns.size === 0) return 0;\n let leastRecentProgress = Number.POSITIVE_INFINITY;\n for (const { lastProgressAt } of this.turns.values()) {\n leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);\n }\n return Math.max(0, (this.now() - leastRecentProgress) / 1000);\n }\n\n private ensureTimer(): void {\n if (this.timer) {\n return;\n }\n this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);\n this.timer.unref?.();\n }\n\n private stopTimer(): void {\n if (!this.timer) {\n return;\n }\n clearInterval(this.timer);\n this.timer = null;\n }\n\n private now(): number {\n return this.options.now?.() ?? Date.now();\n }\n}\n\nexport function recordTurnsQueuedGauge(observability: Observability, value: number): void {\n observability.setGauge({\n name: \"opengeni_turns_queued\",\n help: \"Current number of queued session turns.\",\n value,\n });\n}\n\n/**\n * Record the authoritative turn activity queue rather than aggregate Postgres\n * prompts. A paused human prompt is durable queue truth but is not runnable and\n * never reaches this Temporal task queue. DescribeTaskQueue's approximate\n * backlog count and age are explicitly documented by Temporal as autoscaling\n * signals.\n */\nexport function recordTurnTaskQueueStats(\n observability: Observability,\n stats: TurnTaskQueueStats,\n): void {\n observability.setGauge({\n name: \"opengeni_turn_eligible_backlog\",\n help: \"Temporal runAgentTurn activity tasks eligible for immediate worker admission.\",\n value: nonnegativeFinite(stats.eligibleBacklog),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_backlog_oldest_age_seconds\",\n help: \"Approximate age of the oldest eligible runAgentTurn activity task.\",\n value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_tasks_add_rate\",\n help: \"Temporal runAgentTurn tasks added per second over its rolling window.\",\n value: nonnegativeFinite(stats.tasksAddRate),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_tasks_dispatch_rate\",\n help: \"Temporal runAgentTurn tasks dispatched per second over its rolling window.\",\n value: nonnegativeFinite(stats.tasksDispatchRate),\n });\n}\n\nexport function normalizeTurnTaskQueueStats(\n stats: TemporalTurnTaskQueueStats | undefined,\n): TurnTaskQueueStats {\n if (!stats) {\n throw new Error(\"Temporal DescribeTaskQueue response omitted required stats\");\n }\n const eligibleBacklog = requiredTemporalNumber(\n stats.approximateBacklogCount,\n \"approximateBacklogCount\",\n { integer: true },\n );\n let oldestBacklogAgeSeconds = 0;\n if (stats.approximateBacklogAge) {\n const seconds = optionalTemporalNumber(\n stats.approximateBacklogAge.seconds,\n \"approximateBacklogAge.seconds\",\n { integer: true },\n );\n const nanos = optionalTemporalNumber(\n stats.approximateBacklogAge.nanos,\n \"approximateBacklogAge.nanos\",\n { integer: true },\n );\n if (nanos >= 1_000_000_000) {\n throw new Error(\"Temporal approximateBacklogAge.nanos must be less than one second\");\n }\n oldestBacklogAgeSeconds = seconds + nanos / 1_000_000_000;\n } else if (eligibleBacklog > 0) {\n throw new Error(\"Temporal stats omitted approximateBacklogAge for a nonzero backlog\");\n }\n return {\n eligibleBacklog,\n oldestBacklogAgeSeconds,\n tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, \"tasksAddRate\"),\n tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, \"tasksDispatchRate\"),\n };\n}\n\nexport function startTurnCapacityMonitor(input: {\n observability: Observability;\n read: () => Promise<TurnTaskQueueStats>;\n intervalMs?: number;\n now?: () => number;\n}): { close: () => Promise<void> } {\n const intervalMs = input.intervalMs ?? 15_000;\n const now = input.now ?? Date.now;\n const startedAt = now();\n let lastSuccessAt: number | null = null;\n let lastReadSucceeded = false;\n let stopped = false;\n let running: Promise<void> | null = null;\n const recordStatus = () => {\n const observedAt = now();\n const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);\n const set = (name: string, help: string, value: number) =>\n input.observability.setGauge({ name, help, value });\n set(\n \"opengeni_turn_capacity_monitor_last_read_success\",\n \"Whether the latest Temporal turn-queue capacity read completed successfully.\",\n lastReadSucceeded ? 1 : 0,\n );\n set(\n \"opengeni_turn_capacity_monitor_last_success_timestamp_seconds\",\n \"Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.\",\n lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,\n );\n set(\n \"opengeni_turn_capacity_monitor_last_success_age_seconds\",\n \"Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.\",\n Math.max(0, successAgeMs / 1_000),\n );\n set(\n \"opengeni_turn_capacity_monitor_fresh\",\n \"Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.\",\n lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,\n );\n };\n const refresh = () => {\n // Advance freshness age even while a Temporal read is hung. Old backlog\n // values remain observable for diagnosis but cease to be authoritative.\n recordStatus();\n if (stopped || running) return;\n running = input\n .read()\n .then((stats) => {\n recordTurnTaskQueueStats(input.observability, stats);\n lastSuccessAt = now();\n lastReadSucceeded = true;\n recordStatus();\n })\n .catch((error) => {\n lastReadSucceeded = false;\n recordStatus();\n input.observability.warn(\"turn capacity monitor: Temporal task-queue stats failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n })\n .finally(() => {\n running = null;\n });\n };\n refresh();\n const timer = setInterval(refresh, intervalMs);\n timer.unref?.();\n return {\n close: async () => {\n stopped = true;\n clearInterval(timer);\n await running;\n },\n };\n}\n\nexport function recordSandboxLeaseGauges(\n observability: Observability,\n counts: Partial<Record<SandboxLeaseLiveness, number>>,\n): void {\n for (const liveness of [\"cold\", \"warming\", \"warm\", \"draining\"] as const) {\n observability.setGauge({\n name: \"opengeni_sandbox_leases\",\n help: \"Current sandbox leases by liveness state.\",\n labels: { liveness },\n value: counts[liveness] ?? 0,\n });\n }\n}\n\nexport const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [\n \"leases\",\n \"checkpoint_artifacts\",\n \"rotation_backlog\",\n \"retained_processes\",\n \"expired_drains\",\n] as const;\n\nexport type SandboxInventoryProjectionDomain =\n (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];\n\nexport function recordSandboxInventoryProjectionSuccess(\n observability: Observability,\n domain: SandboxInventoryProjectionDomain,\n timestampSeconds = Date.now() / 1_000,\n): void {\n observability.setGauge({\n name: \"opengeni_sandbox_inventory_refresh_timestamp_seconds\",\n help: \"Unix timestamp of the last complete successful sandbox inventory projection by domain.\",\n labels: { domain },\n value: timestampSeconds,\n });\n}\n\nexport function recordSandboxInventoryProjectionFailure(\n observability: Observability,\n domain: SandboxInventoryProjectionDomain,\n): void {\n observability.incrementCounter({\n name: \"opengeni_sandbox_inventory_refresh_failures_total\",\n help: \"Failed sandbox inventory projection refreshes by domain.\",\n labels: { domain },\n });\n}\n\nexport function recordCreditBalanceGauges(\n observability: Observability,\n balances: CreditBalanceGauge[],\n): void {\n const previous = creditBalanceGaugeAccounts.get(observability) ?? new Set<string>();\n const current = new Set<string>();\n for (const balance of balances) {\n current.add(balance.accountId);\n observability.setGauge({\n name: \"opengeni_credit_balance_micros\",\n help: \"Current credit balance in micros by account.\",\n labels: { account_id: balance.accountId },\n value: balance.balanceMicros,\n });\n }\n for (const accountId of previous) {\n if (!current.has(accountId)) {\n observability.setGauge({\n name: \"opengeni_credit_balance_micros\",\n help: \"Current credit balance in micros by account.\",\n labels: { account_id: accountId },\n value: 0,\n });\n }\n }\n creditBalanceGaugeAccounts.set(observability, current);\n}\n\nexport function recordSandboxOrphansTerminated(observability: Observability, count: number): void {\n if (count <= 0) {\n return;\n }\n observability.incrementCounter({\n name: \"opengeni_sandbox_orphans_terminated_total\",\n help: \"Total provider-side orphan sandboxes terminated by defensive sweeps.\",\n amount: count,\n });\n}\n\nexport type SandboxCheckpointArtifactMetricState =\n | \"candidate\"\n | \"current\"\n | \"previous\"\n | \"delete_pending\"\n | \"deleting\"\n | \"delete_failed\"\n | \"deleted\";\n\nexport function recordSandboxCheckpointArtifactGauges(\n observability: Observability,\n counts: Record<SandboxCheckpointArtifactMetricState, number>,\n): void {\n for (const state of [\n \"candidate\",\n \"current\",\n \"previous\",\n \"delete_pending\",\n \"deleting\",\n \"delete_failed\",\n \"deleted\",\n ] as const) {\n observability.setGauge({\n name: \"opengeni_sandbox_checkpoint_artifacts\",\n help: \"Current durable provider checkpoint artifacts by lifecycle state.\",\n labels: { state },\n value: counts[state],\n });\n }\n}\n\nexport type SandboxCheckpointArtifactOutcome =\n | \"legacy_adopted\"\n | \"claimed\"\n | \"deleted\"\n | \"delete_failed\"\n | \"tombstone_pruned\";\n\nexport function recordSandboxCheckpointArtifactOutcome(\n observability: Observability,\n outcome: SandboxCheckpointArtifactOutcome,\n count = 1,\n): void {\n if (count <= 0) return;\n observability.incrementCounter({\n name: \"opengeni_sandbox_checkpoint_artifact_operations_total\",\n help: \"Durable provider checkpoint artifact operations by fixed outcome.\",\n labels: { outcome },\n amount: count,\n });\n}\n\nexport function recordSandboxDeadlineRotationsRequested(\n observability: Observability,\n count: number,\n): void {\n if (count <= 0) return;\n observability.incrementCounter({\n name: \"opengeni_sandbox_deadline_rotations_requested_total\",\n help: \"Total finite-lifetime sandbox rotations requested before provider deadline.\",\n amount: count,\n });\n}\n\nexport function recordSandboxRotationBacklogGauges(\n observability: Observability,\n backlog: {\n requested: number;\n overdue: number;\n turnBlocked: number;\n directBlocked: number;\n processBlocked: number;\n },\n): void {\n const values = {\n requested: backlog.requested,\n overdue: backlog.overdue,\n turn_blocked: backlog.turnBlocked,\n direct_blocked: backlog.directBlocked,\n process_blocked: backlog.processBlocked,\n } as const;\n for (const [kind, value] of Object.entries(values)) {\n observability.setGauge({\n name: \"opengeni_sandbox_rotation_backlog\",\n help: \"Current finite-lifetime sandbox rotation backlog by fixed condition.\",\n labels: { kind },\n value,\n });\n }\n}\n\nconst RETAINED_PROCESS_OWNER_STATES = [\n \"direct\",\n \"queued\",\n \"running\",\n \"requires_action\",\n \"recovering\",\n \"waiting_capacity\",\n \"completed\",\n \"failed\",\n \"cancelled\",\n \"superseded\",\n \"withdrawn_for_edit\",\n \"missing\",\n \"unknown\",\n] as const;\n\nconst RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);\n\nexport function recordRetainedProcessInventoryGauges(\n observability: Observability,\n counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,\n): void {\n const normalized = new Map<string, { active: number; terminal: number }>();\n for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {\n normalized.set(ownerState, { active: 0, terminal: 0 });\n }\n for (const count of counts) {\n const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)\n ? count.ownerState\n : \"unknown\";\n const current = normalized.get(ownerState)!;\n current.active += Math.max(0, count.activeCount);\n current.terminal += Math.max(0, count.terminalOwnerCount);\n }\n\n for (const [ownerState, count] of normalized) {\n observability.setGauge({\n name: \"opengeni_retained_processes_active\",\n help: \"Current active retained provider processes by durable owner state.\",\n labels: { owner_state: ownerState },\n value: count.active,\n });\n observability.setGauge({\n name: \"opengeni_retained_processes_terminal_owner_backlog\",\n help: \"Current active retained processes whose exact owner attempt is terminal.\",\n labels: { owner_state: ownerState },\n value: count.terminal,\n });\n }\n}\n\nexport const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, \"unknown\"] as const;\nconst EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);\nexport const EXPIRED_DRAINING_AGE_BUCKETS = [\"lt_5m\", \"5m_1h\", \"1h_1d\", \"gte_1d\"] as const;\n\nexport function recordExpiredDrainingSandboxLeaseGauges(\n observability: Observability,\n counts: Array<{ backend: string; ageBucket: string; count: number }>,\n): void {\n const normalized = new Map<string, number>();\n for (const backend of EXPIRED_DRAINING_BACKENDS) {\n for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {\n normalized.set(`${backend}:${ageBucket}`, 0);\n }\n }\n for (const count of counts) {\n const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : \"unknown\";\n if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;\n const key = `${backend}:${count.ageBucket}`;\n normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));\n }\n for (const [key, value] of normalized) {\n const [backend, ageBucket] = key.split(\":\") as [string, string];\n observability.setGauge({\n name: \"opengeni_sandbox_leases_expired_draining\",\n help: \"Current expired draining sandbox leases by backend and fixed age bucket.\",\n labels: { backend, age_bucket: ageBucket },\n value,\n });\n }\n}\n\nexport const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [\n \"claim_failed\",\n \"proof_exited\",\n \"proof_lost\",\n \"proof_checkpoint_failed\",\n \"settled_exited\",\n \"settled_lost\",\n \"settlement_failed\",\n \"identity_mismatch\",\n \"resume_state_missing\",\n \"backend_unsupported\",\n \"provider_running\",\n \"provider_unknown\",\n \"provider_timeout\",\n \"provider_error\",\n \"provider_binding_missing\",\n \"provider_binding_mismatch\",\n \"provider_binding_adopted\",\n \"defer_failed\",\n] as const;\n\nexport type RetainedProcessReconciliationOutcome =\n (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];\n\nexport function recordRetainedProcessReconciliation(\n observability: Observability,\n outcome: RetainedProcessReconciliationOutcome,\n): void {\n observability.incrementCounter({\n name: \"opengeni_retained_process_reconciliation_total\",\n help: \"Bounded retained-process reconciliation observations by fixed app outcome.\",\n labels: { outcome },\n });\n}\n\nexport function recordCreditMicros(\n observability: Observability | undefined,\n kind: CreditMicrosKind,\n amountMicros: number,\n): void {\n if (!observability || amountMicros <= 0) {\n return;\n }\n observability.incrementCounter({\n name: \"opengeni_credit_micros_total\",\n help: \"Total credit micros recorded by kind.\",\n labels: { kind },\n amount: amountMicros,\n });\n}\n\nconst MODEL_REQUEST_PHASE_BUCKETS = [\n 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5, 10, 30, 60, 300,\n];\n\n/**\n * Record provider lifecycle diagnostics from the synchronous transport observer.\n * Provider ids come from the resolved provider registry; phase/outcome are closed\n * enums. This function never performs I/O outside the in-memory metrics registry.\n */\nexport function recordModelRequestPhase(\n observability: Observability,\n input: {\n provider: string;\n phase: \"headers\" | \"first_byte\" | \"terminal\";\n outcome?: \"completed\" | \"failed\" | \"timed_out\";\n durationSeconds: number;\n },\n): void {\n const outcome = input.outcome ?? \"\";\n observability.incrementCounter({\n name: \"opengeni_model_request_phases_total\",\n help: \"Provider model-request lifecycle phases by bounded phase and outcome.\",\n labels: { provider: input.provider, phase: input.phase, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_model_request_phase_duration_seconds\",\n help: \"Monotonic seconds from provider dispatch to a model-request lifecycle phase.\",\n buckets: MODEL_REQUEST_PHASE_BUCKETS,\n labels: { provider: input.provider, phase: input.phase },\n value: Math.max(0, input.durationSeconds),\n });\n}\n\n// ── Streaming SLIs ────────────────────────────────────────────────────────────\n// Instruments the token-streaming pipeline so \"streaming is sluggish\" is a number,\n// not a vibe. Split across the three attributable stages so an operator can tell\n// WHERE the latency lives: the model (TTFT + inter-delta gaps), our durable write\n// path (append latency), or delivery (publish latency + batcher flush shape). All\n// labels are bounded (provider from the model registry, a two-value delta class) —\n// never a session id or a raw user-supplied model string.\n\nexport type StreamDeltaClass = \"message\" | \"reasoning\";\n\n/** Content-delta classes only. A `null` return means \"not a content delta\" — the\n * event that re-arms the TTFT anchor and closes an inter-delta run. */\nfunction contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {\n if (type === \"agent.message.delta\") {\n return \"message\";\n }\n if (type === \"agent.reasoning.delta\") {\n return \"reasoning\";\n }\n return null;\n}\n\n// TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few\n// seconds), so they get their own SHORT buckets — the default duration buckets\n// (which run to 3600s) would collapse every real streaming value into one bucket.\nconst STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];\nconst STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];\n\n/**\n * Per-turn stream-timing tracker fed every normalized runtime event in push order.\n * It emits two model-responsiveness SLIs from the worker's seat on the stream:\n *\n * - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its\n * first streamed content delta. The anchor starts at construction (≈ runStream\n * start, so the first observation is \"how long until text appears\") and re-arms\n * on every non-content event (a tool call, a completed message, a usage frame),\n * so a post-tool response measures the model's restart latency, NOT our own\n * tool-execution time.\n * - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between\n * consecutive content deltas of the SAME class. The run resets on any\n * non-content event so a gap never spans a tool call or a model boundary — it\n * measures only the choppiness of a live token stream.\n *\n * Purely observational and clock-injectable; it never touches the events it sees.\n */\nexport class StreamTimingMetrics {\n private readonly now: () => number;\n private ttftAnchor: number;\n private ttftArmed = true;\n private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();\n\n constructor(\n private readonly observability: Observability,\n private readonly options: { provider: string; now?: () => number },\n ) {\n this.now = options.now ?? (() => performance.now());\n this.ttftAnchor = this.now();\n }\n\n onEvent(type: SessionEventType): void {\n const deltaClass = contentDeltaClass(type);\n if (deltaClass === null) {\n // A non-content event: the model paused emitting. Re-arm TTFT so the next\n // content delta measures (re)start latency, and close every inter-delta run\n // so no gap spans a tool call / model boundary.\n this.ttftAnchor = this.now();\n this.ttftArmed = true;\n this.lastDeltaAt.clear();\n return;\n }\n const at = this.now();\n if (this.ttftArmed) {\n this.observability.observeHistogram({\n name: \"opengeni_stream_ttft_seconds\",\n help: \"Seconds from a model (re)start to its first streamed content delta.\",\n buckets: STREAM_TTFT_BUCKETS,\n labels: { provider: this.options.provider },\n value: Math.max(0, (at - this.ttftAnchor) / 1000),\n });\n this.ttftArmed = false;\n }\n const last = this.lastDeltaAt.get(deltaClass);\n if (last !== undefined) {\n this.observability.observeHistogram({\n name: \"opengeni_stream_inter_delta_gap_seconds\",\n help: \"Seconds between consecutive streamed content deltas of the same class.\",\n buckets: STREAM_INTER_DELTA_BUCKETS,\n labels: { provider: this.options.provider, class: deltaClass },\n value: Math.max(0, (at - last) / 1000),\n });\n }\n this.lastDeltaAt.set(deltaClass, at);\n }\n}\n\n// Batch shapes: sizes are small integers; durations are the append+publish round\n// trip the flush performs (sub-ms to a couple seconds under contention).\nconst STREAM_BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500];\nconst STREAM_IO_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5];\n\n/** Flush shape of the streaming batcher: how many events coalesced into one flush\n * (the coalescing win) and how long that flush took (append + publish). */\nexport function recordBatchFlush(\n observability: Observability,\n input: { events: number; durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_stream_batch_flush_events\",\n help: \"Session events coalesced into one streaming batcher flush.\",\n buckets: STREAM_BATCH_SIZE_BUCKETS,\n value: input.events,\n });\n observability.observeHistogram({\n name: \"opengeni_stream_batch_flush_duration_seconds\",\n help: \"Duration in seconds of one streaming batcher flush (append + publish).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n/** Latency of the durable `appendSessionEvents` DB write — the write path. A p99\n * climb here is our Postgres, not the model or NATS. */\nexport function recordSessionEventAppendLatency(\n observability: Observability,\n input: { durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_session_event_append_seconds\",\n help: \"Duration in seconds of an appendSessionEvents DB write (the durable write path).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb\n * here (with append healthy) is delivery, not the write path. */\nexport function recordSessionEventPublishLatency(\n observability: Observability,\n input: { durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_session_event_publish_seconds\",\n help: \"Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n// Context tokens per response span a wide range; buckets track the pressure toward\n// a model's window so \"sessions are running hot but never compacting\" is queryable.\nconst MODEL_INPUT_TOKENS_BUCKETS = [\n 1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,\n];\n\n/** Observed input (context) tokens per model response — the context-pressure\n * signal. Paired with `opengeni_context_compactions_total`, it makes the\n * \"compaction never firing while contexts run hot\" failure mode expressible. */\nexport function recordModelInputTokens(\n observability: Observability,\n provider: string,\n inputTokens: number,\n): void {\n const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);\n if (normalizedInputTokens === null || normalizedInputTokens === 0) {\n return;\n }\n observability.observeHistogram({\n name: \"opengeni_model_input_tokens\",\n help: \"Observed input (context) tokens per model response, by provider.\",\n buckets: MODEL_INPUT_TOKENS_BUCKETS,\n labels: { provider },\n value: normalizedInputTokens,\n });\n}\n\n/** A context compaction actually fired, by trigger (operator | overflow | proactive\n * | auto). The rate of this — against the input-tokens histogram above — is how an\n * operator sees compaction working (or silently not). */\nexport function recordContextCompaction(observability: Observability, trigger: string): void {\n observability.incrementCounter({\n name: \"opengeni_context_compactions_total\",\n help: \"Total context compactions performed, by trigger.\",\n labels: { trigger },\n });\n}\n\nconst MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];\n\n/**\n * Metadata-only Company Brain exposure telemetry. Labels are closed enums and\n * never include tenant ids, record ids, titles, descriptions, or content.\n */\nexport function recordCompanyBrainContributions(\n observability: Observability,\n receipt: CompanyBrainContributionReceipt,\n): void {\n for (const contribution of receipt.contributions) {\n const labels = {\n category: contribution.category,\n source: contribution.source,\n inclusion_reason: contribution.inclusionReason,\n authority_scope: contribution.authorityScope,\n session_role: receipt.sessionRole,\n memory_prompt_mode: receipt.memoryPromptMode,\n };\n observability.incrementCounter({\n name: \"opengeni_model_context_contributions_total\",\n help: \"Company Brain contributions exposed to a model, by bounded reason and authority.\",\n labels,\n });\n observability.observeHistogram({\n name: \"opengeni_model_context_contribution_estimated_tokens\",\n help: \"Estimated tokens per model-visible Company Brain contribution.\",\n buckets: MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS,\n labels,\n value: contribution.estimatedTokens,\n });\n }\n}\n\n// ── Prompt-cache efficiency ─────────────────────────────────────────────────\n// Per model-call prompt-cache signal, provider-labelled only (bounded\n// cardinality — never a session id or account). `cached_tokens` is the slice of\n// the prompt the provider served from its prompt cache; the ratio cached/prompt\n// is the efficiency of that call. The account-switch hypothesis (a codex account\n// rotation cold-starts the provider's per-account prompt cache) is tested from\n// the per-call STRUCTURED LOG below — the account id is unbounded, so it is\n// hashed into a log field and NEVER a Prometheus label.\n\n// The hit ratio lives in [0, 1]; bucket tighter around the alerting threshold so\n// a p50 near 40% resolves. The default duration buckets (to 3600) would collapse\n// every ratio into the first bucket.\nconst MODEL_CACHE_HIT_RATIO_BUCKETS = [\n 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1,\n];\n\n/**\n * Prompt-cache efficiency of one model response, by provider. Reads from the SAME\n * usage frame that feeds input-token accounting, so the two are always consistent:\n * - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the\n * provider served from cache. Advanced only when >0, so a provider that never\n * reports cached tokens contributes nothing rather than phantom zero-increments.\n * - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a\n * reporting provider wrote into its prompt cache. Absent telemetry contributes\n * nothing; in particular, Codex subscription usage does not currently expose it.\n * - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,\n * observed only when prompt tokens are known (>0) AND cached tokens are reported.\n * A reported cached-token zero records a real 0%; absent/null telemetry remains\n * unknown and must not fabricate a low-cache signal.\n * - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded\n * availability observation per authoritative call (`reported` or `missing`).\n * Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call\n * with no prompt tokens has no ratio (skipped).\n */\nexport function recordModelCacheTokens(\n observability: Observability,\n provider: string,\n input: {\n cachedTokens: number | null | undefined;\n cacheWriteTokens?: number | null | undefined;\n promptTokens: number | null | undefined;\n },\n): void {\n const cached = modelUsageTokenCountOrNull(input.cachedTokens);\n const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);\n const prompt = modelUsageTokenCountOrNull(input.promptTokens);\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cache_read_telemetry_total\",\n help: \"Authoritative model calls with reported or missing cache-read telemetry, by provider.\",\n provider,\n labels: { provider, status: cached === null ? \"missing\" : \"reported\" },\n amount: 1,\n });\n if (cached !== null && cached > 0) {\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cached_tokens_total\",\n help: \"Total prompt tokens served from the provider's prompt cache, by provider.\",\n provider,\n labels: { provider },\n amount: cached,\n });\n }\n if (cacheWrite !== null && cacheWrite > 0) {\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cache_write_tokens_total\",\n help: \"Total prompt tokens written to the provider's prompt cache, by provider.\",\n provider,\n labels: { provider },\n amount: cacheWrite,\n });\n }\n if (cached !== null && prompt !== null && prompt > 0) {\n observability.observeHistogram({\n name: \"opengeni_model_cache_hit_ratio\",\n help: \"Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.\",\n buckets: MODEL_CACHE_HIT_RATIO_BUCKETS,\n labels: { provider },\n // Clamp to [0, 1]: a provider that (rarely) reports cached >= prompt must\n // not skew the histogram past 1.0.\n value: Math.min(1, cached / prompt),\n });\n }\n}\n\nfunction incrementBoundedModelCacheCounter(\n observability: Observability,\n input: {\n name: string;\n help: string;\n provider: string;\n labels: Record<string, string>;\n amount: number;\n },\n): void {\n const totals = modelCacheCounterTotals.get(observability) ?? new Map<string, number>();\n if (!modelCacheCounterTotals.has(observability)) {\n modelCacheCounterTotals.set(observability, totals);\n }\n const labelKey = Object.entries(input.labels)\n .sort(([left], [right]) => left.localeCompare(right))\n .map(([key, value]) => `${key}=${value}`)\n .join(\",\");\n const key = `${input.name}:${labelKey}`;\n const current = totals.get(key) ?? 0;\n if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {\n observability.warn(\"model cache metric cumulative limit reached\", {\n provider: input.provider,\n metric: input.name,\n });\n return;\n }\n observability.incrementCounter({\n name: input.name,\n help: input.help,\n labels: input.labels,\n amount: input.amount,\n });\n totals.set(key, current + input.amount);\n}\n\nfunction nonnegativeFinite(value: number): number {\n return Number.isFinite(value) && value > 0 ? value : 0;\n}\n\nfunction requiredTemporalNumber(\n value: unknown,\n field: string,\n options: { integer?: boolean } = {},\n): number {\n if (value === null || value === undefined) {\n throw new Error(`Temporal stats omitted required ${field}`);\n }\n const parsed = Number(value);\n if (\n !Number.isFinite(parsed) ||\n parsed < 0 ||\n (options.integer && !Number.isSafeInteger(parsed))\n ) {\n throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);\n }\n return parsed;\n}\n\nfunction optionalTemporalNumber(\n value: unknown,\n field: string,\n options: { integer?: boolean } = {},\n): number {\n return value === null || value === undefined ? 0 : requiredTemporalNumber(value, field, options);\n}\n\n/**\n * An opaque, stable, non-reversible tag for a codex credential (account) — the\n * per-call log dimension the account-switch hypothesis correlates against. It is\n * a hash of the NON-SECRET credential ROW id (never a token/bearer), truncated so\n * it is short in logs while still distinguishing a handful of accounts without\n * collision. A null/absent credential (non-codex turn, no active account) tags as\n * \"none\" so the field is always present and never leaks an id verbatim.\n */\nexport function stableAccountHash(credentialId: string | null | undefined): string {\n if (!credentialId) {\n return \"none\";\n }\n return createHash(\"sha256\").update(credentialId).digest(\"hex\").slice(0, 12);\n}\n\n/**\n * The per-call account dimensions for the usage log: the opaque serving-account\n * tag and whether that account CHANGED versus the session's previous call. Within\n * one turn the serving credential is fixed, so a switch can only surface on the\n * turn's FIRST call (compared against the session's durably-recorded prior\n * credential); later calls in the same turn report `false`. A switch is reported\n * only when there was a KNOWN prior account that differs — a session's very first\n * call (no prior) is a cold start, not a switch.\n */\nexport function modelCallAccountContext(input: {\n servingCredentialId: string | null;\n priorSessionCredentialId: string | null;\n isFirstCallOfTurn: boolean;\n}): { servingAccountHash: string; accountChangedFromPrevCall: boolean } {\n const accountChangedFromPrevCall =\n input.isFirstCallOfTurn &&\n input.servingCredentialId !== null &&\n input.priorSessionCredentialId !== null &&\n input.priorSessionCredentialId !== input.servingCredentialId;\n return {\n servingAccountHash: stableAccountHash(input.servingCredentialId),\n accountChangedFromPrevCall,\n };\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAA6C;AAItD;AAAA,EACE;AAAA,EACA;AAAA,OAIK;AAwBP,IAAM,eAAe,oBAAI,QAA6C;AACtE,IAAM,6BAA6B,oBAAI,QAAoC;AAC3E,IAAM,0BAA0B,oBAAI,QAA4C;AAMhF,IAAM,gCAAgC;AAE/B,SAAS,yBAAyB,eAA2C;AAClF,SAAO;AAAA,IACL,OAAO,CAAC,SAAS,eAAe,cAAc,MAAM,SAAS,gBAAgB,UAAU,CAAC;AAAA,IACxF,MAAM,CAAC,SAAS,eAAe,cAAc,KAAK,SAAS,gBAAgB,UAAU,CAAC;AAAA,EACxF;AACF;AAEA,SAAS,gBAAgB,YAAyE;AAChG,MAAI,CAAC,YAAY;AACf,WAAO;AAAA,EACT;AACA,QAAM,YAAwB,CAAC;AAC/B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,UAAU,GAAG;AACrD,cAAU,GAAG,IAAI,oBAAoB,KAAK;AAAA,EAC5C;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAgC;AAC3D,MACE,UAAU,QACV,UAAU,UACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU,WACjB;AACA,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,oCACd,eACqB;AACrB,SAAO;AAAA,IACL,aAAa,CAAC,EAAE,UAAU,SAAS,gBAAgB,MAAM;AACvD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,UAAU,QAAQ;AAAA,MAC9B,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS;AAAA,QACnB,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,iBAAiB,CAAC,EAAE,SAAS,aAAa,SAAS,gBAAgB,MAAM;AACvE,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,cAAc,aAAa,QAAQ;AAAA,MACxD,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,cAAc,YAAY;AAAA,QAC7C,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,yBAAyB,CAAC,EAAE,SAAS,MAAM,MAAM;AAC/C,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,MAAM;AAAA,MAC3B,CAAC;AAAA,IACH;AAAA,IACA,aAAa,CAAC,EAAE,SAAS,IAAI,SAAS,MAAM,QAAQ,iBAAiB,WAAW,MAAM;AACpF,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,IAAI,SAAS,MAAM,QAAQ,GAAG;AAAA,MACnD,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,GAAG;AAAA,QACtB,OAAO;AAAA,MACT,CAAC;AAID,UAAI,QAAQ;AACV,sBAAc,iBAAiB;AAAA,UAC7B,MAAM;AAAA,UACN,MAAM;AAAA,UACN,QAAQ,EAAE,SAAS,GAAG;AAAA,QACxB,CAAC;AAAA,MACH;AAEA,UAAI,eAAe,QAAW;AAC5B,sBAAc,iBAAiB;AAAA,UAC7B,MAAM;AAAA,UACN,MAAM;AAAA,UACN,QAAQ,EAAE,SAAS,GAAG;AAAA,UACtB,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AASO,SAAS,+BAA+B,OAAkD;AAC/F,SAAO,CAAC,MAAM;AACZ,UAAM,cAAc;AAAA,MAClB,SAAS;AAAA,MACT,IAAI,EAAE;AAAA,MACN,SAAS,EAAE;AAAA,MACX,QAAQ,EAAE;AAAA,MACV,SAAS,EAAE;AAAA,MACX,iBAAiB,EAAE,aAAa;AAAA,MAChC,GAAI,EAAE,SAAS,SAAY,EAAE,MAAM,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC;AAAA,MAChE,GAAI,EAAE,eAAe,SAAY,EAAE,YAAY,EAAE,WAAW,IAAI,CAAC;AAAA,IACnE,CAAC;AAAA,EACH;AACF;AAYO,SAAS,yBAAyB,GAA0D;AACjG,MACE,EAAE,YAAY,YACd,EAAE,cACF,wCAAwC,IAAI,EAAE,UAAU,GACxD;AACA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,IAAI,EAAE;AAAA,QACN,YAAY,EAAE;AAAA,QACd,UAAU,EAAE;AAAA,QACZ,GAAI,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,IAAI,CAAC;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,YAAY,QAAQ,EAAE,QAAQ;AAClC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,IAAI,EAAE;AAAA,QACN,YAAY,EAAE,cAAc;AAAA,QAC5B,UAAU,EAAE;AAAA,QACZ,GAAI,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,IAAI,CAAC;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AASO,SAAS,sBAAsB,OAGpC;AACA,QAAM,QAAQ,+BAA+B,KAAK;AAClD,QAAM,WAAoC,CAAC;AAC3C,SAAO;AAAA,IACL,UAAU,CAAC,MAAM;AACf,YAAM,CAAC;AACP,YAAM,QAAQ,yBAAyB,CAAC;AACxC,UAAI,OAAO;AACT,iBAAS,KAAK,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,IACA,aAAa,MAAM,SAAS,OAAO,GAAG,SAAS,MAAM;AAAA,EACvD;AACF;AAEO,SAAS,wBAAwB,eAAoD;AAC1F,QAAM,WAAW,aAAa,IAAI,aAAa;AAC/C,MAAI,UAAU;AACZ,WAAO;AAAA,EACT;AACA,QAAM,UAAU,IAAI,qBAAqB,aAAa;AACtD,eAAa,IAAI,eAAe,OAAO;AACvC,SAAO;AACT;AAEO,IAAM,uBAAN,MAA2B;AAAA,EAIhC,YACmB,eACA,UAA8D,CAAC,GAChF;AAFiB;AACA;AAAA,EAChB;AAAA,EANc,QAAQ,oBAAI,IAA2D;AAAA,EAChF,QAA+C;AAAA,EAOvD,MAAM,QAAsB;AAC1B,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,MAAM,IAAI,QAAQ,EAAE,WAAW,KAAK,gBAAgB,IAAI,CAAC;AAC9D,SAAK,YAAY;AACjB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,SAAS,QAAsB;AAC7B,UAAM,OAAO,KAAK,MAAM,IAAI,MAAM;AAClC,QAAI,CAAC,KAAM;AACX,SAAK,iBAAiB,KAAK,IAAI;AAAA,EACjC;AAAA,EAEA,OAAO,QAAgB,SAA6B,iBAAgC;AAClF,UAAM,YAAY,KAAK,MAAM,IAAI,MAAM,GAAG;AAC1C,QAAI,cAAc,QAAW;AAC3B,WAAK,MAAM,OAAO,MAAM;AAAA,IAC1B;AACA,QAAI,SAAS;AACX,YAAM,mBACJ,oBACC,cAAc,SAAY,IAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,aAAa,GAAI;AAC5E,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,QAAQ;AAAA,MACpB,CAAC;AACD,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,QAAQ;AAAA,QAClB,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,SAAK,cAAc;AACnB,QAAI,KAAK,MAAM,SAAS,GAAG;AACzB,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,gBAAsB;AACpB,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,MAAM;AAAA,IACpB,CAAC;AACD,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,yBAAyB;AAAA,IACvC,CAAC;AACD,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,2BAA2B;AAAA,IACzC,CAAC;AAAA,EACH;AAAA,EAEA,OAAa;AACX,SAAK,MAAM,MAAM;AACjB,SAAK,cAAc;AACnB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEQ,2BAAmC;AACzC,QAAI,KAAK,MAAM,SAAS,GAAG;AACzB,aAAO;AAAA,IACT;AACA,QAAI,SAAS,OAAO;AACpB,eAAW,EAAE,UAAU,KAAK,KAAK,MAAM,OAAO,GAAG;AAC/C,eAAS,KAAK,IAAI,QAAQ,SAAS;AAAA,IACrC;AACA,WAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,UAAU,GAAI;AAAA,EACjD;AAAA,EAEQ,6BAAqC;AAC3C,QAAI,KAAK,MAAM,SAAS,EAAG,QAAO;AAClC,QAAI,sBAAsB,OAAO;AACjC,eAAW,EAAE,eAAe,KAAK,KAAK,MAAM,OAAO,GAAG;AACpD,4BAAsB,KAAK,IAAI,qBAAqB,cAAc;AAAA,IACpE;AACA,WAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,uBAAuB,GAAI;AAAA,EAC9D;AAAA,EAEQ,cAAoB;AAC1B,QAAI,KAAK,OAAO;AACd;AAAA,IACF;AACA,SAAK,QAAQ,YAAY,MAAM,KAAK,cAAc,GAAG,KAAK,QAAQ,qBAAqB,IAAM;AAC7F,SAAK,MAAM,QAAQ;AAAA,EACrB;AAAA,EAEQ,YAAkB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf;AAAA,IACF;AACA,kBAAc,KAAK,KAAK;AACxB,SAAK,QAAQ;AAAA,EACf;AAAA,EAEQ,MAAc;AACpB,WAAO,KAAK,QAAQ,MAAM,KAAK,KAAK,IAAI;AAAA,EAC1C;AACF;AAEO,SAAS,uBAAuB,eAA8B,OAAqB;AACxF,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN;AAAA,EACF,CAAC;AACH;AASO,SAAS,yBACd,eACA,OACM;AACN,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,eAAe;AAAA,EAChD,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,uBAAuB;AAAA,EACxD,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,YAAY;AAAA,EAC7C,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,iBAAiB;AAAA,EAClD,CAAC;AACH;AAEO,SAAS,4BACd,OACoB;AACpB,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,4DAA4D;AAAA,EAC9E;AACA,QAAM,kBAAkB;AAAA,IACtB,MAAM;AAAA,IACN;AAAA,IACA,EAAE,SAAS,KAAK;AAAA,EAClB;AACA,MAAI,0BAA0B;AAC9B,MAAI,MAAM,uBAAuB;AAC/B,UAAM,UAAU;AAAA,MACd,MAAM,sBAAsB;AAAA,MAC5B;AAAA,MACA,EAAE,SAAS,KAAK;AAAA,IAClB;AACA,UAAM,QAAQ;AAAA,MACZ,MAAM,sBAAsB;AAAA,MAC5B;AAAA,MACA,EAAE,SAAS,KAAK;AAAA,IAClB;AACA,QAAI,SAAS,KAAe;AAC1B,YAAM,IAAI,MAAM,mEAAmE;AAAA,IACrF;AACA,8BAA0B,UAAU,QAAQ;AAAA,EAC9C,WAAW,kBAAkB,GAAG;AAC9B,UAAM,IAAI,MAAM,oEAAoE;AAAA,EACtF;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,uBAAuB,MAAM,cAAc,cAAc;AAAA,IACvE,mBAAmB,uBAAuB,MAAM,mBAAmB,mBAAmB;AAAA,EACxF;AACF;AAEO,SAAS,yBAAyB,OAKN;AACjC,QAAM,aAAa,MAAM,cAAc;AACvC,QAAM,MAAM,MAAM,OAAO,KAAK;AAC9B,QAAM,YAAY,IAAI;AACtB,MAAI,gBAA+B;AACnC,MAAI,oBAAoB;AACxB,MAAI,UAAU;AACd,MAAI,UAAgC;AACpC,QAAM,eAAe,MAAM;AACzB,UAAM,aAAa,IAAI;AACvB,UAAM,eAAe,cAAc,iBAAiB;AACpD,UAAM,MAAM,CAAC,MAAc,MAAc,UACvC,MAAM,cAAc,SAAS,EAAE,MAAM,MAAM,MAAM,CAAC;AACpD;AAAA,MACE;AAAA,MACA;AAAA,MACA,oBAAoB,IAAI;AAAA,IAC1B;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,kBAAkB,OAAO,IAAI,gBAAgB;AAAA,IAC/C;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,KAAK,IAAI,GAAG,eAAe,GAAK;AAAA,IAClC;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,qBAAqB,kBAAkB,QAAQ,gBAAgB,aAAa,IAAI,IAAI;AAAA,IACtF;AAAA,EACF;AACA,QAAM,UAAU,MAAM;AAGpB,iBAAa;AACb,QAAI,WAAW,QAAS;AACxB,cAAU,MACP,KAAK,EACL,KAAK,CAAC,UAAU;AACf,+BAAyB,MAAM,eAAe,KAAK;AACnD,sBAAgB,IAAI;AACpB,0BAAoB;AACpB,mBAAa;AAAA,IACf,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,0BAAoB;AACpB,mBAAa;AACb,YAAM,cAAc,KAAK,2DAA2D;AAAA,QAClF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH,CAAC,EACA,QAAQ,MAAM;AACb,gBAAU;AAAA,IACZ,CAAC;AAAA,EACL;AACA,UAAQ;AACR,QAAM,QAAQ,YAAY,SAAS,UAAU;AAC7C,QAAM,QAAQ;AACd,SAAO;AAAA,IACL,OAAO,YAAY;AACjB,gBAAU;AACV,oBAAc,KAAK;AACnB,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEO,SAAS,yBACd,eACA,QACM;AACN,aAAW,YAAY,CAAC,QAAQ,WAAW,QAAQ,UAAU,GAAY;AACvE,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,SAAS;AAAA,MACnB,OAAO,OAAO,QAAQ,KAAK;AAAA,IAC7B,CAAC;AAAA,EACH;AACF;AAaO,SAAS,wCACd,eACA,QACA,mBAAmB,KAAK,IAAI,IAAI,KAC1B;AACN,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,OAAO;AAAA,IACjB,OAAO;AAAA,EACT,CAAC;AACH;AAEO,SAAS,wCACd,eACA,QACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,OAAO;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,0BACd,eACA,UACM;AACN,QAAM,WAAW,2BAA2B,IAAI,aAAa,KAAK,oBAAI,IAAY;AAClF,QAAM,UAAU,oBAAI,IAAY;AAChC,aAAW,WAAW,UAAU;AAC9B,YAAQ,IAAI,QAAQ,SAAS;AAC7B,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,YAAY,QAAQ,UAAU;AAAA,MACxC,OAAO,QAAQ;AAAA,IACjB,CAAC;AAAA,EACH;AACA,aAAW,aAAa,UAAU;AAChC,QAAI,CAAC,QAAQ,IAAI,SAAS,GAAG;AAC3B,oBAAc,SAAS;AAAA,QACrB,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,YAAY,UAAU;AAAA,QAChC,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,EACF;AACA,6BAA2B,IAAI,eAAe,OAAO;AACvD;AAEO,SAAS,+BAA+B,eAA8B,OAAqB;AAChG,MAAI,SAAS,GAAG;AACd;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AACH;AAWO,SAAS,sCACd,eACA,QACM;AACN,aAAW,SAAS;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAAY;AACV,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,MAAM;AAAA,MAChB,OAAO,OAAO,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AASO,SAAS,uCACd,eACA,SACA,QAAQ,GACF;AACN,MAAI,SAAS,EAAG;AAChB,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,IAClB,QAAQ;AAAA,EACV,CAAC;AACH;AAEO,SAAS,wCACd,eACA,OACM;AACN,MAAI,SAAS,EAAG;AAChB,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AACH;AAEO,SAAS,mCACd,eACA,SAOM;AACN,QAAM,SAAS;AAAA,IACb,WAAW,QAAQ;AAAA,IACnB,SAAS,QAAQ;AAAA,IACjB,cAAc,QAAQ;AAAA,IACtB,gBAAgB,QAAQ;AAAA,IACxB,iBAAiB,QAAQ;AAAA,EAC3B;AACA,aAAW,CAAC,MAAM,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AAClD,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,KAAK;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,IAAM,gCAAgC;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,mCAAmC,IAAI,IAAY,6BAA6B;AAE/E,SAAS,qCACd,eACA,QACM;AACN,QAAM,aAAa,oBAAI,IAAkD;AACzE,aAAW,cAAc,+BAA+B;AACtD,eAAW,IAAI,YAAY,EAAE,QAAQ,GAAG,UAAU,EAAE,CAAC;AAAA,EACvD;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,iCAAiC,IAAI,MAAM,UAAU,IACpE,MAAM,aACN;AACJ,UAAM,UAAU,WAAW,IAAI,UAAU;AACzC,YAAQ,UAAU,KAAK,IAAI,GAAG,MAAM,WAAW;AAC/C,YAAQ,YAAY,KAAK,IAAI,GAAG,MAAM,kBAAkB;AAAA,EAC1D;AAEA,aAAW,CAAC,YAAY,KAAK,KAAK,YAAY;AAC5C,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,aAAa,WAAW;AAAA,MAClC,OAAO,MAAM;AAAA,IACf,CAAC;AACD,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,aAAa,WAAW;AAAA,MAClC,OAAO,MAAM;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,IAAM,4BAA4B,CAAC,GAAG,eAAe,SAAS,SAAS;AAC9E,IAAM,+BAA+B,IAAI,IAAY,yBAAyB;AACvE,IAAM,+BAA+B,CAAC,SAAS,SAAS,SAAS,QAAQ;AAEzE,SAAS,wCACd,eACA,QACM;AACN,QAAM,aAAa,oBAAI,IAAoB;AAC3C,aAAW,WAAW,2BAA2B;AAC/C,eAAW,aAAa,8BAA8B;AACpD,iBAAW,IAAI,GAAG,OAAO,IAAI,SAAS,IAAI,CAAC;AAAA,IAC7C;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,UAAU,6BAA6B,IAAI,MAAM,OAAO,IAAI,MAAM,UAAU;AAClF,QAAI,CAAC,6BAA6B,SAAS,MAAM,SAAkB,EAAG;AACtE,UAAM,MAAM,GAAG,OAAO,IAAI,MAAM,SAAS;AACzC,eAAW,IAAI,MAAM,WAAW,IAAI,GAAG,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM,KAAK,CAAC;AAAA,EAC3E;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,YAAY;AACrC,UAAM,CAAC,SAAS,SAAS,IAAI,IAAI,MAAM,GAAG;AAC1C,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,SAAS,YAAY,UAAU;AAAA,MACzC;AAAA,IACF,CAAC;AAAA,EACH;AACF;AA0BO,SAAS,oCACd,eACA,SACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAEO,SAAS,mBACd,eACA,MACA,cACM;AACN,MAAI,CAAC,iBAAiB,gBAAgB,GAAG;AACvC;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,KAAK;AAAA,IACf,QAAQ;AAAA,EACV,CAAC;AACH;AAEA,IAAM,8BAA8B;AAAA,EAClC;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAK;AAAA,EAAG;AAAA,EAAG;AAAA,EAAG;AAAA,EAAI;AAAA,EAAI;AAAA,EAAI;AACtE;AAOO,SAAS,wBACd,eACA,OAMM;AACN,QAAM,UAAU,MAAM,WAAW;AACjC,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,UAAU,MAAM,UAAU,OAAO,MAAM,OAAO,QAAQ;AAAA,EAClE,CAAC;AACD,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,QAAQ,EAAE,UAAU,MAAM,UAAU,OAAO,MAAM,MAAM;AAAA,IACvD,OAAO,KAAK,IAAI,GAAG,MAAM,eAAe;AAAA,EAC1C,CAAC;AACH;AAcA,SAAS,kBAAkB,MAAiD;AAC1E,MAAI,SAAS,uBAAuB;AAClC,WAAO;AAAA,EACT;AACA,MAAI,SAAS,yBAAyB;AACpC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAKA,IAAM,sBAAsB,CAAC,MAAM,MAAM,KAAK,KAAK,MAAM,KAAK,MAAM,GAAG,KAAK,GAAG,GAAG,GAAG,EAAE;AACvF,IAAM,6BAA6B,CAAC,MAAO,MAAM,OAAO,MAAM,KAAK,KAAK,MAAM,KAAK,GAAG,GAAG,CAAC;AAmBnF,IAAM,sBAAN,MAA0B;AAAA,EAM/B,YACmB,eACA,SACjB;AAFiB;AACA;AAEjB,SAAK,MAAM,QAAQ,QAAQ,MAAM,YAAY,IAAI;AACjD,SAAK,aAAa,KAAK,IAAI;AAAA,EAC7B;AAAA,EAXiB;AAAA,EACT;AAAA,EACA,YAAY;AAAA,EACH,cAAc,oBAAI,IAA8B;AAAA,EAUjE,QAAQ,MAA8B;AACpC,UAAM,aAAa,kBAAkB,IAAI;AACzC,QAAI,eAAe,MAAM;AAIvB,WAAK,aAAa,KAAK,IAAI;AAC3B,WAAK,YAAY;AACjB,WAAK,YAAY,MAAM;AACvB;AAAA,IACF;AACA,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI,KAAK,WAAW;AAClB,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,SAAS;AAAA,QACT,QAAQ,EAAE,UAAU,KAAK,QAAQ,SAAS;AAAA,QAC1C,OAAO,KAAK,IAAI,IAAI,KAAK,KAAK,cAAc,GAAI;AAAA,MAClD,CAAC;AACD,WAAK,YAAY;AAAA,IACnB;AACA,UAAM,OAAO,KAAK,YAAY,IAAI,UAAU;AAC5C,QAAI,SAAS,QAAW;AACtB,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,SAAS;AAAA,QACT,QAAQ,EAAE,UAAU,KAAK,QAAQ,UAAU,OAAO,WAAW;AAAA,QAC7D,OAAO,KAAK,IAAI,IAAI,KAAK,QAAQ,GAAI;AAAA,MACvC,CAAC;AAAA,IACH;AACA,SAAK,YAAY,IAAI,YAAY,EAAE;AAAA,EACrC;AACF;AAIA,IAAM,4BAA4B,CAAC,GAAG,GAAG,GAAG,IAAI,IAAI,IAAI,KAAK,KAAK,GAAG;AACrE,IAAM,oBAAoB,CAAC,MAAO,OAAQ,MAAO,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,GAAG,KAAK,CAAC;AAItF,SAAS,iBACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACD,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIO,SAAS,gCACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIO,SAAS,iCACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIA,IAAM,6BAA6B;AAAA,EACjC;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AACrF;AAKO,SAAS,uBACd,eACA,UACA,aACM;AACN,QAAM,wBAAwB,2BAA2B,WAAW;AACpE,MAAI,0BAA0B,QAAQ,0BAA0B,GAAG;AACjE;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,QAAQ,EAAE,SAAS;AAAA,IACnB,OAAO;AAAA,EACT,CAAC;AACH;AAKO,SAAS,wBAAwB,eAA8B,SAAuB;AAC3F,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAEA,IAAM,2CAA2C,CAAC,GAAG,GAAG,IAAI,KAAK,KAAK,MAAO,IAAK;AAM3E,SAAS,gCACd,eACA,SACM;AACN,aAAW,gBAAgB,QAAQ,eAAe;AAChD,UAAM,SAAS;AAAA,MACb,UAAU,aAAa;AAAA,MACvB,QAAQ,aAAa;AAAA,MACrB,kBAAkB,aAAa;AAAA,MAC/B,iBAAiB,aAAa;AAAA,MAC9B,cAAc,QAAQ;AAAA,MACtB,oBAAoB,QAAQ;AAAA,IAC9B;AACA,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IACF,CAAC;AACD,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,MACT;AAAA,MACA,OAAO,aAAa;AAAA,IACtB,CAAC;AAAA,EACH;AACF;AAcA,IAAM,gCAAgC;AAAA,EACpC;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAM;AACjE;AAoBO,SAAS,uBACd,eACA,UACA,OAKM;AACN,QAAM,SAAS,2BAA2B,MAAM,YAAY;AAC5D,QAAM,aAAa,2BAA2B,MAAM,gBAAgB;AACpE,QAAM,SAAS,2BAA2B,MAAM,YAAY;AAC5D,oCAAkC,eAAe;AAAA,IAC/C,MAAM;AAAA,IACN,MAAM;AAAA,IACN;AAAA,IACA,QAAQ,EAAE,UAAU,QAAQ,WAAW,OAAO,YAAY,WAAW;AAAA,IACrE,QAAQ;AAAA,EACV,CAAC;AACD,MAAI,WAAW,QAAQ,SAAS,GAAG;AACjC,sCAAkC,eAAe;AAAA,MAC/C,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,MACA,QAAQ,EAAE,SAAS;AAAA,MACnB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,eAAe,QAAQ,aAAa,GAAG;AACzC,sCAAkC,eAAe;AAAA,MAC/C,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,MACA,QAAQ,EAAE,SAAS;AAAA,MACnB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,WAAW,QAAQ,WAAW,QAAQ,SAAS,GAAG;AACpD,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,MACT,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA,MAGnB,OAAO,KAAK,IAAI,GAAG,SAAS,MAAM;AAAA,IACpC,CAAC;AAAA,EACH;AACF;AAEA,SAAS,kCACP,eACA,OAOM;AACN,QAAM,SAAS,wBAAwB,IAAI,aAAa,KAAK,oBAAI,IAAoB;AACrF,MAAI,CAAC,wBAAwB,IAAI,aAAa,GAAG;AAC/C,4BAAwB,IAAI,eAAe,MAAM;AAAA,EACnD;AACA,QAAM,WAAW,OAAO,QAAQ,MAAM,MAAM,EACzC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,MAAM,KAAK,cAAc,KAAK,CAAC,EACnD,IAAI,CAAC,CAACA,MAAK,KAAK,MAAM,GAAGA,IAAG,IAAI,KAAK,EAAE,EACvC,KAAK,GAAG;AACX,QAAM,MAAM,GAAG,MAAM,IAAI,IAAI,QAAQ;AACrC,QAAM,UAAU,OAAO,IAAI,GAAG,KAAK;AACnC,MAAI,MAAM,SAAS,gCAAgC,SAAS;AAC1D,kBAAc,KAAK,+CAA+C;AAAA,MAChE,UAAU,MAAM;AAAA,MAChB,QAAQ,MAAM;AAAA,IAChB,CAAC;AACD;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,IACZ,QAAQ,MAAM;AAAA,IACd,QAAQ,MAAM;AAAA,EAChB,CAAC;AACD,SAAO,IAAI,KAAK,UAAU,MAAM,MAAM;AACxC;AAEA,SAAS,kBAAkB,OAAuB;AAChD,SAAO,OAAO,SAAS,KAAK,KAAK,QAAQ,IAAI,QAAQ;AACvD;AAEA,SAAS,uBACP,OACA,OACA,UAAiC,CAAC,GAC1B;AACR,MAAI,UAAU,QAAQ,UAAU,QAAW;AACzC,UAAM,IAAI,MAAM,mCAAmC,KAAK,EAAE;AAAA,EAC5D;AACA,QAAM,SAAS,OAAO,KAAK;AAC3B,MACE,CAAC,OAAO,SAAS,MAAM,KACvB,SAAS,KACR,QAAQ,WAAW,CAAC,OAAO,cAAc,MAAM,GAChD;AACA,UAAM,IAAI,MAAM,mCAAmC,KAAK,KAAK,OAAO,KAAK,CAAC,EAAE;AAAA,EAC9E;AACA,SAAO;AACT;AAEA,SAAS,uBACP,OACA,OACA,UAAiC,CAAC,GAC1B;AACR,SAAO,UAAU,QAAQ,UAAU,SAAY,IAAI,uBAAuB,OAAO,OAAO,OAAO;AACjG;AAUO,SAAS,kBAAkB,cAAiD;AACjF,MAAI,CAAC,cAAc;AACjB,WAAO;AAAA,EACT;AACA,SAAO,WAAW,QAAQ,EAAE,OAAO,YAAY,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AAC5E;AAWO,SAAS,wBAAwB,OAIgC;AACtE,QAAM,6BACJ,MAAM,qBACN,MAAM,wBAAwB,QAC9B,MAAM,6BAA6B,QACnC,MAAM,6BAA6B,MAAM;AAC3C,SAAO;AAAA,IACL,oBAAoB,kBAAkB,MAAM,mBAAmB;AAAA,IAC/D;AAAA,EACF;AACF;","names":["key"]}
package/dist/index.d.ts CHANGED
@@ -13,6 +13,8 @@ import { type WorkerRunTarget } from "./worker-service-lifecycle.js";
13
13
  export { createHostExportPump, type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, type HostEventExport, type HostEventExportBatch, type HostEventSink, type HostUsageExport, type HostUsageExportBatch, type HostUsageSink, } from "./host-export.js";
14
14
  export declare const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
15
15
  export declare const FILE_UPLOAD_REAPER_PERIOD_MS: number;
16
+ export declare const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
17
+ export declare const SITE_AUTH_MAINTENANCE_PERIOD_MS: number;
16
18
  export type OpenGeniWorkerRole = "control" | "turn";
17
19
  export type WorkerOptions = {
18
20
  role: OpenGeniWorkerRole;
@@ -46,12 +48,14 @@ export declare function temporalWorkflowExecutionNotFound(error: unknown): boole
46
48
  * until an activity first resolves its injected services. */
47
49
  export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole, dependencies?: ActivityDependencies): Promise<{
48
50
  getCodexCapacityWait: (input: import("./activities.js").GetCodexCapacityWaitInput) => Promise<{
51
+ provider?: "xai";
49
52
  waiterId: string;
50
53
  generation: number;
51
54
  nextCheckAt: string;
52
55
  wakeRevision: number;
53
56
  } | null>;
54
57
  reconcileCodexCapacityWait: (input: import("./activities.js").ReconcileCodexCapacityWaitInput) => Promise<import("./activities.js").ReconcileCodexCapacityWaitResult>;
58
+ maintainBrowserStateArtifacts: () => Promise<import("./activities/browser-state-artifact-reaper.js").MaintainBrowserStateArtifactsResult>;
55
59
  indexDocument: (input: import("./activities.js").IndexDocumentInput) => Promise<{
56
60
  id: string;
57
61
  workspaceId: string;
@@ -175,6 +179,7 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
175
179
  maintainSandboxLeaseSweep: (input: import("./sandbox-reaper-contract.js").SandboxLeaseSweepMaintenanceInput) => Promise<import("./activities/sandbox-lease.js").SandboxLeaseSweepMaintenanceResult>;
176
180
  reapSandboxLeases: () => Promise<import("./activities/sandbox-lease.js").ReapSandboxLeasesResult>;
177
181
  dispatchScheduledTaskRun: (input: import("./activities.js").DispatchScheduledTaskRunInput) => Promise<import("./activities.js").DispatchScheduledTaskRunResult>;
182
+ maintainSiteAuthConnections: () => Promise<import("./activities/site-auth-maintenance.js").MaintainSiteAuthConnectionsResult>;
178
183
  failSessionAttempt: (input: import("./activities/types.js").FailSessionAttemptInput) => Promise<void>;
179
184
  settleSessionInterruptions: (input: import("./activities/types.js").SettleSessionInterruptionsInput) => Promise<{
180
185
  action: "paused" | "continue" | "stale";
@@ -185,6 +190,7 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
185
190
  recoverEscapedMcpTimeout: (input: import("./activities.js").RecoverEscapedMcpTimeoutInput) => Promise<import("./activities.js").RecoverEscapedMcpTimeoutResult>;
186
191
  peekSessionWork: (input: import("./activities/types.js").PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
187
192
  expireSessionHumanInput: (input: import("./activities.js").ExpireSessionHumanInputInput) => Promise<import("./activities.js").ExpireSessionHumanInputResult>;
193
+ expireSessionInteractionIntervention: (input: import("./activities/types.js").ExpireSessionInteractionInterventionInput) => Promise<import("./activities/types.js").ExpireSessionInteractionInterventionResult>;
188
194
  markSessionIdle: (input: import("./activities/types.js").MarkSessionIdleInput) => Promise<void>;
189
195
  dispatchSessionWorkflowWakes(): Promise<import("./activities/workflow-wake.js").DispatchSessionWorkflowWakesResult>;
190
196
  } | {
@@ -258,6 +264,13 @@ export declare function registerFileUploadReaperSchedule(settings: Settings, obs
258
264
  registered: boolean;
259
265
  close: () => Promise<void>;
260
266
  }>;
267
+ /** Register the deployment-wide maintained-auth dispatcher. Its one-minute
268
+ * cadence matches the minimum public policy interval; DB claims own overlap,
269
+ * crash recovery, and exact session idempotency. */
270
+ export declare function registerSiteAuthMaintenanceSchedule(settings: Settings, observability: Observability): Promise<{
271
+ registered: boolean;
272
+ close: () => Promise<void>;
273
+ }>;
261
274
  /**
262
275
  * Register the one repair cadence for committed workflow-wake revisions. The
263
276
  * activity only reads the transactional outbox and sends revision-scoped
package/dist/index.js CHANGED
@@ -15,7 +15,7 @@ import {
15
15
  normalizeTurnTaskQueueStats,
16
16
  observabilityEventLogger,
17
17
  startTurnCapacityMonitor
18
- } from "./chunk-E3RI3I33.js";
18
+ } from "./chunk-TYNPCG4U.js";
19
19
 
20
20
  // src/index.ts
21
21
  import {
@@ -695,6 +695,8 @@ function safeWarn(observability, message, attributes) {
695
695
  var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
696
696
  var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
697
697
  var FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1e3;
698
+ var SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
699
+ var SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1e3;
698
700
  function temporalTimeMs(value) {
699
701
  if (value?.seconds === void 0 || value.seconds === null) return null;
700
702
  return Number(String(value.seconds)) * 1e3 + Math.floor((value.nanos ?? 0) / 1e6);
@@ -709,7 +711,7 @@ function temporalWorkflowExecutionNotFound(error) {
709
711
  return isGrpcServiceError(error) && error.code === 5;
710
712
  }
711
713
  async function createDefaultWorkerActivities(role, dependencies = {}) {
712
- return role === "control" ? (await import("./activities-control-44KBWLSM.js")).createControlActivities(dependencies) : (await import("./activities-turn-K4CEDEGJ.js")).createTurnActivities(dependencies);
714
+ return role === "control" ? (await import("./activities-control-XURE4VKN.js")).createControlActivities(dependencies) : (await import("./activities-turn-6EBXGGLS.js")).createTurnActivities(dependencies);
713
715
  }
714
716
  function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
715
717
  const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
@@ -1075,6 +1077,41 @@ async function registerFileUploadReaperSchedule(settings, observability) {
1075
1077
  throw error;
1076
1078
  }
1077
1079
  }
1080
+ async function registerSiteAuthMaintenanceSchedule(settings, observability) {
1081
+ const connection = await Connection.connect(temporalConnectionOptions(settings));
1082
+ const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
1083
+ try {
1084
+ await temporal.schedule.create({
1085
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1086
+ spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
1087
+ action: {
1088
+ type: "startWorkflow",
1089
+ workflowType: "siteAuthMaintenanceWorkflow",
1090
+ taskQueue: settings.temporalTaskQueue,
1091
+ args: []
1092
+ },
1093
+ policies: {
1094
+ overlap: ScheduleOverlapPolicy.SKIP,
1095
+ catchupWindow: "1m",
1096
+ pauseOnFailure: false
1097
+ }
1098
+ });
1099
+ observability.info("Registered the global site-auth maintenance Schedule", {
1100
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1101
+ maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS
1102
+ });
1103
+ return { registered: true, close: async () => connection.close() };
1104
+ } catch (error) {
1105
+ if (error instanceof ScheduleAlreadyRunning) {
1106
+ observability.info("Global site-auth maintenance Schedule already registered", {
1107
+ scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID
1108
+ });
1109
+ return { registered: false, close: async () => connection.close() };
1110
+ }
1111
+ await connection.close().catch(() => void 0);
1112
+ throw error;
1113
+ }
1114
+ }
1078
1115
  async function registerSessionWorkflowWakeDispatcherSchedule(settings, observability) {
1079
1116
  const connection = await Connection.connect(temporalConnectionOptions(settings));
1080
1117
  const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
@@ -1184,6 +1221,13 @@ async function createOpenGeniWorkerService(options) {
1184
1221
  { ...retryOptions, onRetry }
1185
1222
  )
1186
1223
  );
1224
+ schedules.push(
1225
+ await retryStartupDependency(
1226
+ "Temporal schedule (site auth maintenance)",
1227
+ () => registerSiteAuthMaintenanceSchedule(settings, observability),
1228
+ { ...retryOptions, onRetry }
1229
+ )
1230
+ );
1187
1231
  schedules.push(
1188
1232
  await retryStartupDependency(
1189
1233
  "Temporal schedule (session-workflow wake dispatcher)",
@@ -1352,6 +1396,8 @@ if (import.meta.main) {
1352
1396
  export {
1353
1397
  FILE_UPLOAD_REAPER_PERIOD_MS,
1354
1398
  FILE_UPLOAD_REAPER_SCHEDULE_ID,
1399
+ SITE_AUTH_MAINTENANCE_PERIOD_MS,
1400
+ SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
1355
1401
  createDefaultWorkerActivities,
1356
1402
  createHostExportPump,
1357
1403
  createOpenGeniWorker,
@@ -1360,6 +1406,7 @@ export {
1360
1406
  registerFileUploadReaperSchedule,
1361
1407
  registerSandboxReaperSchedule,
1362
1408
  registerSessionWorkflowWakeDispatcherSchedule,
1409
+ registerSiteAuthMaintenanceSchedule,
1363
1410
  resolveOpenGeniWorkflowDefinition,
1364
1411
  runOpenGeniWorker,
1365
1412
  startWorker,