@opengeni/worker-bundle 0.17.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +12 -0
- package/dist/activities/agent-turn.d.ts +25 -0
- package/dist/activities/browser-state-artifact-reaper.d.ts +29 -0
- package/dist/activities/capabilities.d.ts +6 -0
- package/dist/activities/codex-capacity.d.ts +1 -0
- package/dist/activities/environment.d.ts +7 -1
- package/dist/activities/packs.d.ts +8 -10
- package/dist/activities/session-state.d.ts +4 -2
- package/dist/activities/site-auth-maintenance.d.ts +27 -0
- package/dist/activities/types.d.ts +16 -0
- package/dist/activities/video-generation-admission.d.ts +9 -4
- package/dist/activities/video-generation-credential.d.ts +17 -0
- package/dist/activities/video-reference-staging.d.ts +7 -1
- package/dist/activities/xai-auth.d.ts +23 -0
- package/dist/activities/xai-image-generation.d.ts +26 -0
- package/dist/activities/xai-video-generation.d.ts +18 -0
- package/dist/{activities-control-44KBWLSM.js → activities-control-XURE4VKN.js} +387 -51
- package/dist/activities-control-XURE4VKN.js.map +1 -0
- package/dist/activities-control.d.ts +8 -0
- package/dist/{activities-turn-K4CEDEGJ.js → activities-turn-6EBXGGLS.js} +5312 -3951
- package/dist/activities-turn-6EBXGGLS.js.map +1 -0
- package/dist/activities.d.ts +9 -1
- package/dist/{chunk-KKCRGAGZ.js → chunk-5GRJIP5H.js} +99 -98
- package/dist/chunk-5GRJIP5H.js.map +1 -0
- package/dist/{chunk-E3RI3I33.js → chunk-TYNPCG4U.js} +88 -10
- package/dist/chunk-TYNPCG4U.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +49 -2
- package/dist/index.js.map +1 -1
- package/dist/model-checkpoint-memory-collector.d.ts +1 -0
- package/dist/model-context-contributions.d.ts +40 -0
- package/dist/observability-metrics.d.ts +20 -1
- package/dist/sandbox-routing.d.ts +4 -0
- package/dist/workflow-bundle.js +59 -15
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/site-auth-maintenance.d.ts +2 -0
- package/dist/workflows.d.ts +1 -0
- package/package.json +16 -15
- package/src/activities/agent-run-admission.ts +76 -0
- package/src/activities/agent-turn.ts +1178 -142
- package/src/activities/browser-state-artifact-reaper.ts +120 -0
- package/src/activities/capabilities.ts +10 -0
- package/src/activities/codex-capacity.ts +48 -1
- package/src/activities/environment.ts +14 -8
- package/src/activities/goals.ts +2 -1
- package/src/activities/packs.ts +45 -122
- package/src/activities/parent-wake.ts +13 -1
- package/src/activities/scheduled-tasks.ts +30 -73
- package/src/activities/session-state.ts +18 -0
- package/src/activities/site-auth-maintenance.ts +269 -0
- package/src/activities/types.ts +17 -0
- package/src/activities/video-generation-admission.ts +37 -4
- package/src/activities/video-generation-credential.ts +84 -6
- package/src/activities/video-generation-reconciliation.ts +212 -19
- package/src/activities/video-media-validation.ts +8 -2
- package/src/activities/video-reference-staging.ts +121 -34
- package/src/activities/xai-auth.ts +122 -0
- package/src/activities/xai-image-generation.ts +86 -0
- package/src/activities/xai-video-generation.ts +69 -0
- package/src/activities-control.ts +4 -0
- package/src/activities.ts +5 -0
- package/src/index.ts +51 -0
- package/src/model-checkpoint-memory-collector.ts +8 -1
- package/src/model-context-contributions.ts +159 -0
- package/src/observability-metrics.ts +98 -9
- package/src/sandbox-routing.ts +13 -0
- package/src/workflows/activities.ts +1 -0
- package/src/workflows/file-upload-reaper.ts +8 -2
- package/src/workflows/session.ts +20 -6
- package/src/workflows/site-auth-maintenance.ts +14 -0
- package/src/workflows.ts +1 -0
- package/dist/activities-control-44KBWLSM.js.map +0 -1
- package/dist/activities-turn-K4CEDEGJ.js.map +0 -1
- package/dist/chunk-E3RI3I33.js.map +0 -1
- package/dist/chunk-KKCRGAGZ.js.map +0 -1
|
@@ -168,17 +168,23 @@ var TurnLifecycleMetrics = class {
|
|
|
168
168
|
this.observability = observability;
|
|
169
169
|
this.options = options;
|
|
170
170
|
}
|
|
171
|
-
|
|
171
|
+
turns = /* @__PURE__ */ new Map();
|
|
172
172
|
timer = null;
|
|
173
173
|
start(turnId) {
|
|
174
|
-
|
|
174
|
+
const now = this.now();
|
|
175
|
+
this.turns.set(turnId, { startedAt: now, lastProgressAt: now });
|
|
175
176
|
this.ensureTimer();
|
|
176
177
|
this.refreshGauges();
|
|
177
178
|
}
|
|
179
|
+
progress(turnId) {
|
|
180
|
+
const turn = this.turns.get(turnId);
|
|
181
|
+
if (!turn) return;
|
|
182
|
+
turn.lastProgressAt = this.now();
|
|
183
|
+
}
|
|
178
184
|
finish(turnId, outcome, durationSeconds) {
|
|
179
|
-
const startedAt = this.
|
|
185
|
+
const startedAt = this.turns.get(turnId)?.startedAt;
|
|
180
186
|
if (startedAt !== void 0) {
|
|
181
|
-
this.
|
|
187
|
+
this.turns.delete(turnId);
|
|
182
188
|
}
|
|
183
189
|
if (outcome) {
|
|
184
190
|
const observedDuration = durationSeconds ?? (startedAt === void 0 ? 0 : Math.max(0, (this.now() - startedAt) / 1e3));
|
|
@@ -195,7 +201,7 @@ var TurnLifecycleMetrics = class {
|
|
|
195
201
|
});
|
|
196
202
|
}
|
|
197
203
|
this.refreshGauges();
|
|
198
|
-
if (this.
|
|
204
|
+
if (this.turns.size === 0) {
|
|
199
205
|
this.stopTimer();
|
|
200
206
|
}
|
|
201
207
|
}
|
|
@@ -203,29 +209,42 @@ var TurnLifecycleMetrics = class {
|
|
|
203
209
|
this.observability.setGauge({
|
|
204
210
|
name: "opengeni_turns_inflight",
|
|
205
211
|
help: "Current number of in-flight agent turns in this worker process.",
|
|
206
|
-
value: this.
|
|
212
|
+
value: this.turns.size
|
|
207
213
|
});
|
|
208
214
|
this.observability.setGauge({
|
|
209
215
|
name: "opengeni_turn_oldest_inflight_age_seconds",
|
|
210
216
|
help: "Age in seconds of the oldest in-flight agent turn in this worker process.",
|
|
211
217
|
value: this.oldestInflightAgeSeconds()
|
|
212
218
|
});
|
|
219
|
+
this.observability.setGauge({
|
|
220
|
+
name: "opengeni_turn_oldest_no_progress_age_seconds",
|
|
221
|
+
help: "Seconds since durable progress for the least recently progressing in-flight turn.",
|
|
222
|
+
value: this.oldestNoProgressAgeSeconds()
|
|
223
|
+
});
|
|
213
224
|
}
|
|
214
225
|
stop() {
|
|
215
|
-
this.
|
|
226
|
+
this.turns.clear();
|
|
216
227
|
this.refreshGauges();
|
|
217
228
|
this.stopTimer();
|
|
218
229
|
}
|
|
219
230
|
oldestInflightAgeSeconds() {
|
|
220
|
-
if (this.
|
|
231
|
+
if (this.turns.size === 0) {
|
|
221
232
|
return 0;
|
|
222
233
|
}
|
|
223
234
|
let oldest = Number.POSITIVE_INFINITY;
|
|
224
|
-
for (const startedAt of this.
|
|
235
|
+
for (const { startedAt } of this.turns.values()) {
|
|
225
236
|
oldest = Math.min(oldest, startedAt);
|
|
226
237
|
}
|
|
227
238
|
return Math.max(0, (this.now() - oldest) / 1e3);
|
|
228
239
|
}
|
|
240
|
+
oldestNoProgressAgeSeconds() {
|
|
241
|
+
if (this.turns.size === 0) return 0;
|
|
242
|
+
let leastRecentProgress = Number.POSITIVE_INFINITY;
|
|
243
|
+
for (const { lastProgressAt } of this.turns.values()) {
|
|
244
|
+
leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);
|
|
245
|
+
}
|
|
246
|
+
return Math.max(0, (this.now() - leastRecentProgress) / 1e3);
|
|
247
|
+
}
|
|
229
248
|
ensureTimer() {
|
|
230
249
|
if (this.timer) {
|
|
231
250
|
return;
|
|
@@ -567,6 +586,38 @@ function recordCreditMicros(observability, kind, amountMicros) {
|
|
|
567
586
|
amount: amountMicros
|
|
568
587
|
});
|
|
569
588
|
}
|
|
589
|
+
var MODEL_REQUEST_PHASE_BUCKETS = [
|
|
590
|
+
5e-3,
|
|
591
|
+
0.01,
|
|
592
|
+
0.025,
|
|
593
|
+
0.05,
|
|
594
|
+
0.1,
|
|
595
|
+
0.2,
|
|
596
|
+
0.35,
|
|
597
|
+
0.5,
|
|
598
|
+
1,
|
|
599
|
+
2,
|
|
600
|
+
5,
|
|
601
|
+
10,
|
|
602
|
+
30,
|
|
603
|
+
60,
|
|
604
|
+
300
|
|
605
|
+
];
|
|
606
|
+
function recordModelRequestPhase(observability, input) {
|
|
607
|
+
const outcome = input.outcome ?? "";
|
|
608
|
+
observability.incrementCounter({
|
|
609
|
+
name: "opengeni_model_request_phases_total",
|
|
610
|
+
help: "Provider model-request lifecycle phases by bounded phase and outcome.",
|
|
611
|
+
labels: { provider: input.provider, phase: input.phase, outcome }
|
|
612
|
+
});
|
|
613
|
+
observability.observeHistogram({
|
|
614
|
+
name: "opengeni_model_request_phase_duration_seconds",
|
|
615
|
+
help: "Monotonic seconds from provider dispatch to a model-request lifecycle phase.",
|
|
616
|
+
buckets: MODEL_REQUEST_PHASE_BUCKETS,
|
|
617
|
+
labels: { provider: input.provider, phase: input.phase },
|
|
618
|
+
value: Math.max(0, input.durationSeconds)
|
|
619
|
+
});
|
|
620
|
+
}
|
|
570
621
|
function contentDeltaClass(type) {
|
|
571
622
|
if (type === "agent.message.delta") {
|
|
572
623
|
return "message";
|
|
@@ -686,6 +737,31 @@ function recordContextCompaction(observability, trigger) {
|
|
|
686
737
|
labels: { trigger }
|
|
687
738
|
});
|
|
688
739
|
}
|
|
740
|
+
var MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2048, 8192];
|
|
741
|
+
function recordCompanyBrainContributions(observability, receipt) {
|
|
742
|
+
for (const contribution of receipt.contributions) {
|
|
743
|
+
const labels = {
|
|
744
|
+
category: contribution.category,
|
|
745
|
+
source: contribution.source,
|
|
746
|
+
inclusion_reason: contribution.inclusionReason,
|
|
747
|
+
authority_scope: contribution.authorityScope,
|
|
748
|
+
session_role: receipt.sessionRole,
|
|
749
|
+
memory_prompt_mode: receipt.memoryPromptMode
|
|
750
|
+
};
|
|
751
|
+
observability.incrementCounter({
|
|
752
|
+
name: "opengeni_model_context_contributions_total",
|
|
753
|
+
help: "Company Brain contributions exposed to a model, by bounded reason and authority.",
|
|
754
|
+
labels
|
|
755
|
+
});
|
|
756
|
+
observability.observeHistogram({
|
|
757
|
+
name: "opengeni_model_context_contribution_estimated_tokens",
|
|
758
|
+
help: "Estimated tokens per model-visible Company Brain contribution.",
|
|
759
|
+
buckets: MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS,
|
|
760
|
+
labels,
|
|
761
|
+
value: contribution.estimatedTokens
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
}
|
|
689
765
|
var MODEL_CACHE_HIT_RATIO_BUCKETS = [
|
|
690
766
|
0.05,
|
|
691
767
|
0.1,
|
|
@@ -816,13 +892,15 @@ export {
|
|
|
816
892
|
recordExpiredDrainingSandboxLeaseGauges,
|
|
817
893
|
recordRetainedProcessReconciliation,
|
|
818
894
|
recordCreditMicros,
|
|
895
|
+
recordModelRequestPhase,
|
|
819
896
|
StreamTimingMetrics,
|
|
820
897
|
recordBatchFlush,
|
|
821
898
|
recordSessionEventAppendLatency,
|
|
822
899
|
recordSessionEventPublishLatency,
|
|
823
900
|
recordModelInputTokens,
|
|
824
901
|
recordContextCompaction,
|
|
902
|
+
recordCompanyBrainContributions,
|
|
825
903
|
recordModelCacheTokens,
|
|
826
904
|
modelCallAccountContext
|
|
827
905
|
};
|
|
828
|
-
//# sourceMappingURL=chunk-
|
|
906
|
+
//# sourceMappingURL=chunk-TYNPCG4U.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/observability-metrics.ts"],"sourcesContent":["import { createHash } from \"node:crypto\";\nimport { errorCodeToJSON } from \"@opengeni/agent-proto\";\nimport { SandboxBackend, type SessionEventType } from \"@opengeni/contracts\";\nimport type { EventLogger } from \"@opengeni/events\";\nimport type { Attributes, AttributeValue, Observability } from \"@opengeni/observability\";\nimport type { CompanyBrainContributionReceipt } from \"./model-context-contributions\";\nimport {\n SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES,\n modelUsageTokenCountOrNull,\n type RuntimeMetricsHooks,\n type SelfhostedOpObservation,\n type SelfhostedOpObserver,\n} from \"@opengeni/runtime\";\n\nexport type TurnOutcome = \"completed\" | \"failed\" | \"cancelled\" | \"recovering\";\nexport type CreditMicrosKind = \"usage\" | \"grant\" | \"topup\" | \"refund\";\nexport type SandboxLeaseLiveness = \"cold\" | \"warming\" | \"warm\" | \"draining\";\nexport type CreditBalanceGauge = { accountId: string; balanceMicros: number };\nexport type TurnTaskQueueStats = {\n /** Temporal activity tasks ready for a turn worker to accept. */\n eligibleBacklog: number;\n oldestBacklogAgeSeconds: number;\n tasksAddRate: number;\n tasksDispatchRate: number;\n};\n\nexport type TemporalTurnTaskQueueStats = {\n approximateBacklogCount?: unknown;\n approximateBacklogAge?: {\n seconds?: unknown;\n nanos?: unknown;\n } | null;\n tasksAddRate?: unknown;\n tasksDispatchRate?: unknown;\n} | null;\n\nconst turnTrackers = new WeakMap<Observability, TurnLifecycleMetrics>();\nconst creditBalanceGaugeAccounts = new WeakMap<Observability, Set<string>>();\nconst modelCacheCounterTotals = new WeakMap<Observability, Map<string, number>>();\n\n// A worker process reaching one trillion tokens in one provider/cache counter is\n// already far outside an ordinary scrape lifetime. Refuse further increments\n// before floating-point precision can degrade; a restart resets both the\n// process-local Prometheus registry and this guard together.\nconst MAX_MODEL_CACHE_COUNTER_TOTAL = 1_000_000_000_000;\n\nexport function observabilityEventLogger(observability: Observability): EventLogger {\n return {\n debug: (message, attributes) => observability.debug(message, eventAttributes(attributes)),\n warn: (message, attributes) => observability.warn(message, eventAttributes(attributes)),\n };\n}\n\nfunction eventAttributes(attributes: Record<string, unknown> | undefined): Attributes | undefined {\n if (!attributes) {\n return undefined;\n }\n const projected: Attributes = {};\n for (const [key, value] of Object.entries(attributes)) {\n projected[key] = eventAttributeValue(value);\n }\n return projected;\n}\n\nfunction eventAttributeValue(value: unknown): AttributeValue {\n if (\n value === null ||\n value === undefined ||\n typeof value === \"string\" ||\n typeof value === \"number\" ||\n typeof value === \"boolean\"\n ) {\n return value;\n }\n try {\n return JSON.stringify(value);\n } catch {\n return String(value);\n }\n}\n\nexport function runtimeMetricsHooksForObservability(\n observability: Observability,\n): RuntimeMetricsHooks {\n return {\n onModelCall: ({ provider, outcome, durationSeconds }) => {\n observability.incrementCounter({\n name: \"opengeni_model_calls_total\",\n help: \"Total model calls by provider and outcome.\",\n labels: { provider, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_model_call_duration_seconds\",\n help: \"Model call duration in seconds by provider.\",\n labels: { provider },\n value: durationSeconds,\n });\n },\n onSandboxCreate: ({ backend, imageSource, outcome, durationSeconds }) => {\n observability.incrementCounter({\n name: \"opengeni_sandbox_creates_total\",\n help: \"Total sandbox create attempts by backend and outcome.\",\n labels: { backend, image_source: imageSource, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_sandbox_create_duration_seconds\",\n help: \"Sandbox create duration in seconds by backend.\",\n labels: { backend, image_source: imageSource },\n value: durationSeconds,\n });\n },\n onSandboxWarmingTimeout: ({ backend, stage }) => {\n observability.incrementCounter({\n name: \"opengeni_sandbox_warming_timeouts_total\",\n help: \"Total sandbox warming timeouts.\",\n labels: { backend, stage },\n });\n },\n onSandboxOp: ({ backend, op, outcome, code, healed, durationSeconds, replyBytes }) => {\n observability.incrementCounter({\n name: \"opengeni_machine_op_total\",\n help: \"Total Connected Machine control ops by op, outcome, and typed fault code.\",\n labels: { backend, op, outcome, code: code ?? \"\" },\n });\n observability.observeHistogram({\n name: \"opengeni_machine_op_duration_seconds\",\n help: \"Connected Machine control-op duration in seconds by op.\",\n labels: { backend, op },\n value: durationSeconds,\n });\n // The healed-fault leading indicator: an op that only succeeded after a retry\n // (a blip/backpressure the transport absorbed). The doctrine: healed faults are\n // the leading indicator of the next unhealed one, so they are always recorded.\n if (healed) {\n observability.incrementCounter({\n name: \"opengeni_machine_op_healed_total\",\n help: \"Connected Machine ops that succeeded only after ≥1 in-call retry.\",\n labels: { backend, op },\n });\n }\n // The payload-wall indicator (bytes known only on a PAYLOAD_TOO_LARGE fault today).\n if (replyBytes !== undefined) {\n observability.observeHistogram({\n name: \"opengeni_machine_op_reply_bytes\",\n help: \"Connected Machine control-op reply size in bytes (payload-wall indicator).\",\n labels: { backend, op },\n value: replyBytes,\n });\n }\n },\n };\n}\n\n/**\n * Adapt the runtime's transport-agnostic `SelfhostedOpObserver` to the metrics\n * hooks: map a completed-op observation onto `onSandboxOp`, converting the wire\n * `ErrorCode` to its stable enum-name string (a bounded metric label) and the\n * duration to seconds. Wired into the selfhosted session build so every Connected\n * Machine control op meters.\n */\nexport function selfhostedOpObserverForMetrics(hooks: RuntimeMetricsHooks): SelfhostedOpObserver {\n return (o) => {\n hooks.onSandboxOp?.({\n backend: \"selfhosted\",\n op: o.op,\n outcome: o.outcome,\n healed: o.healed,\n retries: o.retries,\n durationSeconds: o.durationMs / 1000,\n ...(o.code !== undefined ? { code: errorCodeToJSON(o.code) } : {}),\n ...(o.replyBytes !== undefined ? { replyBytes: o.replyBytes } : {}),\n });\n };\n}\n\n/** A session-scoped `machine.op.*` event mapped from a completed-op observation. */\nexport type MachineOpSessionEvent = {\n type: \"machine.op.failed\" | \"machine.op.recovered\";\n payload: { op: string; faultClass: string; attempts: number; machineId?: string };\n};\n\n/** Map an observation to a `machine.op.*` session event, or null if it is not\n * eventable. `machine.op.failed` fires ONLY for infrastructure fault classes (a\n * semantic miss the model asked about is an outcome, not an infra fault);\n * `machine.op.recovered` fires for a healed op (success after ≥1 retry). */\nexport function machineOpSessionEventFor(o: SelfhostedOpObservation): MachineOpSessionEvent | null {\n if (\n o.outcome === \"failed\" &&\n o.faultClass &&\n SELFHOSTED_INFRASTRUCTURE_FAULT_CLASSES.has(o.faultClass)\n ) {\n return {\n type: \"machine.op.failed\",\n payload: {\n op: o.op,\n faultClass: o.faultClass,\n attempts: o.retries,\n ...(o.machineId ? { machineId: o.machineId } : {}),\n },\n };\n }\n if (o.outcome === \"ok\" && o.healed) {\n return {\n type: \"machine.op.recovered\",\n payload: {\n op: o.op,\n faultClass: o.faultClass ?? \"unknown\",\n attempts: o.retries,\n ...(o.machineId ? { machineId: o.machineId } : {}),\n },\n };\n }\n return null;\n}\n\n/**\n * The Connected Machine op observer wired into a turn: it meters EVERY op (the\n * metrics sink) and BUFFERS the eventable ops (infra failures + healed recoveries)\n * as `machine.op.*` session events. The observer is SYNC (fire-and-forget), so the\n * turn drains the buffer to durable session events at a known checkpoint (turn end),\n * awaited — never an unawaited DB write inside the Temporal activity.\n */\nexport function makeMachineOpObserver(hooks: RuntimeMetricsHooks): {\n observer: SelfhostedOpObserver;\n drainEvents(): MachineOpSessionEvent[];\n} {\n const meter = selfhostedOpObserverForMetrics(hooks);\n const buffered: MachineOpSessionEvent[] = [];\n return {\n observer: (o) => {\n meter(o);\n const event = machineOpSessionEventFor(o);\n if (event) {\n buffered.push(event);\n }\n },\n drainEvents: () => buffered.splice(0, buffered.length),\n };\n}\n\nexport function turnLifecycleMetricsFor(observability: Observability): TurnLifecycleMetrics {\n const existing = turnTrackers.get(observability);\n if (existing) {\n return existing;\n }\n const tracker = new TurnLifecycleMetrics(observability);\n turnTrackers.set(observability, tracker);\n return tracker;\n}\n\nexport class TurnLifecycleMetrics {\n private readonly turns = new Map<string, { startedAt: number; lastProgressAt: number }>();\n private timer: ReturnType<typeof setInterval> | null = null;\n\n constructor(\n private readonly observability: Observability,\n private readonly options: { now?: () => number; refreshIntervalMs?: number } = {},\n ) {}\n\n start(turnId: string): void {\n const now = this.now();\n this.turns.set(turnId, { startedAt: now, lastProgressAt: now });\n this.ensureTimer();\n this.refreshGauges();\n }\n\n progress(turnId: string): void {\n const turn = this.turns.get(turnId);\n if (!turn) return;\n turn.lastProgressAt = this.now();\n }\n\n finish(turnId: string, outcome: TurnOutcome | null, durationSeconds?: number): void {\n const startedAt = this.turns.get(turnId)?.startedAt;\n if (startedAt !== undefined) {\n this.turns.delete(turnId);\n }\n if (outcome) {\n const observedDuration =\n durationSeconds ??\n (startedAt === undefined ? 0 : Math.max(0, (this.now() - startedAt) / 1000));\n this.observability.incrementCounter({\n name: \"opengeni_turns_total\",\n help: \"Total agent turns by terminal outcome.\",\n labels: { outcome },\n });\n this.observability.observeHistogram({\n name: \"opengeni_turn_duration_seconds\",\n help: \"Agent turn duration in seconds by terminal outcome.\",\n labels: { outcome },\n value: observedDuration,\n });\n }\n this.refreshGauges();\n if (this.turns.size === 0) {\n this.stopTimer();\n }\n }\n\n refreshGauges(): void {\n this.observability.setGauge({\n name: \"opengeni_turns_inflight\",\n help: \"Current number of in-flight agent turns in this worker process.\",\n value: this.turns.size,\n });\n this.observability.setGauge({\n name: \"opengeni_turn_oldest_inflight_age_seconds\",\n help: \"Age in seconds of the oldest in-flight agent turn in this worker process.\",\n value: this.oldestInflightAgeSeconds(),\n });\n this.observability.setGauge({\n name: \"opengeni_turn_oldest_no_progress_age_seconds\",\n help: \"Seconds since durable progress for the least recently progressing in-flight turn.\",\n value: this.oldestNoProgressAgeSeconds(),\n });\n }\n\n stop(): void {\n this.turns.clear();\n this.refreshGauges();\n this.stopTimer();\n }\n\n private oldestInflightAgeSeconds(): number {\n if (this.turns.size === 0) {\n return 0;\n }\n let oldest = Number.POSITIVE_INFINITY;\n for (const { startedAt } of this.turns.values()) {\n oldest = Math.min(oldest, startedAt);\n }\n return Math.max(0, (this.now() - oldest) / 1000);\n }\n\n private oldestNoProgressAgeSeconds(): number {\n if (this.turns.size === 0) return 0;\n let leastRecentProgress = Number.POSITIVE_INFINITY;\n for (const { lastProgressAt } of this.turns.values()) {\n leastRecentProgress = Math.min(leastRecentProgress, lastProgressAt);\n }\n return Math.max(0, (this.now() - leastRecentProgress) / 1000);\n }\n\n private ensureTimer(): void {\n if (this.timer) {\n return;\n }\n this.timer = setInterval(() => this.refreshGauges(), this.options.refreshIntervalMs ?? 15_000);\n this.timer.unref?.();\n }\n\n private stopTimer(): void {\n if (!this.timer) {\n return;\n }\n clearInterval(this.timer);\n this.timer = null;\n }\n\n private now(): number {\n return this.options.now?.() ?? Date.now();\n }\n}\n\nexport function recordTurnsQueuedGauge(observability: Observability, value: number): void {\n observability.setGauge({\n name: \"opengeni_turns_queued\",\n help: \"Current number of queued session turns.\",\n value,\n });\n}\n\n/**\n * Record the authoritative turn activity queue rather than aggregate Postgres\n * prompts. A paused human prompt is durable queue truth but is not runnable and\n * never reaches this Temporal task queue. DescribeTaskQueue's approximate\n * backlog count and age are explicitly documented by Temporal as autoscaling\n * signals.\n */\nexport function recordTurnTaskQueueStats(\n observability: Observability,\n stats: TurnTaskQueueStats,\n): void {\n observability.setGauge({\n name: \"opengeni_turn_eligible_backlog\",\n help: \"Temporal runAgentTurn activity tasks eligible for immediate worker admission.\",\n value: nonnegativeFinite(stats.eligibleBacklog),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_backlog_oldest_age_seconds\",\n help: \"Approximate age of the oldest eligible runAgentTurn activity task.\",\n value: nonnegativeFinite(stats.oldestBacklogAgeSeconds),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_tasks_add_rate\",\n help: \"Temporal runAgentTurn tasks added per second over its rolling window.\",\n value: nonnegativeFinite(stats.tasksAddRate),\n });\n observability.setGauge({\n name: \"opengeni_turn_eligible_tasks_dispatch_rate\",\n help: \"Temporal runAgentTurn tasks dispatched per second over its rolling window.\",\n value: nonnegativeFinite(stats.tasksDispatchRate),\n });\n}\n\nexport function normalizeTurnTaskQueueStats(\n stats: TemporalTurnTaskQueueStats | undefined,\n): TurnTaskQueueStats {\n if (!stats) {\n throw new Error(\"Temporal DescribeTaskQueue response omitted required stats\");\n }\n const eligibleBacklog = requiredTemporalNumber(\n stats.approximateBacklogCount,\n \"approximateBacklogCount\",\n { integer: true },\n );\n let oldestBacklogAgeSeconds = 0;\n if (stats.approximateBacklogAge) {\n const seconds = optionalTemporalNumber(\n stats.approximateBacklogAge.seconds,\n \"approximateBacklogAge.seconds\",\n { integer: true },\n );\n const nanos = optionalTemporalNumber(\n stats.approximateBacklogAge.nanos,\n \"approximateBacklogAge.nanos\",\n { integer: true },\n );\n if (nanos >= 1_000_000_000) {\n throw new Error(\"Temporal approximateBacklogAge.nanos must be less than one second\");\n }\n oldestBacklogAgeSeconds = seconds + nanos / 1_000_000_000;\n } else if (eligibleBacklog > 0) {\n throw new Error(\"Temporal stats omitted approximateBacklogAge for a nonzero backlog\");\n }\n return {\n eligibleBacklog,\n oldestBacklogAgeSeconds,\n tasksAddRate: optionalTemporalNumber(stats.tasksAddRate, \"tasksAddRate\"),\n tasksDispatchRate: optionalTemporalNumber(stats.tasksDispatchRate, \"tasksDispatchRate\"),\n };\n}\n\nexport function startTurnCapacityMonitor(input: {\n observability: Observability;\n read: () => Promise<TurnTaskQueueStats>;\n intervalMs?: number;\n now?: () => number;\n}): { close: () => Promise<void> } {\n const intervalMs = input.intervalMs ?? 15_000;\n const now = input.now ?? Date.now;\n const startedAt = now();\n let lastSuccessAt: number | null = null;\n let lastReadSucceeded = false;\n let stopped = false;\n let running: Promise<void> | null = null;\n const recordStatus = () => {\n const observedAt = now();\n const successAgeMs = observedAt - (lastSuccessAt ?? startedAt);\n const set = (name: string, help: string, value: number) =>\n input.observability.setGauge({ name, help, value });\n set(\n \"opengeni_turn_capacity_monitor_last_read_success\",\n \"Whether the latest Temporal turn-queue capacity read completed successfully.\",\n lastReadSucceeded ? 1 : 0,\n );\n set(\n \"opengeni_turn_capacity_monitor_last_success_timestamp_seconds\",\n \"Unix timestamp of the latest successful Temporal turn-queue capacity read, or zero before one succeeds.\",\n lastSuccessAt === null ? 0 : lastSuccessAt / 1_000,\n );\n set(\n \"opengeni_turn_capacity_monitor_last_success_age_seconds\",\n \"Age of the latest successful Temporal turn-queue capacity read, or monitor age before one succeeds.\",\n Math.max(0, successAgeMs / 1_000),\n );\n set(\n \"opengeni_turn_capacity_monitor_fresh\",\n \"Whether eligible-backlog gauges have a successful Temporal read within three monitor intervals.\",\n lastReadSucceeded && lastSuccessAt !== null && successAgeMs <= intervalMs * 3 ? 1 : 0,\n );\n };\n const refresh = () => {\n // Advance freshness age even while a Temporal read is hung. Old backlog\n // values remain observable for diagnosis but cease to be authoritative.\n recordStatus();\n if (stopped || running) return;\n running = input\n .read()\n .then((stats) => {\n recordTurnTaskQueueStats(input.observability, stats);\n lastSuccessAt = now();\n lastReadSucceeded = true;\n recordStatus();\n })\n .catch((error) => {\n lastReadSucceeded = false;\n recordStatus();\n input.observability.warn(\"turn capacity monitor: Temporal task-queue stats failed\", {\n error: error instanceof Error ? error.message : String(error),\n });\n })\n .finally(() => {\n running = null;\n });\n };\n refresh();\n const timer = setInterval(refresh, intervalMs);\n timer.unref?.();\n return {\n close: async () => {\n stopped = true;\n clearInterval(timer);\n await running;\n },\n };\n}\n\nexport function recordSandboxLeaseGauges(\n observability: Observability,\n counts: Partial<Record<SandboxLeaseLiveness, number>>,\n): void {\n for (const liveness of [\"cold\", \"warming\", \"warm\", \"draining\"] as const) {\n observability.setGauge({\n name: \"opengeni_sandbox_leases\",\n help: \"Current sandbox leases by liveness state.\",\n labels: { liveness },\n value: counts[liveness] ?? 0,\n });\n }\n}\n\nexport const SANDBOX_INVENTORY_PROJECTION_DOMAINS = [\n \"leases\",\n \"checkpoint_artifacts\",\n \"rotation_backlog\",\n \"retained_processes\",\n \"expired_drains\",\n] as const;\n\nexport type SandboxInventoryProjectionDomain =\n (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];\n\nexport function recordSandboxInventoryProjectionSuccess(\n observability: Observability,\n domain: SandboxInventoryProjectionDomain,\n timestampSeconds = Date.now() / 1_000,\n): void {\n observability.setGauge({\n name: \"opengeni_sandbox_inventory_refresh_timestamp_seconds\",\n help: \"Unix timestamp of the last complete successful sandbox inventory projection by domain.\",\n labels: { domain },\n value: timestampSeconds,\n });\n}\n\nexport function recordSandboxInventoryProjectionFailure(\n observability: Observability,\n domain: SandboxInventoryProjectionDomain,\n): void {\n observability.incrementCounter({\n name: \"opengeni_sandbox_inventory_refresh_failures_total\",\n help: \"Failed sandbox inventory projection refreshes by domain.\",\n labels: { domain },\n });\n}\n\nexport function recordCreditBalanceGauges(\n observability: Observability,\n balances: CreditBalanceGauge[],\n): void {\n const previous = creditBalanceGaugeAccounts.get(observability) ?? new Set<string>();\n const current = new Set<string>();\n for (const balance of balances) {\n current.add(balance.accountId);\n observability.setGauge({\n name: \"opengeni_credit_balance_micros\",\n help: \"Current credit balance in micros by account.\",\n labels: { account_id: balance.accountId },\n value: balance.balanceMicros,\n });\n }\n for (const accountId of previous) {\n if (!current.has(accountId)) {\n observability.setGauge({\n name: \"opengeni_credit_balance_micros\",\n help: \"Current credit balance in micros by account.\",\n labels: { account_id: accountId },\n value: 0,\n });\n }\n }\n creditBalanceGaugeAccounts.set(observability, current);\n}\n\nexport function recordSandboxOrphansTerminated(observability: Observability, count: number): void {\n if (count <= 0) {\n return;\n }\n observability.incrementCounter({\n name: \"opengeni_sandbox_orphans_terminated_total\",\n help: \"Total provider-side orphan sandboxes terminated by defensive sweeps.\",\n amount: count,\n });\n}\n\nexport type SandboxCheckpointArtifactMetricState =\n | \"candidate\"\n | \"current\"\n | \"previous\"\n | \"delete_pending\"\n | \"deleting\"\n | \"delete_failed\"\n | \"deleted\";\n\nexport function recordSandboxCheckpointArtifactGauges(\n observability: Observability,\n counts: Record<SandboxCheckpointArtifactMetricState, number>,\n): void {\n for (const state of [\n \"candidate\",\n \"current\",\n \"previous\",\n \"delete_pending\",\n \"deleting\",\n \"delete_failed\",\n \"deleted\",\n ] as const) {\n observability.setGauge({\n name: \"opengeni_sandbox_checkpoint_artifacts\",\n help: \"Current durable provider checkpoint artifacts by lifecycle state.\",\n labels: { state },\n value: counts[state],\n });\n }\n}\n\nexport type SandboxCheckpointArtifactOutcome =\n | \"legacy_adopted\"\n | \"claimed\"\n | \"deleted\"\n | \"delete_failed\"\n | \"tombstone_pruned\";\n\nexport function recordSandboxCheckpointArtifactOutcome(\n observability: Observability,\n outcome: SandboxCheckpointArtifactOutcome,\n count = 1,\n): void {\n if (count <= 0) return;\n observability.incrementCounter({\n name: \"opengeni_sandbox_checkpoint_artifact_operations_total\",\n help: \"Durable provider checkpoint artifact operations by fixed outcome.\",\n labels: { outcome },\n amount: count,\n });\n}\n\nexport function recordSandboxDeadlineRotationsRequested(\n observability: Observability,\n count: number,\n): void {\n if (count <= 0) return;\n observability.incrementCounter({\n name: \"opengeni_sandbox_deadline_rotations_requested_total\",\n help: \"Total finite-lifetime sandbox rotations requested before provider deadline.\",\n amount: count,\n });\n}\n\nexport function recordSandboxRotationBacklogGauges(\n observability: Observability,\n backlog: {\n requested: number;\n overdue: number;\n turnBlocked: number;\n directBlocked: number;\n processBlocked: number;\n },\n): void {\n const values = {\n requested: backlog.requested,\n overdue: backlog.overdue,\n turn_blocked: backlog.turnBlocked,\n direct_blocked: backlog.directBlocked,\n process_blocked: backlog.processBlocked,\n } as const;\n for (const [kind, value] of Object.entries(values)) {\n observability.setGauge({\n name: \"opengeni_sandbox_rotation_backlog\",\n help: \"Current finite-lifetime sandbox rotation backlog by fixed condition.\",\n labels: { kind },\n value,\n });\n }\n}\n\nconst RETAINED_PROCESS_OWNER_STATES = [\n \"direct\",\n \"queued\",\n \"running\",\n \"requires_action\",\n \"recovering\",\n \"waiting_capacity\",\n \"completed\",\n \"failed\",\n \"cancelled\",\n \"superseded\",\n \"withdrawn_for_edit\",\n \"missing\",\n \"unknown\",\n] as const;\n\nconst RETAINED_PROCESS_OWNER_STATE_SET = new Set<string>(RETAINED_PROCESS_OWNER_STATES);\n\nexport function recordRetainedProcessInventoryGauges(\n observability: Observability,\n counts: Array<{ ownerState: string; activeCount: number; terminalOwnerCount: number }>,\n): void {\n const normalized = new Map<string, { active: number; terminal: number }>();\n for (const ownerState of RETAINED_PROCESS_OWNER_STATES) {\n normalized.set(ownerState, { active: 0, terminal: 0 });\n }\n for (const count of counts) {\n const ownerState = RETAINED_PROCESS_OWNER_STATE_SET.has(count.ownerState)\n ? count.ownerState\n : \"unknown\";\n const current = normalized.get(ownerState)!;\n current.active += Math.max(0, count.activeCount);\n current.terminal += Math.max(0, count.terminalOwnerCount);\n }\n\n for (const [ownerState, count] of normalized) {\n observability.setGauge({\n name: \"opengeni_retained_processes_active\",\n help: \"Current active retained provider processes by durable owner state.\",\n labels: { owner_state: ownerState },\n value: count.active,\n });\n observability.setGauge({\n name: \"opengeni_retained_processes_terminal_owner_backlog\",\n help: \"Current active retained processes whose exact owner attempt is terminal.\",\n labels: { owner_state: ownerState },\n value: count.terminal,\n });\n }\n}\n\nexport const EXPIRED_DRAINING_BACKENDS = [...SandboxBackend.options, \"unknown\"] as const;\nconst EXPIRED_DRAINING_BACKEND_SET = new Set<string>(EXPIRED_DRAINING_BACKENDS);\nexport const EXPIRED_DRAINING_AGE_BUCKETS = [\"lt_5m\", \"5m_1h\", \"1h_1d\", \"gte_1d\"] as const;\n\nexport function recordExpiredDrainingSandboxLeaseGauges(\n observability: Observability,\n counts: Array<{ backend: string; ageBucket: string; count: number }>,\n): void {\n const normalized = new Map<string, number>();\n for (const backend of EXPIRED_DRAINING_BACKENDS) {\n for (const ageBucket of EXPIRED_DRAINING_AGE_BUCKETS) {\n normalized.set(`${backend}:${ageBucket}`, 0);\n }\n }\n for (const count of counts) {\n const backend = EXPIRED_DRAINING_BACKEND_SET.has(count.backend) ? count.backend : \"unknown\";\n if (!EXPIRED_DRAINING_AGE_BUCKETS.includes(count.ageBucket as never)) continue;\n const key = `${backend}:${count.ageBucket}`;\n normalized.set(key, (normalized.get(key) ?? 0) + Math.max(0, count.count));\n }\n for (const [key, value] of normalized) {\n const [backend, ageBucket] = key.split(\":\") as [string, string];\n observability.setGauge({\n name: \"opengeni_sandbox_leases_expired_draining\",\n help: \"Current expired draining sandbox leases by backend and fixed age bucket.\",\n labels: { backend, age_bucket: ageBucket },\n value,\n });\n }\n}\n\nexport const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [\n \"claim_failed\",\n \"proof_exited\",\n \"proof_lost\",\n \"proof_checkpoint_failed\",\n \"settled_exited\",\n \"settled_lost\",\n \"settlement_failed\",\n \"identity_mismatch\",\n \"resume_state_missing\",\n \"backend_unsupported\",\n \"provider_running\",\n \"provider_unknown\",\n \"provider_timeout\",\n \"provider_error\",\n \"provider_binding_missing\",\n \"provider_binding_mismatch\",\n \"provider_binding_adopted\",\n \"defer_failed\",\n] as const;\n\nexport type RetainedProcessReconciliationOutcome =\n (typeof RETAINED_PROCESS_RECONCILIATION_OUTCOMES)[number];\n\nexport function recordRetainedProcessReconciliation(\n observability: Observability,\n outcome: RetainedProcessReconciliationOutcome,\n): void {\n observability.incrementCounter({\n name: \"opengeni_retained_process_reconciliation_total\",\n help: \"Bounded retained-process reconciliation observations by fixed app outcome.\",\n labels: { outcome },\n });\n}\n\nexport function recordCreditMicros(\n observability: Observability | undefined,\n kind: CreditMicrosKind,\n amountMicros: number,\n): void {\n if (!observability || amountMicros <= 0) {\n return;\n }\n observability.incrementCounter({\n name: \"opengeni_credit_micros_total\",\n help: \"Total credit micros recorded by kind.\",\n labels: { kind },\n amount: amountMicros,\n });\n}\n\nconst MODEL_REQUEST_PHASE_BUCKETS = [\n 0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5, 10, 30, 60, 300,\n];\n\n/**\n * Record provider lifecycle diagnostics from the synchronous transport observer.\n * Provider ids come from the resolved provider registry; phase/outcome are closed\n * enums. This function never performs I/O outside the in-memory metrics registry.\n */\nexport function recordModelRequestPhase(\n observability: Observability,\n input: {\n provider: string;\n phase: \"headers\" | \"first_byte\" | \"terminal\";\n outcome?: \"completed\" | \"failed\" | \"timed_out\";\n durationSeconds: number;\n },\n): void {\n const outcome = input.outcome ?? \"\";\n observability.incrementCounter({\n name: \"opengeni_model_request_phases_total\",\n help: \"Provider model-request lifecycle phases by bounded phase and outcome.\",\n labels: { provider: input.provider, phase: input.phase, outcome },\n });\n observability.observeHistogram({\n name: \"opengeni_model_request_phase_duration_seconds\",\n help: \"Monotonic seconds from provider dispatch to a model-request lifecycle phase.\",\n buckets: MODEL_REQUEST_PHASE_BUCKETS,\n labels: { provider: input.provider, phase: input.phase },\n value: Math.max(0, input.durationSeconds),\n });\n}\n\n// ── Streaming SLIs ────────────────────────────────────────────────────────────\n// Instruments the token-streaming pipeline so \"streaming is sluggish\" is a number,\n// not a vibe. Split across the three attributable stages so an operator can tell\n// WHERE the latency lives: the model (TTFT + inter-delta gaps), our durable write\n// path (append latency), or delivery (publish latency + batcher flush shape). All\n// labels are bounded (provider from the model registry, a two-value delta class) —\n// never a session id or a raw user-supplied model string.\n\nexport type StreamDeltaClass = \"message\" | \"reasoning\";\n\n/** Content-delta classes only. A `null` return means \"not a content delta\" — the\n * event that re-arms the TTFT anchor and closes an inter-delta run. */\nfunction contentDeltaClass(type: SessionEventType): StreamDeltaClass | null {\n if (type === \"agent.message.delta\") {\n return \"message\";\n }\n if (type === \"agent.reasoning.delta\") {\n return \"reasoning\";\n }\n return null;\n}\n\n// TTFT and inter-delta live on a human-perceptible scale (tens of ms to a few\n// seconds), so they get their own SHORT buckets — the default duration buckets\n// (which run to 3600s) would collapse every real streaming value into one bucket.\nconst STREAM_TTFT_BUCKETS = [0.02, 0.05, 0.1, 0.2, 0.35, 0.5, 0.75, 1, 1.5, 2, 3, 5, 10];\nconst STREAM_INTER_DELTA_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.35, 0.5, 1, 2, 5];\n\n/**\n * Per-turn stream-timing tracker fed every normalized runtime event in push order.\n * It emits two model-responsiveness SLIs from the worker's seat on the stream:\n *\n * - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its\n * first streamed content delta. The anchor starts at construction (≈ runStream\n * start, so the first observation is \"how long until text appears\") and re-arms\n * on every non-content event (a tool call, a completed message, a usage frame),\n * so a post-tool response measures the model's restart latency, NOT our own\n * tool-execution time.\n * - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between\n * consecutive content deltas of the SAME class. The run resets on any\n * non-content event so a gap never spans a tool call or a model boundary — it\n * measures only the choppiness of a live token stream.\n *\n * Purely observational and clock-injectable; it never touches the events it sees.\n */\nexport class StreamTimingMetrics {\n private readonly now: () => number;\n private ttftAnchor: number;\n private ttftArmed = true;\n private readonly lastDeltaAt = new Map<StreamDeltaClass, number>();\n\n constructor(\n private readonly observability: Observability,\n private readonly options: { provider: string; now?: () => number },\n ) {\n this.now = options.now ?? (() => performance.now());\n this.ttftAnchor = this.now();\n }\n\n onEvent(type: SessionEventType): void {\n const deltaClass = contentDeltaClass(type);\n if (deltaClass === null) {\n // A non-content event: the model paused emitting. Re-arm TTFT so the next\n // content delta measures (re)start latency, and close every inter-delta run\n // so no gap spans a tool call / model boundary.\n this.ttftAnchor = this.now();\n this.ttftArmed = true;\n this.lastDeltaAt.clear();\n return;\n }\n const at = this.now();\n if (this.ttftArmed) {\n this.observability.observeHistogram({\n name: \"opengeni_stream_ttft_seconds\",\n help: \"Seconds from a model (re)start to its first streamed content delta.\",\n buckets: STREAM_TTFT_BUCKETS,\n labels: { provider: this.options.provider },\n value: Math.max(0, (at - this.ttftAnchor) / 1000),\n });\n this.ttftArmed = false;\n }\n const last = this.lastDeltaAt.get(deltaClass);\n if (last !== undefined) {\n this.observability.observeHistogram({\n name: \"opengeni_stream_inter_delta_gap_seconds\",\n help: \"Seconds between consecutive streamed content deltas of the same class.\",\n buckets: STREAM_INTER_DELTA_BUCKETS,\n labels: { provider: this.options.provider, class: deltaClass },\n value: Math.max(0, (at - last) / 1000),\n });\n }\n this.lastDeltaAt.set(deltaClass, at);\n }\n}\n\n// Batch shapes: sizes are small integers; durations are the append+publish round\n// trip the flush performs (sub-ms to a couple seconds under contention).\nconst STREAM_BATCH_SIZE_BUCKETS = [1, 2, 5, 10, 20, 50, 100, 200, 500];\nconst STREAM_IO_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5];\n\n/** Flush shape of the streaming batcher: how many events coalesced into one flush\n * (the coalescing win) and how long that flush took (append + publish). */\nexport function recordBatchFlush(\n observability: Observability,\n input: { events: number; durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_stream_batch_flush_events\",\n help: \"Session events coalesced into one streaming batcher flush.\",\n buckets: STREAM_BATCH_SIZE_BUCKETS,\n value: input.events,\n });\n observability.observeHistogram({\n name: \"opengeni_stream_batch_flush_duration_seconds\",\n help: \"Duration in seconds of one streaming batcher flush (append + publish).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n/** Latency of the durable `appendSessionEvents` DB write — the write path. A p99\n * climb here is our Postgres, not the model or NATS. */\nexport function recordSessionEventAppendLatency(\n observability: Observability,\n input: { durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_session_event_append_seconds\",\n help: \"Duration in seconds of an appendSessionEvents DB write (the durable write path).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n/** Latency of the best-effort NATS live fan-out — the delivery path. A p99 climb\n * here (with append healthy) is delivery, not the write path. */\nexport function recordSessionEventPublishLatency(\n observability: Observability,\n input: { durationSeconds: number },\n): void {\n observability.observeHistogram({\n name: \"opengeni_session_event_publish_seconds\",\n help: \"Duration in seconds of the best-effort NATS live fan-out publish (the delivery path).\",\n buckets: STREAM_IO_BUCKETS,\n value: input.durationSeconds,\n });\n}\n\n// Context tokens per response span a wide range; buckets track the pressure toward\n// a model's window so \"sessions are running hot but never compacting\" is queryable.\nconst MODEL_INPUT_TOKENS_BUCKETS = [\n 1_000, 5_000, 10_000, 25_000, 50_000, 100_000, 150_000, 200_000, 300_000, 500_000, 1_000_000,\n];\n\n/** Observed input (context) tokens per model response — the context-pressure\n * signal. Paired with `opengeni_context_compactions_total`, it makes the\n * \"compaction never firing while contexts run hot\" failure mode expressible. */\nexport function recordModelInputTokens(\n observability: Observability,\n provider: string,\n inputTokens: number,\n): void {\n const normalizedInputTokens = modelUsageTokenCountOrNull(inputTokens);\n if (normalizedInputTokens === null || normalizedInputTokens === 0) {\n return;\n }\n observability.observeHistogram({\n name: \"opengeni_model_input_tokens\",\n help: \"Observed input (context) tokens per model response, by provider.\",\n buckets: MODEL_INPUT_TOKENS_BUCKETS,\n labels: { provider },\n value: normalizedInputTokens,\n });\n}\n\n/** A context compaction actually fired, by trigger (operator | overflow | proactive\n * | auto). The rate of this — against the input-tokens histogram above — is how an\n * operator sees compaction working (or silently not). */\nexport function recordContextCompaction(observability: Observability, trigger: string): void {\n observability.incrementCounter({\n name: \"opengeni_context_compactions_total\",\n help: \"Total context compactions performed, by trigger.\",\n labels: { trigger },\n });\n}\n\nconst MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS = [1, 8, 32, 128, 512, 2_048, 8_192];\n\n/**\n * Metadata-only Company Brain exposure telemetry. Labels are closed enums and\n * never include tenant ids, record ids, titles, descriptions, or content.\n */\nexport function recordCompanyBrainContributions(\n observability: Observability,\n receipt: CompanyBrainContributionReceipt,\n): void {\n for (const contribution of receipt.contributions) {\n const labels = {\n category: contribution.category,\n source: contribution.source,\n inclusion_reason: contribution.inclusionReason,\n authority_scope: contribution.authorityScope,\n session_role: receipt.sessionRole,\n memory_prompt_mode: receipt.memoryPromptMode,\n };\n observability.incrementCounter({\n name: \"opengeni_model_context_contributions_total\",\n help: \"Company Brain contributions exposed to a model, by bounded reason and authority.\",\n labels,\n });\n observability.observeHistogram({\n name: \"opengeni_model_context_contribution_estimated_tokens\",\n help: \"Estimated tokens per model-visible Company Brain contribution.\",\n buckets: MODEL_CONTEXT_CONTRIBUTION_TOKEN_BUCKETS,\n labels,\n value: contribution.estimatedTokens,\n });\n }\n}\n\n// ── Prompt-cache efficiency ─────────────────────────────────────────────────\n// Per model-call prompt-cache signal, provider-labelled only (bounded\n// cardinality — never a session id or account). `cached_tokens` is the slice of\n// the prompt the provider served from its prompt cache; the ratio cached/prompt\n// is the efficiency of that call. The account-switch hypothesis (a codex account\n// rotation cold-starts the provider's per-account prompt cache) is tested from\n// the per-call STRUCTURED LOG below — the account id is unbounded, so it is\n// hashed into a log field and NEVER a Prometheus label.\n\n// The hit ratio lives in [0, 1]; bucket tighter around the alerting threshold so\n// a p50 near 40% resolves. The default duration buckets (to 3600) would collapse\n// every ratio into the first bucket.\nconst MODEL_CACHE_HIT_RATIO_BUCKETS = [\n 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.95, 0.99, 1,\n];\n\n/**\n * Prompt-cache efficiency of one model response, by provider. Reads from the SAME\n * usage frame that feeds input-token accounting, so the two are always consistent:\n * - `opengeni_model_cached_tokens_total{provider}` — cumulative prompt tokens the\n * provider served from cache. Advanced only when >0, so a provider that never\n * reports cached tokens contributes nothing rather than phantom zero-increments.\n * - `opengeni_model_cache_write_tokens_total{provider}` — cumulative tokens a\n * reporting provider wrote into its prompt cache. Absent telemetry contributes\n * nothing; in particular, Codex subscription usage does not currently expose it.\n * - `opengeni_model_cache_hit_ratio{provider}` — cached/prompt for the call,\n * observed only when prompt tokens are known (>0) AND cached tokens are reported.\n * A reported cached-token zero records a real 0%; absent/null telemetry remains\n * unknown and must not fabricate a low-cache signal.\n * - `opengeni_model_cache_read_telemetry_total{provider,status}` — one bounded\n * availability observation per authoritative call (`reported` or `missing`).\n * Invalid, fractional, unsafe, or over-contract token telemetry is rejected. A call\n * with no prompt tokens has no ratio (skipped).\n */\nexport function recordModelCacheTokens(\n observability: Observability,\n provider: string,\n input: {\n cachedTokens: number | null | undefined;\n cacheWriteTokens?: number | null | undefined;\n promptTokens: number | null | undefined;\n },\n): void {\n const cached = modelUsageTokenCountOrNull(input.cachedTokens);\n const cacheWrite = modelUsageTokenCountOrNull(input.cacheWriteTokens);\n const prompt = modelUsageTokenCountOrNull(input.promptTokens);\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cache_read_telemetry_total\",\n help: \"Authoritative model calls with reported or missing cache-read telemetry, by provider.\",\n provider,\n labels: { provider, status: cached === null ? \"missing\" : \"reported\" },\n amount: 1,\n });\n if (cached !== null && cached > 0) {\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cached_tokens_total\",\n help: \"Total prompt tokens served from the provider's prompt cache, by provider.\",\n provider,\n labels: { provider },\n amount: cached,\n });\n }\n if (cacheWrite !== null && cacheWrite > 0) {\n incrementBoundedModelCacheCounter(observability, {\n name: \"opengeni_model_cache_write_tokens_total\",\n help: \"Total prompt tokens written to the provider's prompt cache, by provider.\",\n provider,\n labels: { provider },\n amount: cacheWrite,\n });\n }\n if (cached !== null && prompt !== null && prompt > 0) {\n observability.observeHistogram({\n name: \"opengeni_model_cache_hit_ratio\",\n help: \"Per-call prompt-cache hit ratio (cached/prompt tokens) by provider.\",\n buckets: MODEL_CACHE_HIT_RATIO_BUCKETS,\n labels: { provider },\n // Clamp to [0, 1]: a provider that (rarely) reports cached >= prompt must\n // not skew the histogram past 1.0.\n value: Math.min(1, cached / prompt),\n });\n }\n}\n\nfunction incrementBoundedModelCacheCounter(\n observability: Observability,\n input: {\n name: string;\n help: string;\n provider: string;\n labels: Record<string, string>;\n amount: number;\n },\n): void {\n const totals = modelCacheCounterTotals.get(observability) ?? new Map<string, number>();\n if (!modelCacheCounterTotals.has(observability)) {\n modelCacheCounterTotals.set(observability, totals);\n }\n const labelKey = Object.entries(input.labels)\n .sort(([left], [right]) => left.localeCompare(right))\n .map(([key, value]) => `${key}=${value}`)\n .join(\",\");\n const key = `${input.name}:${labelKey}`;\n const current = totals.get(key) ?? 0;\n if (input.amount > MAX_MODEL_CACHE_COUNTER_TOTAL - current) {\n observability.warn(\"model cache metric cumulative limit reached\", {\n provider: input.provider,\n metric: input.name,\n });\n return;\n }\n observability.incrementCounter({\n name: input.name,\n help: input.help,\n labels: input.labels,\n amount: input.amount,\n });\n totals.set(key, current + input.amount);\n}\n\nfunction nonnegativeFinite(value: number): number {\n return Number.isFinite(value) && value > 0 ? value : 0;\n}\n\nfunction requiredTemporalNumber(\n value: unknown,\n field: string,\n options: { integer?: boolean } = {},\n): number {\n if (value === null || value === undefined) {\n throw new Error(`Temporal stats omitted required ${field}`);\n }\n const parsed = Number(value);\n if (\n !Number.isFinite(parsed) ||\n parsed < 0 ||\n (options.integer && !Number.isSafeInteger(parsed))\n ) {\n throw new Error(`Temporal stats returned invalid ${field}: ${String(value)}`);\n }\n return parsed;\n}\n\nfunction optionalTemporalNumber(\n value: unknown,\n field: string,\n options: { integer?: boolean } = {},\n): number {\n return value === null || value === undefined ? 0 : requiredTemporalNumber(value, field, options);\n}\n\n/**\n * An opaque, stable, non-reversible tag for a codex credential (account) — the\n * per-call log dimension the account-switch hypothesis correlates against. It is\n * a hash of the NON-SECRET credential ROW id (never a token/bearer), truncated so\n * it is short in logs while still distinguishing a handful of accounts without\n * collision. A null/absent credential (non-codex turn, no active account) tags as\n * \"none\" so the field is always present and never leaks an id verbatim.\n */\nexport function stableAccountHash(credentialId: string | null | undefined): string {\n if (!credentialId) {\n return \"none\";\n }\n return createHash(\"sha256\").update(credentialId).digest(\"hex\").slice(0, 12);\n}\n\n/**\n * The per-call account dimensions for the usage log: the opaque serving-account\n * tag and whether that account CHANGED versus the session's previous call. Within\n * one turn the serving credential is fixed, so a switch can only surface on the\n * turn's FIRST call (compared against the session's durably-recorded prior\n * credential); later calls in the same turn report `false`. A switch is reported\n * only when there was a KNOWN prior account that differs — a session's very first\n * call (no prior) is a cold start, not a switch.\n */\nexport function modelCallAccountContext(input: {\n servingCredentialId: string | null;\n priorSessionCredentialId: string | null;\n isFirstCallOfTurn: boolean;\n}): { servingAccountHash: string; accountChangedFromPrevCall: boolean } {\n const accountChangedFromPrevCall =\n input.isFirstCallOfTurn &&\n input.servingCredentialId !== null &&\n input.priorSessionCredentialId !== null &&\n input.priorSessionCredentialId !== input.servingCredentialId;\n return {\n servingAccountHash: stableAccountHash(input.servingCredentialId),\n accountChangedFromPrevCall,\n };\n}\n"],"mappings":";AAAA,SAAS,kBAAkB;AAC3B,SAAS,uBAAuB;AAChC,SAAS,sBAA6C;AAItD;AAAA,EACE;AAAA,EACA;AAAA,OAIK;AAwBP,IAAM,eAAe,oBAAI,QAA6C;AACtE,IAAM,6BAA6B,oBAAI,QAAoC;AAC3E,IAAM,0BAA0B,oBAAI,QAA4C;AAMhF,IAAM,gCAAgC;AAE/B,SAAS,yBAAyB,eAA2C;AAClF,SAAO;AAAA,IACL,OAAO,CAAC,SAAS,eAAe,cAAc,MAAM,SAAS,gBAAgB,UAAU,CAAC;AAAA,IACxF,MAAM,CAAC,SAAS,eAAe,cAAc,KAAK,SAAS,gBAAgB,UAAU,CAAC;AAAA,EACxF;AACF;AAEA,SAAS,gBAAgB,YAAyE;AAChG,MAAI,CAAC,YAAY;AACf,WAAO;AAAA,EACT;AACA,QAAM,YAAwB,CAAC;AAC/B,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,UAAU,GAAG;AACrD,cAAU,GAAG,IAAI,oBAAoB,KAAK;AAAA,EAC5C;AACA,SAAO;AACT;AAEA,SAAS,oBAAoB,OAAgC;AAC3D,MACE,UAAU,QACV,UAAU,UACV,OAAO,UAAU,YACjB,OAAO,UAAU,YACjB,OAAO,UAAU,WACjB;AACA,WAAO;AAAA,EACT;AACA,MAAI;AACF,WAAO,KAAK,UAAU,KAAK;AAAA,EAC7B,QAAQ;AACN,WAAO,OAAO,KAAK;AAAA,EACrB;AACF;AAEO,SAAS,oCACd,eACqB;AACrB,SAAO;AAAA,IACL,aAAa,CAAC,EAAE,UAAU,SAAS,gBAAgB,MAAM;AACvD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,UAAU,QAAQ;AAAA,MAC9B,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS;AAAA,QACnB,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,iBAAiB,CAAC,EAAE,SAAS,aAAa,SAAS,gBAAgB,MAAM;AACvE,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,cAAc,aAAa,QAAQ;AAAA,MACxD,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,cAAc,YAAY;AAAA,QAC7C,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,IACA,yBAAyB,CAAC,EAAE,SAAS,MAAM,MAAM;AAC/C,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,MAAM;AAAA,MAC3B,CAAC;AAAA,IACH;AAAA,IACA,aAAa,CAAC,EAAE,SAAS,IAAI,SAAS,MAAM,QAAQ,iBAAiB,WAAW,MAAM;AACpF,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,IAAI,SAAS,MAAM,QAAQ,GAAG;AAAA,MACnD,CAAC;AACD,oBAAc,iBAAiB;AAAA,QAC7B,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,SAAS,GAAG;AAAA,QACtB,OAAO;AAAA,MACT,CAAC;AAID,UAAI,QAAQ;AACV,sBAAc,iBAAiB;AAAA,UAC7B,MAAM;AAAA,UACN,MAAM;AAAA,UACN,QAAQ,EAAE,SAAS,GAAG;AAAA,QACxB,CAAC;AAAA,MACH;AAEA,UAAI,eAAe,QAAW;AAC5B,sBAAc,iBAAiB;AAAA,UAC7B,MAAM;AAAA,UACN,MAAM;AAAA,UACN,QAAQ,EAAE,SAAS,GAAG;AAAA,UACtB,OAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAAA,EACF;AACF;AASO,SAAS,+BAA+B,OAAkD;AAC/F,SAAO,CAAC,MAAM;AACZ,UAAM,cAAc;AAAA,MAClB,SAAS;AAAA,MACT,IAAI,EAAE;AAAA,MACN,SAAS,EAAE;AAAA,MACX,QAAQ,EAAE;AAAA,MACV,SAAS,EAAE;AAAA,MACX,iBAAiB,EAAE,aAAa;AAAA,MAChC,GAAI,EAAE,SAAS,SAAY,EAAE,MAAM,gBAAgB,EAAE,IAAI,EAAE,IAAI,CAAC;AAAA,MAChE,GAAI,EAAE,eAAe,SAAY,EAAE,YAAY,EAAE,WAAW,IAAI,CAAC;AAAA,IACnE,CAAC;AAAA,EACH;AACF;AAYO,SAAS,yBAAyB,GAA0D;AACjG,MACE,EAAE,YAAY,YACd,EAAE,cACF,wCAAwC,IAAI,EAAE,UAAU,GACxD;AACA,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,IAAI,EAAE;AAAA,QACN,YAAY,EAAE;AAAA,QACd,UAAU,EAAE;AAAA,QACZ,GAAI,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,IAAI,CAAC;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AACA,MAAI,EAAE,YAAY,QAAQ,EAAE,QAAQ;AAClC,WAAO;AAAA,MACL,MAAM;AAAA,MACN,SAAS;AAAA,QACP,IAAI,EAAE;AAAA,QACN,YAAY,EAAE,cAAc;AAAA,QAC5B,UAAU,EAAE;AAAA,QACZ,GAAI,EAAE,YAAY,EAAE,WAAW,EAAE,UAAU,IAAI,CAAC;AAAA,MAClD;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AASO,SAAS,sBAAsB,OAGpC;AACA,QAAM,QAAQ,+BAA+B,KAAK;AAClD,QAAM,WAAoC,CAAC;AAC3C,SAAO;AAAA,IACL,UAAU,CAAC,MAAM;AACf,YAAM,CAAC;AACP,YAAM,QAAQ,yBAAyB,CAAC;AACxC,UAAI,OAAO;AACT,iBAAS,KAAK,KAAK;AAAA,MACrB;AAAA,IACF;AAAA,IACA,aAAa,MAAM,SAAS,OAAO,GAAG,SAAS,MAAM;AAAA,EACvD;AACF;AAEO,SAAS,wBAAwB,eAAoD;AAC1F,QAAM,WAAW,aAAa,IAAI,aAAa;AAC/C,MAAI,UAAU;AACZ,WAAO;AAAA,EACT;AACA,QAAM,UAAU,IAAI,qBAAqB,aAAa;AACtD,eAAa,IAAI,eAAe,OAAO;AACvC,SAAO;AACT;AAEO,IAAM,uBAAN,MAA2B;AAAA,EAIhC,YACmB,eACA,UAA8D,CAAC,GAChF;AAFiB;AACA;AAAA,EAChB;AAAA,EANc,QAAQ,oBAAI,IAA2D;AAAA,EAChF,QAA+C;AAAA,EAOvD,MAAM,QAAsB;AAC1B,UAAM,MAAM,KAAK,IAAI;AACrB,SAAK,MAAM,IAAI,QAAQ,EAAE,WAAW,KAAK,gBAAgB,IAAI,CAAC;AAC9D,SAAK,YAAY;AACjB,SAAK,cAAc;AAAA,EACrB;AAAA,EAEA,SAAS,QAAsB;AAC7B,UAAM,OAAO,KAAK,MAAM,IAAI,MAAM;AAClC,QAAI,CAAC,KAAM;AACX,SAAK,iBAAiB,KAAK,IAAI;AAAA,EACjC;AAAA,EAEA,OAAO,QAAgB,SAA6B,iBAAgC;AAClF,UAAM,YAAY,KAAK,MAAM,IAAI,MAAM,GAAG;AAC1C,QAAI,cAAc,QAAW;AAC3B,WAAK,MAAM,OAAO,MAAM;AAAA,IAC1B;AACA,QAAI,SAAS;AACX,YAAM,mBACJ,oBACC,cAAc,SAAY,IAAI,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,aAAa,GAAI;AAC5E,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,QAAQ;AAAA,MACpB,CAAC;AACD,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,QAAQ;AAAA,QAClB,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AACA,SAAK,cAAc;AACnB,QAAI,KAAK,MAAM,SAAS,GAAG;AACzB,WAAK,UAAU;AAAA,IACjB;AAAA,EACF;AAAA,EAEA,gBAAsB;AACpB,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,MAAM;AAAA,IACpB,CAAC;AACD,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,yBAAyB;AAAA,IACvC,CAAC;AACD,SAAK,cAAc,SAAS;AAAA,MAC1B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,OAAO,KAAK,2BAA2B;AAAA,IACzC,CAAC;AAAA,EACH;AAAA,EAEA,OAAa;AACX,SAAK,MAAM,MAAM;AACjB,SAAK,cAAc;AACnB,SAAK,UAAU;AAAA,EACjB;AAAA,EAEQ,2BAAmC;AACzC,QAAI,KAAK,MAAM,SAAS,GAAG;AACzB,aAAO;AAAA,IACT;AACA,QAAI,SAAS,OAAO;AACpB,eAAW,EAAE,UAAU,KAAK,KAAK,MAAM,OAAO,GAAG;AAC/C,eAAS,KAAK,IAAI,QAAQ,SAAS;AAAA,IACrC;AACA,WAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,UAAU,GAAI;AAAA,EACjD;AAAA,EAEQ,6BAAqC;AAC3C,QAAI,KAAK,MAAM,SAAS,EAAG,QAAO;AAClC,QAAI,sBAAsB,OAAO;AACjC,eAAW,EAAE,eAAe,KAAK,KAAK,MAAM,OAAO,GAAG;AACpD,4BAAsB,KAAK,IAAI,qBAAqB,cAAc;AAAA,IACpE;AACA,WAAO,KAAK,IAAI,IAAI,KAAK,IAAI,IAAI,uBAAuB,GAAI;AAAA,EAC9D;AAAA,EAEQ,cAAoB;AAC1B,QAAI,KAAK,OAAO;AACd;AAAA,IACF;AACA,SAAK,QAAQ,YAAY,MAAM,KAAK,cAAc,GAAG,KAAK,QAAQ,qBAAqB,IAAM;AAC7F,SAAK,MAAM,QAAQ;AAAA,EACrB;AAAA,EAEQ,YAAkB;AACxB,QAAI,CAAC,KAAK,OAAO;AACf;AAAA,IACF;AACA,kBAAc,KAAK,KAAK;AACxB,SAAK,QAAQ;AAAA,EACf;AAAA,EAEQ,MAAc;AACpB,WAAO,KAAK,QAAQ,MAAM,KAAK,KAAK,IAAI;AAAA,EAC1C;AACF;AAEO,SAAS,uBAAuB,eAA8B,OAAqB;AACxF,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN;AAAA,EACF,CAAC;AACH;AASO,SAAS,yBACd,eACA,OACM;AACN,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,eAAe;AAAA,EAChD,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,uBAAuB;AAAA,EACxD,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,YAAY;AAAA,EAC7C,CAAC;AACD,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,OAAO,kBAAkB,MAAM,iBAAiB;AAAA,EAClD,CAAC;AACH;AAEO,SAAS,4BACd,OACoB;AACpB,MAAI,CAAC,OAAO;AACV,UAAM,IAAI,MAAM,4DAA4D;AAAA,EAC9E;AACA,QAAM,kBAAkB;AAAA,IACtB,MAAM;AAAA,IACN;AAAA,IACA,EAAE,SAAS,KAAK;AAAA,EAClB;AACA,MAAI,0BAA0B;AAC9B,MAAI,MAAM,uBAAuB;AAC/B,UAAM,UAAU;AAAA,MACd,MAAM,sBAAsB;AAAA,MAC5B;AAAA,MACA,EAAE,SAAS,KAAK;AAAA,IAClB;AACA,UAAM,QAAQ;AAAA,MACZ,MAAM,sBAAsB;AAAA,MAC5B;AAAA,MACA,EAAE,SAAS,KAAK;AAAA,IAClB;AACA,QAAI,SAAS,KAAe;AAC1B,YAAM,IAAI,MAAM,mEAAmE;AAAA,IACrF;AACA,8BAA0B,UAAU,QAAQ;AAAA,EAC9C,WAAW,kBAAkB,GAAG;AAC9B,UAAM,IAAI,MAAM,oEAAoE;AAAA,EACtF;AACA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,cAAc,uBAAuB,MAAM,cAAc,cAAc;AAAA,IACvE,mBAAmB,uBAAuB,MAAM,mBAAmB,mBAAmB;AAAA,EACxF;AACF;AAEO,SAAS,yBAAyB,OAKN;AACjC,QAAM,aAAa,MAAM,cAAc;AACvC,QAAM,MAAM,MAAM,OAAO,KAAK;AAC9B,QAAM,YAAY,IAAI;AACtB,MAAI,gBAA+B;AACnC,MAAI,oBAAoB;AACxB,MAAI,UAAU;AACd,MAAI,UAAgC;AACpC,QAAM,eAAe,MAAM;AACzB,UAAM,aAAa,IAAI;AACvB,UAAM,eAAe,cAAc,iBAAiB;AACpD,UAAM,MAAM,CAAC,MAAc,MAAc,UACvC,MAAM,cAAc,SAAS,EAAE,MAAM,MAAM,MAAM,CAAC;AACpD;AAAA,MACE;AAAA,MACA;AAAA,MACA,oBAAoB,IAAI;AAAA,IAC1B;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,kBAAkB,OAAO,IAAI,gBAAgB;AAAA,IAC/C;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,KAAK,IAAI,GAAG,eAAe,GAAK;AAAA,IAClC;AACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,qBAAqB,kBAAkB,QAAQ,gBAAgB,aAAa,IAAI,IAAI;AAAA,IACtF;AAAA,EACF;AACA,QAAM,UAAU,MAAM;AAGpB,iBAAa;AACb,QAAI,WAAW,QAAS;AACxB,cAAU,MACP,KAAK,EACL,KAAK,CAAC,UAAU;AACf,+BAAyB,MAAM,eAAe,KAAK;AACnD,sBAAgB,IAAI;AACpB,0BAAoB;AACpB,mBAAa;AAAA,IACf,CAAC,EACA,MAAM,CAAC,UAAU;AAChB,0BAAoB;AACpB,mBAAa;AACb,YAAM,cAAc,KAAK,2DAA2D;AAAA,QAClF,OAAO,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;AAAA,MAC9D,CAAC;AAAA,IACH,CAAC,EACA,QAAQ,MAAM;AACb,gBAAU;AAAA,IACZ,CAAC;AAAA,EACL;AACA,UAAQ;AACR,QAAM,QAAQ,YAAY,SAAS,UAAU;AAC7C,QAAM,QAAQ;AACd,SAAO;AAAA,IACL,OAAO,YAAY;AACjB,gBAAU;AACV,oBAAc,KAAK;AACnB,YAAM;AAAA,IACR;AAAA,EACF;AACF;AAEO,SAAS,yBACd,eACA,QACM;AACN,aAAW,YAAY,CAAC,QAAQ,WAAW,QAAQ,UAAU,GAAY;AACvE,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,SAAS;AAAA,MACnB,OAAO,OAAO,QAAQ,KAAK;AAAA,IAC7B,CAAC;AAAA,EACH;AACF;AAaO,SAAS,wCACd,eACA,QACA,mBAAmB,KAAK,IAAI,IAAI,KAC1B;AACN,gBAAc,SAAS;AAAA,IACrB,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,OAAO;AAAA,IACjB,OAAO;AAAA,EACT,CAAC;AACH;AAEO,SAAS,wCACd,eACA,QACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,OAAO;AAAA,EACnB,CAAC;AACH;AAEO,SAAS,0BACd,eACA,UACM;AACN,QAAM,WAAW,2BAA2B,IAAI,aAAa,KAAK,oBAAI,IAAY;AAClF,QAAM,UAAU,oBAAI,IAAY;AAChC,aAAW,WAAW,UAAU;AAC9B,YAAQ,IAAI,QAAQ,SAAS;AAC7B,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,YAAY,QAAQ,UAAU;AAAA,MACxC,OAAO,QAAQ;AAAA,IACjB,CAAC;AAAA,EACH;AACA,aAAW,aAAa,UAAU;AAChC,QAAI,CAAC,QAAQ,IAAI,SAAS,GAAG;AAC3B,oBAAc,SAAS;AAAA,QACrB,MAAM;AAAA,QACN,MAAM;AAAA,QACN,QAAQ,EAAE,YAAY,UAAU;AAAA,QAChC,OAAO;AAAA,MACT,CAAC;AAAA,IACH;AAAA,EACF;AACA,6BAA2B,IAAI,eAAe,OAAO;AACvD;AAEO,SAAS,+BAA+B,eAA8B,OAAqB;AAChG,MAAI,SAAS,GAAG;AACd;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AACH;AAWO,SAAS,sCACd,eACA,QACM;AACN,aAAW,SAAS;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,GAAY;AACV,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,MAAM;AAAA,MAChB,OAAO,OAAO,KAAK;AAAA,IACrB,CAAC;AAAA,EACH;AACF;AASO,SAAS,uCACd,eACA,SACA,QAAQ,GACF;AACN,MAAI,SAAS,EAAG;AAChB,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,IAClB,QAAQ;AAAA,EACV,CAAC;AACH;AAEO,SAAS,wCACd,eACA,OACM;AACN,MAAI,SAAS,EAAG;AAChB,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ;AAAA,EACV,CAAC;AACH;AAEO,SAAS,mCACd,eACA,SAOM;AACN,QAAM,SAAS;AAAA,IACb,WAAW,QAAQ;AAAA,IACnB,SAAS,QAAQ;AAAA,IACjB,cAAc,QAAQ;AAAA,IACtB,gBAAgB,QAAQ;AAAA,IACxB,iBAAiB,QAAQ;AAAA,EAC3B;AACA,aAAW,CAAC,MAAM,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AAClD,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,KAAK;AAAA,MACf;AAAA,IACF,CAAC;AAAA,EACH;AACF;AAEA,IAAM,gCAAgC;AAAA,EACpC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,IAAM,mCAAmC,IAAI,IAAY,6BAA6B;AAE/E,SAAS,qCACd,eACA,QACM;AACN,QAAM,aAAa,oBAAI,IAAkD;AACzE,aAAW,cAAc,+BAA+B;AACtD,eAAW,IAAI,YAAY,EAAE,QAAQ,GAAG,UAAU,EAAE,CAAC;AAAA,EACvD;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,aAAa,iCAAiC,IAAI,MAAM,UAAU,IACpE,MAAM,aACN;AACJ,UAAM,UAAU,WAAW,IAAI,UAAU;AACzC,YAAQ,UAAU,KAAK,IAAI,GAAG,MAAM,WAAW;AAC/C,YAAQ,YAAY,KAAK,IAAI,GAAG,MAAM,kBAAkB;AAAA,EAC1D;AAEA,aAAW,CAAC,YAAY,KAAK,KAAK,YAAY;AAC5C,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,aAAa,WAAW;AAAA,MAClC,OAAO,MAAM;AAAA,IACf,CAAC;AACD,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,aAAa,WAAW;AAAA,MAClC,OAAO,MAAM;AAAA,IACf,CAAC;AAAA,EACH;AACF;AAEO,IAAM,4BAA4B,CAAC,GAAG,eAAe,SAAS,SAAS;AAC9E,IAAM,+BAA+B,IAAI,IAAY,yBAAyB;AACvE,IAAM,+BAA+B,CAAC,SAAS,SAAS,SAAS,QAAQ;AAEzE,SAAS,wCACd,eACA,QACM;AACN,QAAM,aAAa,oBAAI,IAAoB;AAC3C,aAAW,WAAW,2BAA2B;AAC/C,eAAW,aAAa,8BAA8B;AACpD,iBAAW,IAAI,GAAG,OAAO,IAAI,SAAS,IAAI,CAAC;AAAA,IAC7C;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,UAAM,UAAU,6BAA6B,IAAI,MAAM,OAAO,IAAI,MAAM,UAAU;AAClF,QAAI,CAAC,6BAA6B,SAAS,MAAM,SAAkB,EAAG;AACtE,UAAM,MAAM,GAAG,OAAO,IAAI,MAAM,SAAS;AACzC,eAAW,IAAI,MAAM,WAAW,IAAI,GAAG,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM,KAAK,CAAC;AAAA,EAC3E;AACA,aAAW,CAAC,KAAK,KAAK,KAAK,YAAY;AACrC,UAAM,CAAC,SAAS,SAAS,IAAI,IAAI,MAAM,GAAG;AAC1C,kBAAc,SAAS;AAAA,MACrB,MAAM;AAAA,MACN,MAAM;AAAA,MACN,QAAQ,EAAE,SAAS,YAAY,UAAU;AAAA,MACzC;AAAA,IACF,CAAC;AAAA,EACH;AACF;AA0BO,SAAS,oCACd,eACA,SACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAEO,SAAS,mBACd,eACA,MACA,cACM;AACN,MAAI,CAAC,iBAAiB,gBAAgB,GAAG;AACvC;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,KAAK;AAAA,IACf,QAAQ;AAAA,EACV,CAAC;AACH;AAEA,IAAM,8BAA8B;AAAA,EAClC;AAAA,EAAO;AAAA,EAAM;AAAA,EAAO;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAK;AAAA,EAAG;AAAA,EAAG;AAAA,EAAG;AAAA,EAAI;AAAA,EAAI;AAAA,EAAI;AACtE;AAOO,SAAS,wBACd,eACA,OAMM;AACN,QAAM,UAAU,MAAM,WAAW;AACjC,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,UAAU,MAAM,UAAU,OAAO,MAAM,OAAO,QAAQ;AAAA,EAClE,CAAC;AACD,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,QAAQ,EAAE,UAAU,MAAM,UAAU,OAAO,MAAM,MAAM;AAAA,IACvD,OAAO,KAAK,IAAI,GAAG,MAAM,eAAe;AAAA,EAC1C,CAAC;AACH;AAcA,SAAS,kBAAkB,MAAiD;AAC1E,MAAI,SAAS,uBAAuB;AAClC,WAAO;AAAA,EACT;AACA,MAAI,SAAS,yBAAyB;AACpC,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAKA,IAAM,sBAAsB,CAAC,MAAM,MAAM,KAAK,KAAK,MAAM,KAAK,MAAM,GAAG,KAAK,GAAG,GAAG,GAAG,EAAE;AACvF,IAAM,6BAA6B,CAAC,MAAO,MAAM,OAAO,MAAM,KAAK,KAAK,MAAM,KAAK,GAAG,GAAG,CAAC;AAmBnF,IAAM,sBAAN,MAA0B;AAAA,EAM/B,YACmB,eACA,SACjB;AAFiB;AACA;AAEjB,SAAK,MAAM,QAAQ,QAAQ,MAAM,YAAY,IAAI;AACjD,SAAK,aAAa,KAAK,IAAI;AAAA,EAC7B;AAAA,EAXiB;AAAA,EACT;AAAA,EACA,YAAY;AAAA,EACH,cAAc,oBAAI,IAA8B;AAAA,EAUjE,QAAQ,MAA8B;AACpC,UAAM,aAAa,kBAAkB,IAAI;AACzC,QAAI,eAAe,MAAM;AAIvB,WAAK,aAAa,KAAK,IAAI;AAC3B,WAAK,YAAY;AACjB,WAAK,YAAY,MAAM;AACvB;AAAA,IACF;AACA,UAAM,KAAK,KAAK,IAAI;AACpB,QAAI,KAAK,WAAW;AAClB,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,SAAS;AAAA,QACT,QAAQ,EAAE,UAAU,KAAK,QAAQ,SAAS;AAAA,QAC1C,OAAO,KAAK,IAAI,IAAI,KAAK,KAAK,cAAc,GAAI;AAAA,MAClD,CAAC;AACD,WAAK,YAAY;AAAA,IACnB;AACA,UAAM,OAAO,KAAK,YAAY,IAAI,UAAU;AAC5C,QAAI,SAAS,QAAW;AACtB,WAAK,cAAc,iBAAiB;AAAA,QAClC,MAAM;AAAA,QACN,MAAM;AAAA,QACN,SAAS;AAAA,QACT,QAAQ,EAAE,UAAU,KAAK,QAAQ,UAAU,OAAO,WAAW;AAAA,QAC7D,OAAO,KAAK,IAAI,IAAI,KAAK,QAAQ,GAAI;AAAA,MACvC,CAAC;AAAA,IACH;AACA,SAAK,YAAY,IAAI,YAAY,EAAE;AAAA,EACrC;AACF;AAIA,IAAM,4BAA4B,CAAC,GAAG,GAAG,GAAG,IAAI,IAAI,IAAI,KAAK,KAAK,GAAG;AACrE,IAAM,oBAAoB,CAAC,MAAO,OAAQ,MAAO,MAAM,OAAO,MAAM,KAAK,MAAM,KAAK,GAAG,KAAK,CAAC;AAItF,SAAS,iBACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACD,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIO,SAAS,gCACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIO,SAAS,iCACd,eACA,OACM;AACN,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,OAAO,MAAM;AAAA,EACf,CAAC;AACH;AAIA,IAAM,6BAA6B;AAAA,EACjC;AAAA,EAAO;AAAA,EAAO;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAQ;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AAAA,EAAS;AACrF;AAKO,SAAS,uBACd,eACA,UACA,aACM;AACN,QAAM,wBAAwB,2BAA2B,WAAW;AACpE,MAAI,0BAA0B,QAAQ,0BAA0B,GAAG;AACjE;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,SAAS;AAAA,IACT,QAAQ,EAAE,SAAS;AAAA,IACnB,OAAO;AAAA,EACT,CAAC;AACH;AAKO,SAAS,wBAAwB,eAA8B,SAAuB;AAC3F,gBAAc,iBAAiB;AAAA,IAC7B,MAAM;AAAA,IACN,MAAM;AAAA,IACN,QAAQ,EAAE,QAAQ;AAAA,EACpB,CAAC;AACH;AAEA,IAAM,2CAA2C,CAAC,GAAG,GAAG,IAAI,KAAK,KAAK,MAAO,IAAK;AAM3E,SAAS,gCACd,eACA,SACM;AACN,aAAW,gBAAgB,QAAQ,eAAe;AAChD,UAAM,SAAS;AAAA,MACb,UAAU,aAAa;AAAA,MACvB,QAAQ,aAAa;AAAA,MACrB,kBAAkB,aAAa;AAAA,MAC/B,iBAAiB,aAAa;AAAA,MAC9B,cAAc,QAAQ;AAAA,MACtB,oBAAoB,QAAQ;AAAA,IAC9B;AACA,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,IACF,CAAC;AACD,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,MACT;AAAA,MACA,OAAO,aAAa;AAAA,IACtB,CAAC;AAAA,EACH;AACF;AAcA,IAAM,gCAAgC;AAAA,EACpC;AAAA,EAAM;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAK;AAAA,EAAM;AAAA,EAAM;AACjE;AAoBO,SAAS,uBACd,eACA,UACA,OAKM;AACN,QAAM,SAAS,2BAA2B,MAAM,YAAY;AAC5D,QAAM,aAAa,2BAA2B,MAAM,gBAAgB;AACpE,QAAM,SAAS,2BAA2B,MAAM,YAAY;AAC5D,oCAAkC,eAAe;AAAA,IAC/C,MAAM;AAAA,IACN,MAAM;AAAA,IACN;AAAA,IACA,QAAQ,EAAE,UAAU,QAAQ,WAAW,OAAO,YAAY,WAAW;AAAA,IACrE,QAAQ;AAAA,EACV,CAAC;AACD,MAAI,WAAW,QAAQ,SAAS,GAAG;AACjC,sCAAkC,eAAe;AAAA,MAC/C,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,MACA,QAAQ,EAAE,SAAS;AAAA,MACnB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,eAAe,QAAQ,aAAa,GAAG;AACzC,sCAAkC,eAAe;AAAA,MAC/C,MAAM;AAAA,MACN,MAAM;AAAA,MACN;AAAA,MACA,QAAQ,EAAE,SAAS;AAAA,MACnB,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,WAAW,QAAQ,WAAW,QAAQ,SAAS,GAAG;AACpD,kBAAc,iBAAiB;AAAA,MAC7B,MAAM;AAAA,MACN,MAAM;AAAA,MACN,SAAS;AAAA,MACT,QAAQ,EAAE,SAAS;AAAA;AAAA;AAAA,MAGnB,OAAO,KAAK,IAAI,GAAG,SAAS,MAAM;AAAA,IACpC,CAAC;AAAA,EACH;AACF;AAEA,SAAS,kCACP,eACA,OAOM;AACN,QAAM,SAAS,wBAAwB,IAAI,aAAa,KAAK,oBAAI,IAAoB;AACrF,MAAI,CAAC,wBAAwB,IAAI,aAAa,GAAG;AAC/C,4BAAwB,IAAI,eAAe,MAAM;AAAA,EACnD;AACA,QAAM,WAAW,OAAO,QAAQ,MAAM,MAAM,EACzC,KAAK,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,MAAM,KAAK,cAAc,KAAK,CAAC,EACnD,IAAI,CAAC,CAACA,MAAK,KAAK,MAAM,GAAGA,IAAG,IAAI,KAAK,EAAE,EACvC,KAAK,GAAG;AACX,QAAM,MAAM,GAAG,MAAM,IAAI,IAAI,QAAQ;AACrC,QAAM,UAAU,OAAO,IAAI,GAAG,KAAK;AACnC,MAAI,MAAM,SAAS,gCAAgC,SAAS;AAC1D,kBAAc,KAAK,+CAA+C;AAAA,MAChE,UAAU,MAAM;AAAA,MAChB,QAAQ,MAAM;AAAA,IAChB,CAAC;AACD;AAAA,EACF;AACA,gBAAc,iBAAiB;AAAA,IAC7B,MAAM,MAAM;AAAA,IACZ,MAAM,MAAM;AAAA,IACZ,QAAQ,MAAM;AAAA,IACd,QAAQ,MAAM;AAAA,EAChB,CAAC;AACD,SAAO,IAAI,KAAK,UAAU,MAAM,MAAM;AACxC;AAEA,SAAS,kBAAkB,OAAuB;AAChD,SAAO,OAAO,SAAS,KAAK,KAAK,QAAQ,IAAI,QAAQ;AACvD;AAEA,SAAS,uBACP,OACA,OACA,UAAiC,CAAC,GAC1B;AACR,MAAI,UAAU,QAAQ,UAAU,QAAW;AACzC,UAAM,IAAI,MAAM,mCAAmC,KAAK,EAAE;AAAA,EAC5D;AACA,QAAM,SAAS,OAAO,KAAK;AAC3B,MACE,CAAC,OAAO,SAAS,MAAM,KACvB,SAAS,KACR,QAAQ,WAAW,CAAC,OAAO,cAAc,MAAM,GAChD;AACA,UAAM,IAAI,MAAM,mCAAmC,KAAK,KAAK,OAAO,KAAK,CAAC,EAAE;AAAA,EAC9E;AACA,SAAO;AACT;AAEA,SAAS,uBACP,OACA,OACA,UAAiC,CAAC,GAC1B;AACR,SAAO,UAAU,QAAQ,UAAU,SAAY,IAAI,uBAAuB,OAAO,OAAO,OAAO;AACjG;AAUO,SAAS,kBAAkB,cAAiD;AACjF,MAAI,CAAC,cAAc;AACjB,WAAO;AAAA,EACT;AACA,SAAO,WAAW,QAAQ,EAAE,OAAO,YAAY,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE;AAC5E;AAWO,SAAS,wBAAwB,OAIgC;AACtE,QAAM,6BACJ,MAAM,qBACN,MAAM,wBAAwB,QAC9B,MAAM,6BAA6B,QACnC,MAAM,6BAA6B,MAAM;AAC3C,SAAO;AAAA,IACL,oBAAoB,kBAAkB,MAAM,mBAAmB;AAAA,IAC/D;AAAA,EACF;AACF;","names":["key"]}
|
package/dist/index.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ import { type WorkerRunTarget } from "./worker-service-lifecycle.js";
|
|
|
13
13
|
export { createHostExportPump, type HostExportDrainResult, type HostExportPump, type HostExportPumpOptions, type HostEventExport, type HostEventExportBatch, type HostEventSink, type HostUsageExport, type HostUsageExportBatch, type HostUsageSink, } from "./host-export.js";
|
|
14
14
|
export declare const FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
15
15
|
export declare const FILE_UPLOAD_REAPER_PERIOD_MS: number;
|
|
16
|
+
export declare const SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
|
|
17
|
+
export declare const SITE_AUTH_MAINTENANCE_PERIOD_MS: number;
|
|
16
18
|
export type OpenGeniWorkerRole = "control" | "turn";
|
|
17
19
|
export type WorkerOptions = {
|
|
18
20
|
role: OpenGeniWorkerRole;
|
|
@@ -46,12 +48,14 @@ export declare function temporalWorkflowExecutionNotFound(error: unknown): boole
|
|
|
46
48
|
* until an activity first resolves its injected services. */
|
|
47
49
|
export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole, dependencies?: ActivityDependencies): Promise<{
|
|
48
50
|
getCodexCapacityWait: (input: import("./activities.js").GetCodexCapacityWaitInput) => Promise<{
|
|
51
|
+
provider?: "xai";
|
|
49
52
|
waiterId: string;
|
|
50
53
|
generation: number;
|
|
51
54
|
nextCheckAt: string;
|
|
52
55
|
wakeRevision: number;
|
|
53
56
|
} | null>;
|
|
54
57
|
reconcileCodexCapacityWait: (input: import("./activities.js").ReconcileCodexCapacityWaitInput) => Promise<import("./activities.js").ReconcileCodexCapacityWaitResult>;
|
|
58
|
+
maintainBrowserStateArtifacts: () => Promise<import("./activities/browser-state-artifact-reaper.js").MaintainBrowserStateArtifactsResult>;
|
|
55
59
|
indexDocument: (input: import("./activities.js").IndexDocumentInput) => Promise<{
|
|
56
60
|
id: string;
|
|
57
61
|
workspaceId: string;
|
|
@@ -175,6 +179,7 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
|
|
|
175
179
|
maintainSandboxLeaseSweep: (input: import("./sandbox-reaper-contract.js").SandboxLeaseSweepMaintenanceInput) => Promise<import("./activities/sandbox-lease.js").SandboxLeaseSweepMaintenanceResult>;
|
|
176
180
|
reapSandboxLeases: () => Promise<import("./activities/sandbox-lease.js").ReapSandboxLeasesResult>;
|
|
177
181
|
dispatchScheduledTaskRun: (input: import("./activities.js").DispatchScheduledTaskRunInput) => Promise<import("./activities.js").DispatchScheduledTaskRunResult>;
|
|
182
|
+
maintainSiteAuthConnections: () => Promise<import("./activities/site-auth-maintenance.js").MaintainSiteAuthConnectionsResult>;
|
|
178
183
|
failSessionAttempt: (input: import("./activities/types.js").FailSessionAttemptInput) => Promise<void>;
|
|
179
184
|
settleSessionInterruptions: (input: import("./activities/types.js").SettleSessionInterruptionsInput) => Promise<{
|
|
180
185
|
action: "paused" | "continue" | "stale";
|
|
@@ -185,6 +190,7 @@ export declare function createDefaultWorkerActivities(role: OpenGeniWorkerRole,
|
|
|
185
190
|
recoverEscapedMcpTimeout: (input: import("./activities.js").RecoverEscapedMcpTimeoutInput) => Promise<import("./activities.js").RecoverEscapedMcpTimeoutResult>;
|
|
186
191
|
peekSessionWork: (input: import("./activities/types.js").PeekSessionWorkInput) => Promise<import("@opengeni/db").SessionWorkPeek>;
|
|
187
192
|
expireSessionHumanInput: (input: import("./activities.js").ExpireSessionHumanInputInput) => Promise<import("./activities.js").ExpireSessionHumanInputResult>;
|
|
193
|
+
expireSessionInteractionIntervention: (input: import("./activities/types.js").ExpireSessionInteractionInterventionInput) => Promise<import("./activities/types.js").ExpireSessionInteractionInterventionResult>;
|
|
188
194
|
markSessionIdle: (input: import("./activities/types.js").MarkSessionIdleInput) => Promise<void>;
|
|
189
195
|
dispatchSessionWorkflowWakes(): Promise<import("./activities/workflow-wake.js").DispatchSessionWorkflowWakesResult>;
|
|
190
196
|
} | {
|
|
@@ -258,6 +264,13 @@ export declare function registerFileUploadReaperSchedule(settings: Settings, obs
|
|
|
258
264
|
registered: boolean;
|
|
259
265
|
close: () => Promise<void>;
|
|
260
266
|
}>;
|
|
267
|
+
/** Register the deployment-wide maintained-auth dispatcher. Its one-minute
|
|
268
|
+
* cadence matches the minimum public policy interval; DB claims own overlap,
|
|
269
|
+
* crash recovery, and exact session idempotency. */
|
|
270
|
+
export declare function registerSiteAuthMaintenanceSchedule(settings: Settings, observability: Observability): Promise<{
|
|
271
|
+
registered: boolean;
|
|
272
|
+
close: () => Promise<void>;
|
|
273
|
+
}>;
|
|
261
274
|
/**
|
|
262
275
|
* Register the one repair cadence for committed workflow-wake revisions. The
|
|
263
276
|
* activity only reads the transactional outbox and sends revision-scoped
|
package/dist/index.js
CHANGED
|
@@ -15,7 +15,7 @@ import {
|
|
|
15
15
|
normalizeTurnTaskQueueStats,
|
|
16
16
|
observabilityEventLogger,
|
|
17
17
|
startTurnCapacityMonitor
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-TYNPCG4U.js";
|
|
19
19
|
|
|
20
20
|
// src/index.ts
|
|
21
21
|
import {
|
|
@@ -695,6 +695,8 @@ function safeWarn(observability, message, attributes) {
|
|
|
695
695
|
var SANDBOX_REAPER_SCHEDULE_ID = "opengeni-sandbox-lease-reaper";
|
|
696
696
|
var FILE_UPLOAD_REAPER_SCHEDULE_ID = "opengeni-file-upload-reaper";
|
|
697
697
|
var FILE_UPLOAD_REAPER_PERIOD_MS = 15 * 60 * 1e3;
|
|
698
|
+
var SITE_AUTH_MAINTENANCE_SCHEDULE_ID = "opengeni-site-auth-maintenance";
|
|
699
|
+
var SITE_AUTH_MAINTENANCE_PERIOD_MS = 60 * 1e3;
|
|
698
700
|
function temporalTimeMs(value) {
|
|
699
701
|
if (value?.seconds === void 0 || value.seconds === null) return null;
|
|
700
702
|
return Number(String(value.seconds)) * 1e3 + Math.floor((value.nanos ?? 0) / 1e6);
|
|
@@ -709,7 +711,7 @@ function temporalWorkflowExecutionNotFound(error) {
|
|
|
709
711
|
return isGrpcServiceError(error) && error.code === 5;
|
|
710
712
|
}
|
|
711
713
|
async function createDefaultWorkerActivities(role, dependencies = {}) {
|
|
712
|
-
return role === "control" ? (await import("./activities-control-
|
|
714
|
+
return role === "control" ? (await import("./activities-control-XURE4VKN.js")).createControlActivities(dependencies) : (await import("./activities-turn-6EBXGGLS.js")).createTurnActivities(dependencies);
|
|
713
715
|
}
|
|
714
716
|
function resolveOpenGeniWorkflowDefinition(moduleUrl = import.meta.url) {
|
|
715
717
|
const modulePath = fileURLToPath(moduleUrl).replaceAll("\\", "/");
|
|
@@ -1075,6 +1077,41 @@ async function registerFileUploadReaperSchedule(settings, observability) {
|
|
|
1075
1077
|
throw error;
|
|
1076
1078
|
}
|
|
1077
1079
|
}
|
|
1080
|
+
async function registerSiteAuthMaintenanceSchedule(settings, observability) {
|
|
1081
|
+
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
1082
|
+
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
1083
|
+
try {
|
|
1084
|
+
await temporal.schedule.create({
|
|
1085
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
1086
|
+
spec: { intervals: [{ every: SITE_AUTH_MAINTENANCE_PERIOD_MS }] },
|
|
1087
|
+
action: {
|
|
1088
|
+
type: "startWorkflow",
|
|
1089
|
+
workflowType: "siteAuthMaintenanceWorkflow",
|
|
1090
|
+
taskQueue: settings.temporalTaskQueue,
|
|
1091
|
+
args: []
|
|
1092
|
+
},
|
|
1093
|
+
policies: {
|
|
1094
|
+
overlap: ScheduleOverlapPolicy.SKIP,
|
|
1095
|
+
catchupWindow: "1m",
|
|
1096
|
+
pauseOnFailure: false
|
|
1097
|
+
}
|
|
1098
|
+
});
|
|
1099
|
+
observability.info("Registered the global site-auth maintenance Schedule", {
|
|
1100
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
1101
|
+
maintenancePeriodMs: SITE_AUTH_MAINTENANCE_PERIOD_MS
|
|
1102
|
+
});
|
|
1103
|
+
return { registered: true, close: async () => connection.close() };
|
|
1104
|
+
} catch (error) {
|
|
1105
|
+
if (error instanceof ScheduleAlreadyRunning) {
|
|
1106
|
+
observability.info("Global site-auth maintenance Schedule already registered", {
|
|
1107
|
+
scheduleId: SITE_AUTH_MAINTENANCE_SCHEDULE_ID
|
|
1108
|
+
});
|
|
1109
|
+
return { registered: false, close: async () => connection.close() };
|
|
1110
|
+
}
|
|
1111
|
+
await connection.close().catch(() => void 0);
|
|
1112
|
+
throw error;
|
|
1113
|
+
}
|
|
1114
|
+
}
|
|
1078
1115
|
async function registerSessionWorkflowWakeDispatcherSchedule(settings, observability) {
|
|
1079
1116
|
const connection = await Connection.connect(temporalConnectionOptions(settings));
|
|
1080
1117
|
const temporal = new TemporalClient({ connection, namespace: settings.temporalNamespace });
|
|
@@ -1184,6 +1221,13 @@ async function createOpenGeniWorkerService(options) {
|
|
|
1184
1221
|
{ ...retryOptions, onRetry }
|
|
1185
1222
|
)
|
|
1186
1223
|
);
|
|
1224
|
+
schedules.push(
|
|
1225
|
+
await retryStartupDependency(
|
|
1226
|
+
"Temporal schedule (site auth maintenance)",
|
|
1227
|
+
() => registerSiteAuthMaintenanceSchedule(settings, observability),
|
|
1228
|
+
{ ...retryOptions, onRetry }
|
|
1229
|
+
)
|
|
1230
|
+
);
|
|
1187
1231
|
schedules.push(
|
|
1188
1232
|
await retryStartupDependency(
|
|
1189
1233
|
"Temporal schedule (session-workflow wake dispatcher)",
|
|
@@ -1352,6 +1396,8 @@ if (import.meta.main) {
|
|
|
1352
1396
|
export {
|
|
1353
1397
|
FILE_UPLOAD_REAPER_PERIOD_MS,
|
|
1354
1398
|
FILE_UPLOAD_REAPER_SCHEDULE_ID,
|
|
1399
|
+
SITE_AUTH_MAINTENANCE_PERIOD_MS,
|
|
1400
|
+
SITE_AUTH_MAINTENANCE_SCHEDULE_ID,
|
|
1355
1401
|
createDefaultWorkerActivities,
|
|
1356
1402
|
createHostExportPump,
|
|
1357
1403
|
createOpenGeniWorker,
|
|
@@ -1360,6 +1406,7 @@ export {
|
|
|
1360
1406
|
registerFileUploadReaperSchedule,
|
|
1361
1407
|
registerSandboxReaperSchedule,
|
|
1362
1408
|
registerSessionWorkflowWakeDispatcherSchedule,
|
|
1409
|
+
registerSiteAuthMaintenanceSchedule,
|
|
1363
1410
|
resolveOpenGeniWorkflowDefinition,
|
|
1364
1411
|
runOpenGeniWorker,
|
|
1365
1412
|
startWorker,
|