@aglyn/tenant-data-admin 1.0.0-beta.260 → 1.0.0-beta.262

package/src/lib/server/operator-alerts.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/operator-alerts.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { createHash, createHmac } from 'node:crypto'\nimport {\n  effectiveOperatorAlertSetting,\n  OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE,\n  OPERATOR_ALERT_TIER_LABELS,\n  operatorAlertLevel,\n  renderOperatorAlertTemplate,\n  type AglynNotificationType,\n  type NotificationLevel,\n  type OperatorAlertContext,\n  type OperatorAlertDefinition,\n  type OperatorAlertSettings,\n  type OperatorAlertTypeSetting,\n} from '@aglyn/aglyn/server'\nimport {\n  getOperatorAlert,\n  registerOperatorAlerts,\n} from '@aglyn/aglyn/plugin-manager/operator-alerts'\nimport { createResourceUid } from '@aglyn/aglyn/app-utils/create-resource-uid'\nimport { isEmailConfigured, type SendEmailResult } from '@aglyn/shared-util-email'\nimport firebaseAdmin from './firebase-admin'\n\n/**\n * RAISING AN OPERATOR ALERT (AGL-3377): the one server entry point every\n * alert in the registry goes through.\n *\n * In order, for one call:\n *\n * 1. The type is looked up in the registry (core's catalog plus what plugins\n *    registered). A plugin may pass its definition itself, which registers\n *    it in this process if its declarations have not loaded yet.\n * 2. A `dedupeKey` is claimed against `operatorAlertState`, so a condition\n *    that flaps inside the type's window is told once, and a type with\n *    `minOccurrences` fires only once it has recurred that often.\n * 3. The staff console notification is written for every staff member, as\n *    `notifyStaff` always has.\n * 4. Staff's switches (`platformSettings/operatorAlerts`, coded defaults\n *    until changed) decide the rest: off is console only, `digest` queues it\n *    for the once-a-day digest, `immediate` emails the operator now.\n * 5. The email is the `operator-alert` system email, to\n *    `resolveStaffAlertRecipients()`; the optional out-of-band webhook\n *    (`OPERATOR_ALERT_WEBHOOK_URL`) gets the same alert beside it, so a mail\n *    provider that is itself the outage still reaches somebody.\n *\n * Never throws: an alert is raised from inside the failure it reports, and a\n * second failure there must not replace the first.\n */\n\n/** Where staff's per-type switches live. */\nexport const OPERATOR_ALERT_SETTINGS_COLLECTION = 'platformSettings'\nexport const OPERATOR_ALERT_SETTINGS_DOC = 'operatorAlerts'\n/** One document per (type, dedupeKey): when it last fired, how often since. */\nexport const OPERATOR_ALERT_STATE_COLLECTION = 'operatorAlertState'\n/** Alerts waiting for the daily digest. */\nexport const OPERATOR_ALERT_DIGEST_COLLECTION = 'operatorAlertDigest'\n\n/** The hour, UTC, the digest goes out when staff have not chosen one. */\nexport const OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC = 14\n\nexport interface OperatorAlertSettingsDoc extends OperatorAlertSettings {\n  /** The hour of the day, UTC, the digest is sent. */\n  digestHourUtc?: number\n  updatedAtMs?: number\n  updatedByEmail?: string | null\n}\n\nexport interface RaiseOperatorAlertOptions {\n  /**\n   * What makes two raises the SAME alert — a dispute id, an org and month, a\n   * check id. Absent never dedupes: every raise is told.\n   */\n  dedupeKey?: string\n  /** Values for the definition's `{{tokens}}`. Never personal data. */\n  context?: OperatorAlertContext\n  /** A console path or absolute URL; overrides the definition's link. */\n  url?: string\n  /** The title as written by the caller; overrides the rendered one. */\n  subject?: string\n  /** The body as written by the caller; overrides the rendered one. */\n  body?: string\n  /**\n   * The console notification's level as the caller judged it (AGL-3437);\n   * overrides the definition's. For a caller whose one alert type covers\n   * news of more than one tone.\n   */\n  level?: NotificationLevel\n  orgId?: string\n  hostId?: string\n}\n\nexport type OperatorAlertOutcome =\n  | 'deduped'\n  | 'below-threshold'\n  | 'console-only'\n  | 'queued'\n  | 'delivered'\n  | 'undelivered'\n\nexport interface OperatorAlertResult {\n  outcome: OperatorAlertOutcome\n  type: string\n  email?: SendEmailResult\n  webhook?: OperatorAlertWebhookResult\n}\n\nexport type OperatorAlertWebhookResult =\n  | { posted: true; status: number }\n  | { posted: false; reason: 'unconfigured' | 'refused' | 'network'; status?: number }\n\n/** One alert, rendered, as it is delivered and queued. */\nexport interface RenderedOperatorAlert {\n  type: string\n  label: string\n  tier: OperatorAlertDefinition['tier']\n  category: OperatorAlertDefinition['category']\n  title: string\n  body: string\n  /** Console path or absolute URL; `''` when there is none. */\n  link: string\n  notificationType: AglynNotificationType\n  orgId?: string\n  hostId?: string\n}\n\nconst firestore = () => firebaseAdmin.app().firestore()\n\n// ── Settings ────────────────────────────────────────────────────────────────\n\nconst SETTINGS_TTL_MS = 15_000\nlet settingsCache: { at: number; doc: OperatorAlertSettingsDoc } | undefined\n\n/** Drop the in-process settings cache; the console calls it after a write. */\nexport function invalidateOperatorAlertSettingsCache(): void {\n  settingsCache = undefined\n}\n\n/** Clamps a stored settings document to what the pipeline understands. */\nexport function normalizeOperatorAlertSettings(\n  raw: unknown,\n): OperatorAlertSettingsDoc {\n  const source = (raw && typeof raw === 'object' ? raw : {}) as Record<string, unknown>\n  const types: Record<string, OperatorAlertTypeSetting> = {}\n  const rawTypes = source['types']\n  if (rawTypes && typeof rawTypes === 'object') {\n    for (const [type, value] of Object.entries(rawTypes as Record<string, unknown>)) {\n      if (!type || !value || typeof value !== 'object') continue\n      const entry = value as Record<string, unknown>\n      const setting: OperatorAlertTypeSetting = {}\n      if (typeof entry['enabled'] === 'boolean') setting.enabled = entry['enabled']\n      if (entry['delivery'] === 'immediate' || entry['delivery'] === 'digest') {\n        setting.delivery = entry['delivery']\n      }\n      if (Object.keys(setting).length) types[type] = setting\n    }\n  }\n  const hour = Number(source['digestHourUtc'])\n  return {\n    types,\n    digestHourUtc:\n      Number.isInteger(hour) && hour >= 0 && hour <= 23\n        ? hour\n        : OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC,\n    ...(typeof source['updatedAtMs'] === 'number'\n      ? { updatedAtMs: source['updatedAtMs'] as number }\n      : {}),\n    ...(typeof source['updatedByEmail'] === 'string'\n      ? { updatedByEmail: source['updatedByEmail'] as string }\n      : {}),\n  }\n}\n\n/**\n * Staff's switches. An unreadable document is the coded defaults, uncached:\n * an alert must still go out when the settings cannot be read.\n */\nexport async function readOperatorAlertSettings(): Promise<OperatorAlertSettingsDoc> {\n  const now = Date.now()\n  if (settingsCache && now - settingsCache.at < SETTINGS_TTL_MS) return settingsCache.doc\n  try {\n    const snapshot = await firestore()\n      .collection(OPERATOR_ALERT_SETTINGS_COLLECTION)\n      .doc(OPERATOR_ALERT_SETTINGS_DOC)\n      .get()\n    const doc = normalizeOperatorAlertSettings(snapshot.exists ? snapshot.data() : null)\n    settingsCache = { at: now, doc }\n    return doc\n  } catch (error) {\n    console.error('[operator-alerts] settings unreadable; using the defaults', error)\n    return normalizeOperatorAlertSettings(null)\n  }\n}\n\n// ── Dedupe ──────────────────────────────────────────────────────────────────\n\n/** Keys this instance fired, and until when they stay quiet. */\nconst firedLocally = new Map<string, number>()\nconst FIRED_LOCALLY_MAX = 500\n\nfunction stateDocId(type: string, dedupeKey: string): string {\n  return createHash('sha256').update(`${type}\\u0000${dedupeKey}`).digest('hex').slice(0, 40)\n}\n\nexport type OperatorAlertClaim =\n  | { fire: true; repeats: number }\n  | { fire: false; reason: 'deduped' | 'below-threshold' }\n\n/**\n * Whether a raise with this key fires now. One transaction on the key's\n * state document: inside the window after a fire it is counted and\n * suppressed; before it reaches `minOccurrences` it is counted and held.\n * A store that cannot be read FIRES — a duplicate beats a silence.\n */\nexport async function claimOperatorAlert(\n  definition: Pick<OperatorAlertDefinition, 'type' | 'dedupeWindowMinutes' | 'minOccurrences'>,\n  dedupeKey: string,\n  now = Date.now(),\n): Promise<OperatorAlertClaim> {\n  const windowMs = Math.max(0, definition.dedupeWindowMinutes) * 60_000\n  const minOccurrences = Math.max(1, Math.floor(definition.minOccurrences ?? 1))\n  if (!windowMs) return { fire: true, repeats: 0 }\n  const localKey = `${definition.type}\\u0000${dedupeKey}`\n  const quietUntil = firedLocally.get(localKey)\n  if (quietUntil && quietUntil > now) return { fire: false, reason: 'deduped' }\n  try {\n    const ref = firestore()\n      .collection(OPERATOR_ALERT_STATE_COLLECTION)\n      .doc(stateDocId(definition.type, dedupeKey))\n    const claim = await firestore().runTransaction(async (transaction: any) => {\n      const snapshot = await transaction.get(ref)\n      const state = (snapshot.exists ? snapshot.data() : {}) as Record<string, unknown>\n      const lastFiredAtMs = Number(state['lastFiredAtMs'] ?? 0)\n      const base = { type: definition.type, dedupeKey: dedupeKey.slice(0, 300), updatedAtMs: now }\n      if (lastFiredAtMs && now - lastFiredAtMs < windowMs) {\n        transaction.set(ref, { ...base, suppressed: Number(state['suppressed'] ?? 0) + 1 }, { merge: true })\n        return { fire: false, reason: 'deduped' } as const\n      }\n      const windowStartMs = Number(state['windowStartMs'] ?? 0)\n      const inWindow = windowStartMs && now - windowStartMs < windowMs\n      const occurrences = (inWindow ? Number(state['occurrences'] ?? 0) : 0) + 1\n      if (occurrences < minOccurrences) {\n        transaction.set(\n          ref,\n          { ...base, occurrences, windowStartMs: inWindow ? windowStartMs : now },\n          { merge: true },\n        )\n        return { fire: false, reason: 'below-threshold' } as const\n      }\n      transaction.set(\n        ref,\n        {\n          ...base,\n          lastFiredAtMs: now,\n          occurrences: 0,\n          windowStartMs: 0,\n          suppressed: 0,\n          fired: Number(state['fired'] ?? 0) + 1,\n        },\n        { merge: true },\n      )\n      return { fire: true, repeats: Number(state['suppressed'] ?? 0) } as const\n    })\n    if (claim.fire) {\n      if (firedLocally.size >= FIRED_LOCALLY_MAX) firedLocally.clear()\n      firedLocally.set(localKey, now + windowMs)\n    }\n    return claim\n  } catch (error) {\n    console.error(`[operator-alerts] dedupe unavailable for ${definition.type}; firing`, error)\n    return { fire: true, repeats: 0 }\n  }\n}\n\n// ── Out-of-band webhook ─────────────────────────────────────────────────────\n\nconst WEBHOOK_TIMEOUT_MS = 5_000\n\n/** The webhook URL, when one is configured and is http(s). */\nexport function operatorAlertWebhookUrl(): string {\n  const value = String(process.env['OPERATOR_ALERT_WEBHOOK_URL'] ?? '').trim()\n  return /^https?:\\/\\/\\S+$/i.test(value) ? value : ''\n}\n\n/**\n * The JSON the webhook receives. `text` is what a Slack-compatible incoming\n * webhook renders on its own; `alert` carries the fields for anything that\n * wants to route on them.\n */\nexport function operatorAlertWebhookPayload(\n  alerts: readonly RenderedOperatorAlert[],\n  options: { digest?: boolean; now?: number } = {},\n): Record<string, unknown> {\n  const origin = consoleOrigin()\n  const lines = alerts.map((alert) => {\n    const link = operatorAlertAbsoluteLink(alert.link)\n    return [\n      `[${OPERATOR_ALERT_TIER_LABELS[alert.tier]}] ${alert.title}`,\n      alert.body,\n      link,\n    ]\n      .filter(Boolean)\n      .join('\\n')\n  })\n  const text = options.digest\n    ? [`Operator digest: ${alerts.length} alert${alerts.length === 1 ? '' : 's'}`, ...lines].join('\\n\\n')\n    : lines.join('\\n\\n')\n  const occurredAt = new Date(options.now ?? Date.now()).toISOString()\n  const shaped = alerts.map((alert) => ({\n    type: alert.type,\n    tier: alert.tier,\n    category: alert.category,\n    title: alert.title,\n    body: alert.body,\n    url: operatorAlertAbsoluteLink(alert.link) || null,\n  }))\n  return {\n    text,\n    ...(options.digest ? { digest: shaped } : { alert: shaped[0] ?? null }),\n    occurredAt,\n    ...(origin ? { install: origin } : {}),\n  }\n}\n\n/**\n * Posts alerts to `OPERATOR_ALERT_WEBHOOK_URL`, signed when\n * `OPERATOR_ALERT_WEBHOOK_SECRET` is set: `x-operator-alert-timestamp` is the\n * Unix time and `x-operator-alert-signature` is `sha256=` plus the hex HMAC\n * of `{timestamp}.{body}`. Unset means email only. Never throws.\n */\nexport async function postOperatorAlertWebhook(\n  alerts: readonly RenderedOperatorAlert[],\n  options: { digest?: boolean; now?: number } = {},\n): Promise<OperatorAlertWebhookResult> {\n  const url = operatorAlertWebhookUrl()\n  if (!url || !alerts.length) return { posted: false, reason: 'unconfigured' }\n  const body = JSON.stringify(operatorAlertWebhookPayload(alerts, options))\n  const headers: Record<string, string> = { 'content-type': 'application/json' }\n  const secret = String(process.env['OPERATOR_ALERT_WEBHOOK_SECRET'] ?? '').trim()\n  if (secret) {\n    const timestamp = String(Math.floor((options.now ?? Date.now()) / 1000))\n    headers['x-operator-alert-timestamp'] = timestamp\n    headers['x-operator-alert-signature'] =\n      'sha256=' + createHmac('sha256', secret).update(`${timestamp}.${body}`).digest('hex')\n  }\n  try {\n    const response = await fetch(url, {\n      method: 'POST',\n      headers,\n      body,\n      redirect: 'manual',\n      signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),\n    })\n    if (response.ok) return { posted: true, status: response.status }\n    console.error(`[operator-alerts] webhook answered ${response.status}`)\n    return { posted: false, reason: 'refused', status: response.status }\n  } catch (error) {\n    console.error('[operator-alerts] webhook unreachable', error)\n    return { posted: false, reason: 'network' }\n  }\n}\n\n// ── Links ───────────────────────────────────────────────────────────────────\n\nfunction consoleOrigin(): string {\n  return String(process.env['NEXT_PUBLIC_CONSOLE_URL'] ?? '').trim().replace(/\\/+$/, '')\n}\n\n/** A console path as an inbox or webhook can follow it, or `''`. */\nexport function operatorAlertAbsoluteLink(link: string | undefined | null): string {\n  const value = String(link ?? '').trim()\n  if (!value) return ''\n  if (/^https?:\\/\\//i.test(value)) return value\n  const origin = consoleOrigin()\n  return origin && value.startsWith('/') ? `${origin}${value}` : ''\n}\n\n/**\n * The definition's link with its tokens filled, or `''` when a token it\n * needs is missing — `/admin/orgs/` would open the wrong page.\n */\nfunction renderLink(template: string | undefined, context: OperatorAlertContext | undefined): string {\n  if (!template) return ''\n  const missing = [...template.matchAll(/\\{\\{\\s*([\\w.]+)\\s*\\}\\}/g)].some(([, name]) => {\n    const value = context?.[name as string]\n    return value === null || value === undefined || value === ''\n  })\n  return missing ? '' : renderOperatorAlertTemplate(template, context)\n}\n\n// ── Rendering ───────────────────────────────────────────────────────────────\n\n/** Stands in for a type nobody registered, so it is still told. */\nfunction unregisteredDefinition(type: string): OperatorAlertDefinition {\n  return {\n    type,\n    label: type,\n    description: 'An alert type this install has no registry entry for.',\n    tier: 'should',\n    category: 'ops',\n    title: '{{title}}',\n    body: '{{body}}',\n    delivery: 'immediate',\n    dedupeWindowMinutes: 60,\n    defaultEnabled: true,\n  }\n}\n\n/** Resolves the definition a call names, registering a plugin's own. */\nexport function resolveOperatorAlertDefinition(\n  type: string | OperatorAlertDefinition,\n): OperatorAlertDefinition {\n  if (typeof type !== 'string') {\n    const known = getOperatorAlert(type.type)\n    if (known) return known\n    if (type.pluginId) {\n      try {\n        registerOperatorAlerts([type], { pluginId: type.pluginId })\n      } catch (error) {\n        console.error(`[operator-alerts] ${type.type} could not be registered`, error)\n      }\n    }\n    return type\n  }\n  const known = getOperatorAlert(type)\n  if (known) return known\n  console.error(`[operator-alerts] \"${type}\" is not a registered alert type; telling it anyway`)\n  return unregisteredDefinition(type)\n}\n\nexport function renderOperatorAlert(\n  definition: OperatorAlertDefinition,\n  options: RaiseOperatorAlertOptions = {},\n): RenderedOperatorAlert {\n  const context: OperatorAlertContext = {\n    ...(options.subject ? { title: options.subject } : {}),\n    ...(options.body ? { body: options.body } : {}),\n    ...(options.context ?? {}),\n  }\n  const title =\n    String(options.subject ?? '').trim() ||\n    renderOperatorAlertTemplate(definition.title, context) ||\n    definition.label\n  const body =\n    String(options.body ?? '').trim() || renderOperatorAlertTemplate(definition.body, context)\n  return {\n    type: definition.type,\n    label: definition.label,\n    tier: definition.tier,\n    category: definition.category,\n    title,\n    body,\n    link: String(options.url ?? '').trim() || renderLink(definition.link, context),\n    notificationType: definition.notificationType ?? OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE,\n    ...(options.orgId ? { orgId: options.orgId } : {}),\n    ...(options.hostId ? { hostId: options.hostId } : {}),\n  }\n}\n\n// ── The entry point ─────────────────────────────────────────────────────────\n\n/** Whether an email to the operator can go at all on this install. */\nasync function operatorEmailReachable(): Promise<boolean> {\n  if (!isEmailConfigured()) return false\n  const { resolveStaffAlertRecipients } = await import('./staff-alert-email')\n  return (await resolveStaffAlertRecipients()).length > 0\n}\n\n/** See the module comment. Never throws. */\nexport async function raiseOperatorAlert(\n  type: string | OperatorAlertDefinition,\n  options: RaiseOperatorAlertOptions = {},\n): Promise<OperatorAlertResult> {\n  const typeId = typeof type === 'string' ? type : type.type\n  try {\n    const definition = resolveOperatorAlertDefinition(type)\n    const dedupeKey = String(options.dedupeKey ?? '').trim()\n    let repeats = 0\n    if (dedupeKey) {\n      const claim = await claimOperatorAlert(definition, dedupeKey)\n      if (claim.fire === false) return { outcome: claim.reason, type: typeId }\n      repeats = claim.repeats\n    }\n    const alert = renderOperatorAlert(definition, options)\n    if (repeats > 0) {\n      alert.body = [alert.body, `Raised ${repeats} more time${repeats === 1 ? '' : 's'} since it was last told.`]\n        .filter(Boolean)\n        .join(' ')\n    }\n    const settings = await readOperatorAlertSettings()\n    const { enabled, delivery } = effectiveOperatorAlertSetting(definition, settings)\n    const emailNow = enabled && delivery === 'immediate' && (await operatorEmailReachable())\n\n    const { notifyStaffConsole } = await import('./notifications')\n    await notifyStaffConsole(\n      {\n        type: alert.notificationType,\n        title: alert.title,\n        ...(alert.body ? { body: alert.body } : {}),\n        level: options.level ?? operatorAlertLevel(definition),\n        ...(alert.link && alert.link.startsWith('/') ? { link: alert.link } : {}),\n        ...(alert.orgId ? { orgId: alert.orgId } : {}),\n        ...(alert.hostId ? { hostId: alert.hostId } : {}),\n      },\n      // The operator is emailed; a staff member who also switched email on\n      // for this category is not mailed the same alert a second time.\n      emailNow ? { skipEmail: true } : {},\n    )\n    if (!enabled) return { outcome: 'console-only', type: typeId }\n    // Cloud Logging, for the platform's Cloud Monitoring policies (AGL-3683).\n    const { writeOperatorAlertLog } = await import('./operator-alerts-log')\n    await writeOperatorAlertLog(alert)\n    if (delivery === 'digest') {\n      await queueOperatorAlertForDigest(alert)\n      return { outcome: 'queued', type: typeId }\n    }\n    return await deliverOperatorAlert(alert, { emailNow })\n  } catch (error) {\n    console.error(`[operator-alerts] ${typeId} could not be raised`, error)\n    return { outcome: 'undelivered', type: typeId }\n  }\n}\n\n/** Emails and posts one alert now. */\nasync function deliverOperatorAlert(\n  alert: RenderedOperatorAlert,\n  options: { emailNow: boolean },\n): Promise<OperatorAlertResult> {\n  const { sendOperatorAlertEmail } = await import('./staff-alert-email')\n  const [email, webhook] = await Promise.all([\n    options.emailNow\n      ? sendOperatorAlertEmail({\n          title: alert.title,\n          body: alert.body,\n          url: operatorAlertAbsoluteLink(alert.link),\n          context: `operator-alert ${alert.type}`,\n        })\n      : Promise.resolve<SendEmailResult>({ sent: false, reason: 'unconfigured' }),\n    postOperatorAlertWebhook([alert]),\n  ])\n  return {\n    outcome: email.sent || webhook.posted ? 'delivered' : 'undelivered',\n    type: alert.type,\n    email,\n    webhook,\n  }\n}\n\n// ── The digest ──────────────────────────────────────────────────────────────\n\nasync function queueOperatorAlertForDigest(alert: RenderedOperatorAlert): Promise<void> {\n  try {\n    await firestore()\n      .collection(OPERATOR_ALERT_DIGEST_COLLECTION)\n      .doc(createResourceUid())\n      .set({ ...alert, atMs: Date.now() })\n  } catch (error) {\n    // The queue is the only record the digest will have; with it gone, the\n    // alert goes now rather than never.\n    console.error(`[operator-alerts] digest queue unavailable; sending ${alert.type} now`, error)\n    await deliverOperatorAlert(alert, { emailNow: await operatorEmailReachable() })\n  }\n}\n\n/** At most this many alerts are listed in one digest; the rest are counted. */\nexport const OPERATOR_DIGEST_MAX_LISTED = 50\nconst DIGEST_READ_LIMIT = 500\n\nexport interface OperatorDigestResult {\n  /** Whether a digest went out (or, on a dry run, would). */\n  sent: boolean\n  reason?: 'not-due' | 'already-sent' | 'empty' | 'dry-run' | 'unavailable'\n  count: number\n  email?: SendEmailResult\n  webhook?: OperatorAlertWebhookResult\n}\n\nfunction dayKey(now: number): string {\n  return new Date(now).toISOString().slice(0, 10)\n}\n\n/** The state document that claims one UTC day's digest. */\nexport function operatorDigestMarkerId(now: number): string {\n  return `digest-${dayKey(now)}`\n}\n\n/** The digest's body: one paragraph per alert, newest last. */\nexport function composeOperatorDigestBody(\n  alerts: readonly RenderedOperatorAlert[],\n  total = alerts.length,\n): string {\n  const listed = alerts.slice(0, OPERATOR_DIGEST_MAX_LISTED).map((alert) => {\n    const link = operatorAlertAbsoluteLink(alert.link)\n    return [`• ${alert.title}`, alert.body, link].filter(Boolean).join('\\n')\n  })\n  const rest = total - listed.length\n  if (rest > 0) listed.push(`…and ${rest} more in the staff console.`)\n  return listed.join('\\n\\n')\n}\n\n/**\n * Sends the day's digest once the configured hour has passed: every queued\n * alert in one `operator-alert-digest` email and one webhook post, then\n * clears what it sent. Once per UTC day, claimed by creating\n * `operatorAlertState/digest-{day}`, so two ticks never both send it.\n */\nexport async function sendOperatorAlertDigest(\n  options: { now?: number; dryRun?: boolean; force?: boolean } = {},\n): Promise<OperatorDigestResult> {\n  const now = options.now ?? Date.now()\n  try {\n    const settings = await readOperatorAlertSettings()\n    const hour = settings.digestHourUtc ?? OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC\n    if (!options.force && new Date(now).getUTCHours() < hour) {\n      return { sent: false, reason: 'not-due', count: 0 }\n    }\n    const collection = firestore().collection(OPERATOR_ALERT_DIGEST_COLLECTION)\n    const marker = firestore()\n      .collection(OPERATOR_ALERT_STATE_COLLECTION)\n      .doc(operatorDigestMarkerId(now))\n    if (!options.force && (await marker.get()).exists) {\n      return { sent: false, reason: 'already-sent', count: 0 }\n    }\n    const snapshot = await collection.orderBy('atMs', 'asc').limit(DIGEST_READ_LIMIT).get()\n    const queued = snapshot.docs\n    if (!queued.length) return { sent: false, reason: 'empty', count: 0 }\n    if (options.dryRun) return { sent: false, reason: 'dry-run', count: queued.length }\n    if (!options.force) {\n      try {\n        await marker.create({ atMs: now, count: queued.length })\n      } catch {\n        return { sent: false, reason: 'already-sent', count: 0 }\n      }\n    }\n    const alerts = queued.map((doc: any) => doc.data() as RenderedOperatorAlert)\n    const { sendOperatorAlertDigestEmail } = await import('./staff-alert-email')\n    const [email, webhook] = await Promise.all([\n      sendOperatorAlertDigestEmail({\n        date: dayKey(now),\n        count: alerts.length,\n        body: composeOperatorDigestBody(alerts),\n        url: operatorAlertAbsoluteLink('/admin/operator-alerts'),\n      }),\n      postOperatorAlertWebhook(alerts, { digest: true, now }),\n    ])\n    if (email.sent || webhook.posted) {\n      const batch = firestore().batch()\n      for (const doc of queued) batch.delete(doc.ref)\n      await batch.commit()\n    } else if (!options.force) {\n      // Nothing reached anybody: release the day so the next tick retries.\n      await marker.delete().catch(() => undefined)\n    }\n    return { sent: email.sent || webhook.posted, count: alerts.length, email, webhook }\n  } catch (error) {\n    console.error('[operator-alerts] digest failed', error)\n    return { sent: false, reason: 'unavailable', count: 0 }\n  }\n}\n\n\n// ── The staff page ──────────────────────────────────────────────────────────\n\n/** One registry entry as Staff → Operator alerts lists it. */\nexport interface OperatorAlertCatalogRow {\n  type: string\n  label: string\n  description: string\n  tier: OperatorAlertDefinition['tier']\n  category: OperatorAlertDefinition['category']\n  pluginId: string | null\n  defaults: { enabled: boolean; delivery: OperatorAlertDefinition['delivery'] }\n  effective: { enabled: boolean; delivery: OperatorAlertDefinition['delivery'] }\n  /** Whether staff have answered for this type, rather than the default. */\n  overridden: boolean\n}\n\n/** Every registered type with its default and its effective setting. */\nexport function operatorAlertCatalogRows(\n  definitions: readonly OperatorAlertDefinition[],\n  settings: OperatorAlertSettingsDoc,\n): OperatorAlertCatalogRow[] {\n  return definitions.map((definition) => ({\n    type: definition.type,\n    label: definition.label,\n    description: definition.description,\n    tier: definition.tier,\n    category: definition.category,\n    pluginId: definition.pluginId ?? null,\n    defaults: { enabled: definition.defaultEnabled, delivery: definition.delivery },\n    effective: effectiveOperatorAlertSetting(definition, settings),\n    overridden: Boolean(settings.types?.[definition.type]),\n  }))\n}\n\n/**\n * Sends one `[Test]` alert of a type straight to the operator's inbox and\n * webhook, past dedupe and staff's switches, so an operator can prove the\n * channels reach them. Writes no console notification.\n */\nexport async function sendOperatorAlertTest(\n  definition: OperatorAlertDefinition,\n): Promise<{ email: SendEmailResult; webhook: OperatorAlertWebhookResult }> {\n  const sample: Record<string, string> = {}\n  for (const [, name] of `${definition.title} ${definition.body} ${definition.link ?? ''}`.matchAll(\n    /\\{\\{\\s*([\\w.]+)\\s*\\}\\}/g,\n  )) {\n    sample[name as string] = `sample ${name}`\n  }\n  const alert = renderOperatorAlert(definition, { context: sample })\n  alert.title = `[Test] ${alert.title}`\n  alert.body = [\n    alert.body,\n    'This is a test of the operator alert channels; nothing happened.',\n  ].join(' ')\n  const { sendOperatorAlertEmail } = await import('./staff-alert-email')\n  const [email, webhook] = await Promise.all([\n    isEmailConfigured()\n      ? sendOperatorAlertEmail({\n          title: alert.title,\n          body: alert.body,\n          url: operatorAlertAbsoluteLink(alert.link) || operatorAlertAbsoluteLink('/admin/operator-alerts'),\n          context: `operator-alert-test ${definition.type}`,\n        })\n      : Promise.resolve<SendEmailResult>({ sent: false, reason: 'unconfigured' }),\n    postOperatorAlertWebhook([alert]),\n  ])\n  return { email, webhook }\n}\n\n/** Where operator mail goes, named but never listed, for the staff page. */\nexport async function describeOperatorAlertRecipients(): Promise<{\n  source: 'STAFF_ALERT_EMAIL' | 'NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL' | 'staff-accounts' | 'none'\n  count: number\n}> {\n  const { resolveStaffAlertRecipients } = await import('./staff-alert-email')\n  const recipients = await resolveStaffAlertRecipients()\n  if (!recipients.length) return { source: 'none', count: 0 }\n  const configured = String(process.env['STAFF_ALERT_EMAIL'] ?? '').trim().toLowerCase()\n  if (configured.includes('@') && recipients[0] === configured) {\n    return { source: 'STAFF_ALERT_EMAIL', count: 1 }\n  }\n  const support = String(process.env['NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL'] ?? '').trim().toLowerCase()\n  if (support.includes('@') && recipients[0] === support) {\n    return { source: 'NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL', count: 1 }\n  }\n  return { source: 'staff-accounts', count: recipients.length }\n}\n"],"names":["createHash","createHmac","effectiveOperatorAlertSetting","OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE","OPERATOR_ALERT_TIER_LABELS","operatorAlertLevel","renderOperatorAlertTemplate","getOperatorAlert","registerOperatorAlerts","createResourceUid","isEmailConfigured","firebaseAdmin","OPERATOR_ALERT_SETTINGS_COLLECTION","OPERATOR_ALERT_SETTINGS_DOC","OPERATOR_ALERT_STATE_COLLECTION","OPERATOR_ALERT_DIGEST_COLLECTION","OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC","firestore","app","SETTINGS_TTL_MS","settingsCache","invalidateOperatorAlertSettingsCache","undefined","normalizeOperatorAlertSettings","raw","source","types","rawTypes","type","value","Object","entries","entry","setting","enabled","delivery","keys","length","hour","Number","digestHourUtc","isInteger","updatedAtMs","updatedByEmail","readOperatorAlertSettings","now","Date","at","doc","snapshot","collection","get","exists","data","error","console","firedLocally","Map","FIRED_LOCALLY_MAX","stateDocId","dedupeKey","update","digest","slice","claimOperatorAlert","definition","windowMs","Math","max","dedupeWindowMinutes","minOccurrences","floor","fire","repeats","localKey","quietUntil","reason","ref","claim","runTransaction","transaction","state","lastFiredAtMs","base","set","suppressed","merge","windowStartMs","inWindow","occurrences","fired","size","clear","WEBHOOK_TIMEOUT_MS","operatorAlertWebhookUrl","process","String","env","trim","test","operatorAlertWebhookPayload","alerts","options","shaped","origin","consoleOrigin","lines","map","alert","link","operatorAlertAbsoluteLink","tier","title","body","filter","Boolean","join","text","occurredAt","toISOString","category","url","install","postOperatorAlertWebhook","posted","JSON","stringify","headers","secret","timestamp","response","fetch","method","redirect","signal","AbortSignal","timeout","ok","status","replace","startsWith","renderLink","template","context","missing","matchAll","some","name","unregisteredDefinition","label","description","defaultEnabled","resolveOperatorAlertDefinition","known","pluginId","renderOperatorAlert","subject","notificationType","orgId","hostId","operatorEmailReachable","resolveStaffAlertRecipients","raiseOperatorAlert","typeId","outcome","settings","emailNow","notifyStaffConsole","level","skipEmail","writeOperatorAlertLog","queueOperatorAlertForDigest","deliverOperatorAlert","sendOperatorAlertEmail","email","webhook","Promise","all","resolve","sent","atMs","OPERATOR_DIGEST_MAX_LISTED","DIGEST_READ_LIMIT","dayKey","operatorDigestMarkerId","composeOperatorDigestBody","total","listed","rest","push","sendOperatorAlertDigest","force","getUTCHours","count","marker","orderBy","limit","queued","docs","dryRun","create","sendOperatorAlertDigestEmail","date","batch","delete","commit","catch","operatorAlertCatalogRows","definitions","defaults","effective","overridden","sendOperatorAlertTest","sample","describeOperatorAlertRecipients","recipients","configured","toLowerCase","includes","support"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,UAAU,EAAEC,UAAU,QAAQ,cAAa;AACpD,SACEC,6BAA6B,EAC7BC,wCAAwC,EACxCC,0BAA0B,EAC1BC,kBAAkB,EAClBC,2BAA2B,QAOtB,sBAAqB;AAC5B,SACEC,gBAAgB,EAChBC,sBAAsB,QACjB,8CAA6C;AACpD,SAASC,iBAAiB,QAAQ,6CAA4C;AAC9E,SAASC,iBAAiB,QAA8B,2BAA0B;AAClF,OAAOC,mBAAmB,sBAAkB;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;CAwBC,GAED,0CAA0C,GAC1C,OAAO,MAAMC,qCAAqC,mBAAkB;AACpE,OAAO,MAAMC,8BAA8B,iBAAgB;AAC3D,6EAA6E,GAC7E,OAAO,MAAMC,kCAAkC,qBAAoB;AACnE,yCAAyC,GACzC,OAAO,MAAMC,mCAAmC,sBAAqB;AAErE,uEAAuE,GACvE,OAAO,MAAMC,yCAAyC,GAAE;AAmExD,MAAMC,YAAY,IAAMN,cAAcO,GAAG,GAAGD,SAAS;AAErD,+EAA+E;AAE/E,MAAME,kBAAkB;AACxB,IAAIC;AAEJ,4EAA4E,GAC5E,OAAO,SAASC;IACdD,gBAAgBE;AAClB;AAEA,wEAAwE,GACxE,OAAO,SAASC,+BACdC,GAAY;IAEZ,MAAMC,SAAUD,OAAO,OAAOA,QAAQ,WAAWA,MAAM,CAAC;IACxD,MAAME,QAAkD,CAAC;IACzD,MAAMC,WAAWF,MAAM,CAAC,QAAQ;IAChC,IAAIE,YAAY,OAAOA,aAAa,UAAU;QAC5C,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACJ,UAAsC;YAC/E,IAAI,CAACC,QAAQ,CAACC,SAAS,OAAOA,UAAU,UAAU;YAClD,MAAMG,QAAQH;YACd,MAAMI,UAAoC,CAAC;YAC3C,IAAI,OAAOD,KAAK,CAAC,UAAU,KAAK,WAAWC,QAAQC,OAAO,GAAGF,KAAK,CAAC,UAAU;YAC7E,IAAIA,KAAK,CAAC,WAAW,KAAK,eAAeA,KAAK,CAAC,WAAW,KAAK,UAAU;gBACvEC,QAAQE,QAAQ,GAAGH,KAAK,CAAC,WAAW;YACtC;YACA,IAAIF,OAAOM,IAAI,CAACH,SAASI,MAAM,EAAEX,KAAK,CAACE,KAAK,GAAGK;QACjD;IACF;IACA,MAAMK,OAAOC,OAAOd,MAAM,CAAC,gBAAgB;IAC3C,OAAO;QACLC;QACAc,eACED,OAAOE,SAAS,CAACH,SAASA,QAAQ,KAAKA,QAAQ,KAC3CA,OACAtB;OACF,OAAOS,MAAM,CAAC,cAAc,KAAK,WACjC;QAAEiB,aAAajB,MAAM,CAAC,cAAc;IAAW,IAC/C,CAAC,GACD,OAAOA,MAAM,CAAC,iBAAiB,KAAK,WACpC;QAAEkB,gBAAgBlB,MAAM,CAAC,iBAAiB;IAAW,IACrD,CAAC;AAET;AAEA;;;CAGC,GACD,OAAO,eAAemB;IACpB,MAAMC,MAAMC,KAAKD,GAAG;IACpB,IAAIzB,iBAAiByB,MAAMzB,cAAc2B,EAAE,GAAG5B,iBAAiB,OAAOC,cAAc4B,GAAG;IACvF,IAAI;QACF,MAAMC,WAAW,MAAMhC,YACpBiC,UAAU,CAACtC,oCACXoC,GAAG,CAACnC,6BACJsC,GAAG;QACN,MAAMH,MAAMzB,+BAA+B0B,SAASG,MAAM,GAAGH,SAASI,IAAI,KAAK;QAC/EjC,gBAAgB;YAAE2B,IAAIF;YAAKG;QAAI;QAC/B,OAAOA;IACT,EAAE,OAAOM,OAAO;QACdC,QAAQD,KAAK,CAAC,6DAA6DA;QAC3E,OAAO/B,+BAA+B;IACxC;AACF;AAEA,+EAA+E;AAE/E,8DAA8D,GAC9D,MAAMiC,eAAe,IAAIC;AACzB,MAAMC,oBAAoB;AAE1B,SAASC,WAAW/B,IAAY,EAAEgC,SAAiB;IACjD,OAAO5D,WAAW,UAAU6D,MAAM,CAAC,GAAGjC,KAAK,MAAM,EAAEgC,WAAW,EAAEE,MAAM,CAAC,OAAOC,KAAK,CAAC,GAAG;AACzF;AAMA;;;;;CAKC,GACD,OAAO,eAAeC,mBACpBC,UAA4F,EAC5FL,SAAiB,EACjBf,MAAMC,KAAKD,GAAG,EAAE;QAG8BoB;IAD9C,MAAMC,WAAWC,KAAKC,GAAG,CAAC,GAAGH,WAAWI,mBAAmB,IAAI;IAC/D,MAAMC,iBAAiBH,KAAKC,GAAG,CAAC,GAAGD,KAAKI,KAAK,EAACN,6BAAAA,WAAWK,cAAc,YAAzBL,6BAA6B;IAC3E,IAAI,CAACC,UAAU,OAAO;QAAEM,MAAM;QAAMC,SAAS;IAAE;IAC/C,MAAMC,WAAW,GAAGT,WAAWrC,IAAI,CAAC,MAAM,EAAEgC,WAAW;IACvD,MAAMe,aAAanB,aAAaL,GAAG,CAACuB;IACpC,IAAIC,cAAcA,aAAa9B,KAAK,OAAO;QAAE2B,MAAM;QAAOI,QAAQ;IAAU;IAC5E,IAAI;QACF,MAAMC,MAAM5D,YACTiC,UAAU,CAACpC,iCACXkC,GAAG,CAACW,WAAWM,WAAWrC,IAAI,EAAEgC;QACnC,MAAMkB,QAAQ,MAAM7D,YAAY8D,cAAc,CAAC,OAAOC;gBAGvBC,sBAMAA,sBAEUA,oBAiBrBA,cAImBA;YA/BrC,MAAMhC,WAAW,MAAM+B,YAAY7B,GAAG,CAAC0B;YACvC,MAAMI,QAAShC,SAASG,MAAM,GAAGH,SAASI,IAAI,KAAK,CAAC;YACpD,MAAM6B,gBAAgB3C,QAAO0C,uBAAAA,KAAK,CAAC,gBAAgB,YAAtBA,uBAA0B;YACvD,MAAME,OAAO;gBAAEvD,MAAMqC,WAAWrC,IAAI;gBAAEgC,WAAWA,UAAUG,KAAK,CAAC,GAAG;gBAAMrB,aAAaG;YAAI;YAC3F,IAAIqC,iBAAiBrC,MAAMqC,gBAAgBhB,UAAU;oBACAe;gBAAnDD,YAAYI,GAAG,CAACP,KAAK,aAAKM;oBAAME,YAAY9C,QAAO0C,qBAAAA,KAAK,CAAC,aAAa,YAAnBA,qBAAuB,KAAK;oBAAK;oBAAEK,OAAO;gBAAK;gBAClG,OAAO;oBAAEd,MAAM;oBAAOI,QAAQ;gBAAU;YAC1C;YACA,MAAMW,gBAAgBhD,QAAO0C,uBAAAA,KAAK,CAAC,gBAAgB,YAAtBA,uBAA0B;YACvD,MAAMO,WAAWD,iBAAiB1C,MAAM0C,gBAAgBrB;YACxD,MAAMuB,cAAc,AAACD,CAAAA,WAAWjD,QAAO0C,qBAAAA,KAAK,CAAC,cAAc,YAApBA,qBAAwB,KAAK,CAAA,IAAK;YACzE,IAAIQ,cAAcnB,gBAAgB;gBAChCU,YAAYI,GAAG,CACbP,KACA,aAAKM;oBAAMM;oBAAaF,eAAeC,WAAWD,gBAAgB1C;oBAClE;oBAAEyC,OAAO;gBAAK;gBAEhB,OAAO;oBAAEd,MAAM;oBAAOI,QAAQ;gBAAkB;YAClD;YACAI,YAAYI,GAAG,CACbP,KACA,aACKM;gBACHD,eAAerC;gBACf4C,aAAa;gBACbF,eAAe;gBACfF,YAAY;gBACZK,OAAOnD,QAAO0C,eAAAA,KAAK,CAAC,QAAQ,YAAdA,eAAkB,KAAK;gBAEvC;gBAAEK,OAAO;YAAK;YAEhB,OAAO;gBAAEd,MAAM;gBAAMC,SAASlC,QAAO0C,oBAAAA,KAAK,CAAC,aAAa,YAAnBA,oBAAuB;YAAG;QACjE;QACA,IAAIH,MAAMN,IAAI,EAAE;YACd,IAAIhB,aAAamC,IAAI,IAAIjC,mBAAmBF,aAAaoC,KAAK;YAC9DpC,aAAa4B,GAAG,CAACV,UAAU7B,MAAMqB;QACnC;QACA,OAAOY;IACT,EAAE,OAAOxB,OAAO;QACdC,QAAQD,KAAK,CAAC,CAAC,yCAAyC,EAAEW,WAAWrC,IAAI,CAAC,QAAQ,CAAC,EAAE0B;QACrF,OAAO;YAAEkB,MAAM;YAAMC,SAAS;QAAE;IAClC;AACF;AAEA,+EAA+E;AAE/E,MAAMoB,qBAAqB;AAE3B,4DAA4D,GAC5D,OAAO,SAASC;QACOC;IAArB,MAAMlE,QAAQmE,QAAOD,0CAAAA,QAAQE,GAAG,CAAC,6BAA6B,YAAzCF,0CAA6C,IAAIG,IAAI;IAC1E,OAAO,oBAAoBC,IAAI,CAACtE,SAASA,QAAQ;AACnD;AAEA;;;;CAIC,GACD,OAAO,SAASuE,4BACdC,MAAwC,EACxCC,UAA8C,CAAC,CAAC;QAgBpBA,cAWyBC;IAzBrD,MAAMC,SAASC;IACf,MAAMC,QAAQL,OAAOM,GAAG,CAAC,CAACC;QACxB,MAAMC,OAAOC,0BAA0BF,MAAMC,IAAI;QACjD,OAAO;YACL,CAAC,CAAC,EAAEzG,0BAA0B,CAACwG,MAAMG,IAAI,CAAC,CAAC,EAAE,EAAEH,MAAMI,KAAK,EAAE;YAC5DJ,MAAMK,IAAI;YACVJ;SACD,CACEK,MAAM,CAACC,SACPC,IAAI,CAAC;IACV;IACA,MAAMC,OAAOf,QAAQxC,MAAM,GACvB;QAAC,CAAC,iBAAiB,EAAEuC,OAAOhE,MAAM,CAAC,MAAM,EAAEgE,OAAOhE,MAAM,KAAK,IAAI,KAAK,KAAK;WAAKqE;KAAM,CAACU,IAAI,CAAC,UAC5FV,MAAMU,IAAI,CAAC;IACf,MAAME,aAAa,IAAIxE,MAAKwD,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG,IAAI0E,WAAW;IAClE,MAAMhB,SAASF,OAAOM,GAAG,CAAC,CAACC,QAAW,CAAA;YACpChF,MAAMgF,MAAMhF,IAAI;YAChBmF,MAAMH,MAAMG,IAAI;YAChBS,UAAUZ,MAAMY,QAAQ;YACxBR,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI,KAAK;QAChD,CAAA;IACA,OAAO;QACLQ;OACIf,QAAQxC,MAAM,GAAG;QAAEA,QAAQyC;IAAO,IAAI;QAAEK,KAAK,GAAEL,WAAAA,MAAM,CAAC,EAAE,YAATA,WAAa;IAAK;QACrEe;OACId,SAAS;QAAEkB,SAASlB;IAAO,IAAI,CAAC;AAExC;AAEA;;;;;CAKC,GACD,OAAO,eAAemB,yBACpBtB,MAAwC,EACxCC,UAA8C,CAAC,CAAC;QAM1BP;IAJtB,MAAM0B,MAAM3B;IACZ,IAAI,CAAC2B,OAAO,CAACpB,OAAOhE,MAAM,EAAE,OAAO;QAAEuF,QAAQ;QAAOhD,QAAQ;IAAe;IAC3E,MAAMqC,OAAOY,KAAKC,SAAS,CAAC1B,4BAA4BC,QAAQC;IAChE,MAAMyB,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,SAAShC,QAAOD,6CAAAA,QAAQE,GAAG,CAAC,gCAAgC,YAA5CF,6CAAgD,IAAIG,IAAI;IAC9E,IAAI8B,QAAQ;YAC2B1B;QAArC,MAAM2B,YAAYjC,OAAO7B,KAAKI,KAAK,CAAC,EAAC+B,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG,MAAM;QAClEkF,OAAO,CAAC,6BAA6B,GAAGE;QACxCF,OAAO,CAAC,6BAA6B,GACnC,YAAY9H,WAAW,UAAU+H,QAAQnE,MAAM,CAAC,GAAGoE,UAAU,CAAC,EAAEhB,MAAM,EAAEnD,MAAM,CAAC;IACnF;IACA,IAAI;QACF,MAAMoE,WAAW,MAAMC,MAAMV,KAAK;YAChCW,QAAQ;YACRL;YACAd;YACAoB,UAAU;YACVC,QAAQC,YAAYC,OAAO,CAAC3C;QAC9B;QACA,IAAIqC,SAASO,EAAE,EAAE,OAAO;YAAEb,QAAQ;YAAMc,QAAQR,SAASQ,MAAM;QAAC;QAChEnF,QAAQD,KAAK,CAAC,CAAC,mCAAmC,EAAE4E,SAASQ,MAAM,EAAE;QACrE,OAAO;YAAEd,QAAQ;YAAOhD,QAAQ;YAAW8D,QAAQR,SAASQ,MAAM;QAAC;IACrE,EAAE,OAAOpF,OAAO;QACdC,QAAQD,KAAK,CAAC,yCAAyCA;QACvD,OAAO;YAAEsE,QAAQ;YAAOhD,QAAQ;QAAU;IAC5C;AACF;AAEA,+EAA+E;AAE/E,SAAS6B;QACOV;IAAd,OAAOC,QAAOD,uCAAAA,QAAQE,GAAG,CAAC,0BAA0B,YAAtCF,uCAA0C,IAAIG,IAAI,GAAGyC,OAAO,CAAC,QAAQ;AACrF;AAEA,kEAAkE,GAClE,OAAO,SAAS7B,0BAA0BD,IAA+B;IACvE,MAAMhF,QAAQmE,OAAOa,eAAAA,OAAQ,IAAIX,IAAI;IACrC,IAAI,CAACrE,OAAO,OAAO;IACnB,IAAI,gBAAgBsE,IAAI,CAACtE,QAAQ,OAAOA;IACxC,MAAM2E,SAASC;IACf,OAAOD,UAAU3E,MAAM+G,UAAU,CAAC,OAAO,GAAGpC,SAAS3E,OAAO,GAAG;AACjE;AAEA;;;CAGC,GACD,SAASgH,WAAWC,QAA4B,EAAEC,OAAyC;IACzF,IAAI,CAACD,UAAU,OAAO;IACtB,MAAME,UAAU;WAAIF,SAASG,QAAQ,CAAC;KAA2B,CAACC,IAAI,CAAC,CAAC,GAAGC,KAAK;QAC9E,MAAMtH,QAAQkH,2BAAAA,OAAS,CAACI,KAAe;QACvC,OAAOtH,UAAU,QAAQA,UAAUP,aAAaO,UAAU;IAC5D;IACA,OAAOmH,UAAU,KAAK1I,4BAA4BwI,UAAUC;AAC9D;AAEA,+EAA+E;AAE/E,iEAAiE,GACjE,SAASK,uBAAuBxH,IAAY;IAC1C,OAAO;QACLA;QACAyH,OAAOzH;QACP0H,aAAa;QACbvC,MAAM;QACNS,UAAU;QACVR,OAAO;QACPC,MAAM;QACN9E,UAAU;QACVkC,qBAAqB;QACrBkF,gBAAgB;IAClB;AACF;AAEA,sEAAsE,GACtE,OAAO,SAASC,+BACd5H,IAAsC;IAEtC,IAAI,OAAOA,SAAS,UAAU;QAC5B,MAAM6H,QAAQlJ,iBAAiBqB,KAAKA,IAAI;QACxC,IAAI6H,OAAO,OAAOA;QAClB,IAAI7H,KAAK8H,QAAQ,EAAE;YACjB,IAAI;gBACFlJ,uBAAuB;oBAACoB;iBAAK,EAAE;oBAAE8H,UAAU9H,KAAK8H,QAAQ;gBAAC;YAC3D,EAAE,OAAOpG,OAAO;gBACdC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAE1B,KAAKA,IAAI,CAAC,wBAAwB,CAAC,EAAE0B;YAC1E;QACF;QACA,OAAO1B;IACT;IACA,MAAM6H,QAAQlJ,iBAAiBqB;IAC/B,IAAI6H,OAAO,OAAOA;IAClBlG,QAAQD,KAAK,CAAC,CAAC,mBAAmB,EAAE1B,KAAK,mDAAmD,CAAC;IAC7F,OAAOwH,uBAAuBxH;AAChC;AAEA,OAAO,SAAS+H,oBACd1F,UAAmC,EACnCqC,UAAqC,CAAC,CAAC;QAKjCA,kBAGGA,kBAIAA,eAQMA,cACKrC;IAnBpB,MAAM8E,UAAgC,aAChCzC,QAAQsD,OAAO,GAAG;QAAE5C,OAAOV,QAAQsD,OAAO;IAAC,IAAI,CAAC,GAChDtD,QAAQW,IAAI,GAAG;QAAEA,MAAMX,QAAQW,IAAI;IAAC,IAAI,CAAC,IACzCX,mBAAAA,QAAQyC,OAAO,YAAfzC,mBAAmB,CAAC;IAE1B,MAAMU,QACJhB,QAAOM,mBAAAA,QAAQsD,OAAO,YAAftD,mBAAmB,IAAIJ,IAAI,MAClC5F,4BAA4B2D,WAAW+C,KAAK,EAAE+B,YAC9C9E,WAAWoF,KAAK;IAClB,MAAMpC,OACJjB,QAAOM,gBAAAA,QAAQW,IAAI,YAAZX,gBAAgB,IAAIJ,IAAI,MAAM5F,4BAA4B2D,WAAWgD,IAAI,EAAE8B;IACpF,OAAO;QACLnH,MAAMqC,WAAWrC,IAAI;QACrByH,OAAOpF,WAAWoF,KAAK;QACvBtC,MAAM9C,WAAW8C,IAAI;QACrBS,UAAUvD,WAAWuD,QAAQ;QAC7BR;QACAC;QACAJ,MAAMb,QAAOM,eAAAA,QAAQmB,GAAG,YAAXnB,eAAe,IAAIJ,IAAI,MAAM2C,WAAW5E,WAAW4C,IAAI,EAAEkC;QACtEc,gBAAgB,GAAE5F,+BAAAA,WAAW4F,gBAAgB,YAA3B5F,+BAA+B9D;OAC7CmG,QAAQwD,KAAK,GAAG;QAAEA,OAAOxD,QAAQwD,KAAK;IAAC,IAAI,CAAC,GAC5CxD,QAAQyD,MAAM,GAAG;QAAEA,QAAQzD,QAAQyD,MAAM;IAAC,IAAI,CAAC;AAEvD;AAEA,+EAA+E;AAE/E,oEAAoE,GACpE,eAAeC;IACb,IAAI,CAACtJ,qBAAqB,OAAO;IACjC,MAAM,EAAEuJ,2BAA2B,EAAE,GAAG,MAAM,MAAM,CAAC;IACrD,OAAO,AAAC,CAAA,MAAMA,6BAA4B,EAAG5H,MAAM,GAAG;AACxD;AAEA,0CAA0C,GAC1C,OAAO,eAAe6H,mBACpBtI,IAAsC,EACtC0E,UAAqC,CAAC,CAAC;IAEvC,MAAM6D,SAAS,OAAOvI,SAAS,WAAWA,OAAOA,KAAKA,IAAI;IAC1D,IAAI;YAEuB0E,oBAuBdA;QAxBX,MAAMrC,aAAauF,+BAA+B5H;QAClD,MAAMgC,YAAYoC,QAAOM,qBAAAA,QAAQ1C,SAAS,YAAjB0C,qBAAqB,IAAIJ,IAAI;QACtD,IAAIzB,UAAU;QACd,IAAIb,WAAW;YACb,MAAMkB,QAAQ,MAAMd,mBAAmBC,YAAYL;YACnD,IAAIkB,MAAMN,IAAI,KAAK,OAAO,OAAO;gBAAE4F,SAAStF,MAAMF,MAAM;gBAAEhD,MAAMuI;YAAO;YACvE1F,UAAUK,MAAML,OAAO;QACzB;QACA,MAAMmC,QAAQ+C,oBAAoB1F,YAAYqC;QAC9C,IAAI7B,UAAU,GAAG;YACfmC,MAAMK,IAAI,GAAG;gBAACL,MAAMK,IAAI;gBAAE,CAAC,OAAO,EAAExC,QAAQ,UAAU,EAAEA,YAAY,IAAI,KAAK,IAAI,wBAAwB,CAAC;aAAC,CACxGyC,MAAM,CAACC,SACPC,IAAI,CAAC;QACV;QACA,MAAMiD,WAAW,MAAMzH;QACvB,MAAM,EAAEV,OAAO,EAAEC,QAAQ,EAAE,GAAGjC,8BAA8B+D,YAAYoG;QACxE,MAAMC,WAAWpI,WAAWC,aAAa,eAAgB,MAAM6H;QAE/D,MAAM,EAAEO,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;QAC5C,MAAMA,mBACJ;YACE3I,MAAMgF,MAAMiD,gBAAgB;YAC5B7C,OAAOJ,MAAMI,KAAK;WACdJ,MAAMK,IAAI,GAAG;YAAEA,MAAML,MAAMK,IAAI;QAAC,IAAI,CAAC;YACzCuD,KAAK,GAAElE,iBAAAA,QAAQkE,KAAK,YAAblE,iBAAiBjG,mBAAmB4D;WACvC2C,MAAMC,IAAI,IAAID,MAAMC,IAAI,CAAC+B,UAAU,CAAC,OAAO;YAAE/B,MAAMD,MAAMC,IAAI;QAAC,IAAI,CAAC,GACnED,MAAMkD,KAAK,GAAG;YAAEA,OAAOlD,MAAMkD,KAAK;QAAC,IAAI,CAAC,GACxClD,MAAMmD,MAAM,GAAG;YAAEA,QAAQnD,MAAMmD,MAAM;QAAC,IAAI,CAAC,IAEjD,qEAAqE;QACrE,gEAAgE;QAChEO,WAAW;YAAEG,WAAW;QAAK,IAAI,CAAC;QAEpC,IAAI,CAACvI,SAAS,OAAO;YAAEkI,SAAS;YAAgBxI,MAAMuI;QAAO;QAC7D,0EAA0E;QAC1E,MAAM,EAAEO,qBAAqB,EAAE,GAAG,MAAM,MAAM,CAAC;QAC/C,MAAMA,sBAAsB9D;QAC5B,IAAIzE,aAAa,UAAU;YACzB,MAAMwI,4BAA4B/D;YAClC,OAAO;gBAAEwD,SAAS;gBAAUxI,MAAMuI;YAAO;QAC3C;QACA,OAAO,MAAMS,qBAAqBhE,OAAO;YAAE0D;QAAS;IACtD,EAAE,OAAOhH,OAAO;QACdC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAE6G,OAAO,oBAAoB,CAAC,EAAE7G;QACjE,OAAO;YAAE8G,SAAS;YAAexI,MAAMuI;QAAO;IAChD;AACF;AAEA,oCAAoC,GACpC,eAAeS,qBACbhE,KAA4B,EAC5BN,OAA8B;IAE9B,MAAM,EAAEuE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,MAAM,CAACC,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACzC3E,QAAQgE,QAAQ,GACZO,uBAAuB;YACrB7D,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI;YACzCkC,SAAS,CAAC,eAAe,EAAEnC,MAAMhF,IAAI,EAAE;QACzC,KACAoJ,QAAQE,OAAO,CAAkB;YAAEC,MAAM;YAAOvG,QAAQ;QAAe;QAC3E+C,yBAAyB;YAACf;SAAM;KACjC;IACD,OAAO;QACLwD,SAASU,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM,GAAG,cAAc;QACtDhG,MAAMgF,MAAMhF,IAAI;QAChBkJ;QACAC;IACF;AACF;AAEA,+EAA+E;AAE/E,eAAeJ,4BAA4B/D,KAA4B;IACrE,IAAI;QACF,MAAM3F,YACHiC,UAAU,CAACnC,kCACXiC,GAAG,CAACvC,qBACJ2E,GAAG,CAAC,aAAKwB;YAAOwE,MAAMtI,KAAKD,GAAG;;IACnC,EAAE,OAAOS,OAAO;QACd,uEAAuE;QACvE,oCAAoC;QACpCC,QAAQD,KAAK,CAAC,CAAC,oDAAoD,EAAEsD,MAAMhF,IAAI,CAAC,IAAI,CAAC,EAAE0B;QACvF,MAAMsH,qBAAqBhE,OAAO;YAAE0D,UAAU,MAAMN;QAAyB;IAC/E;AACF;AAEA,6EAA6E,GAC7E,OAAO,MAAMqB,6BAA6B,GAAE;AAC5C,MAAMC,oBAAoB;AAW1B,SAASC,OAAO1I,GAAW;IACzB,OAAO,IAAIC,KAAKD,KAAK0E,WAAW,GAAGxD,KAAK,CAAC,GAAG;AAC9C;AAEA,yDAAyD,GACzD,OAAO,SAASyH,uBAAuB3I,GAAW;IAChD,OAAO,CAAC,OAAO,EAAE0I,OAAO1I,MAAM;AAChC;AAEA,6DAA6D,GAC7D,OAAO,SAAS4I,0BACdpF,MAAwC,EACxCqF,QAAQrF,OAAOhE,MAAM;IAErB,MAAMsJ,SAAStF,OAAOtC,KAAK,CAAC,GAAGsH,4BAA4B1E,GAAG,CAAC,CAACC;QAC9D,MAAMC,OAAOC,0BAA0BF,MAAMC,IAAI;QACjD,OAAO;YAAC,CAAC,EAAE,EAAED,MAAMI,KAAK,EAAE;YAAEJ,MAAMK,IAAI;YAAEJ;SAAK,CAACK,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE;IACA,MAAMwE,OAAOF,QAAQC,OAAOtJ,MAAM;IAClC,IAAIuJ,OAAO,GAAGD,OAAOE,IAAI,CAAC,CAAC,KAAK,EAAED,KAAK,2BAA2B,CAAC;IACnE,OAAOD,OAAOvE,IAAI,CAAC;AACrB;AAEA;;;;;CAKC,GACD,OAAO,eAAe0E,wBACpBxF,UAA+D,CAAC,CAAC;QAErDA;IAAZ,MAAMzD,OAAMyD,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG;IACnC,IAAI;YAEWwH;QADb,MAAMA,WAAW,MAAMzH;QACvB,MAAMN,QAAO+H,0BAAAA,SAAS7H,aAAa,YAAtB6H,0BAA0BrJ;QACvC,IAAI,CAACsF,QAAQyF,KAAK,IAAI,IAAIjJ,KAAKD,KAAKmJ,WAAW,KAAK1J,MAAM;YACxD,OAAO;gBAAE6I,MAAM;gBAAOvG,QAAQ;gBAAWqH,OAAO;YAAE;QACpD;QACA,MAAM/I,aAAajC,YAAYiC,UAAU,CAACnC;QAC1C,MAAMmL,SAASjL,YACZiC,UAAU,CAACpC,iCACXkC,GAAG,CAACwI,uBAAuB3I;QAC9B,IAAI,CAACyD,QAAQyF,KAAK,IAAI,AAAC,CAAA,MAAMG,OAAO/I,GAAG,EAAC,EAAGC,MAAM,EAAE;YACjD,OAAO;gBAAE+H,MAAM;gBAAOvG,QAAQ;gBAAgBqH,OAAO;YAAE;QACzD;QACA,MAAMhJ,WAAW,MAAMC,WAAWiJ,OAAO,CAAC,QAAQ,OAAOC,KAAK,CAACd,mBAAmBnI,GAAG;QACrF,MAAMkJ,SAASpJ,SAASqJ,IAAI;QAC5B,IAAI,CAACD,OAAOhK,MAAM,EAAE,OAAO;YAAE8I,MAAM;YAAOvG,QAAQ;YAASqH,OAAO;QAAE;QACpE,IAAI3F,QAAQiG,MAAM,EAAE,OAAO;YAAEpB,MAAM;YAAOvG,QAAQ;YAAWqH,OAAOI,OAAOhK,MAAM;QAAC;QAClF,IAAI,CAACiE,QAAQyF,KAAK,EAAE;YAClB,IAAI;gBACF,MAAMG,OAAOM,MAAM,CAAC;oBAAEpB,MAAMvI;oBAAKoJ,OAAOI,OAAOhK,MAAM;gBAAC;YACxD,EAAE,eAAM;gBACN,OAAO;oBAAE8I,MAAM;oBAAOvG,QAAQ;oBAAgBqH,OAAO;gBAAE;YACzD;QACF;QACA,MAAM5F,SAASgG,OAAO1F,GAAG,CAAC,CAAC3D,MAAaA,IAAIK,IAAI;QAChD,MAAM,EAAEoJ,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC;QACtD,MAAM,CAAC3B,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;YACzCwB,6BAA6B;gBAC3BC,MAAMnB,OAAO1I;gBACboJ,OAAO5F,OAAOhE,MAAM;gBACpB4E,MAAMwE,0BAA0BpF;gBAChCoB,KAAKX,0BAA0B;YACjC;YACAa,yBAAyBtB,QAAQ;gBAAEvC,QAAQ;gBAAMjB;YAAI;SACtD;QACD,IAAIiI,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM,EAAE;YAChC,MAAM+E,QAAQ1L,YAAY0L,KAAK;YAC/B,KAAK,MAAM3J,OAAOqJ,OAAQM,MAAMC,MAAM,CAAC5J,IAAI6B,GAAG;YAC9C,MAAM8H,MAAME,MAAM;QACpB,OAAO,IAAI,CAACvG,QAAQyF,KAAK,EAAE;YACzB,qEAAqE;YACrE,MAAMG,OAAOU,MAAM,GAAGE,KAAK,CAAC,IAAMxL;QACpC;QACA,OAAO;YAAE6J,MAAML,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM;YAAEqE,OAAO5F,OAAOhE,MAAM;YAAEyI;YAAOC;QAAQ;IACpF,EAAE,OAAOzH,OAAO;QACdC,QAAQD,KAAK,CAAC,mCAAmCA;QACjD,OAAO;YAAE6H,MAAM;YAAOvG,QAAQ;YAAeqH,OAAO;QAAE;IACxD;AACF;AAmBA,sEAAsE,GACtE,OAAO,SAASc,yBACdC,WAA+C,EAC/C3C,QAAkC;IAElC,OAAO2C,YAAYrG,GAAG,CAAC,CAAC1C;YAMZA;YAGUoG;eATkB;YACtCzI,MAAMqC,WAAWrC,IAAI;YACrByH,OAAOpF,WAAWoF,KAAK;YACvBC,aAAarF,WAAWqF,WAAW;YACnCvC,MAAM9C,WAAW8C,IAAI;YACrBS,UAAUvD,WAAWuD,QAAQ;YAC7BkC,QAAQ,GAAEzF,uBAAAA,WAAWyF,QAAQ,YAAnBzF,uBAAuB;YACjCgJ,UAAU;gBAAE/K,SAAS+B,WAAWsF,cAAc;gBAAEpH,UAAU8B,WAAW9B,QAAQ;YAAC;YAC9E+K,WAAWhN,8BAA8B+D,YAAYoG;YACrD8C,YAAYhG,SAAQkD,kBAAAA,SAAS3I,KAAK,qBAAd2I,eAAgB,CAACpG,WAAWrC,IAAI,CAAC;QACvD;;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAewL,sBACpBnJ,UAAmC;QAG8BA;IADjE,MAAMoJ,SAAiC,CAAC;IACxC,KAAK,MAAM,GAAGlE,KAAK,IAAI,GAAGlF,WAAW+C,KAAK,CAAC,CAAC,EAAE/C,WAAWgD,IAAI,CAAC,CAAC,GAAEhD,mBAAAA,WAAW4C,IAAI,YAAf5C,mBAAmB,IAAI,CAACgF,QAAQ,CAC/F,2BACC;QACDoE,MAAM,CAAClE,KAAe,GAAG,CAAC,OAAO,EAAEA,MAAM;IAC3C;IACA,MAAMvC,QAAQ+C,oBAAoB1F,YAAY;QAAE8E,SAASsE;IAAO;IAChEzG,MAAMI,KAAK,GAAG,CAAC,OAAO,EAAEJ,MAAMI,KAAK,EAAE;IACrCJ,MAAMK,IAAI,GAAG;QACXL,MAAMK,IAAI;QACV;KACD,CAACG,IAAI,CAAC;IACP,MAAM,EAAEyD,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,MAAM,CAACC,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACzCvK,sBACImK,uBAAuB;YACrB7D,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI,KAAKC,0BAA0B;YACxEiC,SAAS,CAAC,oBAAoB,EAAE9E,WAAWrC,IAAI,EAAE;QACnD,KACAoJ,QAAQE,OAAO,CAAkB;YAAEC,MAAM;YAAOvG,QAAQ;QAAe;QAC3E+C,yBAAyB;YAACf;SAAM;KACjC;IACD,OAAO;QAAEkE;QAAOC;IAAQ;AAC1B;AAEA,0EAA0E,GAC1E,OAAO,eAAeuC;QAOMvH,gCAIHA;IAPvB,MAAM,EAAEkE,2BAA2B,EAAE,GAAG,MAAM,MAAM,CAAC;IACrD,MAAMsD,aAAa,MAAMtD;IACzB,IAAI,CAACsD,WAAWlL,MAAM,EAAE,OAAO;QAAEZ,QAAQ;QAAQwK,OAAO;IAAE;IAC1D,MAAMuB,aAAaxH,QAAOD,iCAAAA,QAAQE,GAAG,CAAC,oBAAoB,YAAhCF,iCAAoC,IAAIG,IAAI,GAAGuH,WAAW;IACpF,IAAID,WAAWE,QAAQ,CAAC,QAAQH,UAAU,CAAC,EAAE,KAAKC,YAAY;QAC5D,OAAO;YAAE/L,QAAQ;YAAqBwK,OAAO;QAAE;IACjD;IACA,MAAM0B,UAAU3H,QAAOD,kDAAAA,QAAQE,GAAG,CAAC,qCAAqC,YAAjDF,kDAAqD,IAAIG,IAAI,GAAGuH,WAAW;IAClG,IAAIE,QAAQD,QAAQ,CAAC,QAAQH,UAAU,CAAC,EAAE,KAAKI,SAAS;QACtD,OAAO;YAAElM,QAAQ;YAAsCwK,OAAO;QAAE;IAClE;IACA,OAAO;QAAExK,QAAQ;QAAkBwK,OAAOsB,WAAWlL,MAAM;IAAC;AAC9D"}
+{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/operator-alerts.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\nimport { createHash, createHmac } from 'node:crypto'\nimport {\n  effectiveOperatorAlertSetting,\n  OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE,\n  OPERATOR_ALERT_TIER_LABELS,\n  operatorAlertLevel,\n  renderOperatorAlertTemplate,\n  type AglynNotificationType,\n  type NotificationLevel,\n  type OperatorAlertContext,\n  type OperatorAlertDefinition,\n  type OperatorAlertSettings,\n  type OperatorAlertTypeSetting,\n} from '@aglyn/aglyn/server'\nimport {\n  getOperatorAlert,\n  registerOperatorAlerts,\n} from '@aglyn/aglyn/plugin-manager/operator-alerts'\nimport { createResourceUid } from '@aglyn/aglyn/app-utils/create-resource-uid'\nimport { isEmailConfigured, type SendEmailResult } from '@aglyn/shared-util-email'\nimport firebaseAdmin from './firebase-admin'\n\n/**\n * RAISING AN OPERATOR ALERT (AGL-3377): the one server entry point every\n * alert in the registry goes through.\n *\n * In order, for one call:\n *\n * 1. The type is looked up in the registry (core's catalog plus what plugins\n *    registered). A plugin may pass its definition itself, which registers\n *    it in this process if its declarations have not loaded yet.\n * 2. A `dedupeKey` is claimed against `operatorAlertState`, so a condition\n *    that flaps inside the type's window is told once, and a type with\n *    `minOccurrences` fires only once it has recurred that often.\n * 3. The staff console notification is written for every staff member, as\n *    `notifyStaff` always has.\n * 4. Staff's switches (`platformSettings/operatorAlerts`, coded defaults\n *    until changed) decide the rest: off is console only, `digest` queues it\n *    for the once-a-day digest, `immediate` emails the operator now.\n * 5. The email is the `operator-alert` system email, to\n *    `resolveStaffAlertRecipients()`; the optional out-of-band webhook\n *    (`OPERATOR_ALERT_WEBHOOK_URL`) gets the same alert beside it, so a mail\n *    provider that is itself the outage still reaches somebody.\n *\n * Never throws: an alert is raised from inside the failure it reports, and a\n * second failure there must not replace the first.\n */\n\n/** Where staff's per-type switches live. */\nexport const OPERATOR_ALERT_SETTINGS_COLLECTION = 'platformSettings'\nexport const OPERATOR_ALERT_SETTINGS_DOC = 'operatorAlerts'\n/** One document per (type, dedupeKey): when it last fired, how often since. */\nexport const OPERATOR_ALERT_STATE_COLLECTION = 'operatorAlertState'\n/** Alerts waiting for the daily digest. */\nexport const OPERATOR_ALERT_DIGEST_COLLECTION = 'operatorAlertDigest'\n\n/** The hour, UTC, the digest goes out when staff have not chosen one. */\nexport const OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC = 14\n\nexport interface OperatorAlertSettingsDoc extends OperatorAlertSettings {\n  /** The hour of the day, UTC, the digest is sent. */\n  digestHourUtc?: number\n  updatedAtMs?: number\n  updatedByEmail?: string | null\n}\n\nexport interface RaiseOperatorAlertOptions {\n  /**\n   * What makes two raises the SAME alert — a dispute id, an org and month, a\n   * check id. Absent never dedupes: every raise is told.\n   */\n  dedupeKey?: string\n  /** Values for the definition's `{{tokens}}`. Never personal data. */\n  context?: OperatorAlertContext\n  /** A console path or absolute URL; overrides the definition's link. */\n  url?: string\n  /** The title as written by the caller; overrides the rendered one. */\n  subject?: string\n  /** The body as written by the caller; overrides the rendered one. */\n  body?: string\n  /**\n   * The console notification's level as the caller judged it (AGL-3437);\n   * overrides the definition's. For a caller whose one alert type covers\n   * news of more than one tone.\n   */\n  level?: NotificationLevel\n  /**\n   * The tier as the caller judged it, for an alert that grows more urgent\n   * the longer its condition lasts (a check still degraded after 2 h). Sets\n   * the `[Must know]` prefix and the tier the webhook routes on; the console\n   * level follows it unless the definition or `level` names one.\n   */\n  tier?: OperatorAlertDefinition['tier']\n  orgId?: string\n  hostId?: string\n}\n\nexport type OperatorAlertOutcome =\n  | 'deduped'\n  | 'below-threshold'\n  | 'console-only'\n  | 'queued'\n  | 'delivered'\n  | 'undelivered'\n\nexport interface OperatorAlertResult {\n  outcome: OperatorAlertOutcome\n  type: string\n  email?: SendEmailResult\n  webhook?: OperatorAlertWebhookResult\n}\n\nexport type OperatorAlertWebhookResult =\n  | { posted: true; status: number }\n  | { posted: false; reason: 'unconfigured' | 'refused' | 'network'; status?: number }\n\n/** One alert, rendered, as it is delivered and queued. */\nexport interface RenderedOperatorAlert {\n  type: string\n  label: string\n  tier: OperatorAlertDefinition['tier']\n  category: OperatorAlertDefinition['category']\n  title: string\n  body: string\n  /** Console path or absolute URL; `''` when there is none. */\n  link: string\n  notificationType: AglynNotificationType\n  orgId?: string\n  hostId?: string\n}\n\nconst firestore = () => firebaseAdmin.app().firestore()\n\n// ── Settings ────────────────────────────────────────────────────────────────\n\nconst SETTINGS_TTL_MS = 15_000\nlet settingsCache: { at: number; doc: OperatorAlertSettingsDoc } | undefined\n\n/** Drop the in-process settings cache; the console calls it after a write. */\nexport function invalidateOperatorAlertSettingsCache(): void {\n  settingsCache = undefined\n}\n\n/** Clamps a stored settings document to what the pipeline understands. */\nexport function normalizeOperatorAlertSettings(\n  raw: unknown,\n): OperatorAlertSettingsDoc {\n  const source = (raw && typeof raw === 'object' ? raw : {}) as Record<string, unknown>\n  const types: Record<string, OperatorAlertTypeSetting> = {}\n  const rawTypes = source['types']\n  if (rawTypes && typeof rawTypes === 'object') {\n    for (const [type, value] of Object.entries(rawTypes as Record<string, unknown>)) {\n      if (!type || !value || typeof value !== 'object') continue\n      const entry = value as Record<string, unknown>\n      const setting: OperatorAlertTypeSetting = {}\n      if (typeof entry['enabled'] === 'boolean') setting.enabled = entry['enabled']\n      if (entry['delivery'] === 'immediate' || entry['delivery'] === 'digest') {\n        setting.delivery = entry['delivery']\n      }\n      if (Object.keys(setting).length) types[type] = setting\n    }\n  }\n  const hour = Number(source['digestHourUtc'])\n  return {\n    types,\n    digestHourUtc:\n      Number.isInteger(hour) && hour >= 0 && hour <= 23\n        ? hour\n        : OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC,\n    ...(typeof source['updatedAtMs'] === 'number'\n      ? { updatedAtMs: source['updatedAtMs'] as number }\n      : {}),\n    ...(typeof source['updatedByEmail'] === 'string'\n      ? { updatedByEmail: source['updatedByEmail'] as string }\n      : {}),\n  }\n}\n\n/**\n * Staff's switches. An unreadable document is the coded defaults, uncached:\n * an alert must still go out when the settings cannot be read.\n */\nexport async function readOperatorAlertSettings(): Promise<OperatorAlertSettingsDoc> {\n  const now = Date.now()\n  if (settingsCache && now - settingsCache.at < SETTINGS_TTL_MS) return settingsCache.doc\n  try {\n    const snapshot = await firestore()\n      .collection(OPERATOR_ALERT_SETTINGS_COLLECTION)\n      .doc(OPERATOR_ALERT_SETTINGS_DOC)\n      .get()\n    const doc = normalizeOperatorAlertSettings(snapshot.exists ? snapshot.data() : null)\n    settingsCache = { at: now, doc }\n    return doc\n  } catch (error) {\n    console.error('[operator-alerts] settings unreadable; using the defaults', error)\n    return normalizeOperatorAlertSettings(null)\n  }\n}\n\n// ── Dedupe ──────────────────────────────────────────────────────────────────\n\n/** Keys this instance fired, and until when they stay quiet. */\nconst firedLocally = new Map<string, number>()\nconst FIRED_LOCALLY_MAX = 500\n\nfunction stateDocId(type: string, dedupeKey: string): string {\n  return createHash('sha256').update(`${type}\\u0000${dedupeKey}`).digest('hex').slice(0, 40)\n}\n\nexport type OperatorAlertClaim =\n  | { fire: true; repeats: number }\n  | { fire: false; reason: 'deduped' | 'below-threshold' }\n\n/**\n * Whether a raise with this key fires now. One transaction on the key's\n * state document: inside the window after a fire it is counted and\n * suppressed; before it reaches `minOccurrences` it is counted and held.\n * A store that cannot be read FIRES — a duplicate beats a silence.\n */\nexport async function claimOperatorAlert(\n  definition: Pick<OperatorAlertDefinition, 'type' | 'dedupeWindowMinutes' | 'minOccurrences'>,\n  dedupeKey: string,\n  now = Date.now(),\n): Promise<OperatorAlertClaim> {\n  const windowMs = Math.max(0, definition.dedupeWindowMinutes) * 60_000\n  const minOccurrences = Math.max(1, Math.floor(definition.minOccurrences ?? 1))\n  if (!windowMs) return { fire: true, repeats: 0 }\n  const localKey = `${definition.type}\\u0000${dedupeKey}`\n  const quietUntil = firedLocally.get(localKey)\n  if (quietUntil && quietUntil > now) return { fire: false, reason: 'deduped' }\n  try {\n    const ref = firestore()\n      .collection(OPERATOR_ALERT_STATE_COLLECTION)\n      .doc(stateDocId(definition.type, dedupeKey))\n    const claim = await firestore().runTransaction(async (transaction: any) => {\n      const snapshot = await transaction.get(ref)\n      const state = (snapshot.exists ? snapshot.data() : {}) as Record<string, unknown>\n      const lastFiredAtMs = Number(state['lastFiredAtMs'] ?? 0)\n      const base = { type: definition.type, dedupeKey: dedupeKey.slice(0, 300), updatedAtMs: now }\n      if (lastFiredAtMs && now - lastFiredAtMs < windowMs) {\n        transaction.set(ref, { ...base, suppressed: Number(state['suppressed'] ?? 0) + 1 }, { merge: true })\n        return { fire: false, reason: 'deduped' } as const\n      }\n      const windowStartMs = Number(state['windowStartMs'] ?? 0)\n      const inWindow = windowStartMs && now - windowStartMs < windowMs\n      const occurrences = (inWindow ? Number(state['occurrences'] ?? 0) : 0) + 1\n      if (occurrences < minOccurrences) {\n        transaction.set(\n          ref,\n          { ...base, occurrences, windowStartMs: inWindow ? windowStartMs : now },\n          { merge: true },\n        )\n        return { fire: false, reason: 'below-threshold' } as const\n      }\n      transaction.set(\n        ref,\n        {\n          ...base,\n          lastFiredAtMs: now,\n          occurrences: 0,\n          windowStartMs: 0,\n          suppressed: 0,\n          fired: Number(state['fired'] ?? 0) + 1,\n        },\n        { merge: true },\n      )\n      return { fire: true, repeats: Number(state['suppressed'] ?? 0) } as const\n    })\n    if (claim.fire) {\n      if (firedLocally.size >= FIRED_LOCALLY_MAX) firedLocally.clear()\n      firedLocally.set(localKey, now + windowMs)\n    }\n    return claim\n  } catch (error) {\n    console.error(`[operator-alerts] dedupe unavailable for ${definition.type}; firing`, error)\n    return { fire: true, repeats: 0 }\n  }\n}\n\n// ── Out-of-band webhook ─────────────────────────────────────────────────────\n\nconst WEBHOOK_TIMEOUT_MS = 5_000\n\n/** The webhook URL, when one is configured and is http(s). */\nexport function operatorAlertWebhookUrl(): string {\n  const value = String(process.env['OPERATOR_ALERT_WEBHOOK_URL'] ?? '').trim()\n  return /^https?:\\/\\/\\S+$/i.test(value) ? value : ''\n}\n\n/**\n * The JSON the webhook receives. `text` is what a Slack-compatible incoming\n * webhook renders on its own; `alert` carries the fields for anything that\n * wants to route on them.\n */\nexport function operatorAlertWebhookPayload(\n  alerts: readonly RenderedOperatorAlert[],\n  options: { digest?: boolean; now?: number } = {},\n): Record<string, unknown> {\n  const origin = consoleOrigin()\n  const lines = alerts.map((alert) => {\n    const link = operatorAlertAbsoluteLink(alert.link)\n    return [\n      `[${OPERATOR_ALERT_TIER_LABELS[alert.tier]}] ${alert.title}`,\n      alert.body,\n      link,\n    ]\n      .filter(Boolean)\n      .join('\\n')\n  })\n  const text = options.digest\n    ? [`Operator digest: ${alerts.length} alert${alerts.length === 1 ? '' : 's'}`, ...lines].join('\\n\\n')\n    : lines.join('\\n\\n')\n  const occurredAt = new Date(options.now ?? Date.now()).toISOString()\n  const shaped = alerts.map((alert) => ({\n    type: alert.type,\n    tier: alert.tier,\n    category: alert.category,\n    title: alert.title,\n    body: alert.body,\n    url: operatorAlertAbsoluteLink(alert.link) || null,\n  }))\n  return {\n    text,\n    ...(options.digest ? { digest: shaped } : { alert: shaped[0] ?? null }),\n    occurredAt,\n    ...(origin ? { install: origin } : {}),\n  }\n}\n\n/**\n * Posts alerts to `OPERATOR_ALERT_WEBHOOK_URL`, signed when\n * `OPERATOR_ALERT_WEBHOOK_SECRET` is set: `x-operator-alert-timestamp` is the\n * Unix time and `x-operator-alert-signature` is `sha256=` plus the hex HMAC\n * of `{timestamp}.{body}`. Unset means email only. Never throws.\n */\nexport async function postOperatorAlertWebhook(\n  alerts: readonly RenderedOperatorAlert[],\n  options: { digest?: boolean; now?: number } = {},\n): Promise<OperatorAlertWebhookResult> {\n  const url = operatorAlertWebhookUrl()\n  if (!url || !alerts.length) return { posted: false, reason: 'unconfigured' }\n  const body = JSON.stringify(operatorAlertWebhookPayload(alerts, options))\n  const headers: Record<string, string> = { 'content-type': 'application/json' }\n  const secret = String(process.env['OPERATOR_ALERT_WEBHOOK_SECRET'] ?? '').trim()\n  if (secret) {\n    const timestamp = String(Math.floor((options.now ?? Date.now()) / 1000))\n    headers['x-operator-alert-timestamp'] = timestamp\n    headers['x-operator-alert-signature'] =\n      'sha256=' + createHmac('sha256', secret).update(`${timestamp}.${body}`).digest('hex')\n  }\n  try {\n    const response = await fetch(url, {\n      method: 'POST',\n      headers,\n      body,\n      redirect: 'manual',\n      signal: AbortSignal.timeout(WEBHOOK_TIMEOUT_MS),\n    })\n    if (response.ok) return { posted: true, status: response.status }\n    console.error(`[operator-alerts] webhook answered ${response.status}`)\n    return { posted: false, reason: 'refused', status: response.status }\n  } catch (error) {\n    console.error('[operator-alerts] webhook unreachable', error)\n    return { posted: false, reason: 'network' }\n  }\n}\n\n// ── Links ───────────────────────────────────────────────────────────────────\n\nfunction consoleOrigin(): string {\n  return String(process.env['NEXT_PUBLIC_CONSOLE_URL'] ?? '').trim().replace(/\\/+$/, '')\n}\n\n/** A console path as an inbox or webhook can follow it, or `''`. */\nexport function operatorAlertAbsoluteLink(link: string | undefined | null): string {\n  const value = String(link ?? '').trim()\n  if (!value) return ''\n  if (/^https?:\\/\\//i.test(value)) return value\n  const origin = consoleOrigin()\n  return origin && value.startsWith('/') ? `${origin}${value}` : ''\n}\n\n/**\n * The definition's link with its tokens filled, or `''` when a token it\n * needs is missing — `/admin/orgs/` would open the wrong page.\n */\nfunction renderLink(template: string | undefined, context: OperatorAlertContext | undefined): string {\n  if (!template) return ''\n  const missing = [...template.matchAll(/\\{\\{\\s*([\\w.]+)\\s*\\}\\}/g)].some(([, name]) => {\n    const value = context?.[name as string]\n    return value === null || value === undefined || value === ''\n  })\n  return missing ? '' : renderOperatorAlertTemplate(template, context)\n}\n\n// ── Rendering ───────────────────────────────────────────────────────────────\n\n/** Stands in for a type nobody registered, so it is still told. */\nfunction unregisteredDefinition(type: string): OperatorAlertDefinition {\n  return {\n    type,\n    label: type,\n    description: 'An alert type this install has no registry entry for.',\n    tier: 'should',\n    category: 'ops',\n    title: '{{title}}',\n    body: '{{body}}',\n    delivery: 'immediate',\n    dedupeWindowMinutes: 60,\n    defaultEnabled: true,\n  }\n}\n\n/** Resolves the definition a call names, registering a plugin's own. */\nexport function resolveOperatorAlertDefinition(\n  type: string | OperatorAlertDefinition,\n): OperatorAlertDefinition {\n  if (typeof type !== 'string') {\n    const known = getOperatorAlert(type.type)\n    if (known) return known\n    if (type.pluginId) {\n      try {\n        registerOperatorAlerts([type], { pluginId: type.pluginId })\n      } catch (error) {\n        console.error(`[operator-alerts] ${type.type} could not be registered`, error)\n      }\n    }\n    return type\n  }\n  const known = getOperatorAlert(type)\n  if (known) return known\n  console.error(`[operator-alerts] \"${type}\" is not a registered alert type; telling it anyway`)\n  return unregisteredDefinition(type)\n}\n\nexport function renderOperatorAlert(\n  definition: OperatorAlertDefinition,\n  options: RaiseOperatorAlertOptions = {},\n): RenderedOperatorAlert {\n  const context: OperatorAlertContext = {\n    ...(options.subject ? { title: options.subject } : {}),\n    ...(options.body ? { body: options.body } : {}),\n    ...(options.context ?? {}),\n  }\n  const title =\n    String(options.subject ?? '').trim() ||\n    renderOperatorAlertTemplate(definition.title, context) ||\n    definition.label\n  const body =\n    String(options.body ?? '').trim() || renderOperatorAlertTemplate(definition.body, context)\n  return {\n    type: definition.type,\n    label: definition.label,\n    tier: options.tier ?? definition.tier,\n    category: definition.category,\n    title,\n    body,\n    link: String(options.url ?? '').trim() || renderLink(definition.link, context),\n    notificationType: definition.notificationType ?? OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE,\n    ...(options.orgId ? { orgId: options.orgId } : {}),\n    ...(options.hostId ? { hostId: options.hostId } : {}),\n  }\n}\n\n// ── The entry point ─────────────────────────────────────────────────────────\n\n/** Whether an email to the operator can go at all on this install. */\nasync function operatorEmailReachable(): Promise<boolean> {\n  if (!isEmailConfigured()) return false\n  const { resolveStaffAlertRecipients } = await import('./staff-alert-email')\n  return (await resolveStaffAlertRecipients()).length > 0\n}\n\n/** See the module comment. Never throws. */\nexport async function raiseOperatorAlert(\n  type: string | OperatorAlertDefinition,\n  options: RaiseOperatorAlertOptions = {},\n): Promise<OperatorAlertResult> {\n  const typeId = typeof type === 'string' ? type : type.type\n  try {\n    const definition = resolveOperatorAlertDefinition(type)\n    const dedupeKey = String(options.dedupeKey ?? '').trim()\n    let repeats = 0\n    if (dedupeKey) {\n      const claim = await claimOperatorAlert(definition, dedupeKey)\n      if (claim.fire === false) return { outcome: claim.reason, type: typeId }\n      repeats = claim.repeats\n    }\n    const alert = renderOperatorAlert(definition, options)\n    if (repeats > 0) {\n      alert.body = [alert.body, `Raised ${repeats} more time${repeats === 1 ? '' : 's'} since it was last told.`]\n        .filter(Boolean)\n        .join(' ')\n    }\n    const settings = await readOperatorAlertSettings()\n    const { enabled, delivery } = effectiveOperatorAlertSetting(definition, settings)\n    const emailNow = enabled && delivery === 'immediate' && (await operatorEmailReachable())\n\n    const { notifyStaffConsole } = await import('./notifications')\n    await notifyStaffConsole(\n      {\n        type: alert.notificationType,\n        title: alert.title,\n        ...(alert.body ? { body: alert.body } : {}),\n        level: options.level ?? operatorAlertLevel({ tier: alert.tier, level: definition.level }),\n        ...(alert.link && alert.link.startsWith('/') ? { link: alert.link } : {}),\n        ...(alert.orgId ? { orgId: alert.orgId } : {}),\n        ...(alert.hostId ? { hostId: alert.hostId } : {}),\n      },\n      // The operator is emailed; a staff member who also switched email on\n      // for this category is not mailed the same alert a second time.\n      emailNow ? { skipEmail: true } : {},\n    )\n    if (!enabled) return { outcome: 'console-only', type: typeId }\n    // Cloud Logging, for the platform's Cloud Monitoring policies (AGL-3683).\n    const { writeOperatorAlertLog } = await import('./operator-alerts-log')\n    await writeOperatorAlertLog(alert)\n    if (delivery === 'digest') {\n      await queueOperatorAlertForDigest(alert)\n      return { outcome: 'queued', type: typeId }\n    }\n    return await deliverOperatorAlert(alert, { emailNow })\n  } catch (error) {\n    console.error(`[operator-alerts] ${typeId} could not be raised`, error)\n    return { outcome: 'undelivered', type: typeId }\n  }\n}\n\n/** Emails and posts one alert now. */\nasync function deliverOperatorAlert(\n  alert: RenderedOperatorAlert,\n  options: { emailNow: boolean },\n): Promise<OperatorAlertResult> {\n  const { sendOperatorAlertEmail } = await import('./staff-alert-email')\n  const [email, webhook] = await Promise.all([\n    options.emailNow\n      ? sendOperatorAlertEmail({\n          title: alert.title,\n          body: alert.body,\n          url: operatorAlertAbsoluteLink(alert.link),\n          context: `operator-alert ${alert.type}`,\n        })\n      : Promise.resolve<SendEmailResult>({ sent: false, reason: 'unconfigured' }),\n    postOperatorAlertWebhook([alert]),\n  ])\n  return {\n    outcome: email.sent || webhook.posted ? 'delivered' : 'undelivered',\n    type: alert.type,\n    email,\n    webhook,\n  }\n}\n\n// ── The digest ──────────────────────────────────────────────────────────────\n\nasync function queueOperatorAlertForDigest(alert: RenderedOperatorAlert): Promise<void> {\n  try {\n    await firestore()\n      .collection(OPERATOR_ALERT_DIGEST_COLLECTION)\n      .doc(createResourceUid())\n      .set({ ...alert, atMs: Date.now() })\n  } catch (error) {\n    // The queue is the only record the digest will have; with it gone, the\n    // alert goes now rather than never.\n    console.error(`[operator-alerts] digest queue unavailable; sending ${alert.type} now`, error)\n    await deliverOperatorAlert(alert, { emailNow: await operatorEmailReachable() })\n  }\n}\n\n/** At most this many alerts are listed in one digest; the rest are counted. */\nexport const OPERATOR_DIGEST_MAX_LISTED = 50\nconst DIGEST_READ_LIMIT = 500\n\nexport interface OperatorDigestResult {\n  /** Whether a digest went out (or, on a dry run, would). */\n  sent: boolean\n  reason?: 'not-due' | 'already-sent' | 'empty' | 'dry-run' | 'unavailable'\n  count: number\n  email?: SendEmailResult\n  webhook?: OperatorAlertWebhookResult\n}\n\nfunction dayKey(now: number): string {\n  return new Date(now).toISOString().slice(0, 10)\n}\n\n/** The state document that claims one UTC day's digest. */\nexport function operatorDigestMarkerId(now: number): string {\n  return `digest-${dayKey(now)}`\n}\n\n/** The digest's body: one paragraph per alert, newest last. */\nexport function composeOperatorDigestBody(\n  alerts: readonly RenderedOperatorAlert[],\n  total = alerts.length,\n): string {\n  const listed = alerts.slice(0, OPERATOR_DIGEST_MAX_LISTED).map((alert) => {\n    const link = operatorAlertAbsoluteLink(alert.link)\n    return [`• ${alert.title}`, alert.body, link].filter(Boolean).join('\\n')\n  })\n  const rest = total - listed.length\n  if (rest > 0) listed.push(`…and ${rest} more in the staff console.`)\n  return listed.join('\\n\\n')\n}\n\n/**\n * Sends the day's digest once the configured hour has passed: every queued\n * alert in one `operator-alert-digest` email and one webhook post, then\n * clears what it sent. Once per UTC day, claimed by creating\n * `operatorAlertState/digest-{day}`, so two ticks never both send it.\n */\nexport async function sendOperatorAlertDigest(\n  options: { now?: number; dryRun?: boolean; force?: boolean } = {},\n): Promise<OperatorDigestResult> {\n  const now = options.now ?? Date.now()\n  try {\n    const settings = await readOperatorAlertSettings()\n    const hour = settings.digestHourUtc ?? OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC\n    if (!options.force && new Date(now).getUTCHours() < hour) {\n      return { sent: false, reason: 'not-due', count: 0 }\n    }\n    const collection = firestore().collection(OPERATOR_ALERT_DIGEST_COLLECTION)\n    const marker = firestore()\n      .collection(OPERATOR_ALERT_STATE_COLLECTION)\n      .doc(operatorDigestMarkerId(now))\n    if (!options.force && (await marker.get()).exists) {\n      return { sent: false, reason: 'already-sent', count: 0 }\n    }\n    const snapshot = await collection.orderBy('atMs', 'asc').limit(DIGEST_READ_LIMIT).get()\n    const queued = snapshot.docs\n    if (!queued.length) return { sent: false, reason: 'empty', count: 0 }\n    if (options.dryRun) return { sent: false, reason: 'dry-run', count: queued.length }\n    if (!options.force) {\n      try {\n        await marker.create({ atMs: now, count: queued.length })\n      } catch {\n        return { sent: false, reason: 'already-sent', count: 0 }\n      }\n    }\n    const alerts = queued.map((doc: any) => doc.data() as RenderedOperatorAlert)\n    const { sendOperatorAlertDigestEmail } = await import('./staff-alert-email')\n    const [email, webhook] = await Promise.all([\n      sendOperatorAlertDigestEmail({\n        date: dayKey(now),\n        count: alerts.length,\n        body: composeOperatorDigestBody(alerts),\n        url: operatorAlertAbsoluteLink('/admin/operator-alerts'),\n      }),\n      postOperatorAlertWebhook(alerts, { digest: true, now }),\n    ])\n    if (email.sent || webhook.posted) {\n      const batch = firestore().batch()\n      for (const doc of queued) batch.delete(doc.ref)\n      await batch.commit()\n    } else if (!options.force) {\n      // Nothing reached anybody: release the day so the next tick retries.\n      await marker.delete().catch(() => undefined)\n    }\n    return { sent: email.sent || webhook.posted, count: alerts.length, email, webhook }\n  } catch (error) {\n    console.error('[operator-alerts] digest failed', error)\n    return { sent: false, reason: 'unavailable', count: 0 }\n  }\n}\n\n\n// ── The staff page ──────────────────────────────────────────────────────────\n\n/** One registry entry as Staff → Operator alerts lists it. */\nexport interface OperatorAlertCatalogRow {\n  type: string\n  label: string\n  description: string\n  tier: OperatorAlertDefinition['tier']\n  category: OperatorAlertDefinition['category']\n  pluginId: string | null\n  defaults: { enabled: boolean; delivery: OperatorAlertDefinition['delivery'] }\n  effective: { enabled: boolean; delivery: OperatorAlertDefinition['delivery'] }\n  /** Whether staff have answered for this type, rather than the default. */\n  overridden: boolean\n}\n\n/** Every registered type with its default and its effective setting. */\nexport function operatorAlertCatalogRows(\n  definitions: readonly OperatorAlertDefinition[],\n  settings: OperatorAlertSettingsDoc,\n): OperatorAlertCatalogRow[] {\n  return definitions.map((definition) => ({\n    type: definition.type,\n    label: definition.label,\n    description: definition.description,\n    tier: definition.tier,\n    category: definition.category,\n    pluginId: definition.pluginId ?? null,\n    defaults: { enabled: definition.defaultEnabled, delivery: definition.delivery },\n    effective: effectiveOperatorAlertSetting(definition, settings),\n    overridden: Boolean(settings.types?.[definition.type]),\n  }))\n}\n\n/**\n * Sends one `[Test]` alert of a type straight to the operator's inbox and\n * webhook, past dedupe and staff's switches, so an operator can prove the\n * channels reach them. Writes no console notification.\n */\nexport async function sendOperatorAlertTest(\n  definition: OperatorAlertDefinition,\n): Promise<{ email: SendEmailResult; webhook: OperatorAlertWebhookResult }> {\n  const sample: Record<string, string> = {}\n  for (const [, name] of `${definition.title} ${definition.body} ${definition.link ?? ''}`.matchAll(\n    /\\{\\{\\s*([\\w.]+)\\s*\\}\\}/g,\n  )) {\n    sample[name as string] = `sample ${name}`\n  }\n  const alert = renderOperatorAlert(definition, { context: sample })\n  alert.title = `[Test] ${alert.title}`\n  alert.body = [\n    alert.body,\n    'This is a test of the operator alert channels; nothing happened.',\n  ].join(' ')\n  const { sendOperatorAlertEmail } = await import('./staff-alert-email')\n  const [email, webhook] = await Promise.all([\n    isEmailConfigured()\n      ? sendOperatorAlertEmail({\n          title: alert.title,\n          body: alert.body,\n          url: operatorAlertAbsoluteLink(alert.link) || operatorAlertAbsoluteLink('/admin/operator-alerts'),\n          context: `operator-alert-test ${definition.type}`,\n        })\n      : Promise.resolve<SendEmailResult>({ sent: false, reason: 'unconfigured' }),\n    postOperatorAlertWebhook([alert]),\n  ])\n  return { email, webhook }\n}\n\n/** Where operator mail goes, named but never listed, for the staff page. */\nexport async function describeOperatorAlertRecipients(): Promise<{\n  source: 'STAFF_ALERT_EMAIL' | 'NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL' | 'staff-accounts' | 'none'\n  count: number\n}> {\n  const { resolveStaffAlertRecipients } = await import('./staff-alert-email')\n  const recipients = await resolveStaffAlertRecipients()\n  if (!recipients.length) return { source: 'none', count: 0 }\n  const configured = String(process.env['STAFF_ALERT_EMAIL'] ?? '').trim().toLowerCase()\n  if (configured.includes('@') && recipients[0] === configured) {\n    return { source: 'STAFF_ALERT_EMAIL', count: 1 }\n  }\n  const support = String(process.env['NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL'] ?? '').trim().toLowerCase()\n  if (support.includes('@') && recipients[0] === support) {\n    return { source: 'NEXT_PUBLIC_OPERATOR_SUPPORT_EMAIL', count: 1 }\n  }\n  return { source: 'staff-accounts', count: recipients.length }\n}\n"],"names":["createHash","createHmac","effectiveOperatorAlertSetting","OPERATOR_ALERT_DEFAULT_NOTIFICATION_TYPE","OPERATOR_ALERT_TIER_LABELS","operatorAlertLevel","renderOperatorAlertTemplate","getOperatorAlert","registerOperatorAlerts","createResourceUid","isEmailConfigured","firebaseAdmin","OPERATOR_ALERT_SETTINGS_COLLECTION","OPERATOR_ALERT_SETTINGS_DOC","OPERATOR_ALERT_STATE_COLLECTION","OPERATOR_ALERT_DIGEST_COLLECTION","OPERATOR_ALERT_DIGEST_DEFAULT_HOUR_UTC","firestore","app","SETTINGS_TTL_MS","settingsCache","invalidateOperatorAlertSettingsCache","undefined","normalizeOperatorAlertSettings","raw","source","types","rawTypes","type","value","Object","entries","entry","setting","enabled","delivery","keys","length","hour","Number","digestHourUtc","isInteger","updatedAtMs","updatedByEmail","readOperatorAlertSettings","now","Date","at","doc","snapshot","collection","get","exists","data","error","console","firedLocally","Map","FIRED_LOCALLY_MAX","stateDocId","dedupeKey","update","digest","slice","claimOperatorAlert","definition","windowMs","Math","max","dedupeWindowMinutes","minOccurrences","floor","fire","repeats","localKey","quietUntil","reason","ref","claim","runTransaction","transaction","state","lastFiredAtMs","base","set","suppressed","merge","windowStartMs","inWindow","occurrences","fired","size","clear","WEBHOOK_TIMEOUT_MS","operatorAlertWebhookUrl","process","String","env","trim","test","operatorAlertWebhookPayload","alerts","options","shaped","origin","consoleOrigin","lines","map","alert","link","operatorAlertAbsoluteLink","tier","title","body","filter","Boolean","join","text","occurredAt","toISOString","category","url","install","postOperatorAlertWebhook","posted","JSON","stringify","headers","secret","timestamp","response","fetch","method","redirect","signal","AbortSignal","timeout","ok","status","replace","startsWith","renderLink","template","context","missing","matchAll","some","name","unregisteredDefinition","label","description","defaultEnabled","resolveOperatorAlertDefinition","known","pluginId","renderOperatorAlert","subject","notificationType","orgId","hostId","operatorEmailReachable","resolveStaffAlertRecipients","raiseOperatorAlert","typeId","outcome","settings","emailNow","notifyStaffConsole","level","skipEmail","writeOperatorAlertLog","queueOperatorAlertForDigest","deliverOperatorAlert","sendOperatorAlertEmail","email","webhook","Promise","all","resolve","sent","atMs","OPERATOR_DIGEST_MAX_LISTED","DIGEST_READ_LIMIT","dayKey","operatorDigestMarkerId","composeOperatorDigestBody","total","listed","rest","push","sendOperatorAlertDigest","force","getUTCHours","count","marker","orderBy","limit","queued","docs","dryRun","create","sendOperatorAlertDigestEmail","date","batch","delete","commit","catch","operatorAlertCatalogRows","definitions","defaults","effective","overridden","sendOperatorAlertTest","sample","describeOperatorAlertRecipients","recipients","configured","toLowerCase","includes","support"],"mappings":";AAAA;;;;;;;;;;;;;;;CAeC,GAED,SAASA,UAAU,EAAEC,UAAU,QAAQ,cAAa;AACpD,SACEC,6BAA6B,EAC7BC,wCAAwC,EACxCC,0BAA0B,EAC1BC,kBAAkB,EAClBC,2BAA2B,QAOtB,sBAAqB;AAC5B,SACEC,gBAAgB,EAChBC,sBAAsB,QACjB,8CAA6C;AACpD,SAASC,iBAAiB,QAAQ,6CAA4C;AAC9E,SAASC,iBAAiB,QAA8B,2BAA0B;AAClF,OAAOC,mBAAmB,sBAAkB;AAE5C;;;;;;;;;;;;;;;;;;;;;;;;CAwBC,GAED,0CAA0C,GAC1C,OAAO,MAAMC,qCAAqC,mBAAkB;AACpE,OAAO,MAAMC,8BAA8B,iBAAgB;AAC3D,6EAA6E,GAC7E,OAAO,MAAMC,kCAAkC,qBAAoB;AACnE,yCAAyC,GACzC,OAAO,MAAMC,mCAAmC,sBAAqB;AAErE,uEAAuE,GACvE,OAAO,MAAMC,yCAAyC,GAAE;AA0ExD,MAAMC,YAAY,IAAMN,cAAcO,GAAG,GAAGD,SAAS;AAErD,+EAA+E;AAE/E,MAAME,kBAAkB;AACxB,IAAIC;AAEJ,4EAA4E,GAC5E,OAAO,SAASC;IACdD,gBAAgBE;AAClB;AAEA,wEAAwE,GACxE,OAAO,SAASC,+BACdC,GAAY;IAEZ,MAAMC,SAAUD,OAAO,OAAOA,QAAQ,WAAWA,MAAM,CAAC;IACxD,MAAME,QAAkD,CAAC;IACzD,MAAMC,WAAWF,MAAM,CAAC,QAAQ;IAChC,IAAIE,YAAY,OAAOA,aAAa,UAAU;QAC5C,KAAK,MAAM,CAACC,MAAMC,MAAM,IAAIC,OAAOC,OAAO,CAACJ,UAAsC;YAC/E,IAAI,CAACC,QAAQ,CAACC,SAAS,OAAOA,UAAU,UAAU;YAClD,MAAMG,QAAQH;YACd,MAAMI,UAAoC,CAAC;YAC3C,IAAI,OAAOD,KAAK,CAAC,UAAU,KAAK,WAAWC,QAAQC,OAAO,GAAGF,KAAK,CAAC,UAAU;YAC7E,IAAIA,KAAK,CAAC,WAAW,KAAK,eAAeA,KAAK,CAAC,WAAW,KAAK,UAAU;gBACvEC,QAAQE,QAAQ,GAAGH,KAAK,CAAC,WAAW;YACtC;YACA,IAAIF,OAAOM,IAAI,CAACH,SAASI,MAAM,EAAEX,KAAK,CAACE,KAAK,GAAGK;QACjD;IACF;IACA,MAAMK,OAAOC,OAAOd,MAAM,CAAC,gBAAgB;IAC3C,OAAO;QACLC;QACAc,eACED,OAAOE,SAAS,CAACH,SAASA,QAAQ,KAAKA,QAAQ,KAC3CA,OACAtB;OACF,OAAOS,MAAM,CAAC,cAAc,KAAK,WACjC;QAAEiB,aAAajB,MAAM,CAAC,cAAc;IAAW,IAC/C,CAAC,GACD,OAAOA,MAAM,CAAC,iBAAiB,KAAK,WACpC;QAAEkB,gBAAgBlB,MAAM,CAAC,iBAAiB;IAAW,IACrD,CAAC;AAET;AAEA;;;CAGC,GACD,OAAO,eAAemB;IACpB,MAAMC,MAAMC,KAAKD,GAAG;IACpB,IAAIzB,iBAAiByB,MAAMzB,cAAc2B,EAAE,GAAG5B,iBAAiB,OAAOC,cAAc4B,GAAG;IACvF,IAAI;QACF,MAAMC,WAAW,MAAMhC,YACpBiC,UAAU,CAACtC,oCACXoC,GAAG,CAACnC,6BACJsC,GAAG;QACN,MAAMH,MAAMzB,+BAA+B0B,SAASG,MAAM,GAAGH,SAASI,IAAI,KAAK;QAC/EjC,gBAAgB;YAAE2B,IAAIF;YAAKG;QAAI;QAC/B,OAAOA;IACT,EAAE,OAAOM,OAAO;QACdC,QAAQD,KAAK,CAAC,6DAA6DA;QAC3E,OAAO/B,+BAA+B;IACxC;AACF;AAEA,+EAA+E;AAE/E,8DAA8D,GAC9D,MAAMiC,eAAe,IAAIC;AACzB,MAAMC,oBAAoB;AAE1B,SAASC,WAAW/B,IAAY,EAAEgC,SAAiB;IACjD,OAAO5D,WAAW,UAAU6D,MAAM,CAAC,GAAGjC,KAAK,MAAM,EAAEgC,WAAW,EAAEE,MAAM,CAAC,OAAOC,KAAK,CAAC,GAAG;AACzF;AAMA;;;;;CAKC,GACD,OAAO,eAAeC,mBACpBC,UAA4F,EAC5FL,SAAiB,EACjBf,MAAMC,KAAKD,GAAG,EAAE;QAG8BoB;IAD9C,MAAMC,WAAWC,KAAKC,GAAG,CAAC,GAAGH,WAAWI,mBAAmB,IAAI;IAC/D,MAAMC,iBAAiBH,KAAKC,GAAG,CAAC,GAAGD,KAAKI,KAAK,EAACN,6BAAAA,WAAWK,cAAc,YAAzBL,6BAA6B;IAC3E,IAAI,CAACC,UAAU,OAAO;QAAEM,MAAM;QAAMC,SAAS;IAAE;IAC/C,MAAMC,WAAW,GAAGT,WAAWrC,IAAI,CAAC,MAAM,EAAEgC,WAAW;IACvD,MAAMe,aAAanB,aAAaL,GAAG,CAACuB;IACpC,IAAIC,cAAcA,aAAa9B,KAAK,OAAO;QAAE2B,MAAM;QAAOI,QAAQ;IAAU;IAC5E,IAAI;QACF,MAAMC,MAAM5D,YACTiC,UAAU,CAACpC,iCACXkC,GAAG,CAACW,WAAWM,WAAWrC,IAAI,EAAEgC;QACnC,MAAMkB,QAAQ,MAAM7D,YAAY8D,cAAc,CAAC,OAAOC;gBAGvBC,sBAMAA,sBAEUA,oBAiBrBA,cAImBA;YA/BrC,MAAMhC,WAAW,MAAM+B,YAAY7B,GAAG,CAAC0B;YACvC,MAAMI,QAAShC,SAASG,MAAM,GAAGH,SAASI,IAAI,KAAK,CAAC;YACpD,MAAM6B,gBAAgB3C,QAAO0C,uBAAAA,KAAK,CAAC,gBAAgB,YAAtBA,uBAA0B;YACvD,MAAME,OAAO;gBAAEvD,MAAMqC,WAAWrC,IAAI;gBAAEgC,WAAWA,UAAUG,KAAK,CAAC,GAAG;gBAAMrB,aAAaG;YAAI;YAC3F,IAAIqC,iBAAiBrC,MAAMqC,gBAAgBhB,UAAU;oBACAe;gBAAnDD,YAAYI,GAAG,CAACP,KAAK,aAAKM;oBAAME,YAAY9C,QAAO0C,qBAAAA,KAAK,CAAC,aAAa,YAAnBA,qBAAuB,KAAK;oBAAK;oBAAEK,OAAO;gBAAK;gBAClG,OAAO;oBAAEd,MAAM;oBAAOI,QAAQ;gBAAU;YAC1C;YACA,MAAMW,gBAAgBhD,QAAO0C,uBAAAA,KAAK,CAAC,gBAAgB,YAAtBA,uBAA0B;YACvD,MAAMO,WAAWD,iBAAiB1C,MAAM0C,gBAAgBrB;YACxD,MAAMuB,cAAc,AAACD,CAAAA,WAAWjD,QAAO0C,qBAAAA,KAAK,CAAC,cAAc,YAApBA,qBAAwB,KAAK,CAAA,IAAK;YACzE,IAAIQ,cAAcnB,gBAAgB;gBAChCU,YAAYI,GAAG,CACbP,KACA,aAAKM;oBAAMM;oBAAaF,eAAeC,WAAWD,gBAAgB1C;oBAClE;oBAAEyC,OAAO;gBAAK;gBAEhB,OAAO;oBAAEd,MAAM;oBAAOI,QAAQ;gBAAkB;YAClD;YACAI,YAAYI,GAAG,CACbP,KACA,aACKM;gBACHD,eAAerC;gBACf4C,aAAa;gBACbF,eAAe;gBACfF,YAAY;gBACZK,OAAOnD,QAAO0C,eAAAA,KAAK,CAAC,QAAQ,YAAdA,eAAkB,KAAK;gBAEvC;gBAAEK,OAAO;YAAK;YAEhB,OAAO;gBAAEd,MAAM;gBAAMC,SAASlC,QAAO0C,oBAAAA,KAAK,CAAC,aAAa,YAAnBA,oBAAuB;YAAG;QACjE;QACA,IAAIH,MAAMN,IAAI,EAAE;YACd,IAAIhB,aAAamC,IAAI,IAAIjC,mBAAmBF,aAAaoC,KAAK;YAC9DpC,aAAa4B,GAAG,CAACV,UAAU7B,MAAMqB;QACnC;QACA,OAAOY;IACT,EAAE,OAAOxB,OAAO;QACdC,QAAQD,KAAK,CAAC,CAAC,yCAAyC,EAAEW,WAAWrC,IAAI,CAAC,QAAQ,CAAC,EAAE0B;QACrF,OAAO;YAAEkB,MAAM;YAAMC,SAAS;QAAE;IAClC;AACF;AAEA,+EAA+E;AAE/E,MAAMoB,qBAAqB;AAE3B,4DAA4D,GAC5D,OAAO,SAASC;QACOC;IAArB,MAAMlE,QAAQmE,QAAOD,0CAAAA,QAAQE,GAAG,CAAC,6BAA6B,YAAzCF,0CAA6C,IAAIG,IAAI;IAC1E,OAAO,oBAAoBC,IAAI,CAACtE,SAASA,QAAQ;AACnD;AAEA;;;;CAIC,GACD,OAAO,SAASuE,4BACdC,MAAwC,EACxCC,UAA8C,CAAC,CAAC;QAgBpBA,cAWyBC;IAzBrD,MAAMC,SAASC;IACf,MAAMC,QAAQL,OAAOM,GAAG,CAAC,CAACC;QACxB,MAAMC,OAAOC,0BAA0BF,MAAMC,IAAI;QACjD,OAAO;YACL,CAAC,CAAC,EAAEzG,0BAA0B,CAACwG,MAAMG,IAAI,CAAC,CAAC,EAAE,EAAEH,MAAMI,KAAK,EAAE;YAC5DJ,MAAMK,IAAI;YACVJ;SACD,CACEK,MAAM,CAACC,SACPC,IAAI,CAAC;IACV;IACA,MAAMC,OAAOf,QAAQxC,MAAM,GACvB;QAAC,CAAC,iBAAiB,EAAEuC,OAAOhE,MAAM,CAAC,MAAM,EAAEgE,OAAOhE,MAAM,KAAK,IAAI,KAAK,KAAK;WAAKqE;KAAM,CAACU,IAAI,CAAC,UAC5FV,MAAMU,IAAI,CAAC;IACf,MAAME,aAAa,IAAIxE,MAAKwD,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG,IAAI0E,WAAW;IAClE,MAAMhB,SAASF,OAAOM,GAAG,CAAC,CAACC,QAAW,CAAA;YACpChF,MAAMgF,MAAMhF,IAAI;YAChBmF,MAAMH,MAAMG,IAAI;YAChBS,UAAUZ,MAAMY,QAAQ;YACxBR,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI,KAAK;QAChD,CAAA;IACA,OAAO;QACLQ;OACIf,QAAQxC,MAAM,GAAG;QAAEA,QAAQyC;IAAO,IAAI;QAAEK,KAAK,GAAEL,WAAAA,MAAM,CAAC,EAAE,YAATA,WAAa;IAAK;QACrEe;OACId,SAAS;QAAEkB,SAASlB;IAAO,IAAI,CAAC;AAExC;AAEA;;;;;CAKC,GACD,OAAO,eAAemB,yBACpBtB,MAAwC,EACxCC,UAA8C,CAAC,CAAC;QAM1BP;IAJtB,MAAM0B,MAAM3B;IACZ,IAAI,CAAC2B,OAAO,CAACpB,OAAOhE,MAAM,EAAE,OAAO;QAAEuF,QAAQ;QAAOhD,QAAQ;IAAe;IAC3E,MAAMqC,OAAOY,KAAKC,SAAS,CAAC1B,4BAA4BC,QAAQC;IAChE,MAAMyB,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,SAAShC,QAAOD,6CAAAA,QAAQE,GAAG,CAAC,gCAAgC,YAA5CF,6CAAgD,IAAIG,IAAI;IAC9E,IAAI8B,QAAQ;YAC2B1B;QAArC,MAAM2B,YAAYjC,OAAO7B,KAAKI,KAAK,CAAC,EAAC+B,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG,MAAM;QAClEkF,OAAO,CAAC,6BAA6B,GAAGE;QACxCF,OAAO,CAAC,6BAA6B,GACnC,YAAY9H,WAAW,UAAU+H,QAAQnE,MAAM,CAAC,GAAGoE,UAAU,CAAC,EAAEhB,MAAM,EAAEnD,MAAM,CAAC;IACnF;IACA,IAAI;QACF,MAAMoE,WAAW,MAAMC,MAAMV,KAAK;YAChCW,QAAQ;YACRL;YACAd;YACAoB,UAAU;YACVC,QAAQC,YAAYC,OAAO,CAAC3C;QAC9B;QACA,IAAIqC,SAASO,EAAE,EAAE,OAAO;YAAEb,QAAQ;YAAMc,QAAQR,SAASQ,MAAM;QAAC;QAChEnF,QAAQD,KAAK,CAAC,CAAC,mCAAmC,EAAE4E,SAASQ,MAAM,EAAE;QACrE,OAAO;YAAEd,QAAQ;YAAOhD,QAAQ;YAAW8D,QAAQR,SAASQ,MAAM;QAAC;IACrE,EAAE,OAAOpF,OAAO;QACdC,QAAQD,KAAK,CAAC,yCAAyCA;QACvD,OAAO;YAAEsE,QAAQ;YAAOhD,QAAQ;QAAU;IAC5C;AACF;AAEA,+EAA+E;AAE/E,SAAS6B;QACOV;IAAd,OAAOC,QAAOD,uCAAAA,QAAQE,GAAG,CAAC,0BAA0B,YAAtCF,uCAA0C,IAAIG,IAAI,GAAGyC,OAAO,CAAC,QAAQ;AACrF;AAEA,kEAAkE,GAClE,OAAO,SAAS7B,0BAA0BD,IAA+B;IACvE,MAAMhF,QAAQmE,OAAOa,eAAAA,OAAQ,IAAIX,IAAI;IACrC,IAAI,CAACrE,OAAO,OAAO;IACnB,IAAI,gBAAgBsE,IAAI,CAACtE,QAAQ,OAAOA;IACxC,MAAM2E,SAASC;IACf,OAAOD,UAAU3E,MAAM+G,UAAU,CAAC,OAAO,GAAGpC,SAAS3E,OAAO,GAAG;AACjE;AAEA;;;CAGC,GACD,SAASgH,WAAWC,QAA4B,EAAEC,OAAyC;IACzF,IAAI,CAACD,UAAU,OAAO;IACtB,MAAME,UAAU;WAAIF,SAASG,QAAQ,CAAC;KAA2B,CAACC,IAAI,CAAC,CAAC,GAAGC,KAAK;QAC9E,MAAMtH,QAAQkH,2BAAAA,OAAS,CAACI,KAAe;QACvC,OAAOtH,UAAU,QAAQA,UAAUP,aAAaO,UAAU;IAC5D;IACA,OAAOmH,UAAU,KAAK1I,4BAA4BwI,UAAUC;AAC9D;AAEA,+EAA+E;AAE/E,iEAAiE,GACjE,SAASK,uBAAuBxH,IAAY;IAC1C,OAAO;QACLA;QACAyH,OAAOzH;QACP0H,aAAa;QACbvC,MAAM;QACNS,UAAU;QACVR,OAAO;QACPC,MAAM;QACN9E,UAAU;QACVkC,qBAAqB;QACrBkF,gBAAgB;IAClB;AACF;AAEA,sEAAsE,GACtE,OAAO,SAASC,+BACd5H,IAAsC;IAEtC,IAAI,OAAOA,SAAS,UAAU;QAC5B,MAAM6H,QAAQlJ,iBAAiBqB,KAAKA,IAAI;QACxC,IAAI6H,OAAO,OAAOA;QAClB,IAAI7H,KAAK8H,QAAQ,EAAE;YACjB,IAAI;gBACFlJ,uBAAuB;oBAACoB;iBAAK,EAAE;oBAAE8H,UAAU9H,KAAK8H,QAAQ;gBAAC;YAC3D,EAAE,OAAOpG,OAAO;gBACdC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAE1B,KAAKA,IAAI,CAAC,wBAAwB,CAAC,EAAE0B;YAC1E;QACF;QACA,OAAO1B;IACT;IACA,MAAM6H,QAAQlJ,iBAAiBqB;IAC/B,IAAI6H,OAAO,OAAOA;IAClBlG,QAAQD,KAAK,CAAC,CAAC,mBAAmB,EAAE1B,KAAK,mDAAmD,CAAC;IAC7F,OAAOwH,uBAAuBxH;AAChC;AAEA,OAAO,SAAS+H,oBACd1F,UAAmC,EACnCqC,UAAqC,CAAC,CAAC;QAKjCA,kBAGGA,kBAIAA,eAIDA,eAIOA,cACKrC;IAnBpB,MAAM8E,UAAgC,aAChCzC,QAAQsD,OAAO,GAAG;QAAE5C,OAAOV,QAAQsD,OAAO;IAAC,IAAI,CAAC,GAChDtD,QAAQW,IAAI,GAAG;QAAEA,MAAMX,QAAQW,IAAI;IAAC,IAAI,CAAC,IACzCX,mBAAAA,QAAQyC,OAAO,YAAfzC,mBAAmB,CAAC;IAE1B,MAAMU,QACJhB,QAAOM,mBAAAA,QAAQsD,OAAO,YAAftD,mBAAmB,IAAIJ,IAAI,MAClC5F,4BAA4B2D,WAAW+C,KAAK,EAAE+B,YAC9C9E,WAAWoF,KAAK;IAClB,MAAMpC,OACJjB,QAAOM,gBAAAA,QAAQW,IAAI,YAAZX,gBAAgB,IAAIJ,IAAI,MAAM5F,4BAA4B2D,WAAWgD,IAAI,EAAE8B;IACpF,OAAO;QACLnH,MAAMqC,WAAWrC,IAAI;QACrByH,OAAOpF,WAAWoF,KAAK;QACvBtC,IAAI,GAAET,gBAAAA,QAAQS,IAAI,YAAZT,gBAAgBrC,WAAW8C,IAAI;QACrCS,UAAUvD,WAAWuD,QAAQ;QAC7BR;QACAC;QACAJ,MAAMb,QAAOM,eAAAA,QAAQmB,GAAG,YAAXnB,eAAe,IAAIJ,IAAI,MAAM2C,WAAW5E,WAAW4C,IAAI,EAAEkC;QACtEc,gBAAgB,GAAE5F,+BAAAA,WAAW4F,gBAAgB,YAA3B5F,+BAA+B9D;OAC7CmG,QAAQwD,KAAK,GAAG;QAAEA,OAAOxD,QAAQwD,KAAK;IAAC,IAAI,CAAC,GAC5CxD,QAAQyD,MAAM,GAAG;QAAEA,QAAQzD,QAAQyD,MAAM;IAAC,IAAI,CAAC;AAEvD;AAEA,+EAA+E;AAE/E,oEAAoE,GACpE,eAAeC;IACb,IAAI,CAACtJ,qBAAqB,OAAO;IACjC,MAAM,EAAEuJ,2BAA2B,EAAE,GAAG,MAAM,MAAM,CAAC;IACrD,OAAO,AAAC,CAAA,MAAMA,6BAA4B,EAAG5H,MAAM,GAAG;AACxD;AAEA,0CAA0C,GAC1C,OAAO,eAAe6H,mBACpBtI,IAAsC,EACtC0E,UAAqC,CAAC,CAAC;IAEvC,MAAM6D,SAAS,OAAOvI,SAAS,WAAWA,OAAOA,KAAKA,IAAI;IAC1D,IAAI;YAEuB0E,oBAuBdA;QAxBX,MAAMrC,aAAauF,+BAA+B5H;QAClD,MAAMgC,YAAYoC,QAAOM,qBAAAA,QAAQ1C,SAAS,YAAjB0C,qBAAqB,IAAIJ,IAAI;QACtD,IAAIzB,UAAU;QACd,IAAIb,WAAW;YACb,MAAMkB,QAAQ,MAAMd,mBAAmBC,YAAYL;YACnD,IAAIkB,MAAMN,IAAI,KAAK,OAAO,OAAO;gBAAE4F,SAAStF,MAAMF,MAAM;gBAAEhD,MAAMuI;YAAO;YACvE1F,UAAUK,MAAML,OAAO;QACzB;QACA,MAAMmC,QAAQ+C,oBAAoB1F,YAAYqC;QAC9C,IAAI7B,UAAU,GAAG;YACfmC,MAAMK,IAAI,GAAG;gBAACL,MAAMK,IAAI;gBAAE,CAAC,OAAO,EAAExC,QAAQ,UAAU,EAAEA,YAAY,IAAI,KAAK,IAAI,wBAAwB,CAAC;aAAC,CACxGyC,MAAM,CAACC,SACPC,IAAI,CAAC;QACV;QACA,MAAMiD,WAAW,MAAMzH;QACvB,MAAM,EAAEV,OAAO,EAAEC,QAAQ,EAAE,GAAGjC,8BAA8B+D,YAAYoG;QACxE,MAAMC,WAAWpI,WAAWC,aAAa,eAAgB,MAAM6H;QAE/D,MAAM,EAAEO,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;QAC5C,MAAMA,mBACJ;YACE3I,MAAMgF,MAAMiD,gBAAgB;YAC5B7C,OAAOJ,MAAMI,KAAK;WACdJ,MAAMK,IAAI,GAAG;YAAEA,MAAML,MAAMK,IAAI;QAAC,IAAI,CAAC;YACzCuD,KAAK,GAAElE,iBAAAA,QAAQkE,KAAK,YAAblE,iBAAiBjG,mBAAmB;gBAAE0G,MAAMH,MAAMG,IAAI;gBAAEyD,OAAOvG,WAAWuG,KAAK;YAAC;WACnF5D,MAAMC,IAAI,IAAID,MAAMC,IAAI,CAAC+B,UAAU,CAAC,OAAO;YAAE/B,MAAMD,MAAMC,IAAI;QAAC,IAAI,CAAC,GACnED,MAAMkD,KAAK,GAAG;YAAEA,OAAOlD,MAAMkD,KAAK;QAAC,IAAI,CAAC,GACxClD,MAAMmD,MAAM,GAAG;YAAEA,QAAQnD,MAAMmD,MAAM;QAAC,IAAI,CAAC,IAEjD,qEAAqE;QACrE,gEAAgE;QAChEO,WAAW;YAAEG,WAAW;QAAK,IAAI,CAAC;QAEpC,IAAI,CAACvI,SAAS,OAAO;YAAEkI,SAAS;YAAgBxI,MAAMuI;QAAO;QAC7D,0EAA0E;QAC1E,MAAM,EAAEO,qBAAqB,EAAE,GAAG,MAAM,MAAM,CAAC;QAC/C,MAAMA,sBAAsB9D;QAC5B,IAAIzE,aAAa,UAAU;YACzB,MAAMwI,4BAA4B/D;YAClC,OAAO;gBAAEwD,SAAS;gBAAUxI,MAAMuI;YAAO;QAC3C;QACA,OAAO,MAAMS,qBAAqBhE,OAAO;YAAE0D;QAAS;IACtD,EAAE,OAAOhH,OAAO;QACdC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAE6G,OAAO,oBAAoB,CAAC,EAAE7G;QACjE,OAAO;YAAE8G,SAAS;YAAexI,MAAMuI;QAAO;IAChD;AACF;AAEA,oCAAoC,GACpC,eAAeS,qBACbhE,KAA4B,EAC5BN,OAA8B;IAE9B,MAAM,EAAEuE,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,MAAM,CAACC,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACzC3E,QAAQgE,QAAQ,GACZO,uBAAuB;YACrB7D,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI;YACzCkC,SAAS,CAAC,eAAe,EAAEnC,MAAMhF,IAAI,EAAE;QACzC,KACAoJ,QAAQE,OAAO,CAAkB;YAAEC,MAAM;YAAOvG,QAAQ;QAAe;QAC3E+C,yBAAyB;YAACf;SAAM;KACjC;IACD,OAAO;QACLwD,SAASU,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM,GAAG,cAAc;QACtDhG,MAAMgF,MAAMhF,IAAI;QAChBkJ;QACAC;IACF;AACF;AAEA,+EAA+E;AAE/E,eAAeJ,4BAA4B/D,KAA4B;IACrE,IAAI;QACF,MAAM3F,YACHiC,UAAU,CAACnC,kCACXiC,GAAG,CAACvC,qBACJ2E,GAAG,CAAC,aAAKwB;YAAOwE,MAAMtI,KAAKD,GAAG;;IACnC,EAAE,OAAOS,OAAO;QACd,uEAAuE;QACvE,oCAAoC;QACpCC,QAAQD,KAAK,CAAC,CAAC,oDAAoD,EAAEsD,MAAMhF,IAAI,CAAC,IAAI,CAAC,EAAE0B;QACvF,MAAMsH,qBAAqBhE,OAAO;YAAE0D,UAAU,MAAMN;QAAyB;IAC/E;AACF;AAEA,6EAA6E,GAC7E,OAAO,MAAMqB,6BAA6B,GAAE;AAC5C,MAAMC,oBAAoB;AAW1B,SAASC,OAAO1I,GAAW;IACzB,OAAO,IAAIC,KAAKD,KAAK0E,WAAW,GAAGxD,KAAK,CAAC,GAAG;AAC9C;AAEA,yDAAyD,GACzD,OAAO,SAASyH,uBAAuB3I,GAAW;IAChD,OAAO,CAAC,OAAO,EAAE0I,OAAO1I,MAAM;AAChC;AAEA,6DAA6D,GAC7D,OAAO,SAAS4I,0BACdpF,MAAwC,EACxCqF,QAAQrF,OAAOhE,MAAM;IAErB,MAAMsJ,SAAStF,OAAOtC,KAAK,CAAC,GAAGsH,4BAA4B1E,GAAG,CAAC,CAACC;QAC9D,MAAMC,OAAOC,0BAA0BF,MAAMC,IAAI;QACjD,OAAO;YAAC,CAAC,EAAE,EAAED,MAAMI,KAAK,EAAE;YAAEJ,MAAMK,IAAI;YAAEJ;SAAK,CAACK,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE;IACA,MAAMwE,OAAOF,QAAQC,OAAOtJ,MAAM;IAClC,IAAIuJ,OAAO,GAAGD,OAAOE,IAAI,CAAC,CAAC,KAAK,EAAED,KAAK,2BAA2B,CAAC;IACnE,OAAOD,OAAOvE,IAAI,CAAC;AACrB;AAEA;;;;;CAKC,GACD,OAAO,eAAe0E,wBACpBxF,UAA+D,CAAC,CAAC;QAErDA;IAAZ,MAAMzD,OAAMyD,eAAAA,QAAQzD,GAAG,YAAXyD,eAAexD,KAAKD,GAAG;IACnC,IAAI;YAEWwH;QADb,MAAMA,WAAW,MAAMzH;QACvB,MAAMN,QAAO+H,0BAAAA,SAAS7H,aAAa,YAAtB6H,0BAA0BrJ;QACvC,IAAI,CAACsF,QAAQyF,KAAK,IAAI,IAAIjJ,KAAKD,KAAKmJ,WAAW,KAAK1J,MAAM;YACxD,OAAO;gBAAE6I,MAAM;gBAAOvG,QAAQ;gBAAWqH,OAAO;YAAE;QACpD;QACA,MAAM/I,aAAajC,YAAYiC,UAAU,CAACnC;QAC1C,MAAMmL,SAASjL,YACZiC,UAAU,CAACpC,iCACXkC,GAAG,CAACwI,uBAAuB3I;QAC9B,IAAI,CAACyD,QAAQyF,KAAK,IAAI,AAAC,CAAA,MAAMG,OAAO/I,GAAG,EAAC,EAAGC,MAAM,EAAE;YACjD,OAAO;gBAAE+H,MAAM;gBAAOvG,QAAQ;gBAAgBqH,OAAO;YAAE;QACzD;QACA,MAAMhJ,WAAW,MAAMC,WAAWiJ,OAAO,CAAC,QAAQ,OAAOC,KAAK,CAACd,mBAAmBnI,GAAG;QACrF,MAAMkJ,SAASpJ,SAASqJ,IAAI;QAC5B,IAAI,CAACD,OAAOhK,MAAM,EAAE,OAAO;YAAE8I,MAAM;YAAOvG,QAAQ;YAASqH,OAAO;QAAE;QACpE,IAAI3F,QAAQiG,MAAM,EAAE,OAAO;YAAEpB,MAAM;YAAOvG,QAAQ;YAAWqH,OAAOI,OAAOhK,MAAM;QAAC;QAClF,IAAI,CAACiE,QAAQyF,KAAK,EAAE;YAClB,IAAI;gBACF,MAAMG,OAAOM,MAAM,CAAC;oBAAEpB,MAAMvI;oBAAKoJ,OAAOI,OAAOhK,MAAM;gBAAC;YACxD,EAAE,eAAM;gBACN,OAAO;oBAAE8I,MAAM;oBAAOvG,QAAQ;oBAAgBqH,OAAO;gBAAE;YACzD;QACF;QACA,MAAM5F,SAASgG,OAAO1F,GAAG,CAAC,CAAC3D,MAAaA,IAAIK,IAAI;QAChD,MAAM,EAAEoJ,4BAA4B,EAAE,GAAG,MAAM,MAAM,CAAC;QACtD,MAAM,CAAC3B,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;YACzCwB,6BAA6B;gBAC3BC,MAAMnB,OAAO1I;gBACboJ,OAAO5F,OAAOhE,MAAM;gBACpB4E,MAAMwE,0BAA0BpF;gBAChCoB,KAAKX,0BAA0B;YACjC;YACAa,yBAAyBtB,QAAQ;gBAAEvC,QAAQ;gBAAMjB;YAAI;SACtD;QACD,IAAIiI,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM,EAAE;YAChC,MAAM+E,QAAQ1L,YAAY0L,KAAK;YAC/B,KAAK,MAAM3J,OAAOqJ,OAAQM,MAAMC,MAAM,CAAC5J,IAAI6B,GAAG;YAC9C,MAAM8H,MAAME,MAAM;QACpB,OAAO,IAAI,CAACvG,QAAQyF,KAAK,EAAE;YACzB,qEAAqE;YACrE,MAAMG,OAAOU,MAAM,GAAGE,KAAK,CAAC,IAAMxL;QACpC;QACA,OAAO;YAAE6J,MAAML,MAAMK,IAAI,IAAIJ,QAAQnD,MAAM;YAAEqE,OAAO5F,OAAOhE,MAAM;YAAEyI;YAAOC;QAAQ;IACpF,EAAE,OAAOzH,OAAO;QACdC,QAAQD,KAAK,CAAC,mCAAmCA;QACjD,OAAO;YAAE6H,MAAM;YAAOvG,QAAQ;YAAeqH,OAAO;QAAE;IACxD;AACF;AAmBA,sEAAsE,GACtE,OAAO,SAASc,yBACdC,WAA+C,EAC/C3C,QAAkC;IAElC,OAAO2C,YAAYrG,GAAG,CAAC,CAAC1C;YAMZA;YAGUoG;eATkB;YACtCzI,MAAMqC,WAAWrC,IAAI;YACrByH,OAAOpF,WAAWoF,KAAK;YACvBC,aAAarF,WAAWqF,WAAW;YACnCvC,MAAM9C,WAAW8C,IAAI;YACrBS,UAAUvD,WAAWuD,QAAQ;YAC7BkC,QAAQ,GAAEzF,uBAAAA,WAAWyF,QAAQ,YAAnBzF,uBAAuB;YACjCgJ,UAAU;gBAAE/K,SAAS+B,WAAWsF,cAAc;gBAAEpH,UAAU8B,WAAW9B,QAAQ;YAAC;YAC9E+K,WAAWhN,8BAA8B+D,YAAYoG;YACrD8C,YAAYhG,SAAQkD,kBAAAA,SAAS3I,KAAK,qBAAd2I,eAAgB,CAACpG,WAAWrC,IAAI,CAAC;QACvD;;AACF;AAEA;;;;CAIC,GACD,OAAO,eAAewL,sBACpBnJ,UAAmC;QAG8BA;IADjE,MAAMoJ,SAAiC,CAAC;IACxC,KAAK,MAAM,GAAGlE,KAAK,IAAI,GAAGlF,WAAW+C,KAAK,CAAC,CAAC,EAAE/C,WAAWgD,IAAI,CAAC,CAAC,GAAEhD,mBAAAA,WAAW4C,IAAI,YAAf5C,mBAAmB,IAAI,CAACgF,QAAQ,CAC/F,2BACC;QACDoE,MAAM,CAAClE,KAAe,GAAG,CAAC,OAAO,EAAEA,MAAM;IAC3C;IACA,MAAMvC,QAAQ+C,oBAAoB1F,YAAY;QAAE8E,SAASsE;IAAO;IAChEzG,MAAMI,KAAK,GAAG,CAAC,OAAO,EAAEJ,MAAMI,KAAK,EAAE;IACrCJ,MAAMK,IAAI,GAAG;QACXL,MAAMK,IAAI;QACV;KACD,CAACG,IAAI,CAAC;IACP,MAAM,EAAEyD,sBAAsB,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,MAAM,CAACC,OAAOC,QAAQ,GAAG,MAAMC,QAAQC,GAAG,CAAC;QACzCvK,sBACImK,uBAAuB;YACrB7D,OAAOJ,MAAMI,KAAK;YAClBC,MAAML,MAAMK,IAAI;YAChBQ,KAAKX,0BAA0BF,MAAMC,IAAI,KAAKC,0BAA0B;YACxEiC,SAAS,CAAC,oBAAoB,EAAE9E,WAAWrC,IAAI,EAAE;QACnD,KACAoJ,QAAQE,OAAO,CAAkB;YAAEC,MAAM;YAAOvG,QAAQ;QAAe;QAC3E+C,yBAAyB;YAACf;SAAM;KACjC;IACD,OAAO;QAAEkE;QAAOC;IAAQ;AAC1B;AAEA,0EAA0E,GAC1E,OAAO,eAAeuC;QAOMvH,gCAIHA;IAPvB,MAAM,EAAEkE,2BAA2B,EAAE,GAAG,MAAM,MAAM,CAAC;IACrD,MAAMsD,aAAa,MAAMtD;IACzB,IAAI,CAACsD,WAAWlL,MAAM,EAAE,OAAO;QAAEZ,QAAQ;QAAQwK,OAAO;IAAE;IAC1D,MAAMuB,aAAaxH,QAAOD,iCAAAA,QAAQE,GAAG,CAAC,oBAAoB,YAAhCF,iCAAoC,IAAIG,IAAI,GAAGuH,WAAW;IACpF,IAAID,WAAWE,QAAQ,CAAC,QAAQH,UAAU,CAAC,EAAE,KAAKC,YAAY;QAC5D,OAAO;YAAE/L,QAAQ;YAAqBwK,OAAO;QAAE;IACjD;IACA,MAAM0B,UAAU3H,QAAOD,kDAAAA,QAAQE,GAAG,CAAC,qCAAqC,YAAjDF,kDAAqD,IAAIG,IAAI,GAAGuH,WAAW;IAClG,IAAIE,QAAQD,QAAQ,CAAC,QAAQH,UAAU,CAAC,EAAE,KAAKI,SAAS;QACtD,OAAO;YAAElM,QAAQ;YAAsCwK,OAAO;QAAE;IAClE;IACA,OAAO;QAAExK,QAAQ;QAAkBwK,OAAOsB,WAAWlL,MAAM;IAAC;AAC9D"}

package/src/lib/server/operator-health.d.ts

@@ -14,6 +14,36 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+/**
+ * HEALTH TRANSITIONS (AGL-3377): the health checks, remembered.
+ *
+ * Every `/api/health/*` endpoint answers 200 or 503 to whoever asks and
+ * forgets the answer. An install with an external uptime monitor hears the
+ * 503; an install without one — most self-hosted ones — hears nothing, and
+ * a dead cron or a stale backup is found on the day it is needed.
+ *
+ * `recordHealthState` keeps the last verdict per check in
+ * `operatorHealthState/{checkId}` and raises an operator alert on the EDGE:
+ * `system.healthDegraded` when a healthy (or never-seen) check goes
+ * degraded, `system.healthRecovered` when it comes back. Once each, however
+ * many instances and pollers see the same state, because the edge is decided
+ * inside a transaction on the check's document.
+ *
+ * Cheap on the hot path: each instance remembers the last state it wrote, so
+ * a poll that finds nothing changed costs no Firestore round trip at all. An
+ * instance's FIRST poll of a check reads the document outside a transaction
+ * and writes nothing when the stored status already agrees: serverless
+ * instances come and go all day, and rewriting `detail` on every cold start
+ * was a write per check per instance that nothing reads.
+ * The Admin SDK is loaded on the first write rather than at import, so every
+ * health route can import this without initializing anything. Never throws.
+ *
+ * The edge is told once, and a check that STAYS degraded must not be
+ * forgotten after it (AGL-3860). `escalateStillDegradedHealth`, run by the
+ * operator alerts tick, raises `system.healthStillDegraded` at 1 h and every
+ * 2 h after, as Must know from 2 h on.
+ */
+import type { OperatorAlertOutcome } from './operator-alerts';
 export declare const OPERATOR_HEALTH_COLLECTION = "operatorHealthState";
 export type OperatorHealthStatus = 'ok' | 'degraded';
 export interface OperatorHealthStateDoc {
@@ -24,6 +54,13 @@ export interface OperatorHealthStateDoc {
     sinceMs: number;
     detail: string;
     updatedAtMs: number;
+    /**
+     * The last still-degraded reminder told for THIS incident (AGL-3860), as
+     * its step: 1 at 1 h, 2 at 3 h, 3 at 5 h. A transition rewrites the whole
+     * document, so a new incident starts again from none.
+     */
+    escalatedStep?: number;
+    lastEscalatedMs?: number;
 }
 export type OperatorHealthTransition = 'degraded' | 'recovered' | null;
 /** Test seam. */
@@ -36,6 +73,38 @@ export declare function recordHealthState(checkId: string, status: OperatorHealt
 }): Promise<OperatorHealthTransition>;
 /** Every recorded check, for the staff page and the sweep's report. */
 export declare function listHealthStates(): Promise<OperatorHealthStateDoc[]>;
+/** The first still-degraded reminder, after this long degraded. */
+export declare const HEALTH_ESCALATION_FIRST_MS: number;
+/** Each reminder after the first, this long after the one before. */
+export declare const HEALTH_ESCALATION_EVERY_MS: number;
+/** From this long degraded on, a reminder is Must know. */
+export declare const HEALTH_ESCALATION_MUST_AFTER_MS: number;
+/**
+ * The reminder a check degraded this long is owed: 0 before 1 h, 1 from
+ * 1 h, 2 from 3 h, 3 from 5 h. A tick that missed a step owes only the
+ * latest, so a sweep that was down for hours tells it once, not N times.
+ */
+export declare function healthEscalationStep(degradedForMs: number): number;
+export interface HealthEscalation {
+    checkId: string;
+    step: number;
+    tier: 'should' | 'must';
+    /** What raising it did; `dry-run` when nothing was raised on purpose. */
+    outcome: OperatorAlertOutcome | 'dry-run';
+}
+/**
+ * Raises `system.healthStillDegraded` for every recorded check that has
+ * been degraded long enough to be owed its next reminder (AGL-3860). The
+ * step is claimed in a transaction on the check's own state document, and
+ * only while the document still holds the same incident (`sinceMs`), so any
+ * number of ticks on any number of instances tell each step once, and a
+ * check that recovered in between is not told at all. Never throws.
+ */
+export declare function escalateStillDegradedHealth(options?: {
+    now?: number;
+    dryRun?: boolean;
+    states?: readonly OperatorHealthStateDoc[];
+}): Promise<HealthEscalation[]>;
 /**
  * One health endpoint's body as a state: `status` is the endpoint's own
  * verdict, and the detail names each failing check and its code.

package/src/lib/server/operator-health.js

@@ -36,6 +36,11 @@
  * was a write per check per instance that nothing reads.
  * The Admin SDK is loaded on the first write rather than at import, so every
  * health route can import this without initializing anything. Never throws.
+ *
+ * The edge is told once, and a check that STAYS degraded must not be
+ * forgotten after it (AGL-3860). `escalateStillDegradedHealth`, run by the
+ * operator alerts tick, raises `system.healthStillDegraded` at 1 h and every
+ * 2 h after, as Must know from 2 h on.
  */ async function firestore() {
     const { default: firebaseAdmin } = await import("./firebase-admin.js");
     return firebaseAdmin.app().firestore();
@@ -138,6 +143,88 @@ export async function recordHealthState(checkId, status, detail, options = {}) {
         return [];
     }
 }
+/** The first still-degraded reminder, after this long degraded. */ export const HEALTH_ESCALATION_FIRST_MS = 60 * 60000;
+/** Each reminder after the first, this long after the one before. */ export const HEALTH_ESCALATION_EVERY_MS = 2 * 60 * 60000;
+/** From this long degraded on, a reminder is Must know. */ export const HEALTH_ESCALATION_MUST_AFTER_MS = 2 * 60 * 60000;
+/**
+ * The reminder a check degraded this long is owed: 0 before 1 h, 1 from
+ * 1 h, 2 from 3 h, 3 from 5 h. A tick that missed a step owes only the
+ * latest, so a sweep that was down for hours tells it once, not N times.
+ */ export function healthEscalationStep(degradedForMs) {
+    if (!(degradedForMs >= HEALTH_ESCALATION_FIRST_MS)) return 0;
+    return 1 + Math.floor((degradedForMs - HEALTH_ESCALATION_FIRST_MS) / HEALTH_ESCALATION_EVERY_MS);
+}
+/**
+ * Raises `system.healthStillDegraded` for every recorded check that has
+ * been degraded long enough to be owed its next reminder (AGL-3860). The
+ * step is claimed in a transaction on the check's own state document, and
+ * only while the document still holds the same incident (`sinceMs`), so any
+ * number of ticks on any number of instances tell each step once, and a
+ * check that recovered in between is not told at all. Never throws.
+ */ export async function escalateStillDegradedHealth(options = {}) {
+    var _options_now, _options_states;
+    const now = (_options_now = options.now) != null ? _options_now : Date.now();
+    const states = (_options_states = options.states) != null ? _options_states : await listHealthStates();
+    const told = [];
+    for (const state of states){
+        var _state_escalatedStep;
+        if ((state == null ? void 0 : state.status) !== 'degraded' || !state.checkId) continue;
+        const sinceMs = Number(state.sinceMs);
+        if (!Number.isFinite(sinceMs)) continue;
+        const degradedForMs = now - sinceMs;
+        const step = healthEscalationStep(degradedForMs);
+        if (!step || ((_state_escalatedStep = state.escalatedStep) != null ? _state_escalatedStep : 0) >= step) continue;
+        const tier = degradedForMs >= HEALTH_ESCALATION_MUST_AFTER_MS ? 'must' : 'should';
+        if (options.dryRun) {
+            told.push({
+                checkId: state.checkId,
+                step,
+                tier,
+                outcome: 'dry-run'
+            });
+            continue;
+        }
+        try {
+            var _state_label, _state_detail;
+            const db = await firestore();
+            const ref = db.collection(OPERATOR_HEALTH_COLLECTION).doc(state.checkId);
+            const claimed = await db.runTransaction(async (transaction)=>{
+                var _current_escalatedStep;
+                const snapshot = await transaction.get(ref);
+                const current = snapshot.exists ? snapshot.data() : null;
+                if ((current == null ? void 0 : current.status) !== 'degraded' || current.sinceMs !== sinceMs || ((_current_escalatedStep = current.escalatedStep) != null ? _current_escalatedStep : 0) >= step) {
+                    return false;
+                }
+                transaction.update(ref, {
+                    escalatedStep: step,
+                    lastEscalatedMs: now
+                });
+                return true;
+            });
+            if (!claimed) continue;
+            const { raiseOperatorAlert } = await import("./operator-alerts.js");
+            const result = await raiseOperatorAlert('system.healthStillDegraded', {
+                dedupeKey: `${state.checkId}:${step}`,
+                tier,
+                context: {
+                    check: String((_state_label = state.label) != null ? _state_label : '').trim() || state.checkId,
+                    duration: describeHealthDuration(degradedForMs),
+                    detail: String((_state_detail = state.detail) != null ? _state_detail : '').trim()
+                }
+            });
+            told.push({
+                checkId: state.checkId,
+                step,
+                tier,
+                outcome: result.outcome
+            });
+        } catch (error) {
+            // The next tick tries the same step again: it was never claimed.
+            console.error(`[operator-health] ${state.checkId} could not be escalated`, error);
+        }
+    }
+    return told;
+}
 /**
  * One health endpoint's body as a state: `status` is the endpoint's own
  * verdict, and the detail names each failing check and its code.

package/src/lib/server/operator-health.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/operator-health.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * HEALTH TRANSITIONS (AGL-3377): the health checks, remembered.\n *\n * Every `/api/health/*` endpoint answers 200 or 503 to whoever asks and\n * forgets the answer. An install with an external uptime monitor hears the\n * 503; an install without one — most self-hosted ones — hears nothing, and\n * a dead cron or a stale backup is found on the day it is needed.\n *\n * `recordHealthState` keeps the last verdict per check in\n * `operatorHealthState/{checkId}` and raises an operator alert on the EDGE:\n * `system.healthDegraded` when a healthy (or never-seen) check goes\n * degraded, `system.healthRecovered` when it comes back. Once each, however\n * many instances and pollers see the same state, because the edge is decided\n * inside a transaction on the check's document.\n *\n * Cheap on the hot path: each instance remembers the last state it wrote, so\n * a poll that finds nothing changed costs no Firestore round trip at all. An\n * instance's FIRST poll of a check reads the document outside a transaction\n * and writes nothing when the stored status already agrees: serverless\n * instances come and go all day, and rewriting `detail` on every cold start\n * was a write per check per instance that nothing reads.\n * The Admin SDK is loaded on the first write rather than at import, so every\n * health route can import this without initializing anything. Never throws.\n */\n\nasync function firestore(): Promise<any> {\n  const { default: firebaseAdmin } = await import('./firebase-admin')\n  return firebaseAdmin.app().firestore()\n}\n\nexport const OPERATOR_HEALTH_COLLECTION = 'operatorHealthState'\n\nexport type OperatorHealthStatus = 'ok' | 'degraded'\n\nexport interface OperatorHealthStateDoc {\n  checkId: string\n  label: string\n  status: OperatorHealthStatus\n  /** When the check entered `status`. */\n  sinceMs: number\n  detail: string\n  updatedAtMs: number\n}\n\nexport type OperatorHealthTransition = 'degraded' | 'recovered' | null\n\n/** The last status this instance wrote per check. */\nconst lastWritten = new Map<string, OperatorHealthStatus>()\n\n/** Test seam. */\nexport function resetOperatorHealthCacheForTests(): void {\n  lastWritten.clear()\n}\n\n/** \"3 h 5 min\", for a recovery's duration. */\nexport function describeHealthDuration(ms: number): string {\n  const minutes = Math.max(1, Math.round(ms / 60_000))\n  if (minutes < 60) return `${minutes} min`\n  const hours = Math.floor(minutes / 60)\n  const rest = minutes % 60\n  if (hours < 48) return rest ? `${hours} h ${rest} min` : `${hours} h`\n  return `${Math.round(hours / 24)} days`\n}\n\nexport async function recordHealthState(\n  checkId: string,\n  status: OperatorHealthStatus,\n  detail: string,\n  options: { label?: string; now?: number } = {},\n): Promise<OperatorHealthTransition> {\n  const id = String(checkId ?? '').trim()\n  if (!id) return null\n  if (lastWritten.get(id) === status) return null\n  const now = options.now ?? Date.now()\n  const label = String(options.label ?? '').trim() || id\n  const summary = String(detail ?? '').trim().slice(0, 1_000)\n  try {\n    const db = await firestore()\n    const ref = db.collection(OPERATOR_HEALTH_COLLECTION).doc(id)\n    const stored = await ref.get()\n    if (stored.exists && stored.get('status') === status) {\n      lastWritten.set(id, status)\n      return null\n    }\n    const edge = await db.runTransaction(async (transaction: any) => {\n      const snapshot = await transaction.get(ref)\n      const prior = (snapshot.exists ? snapshot.data() : null) as\n        | Partial<OperatorHealthStateDoc>\n        | null\n      if (prior?.status === status) {\n        return { transition: null, priorSinceMs: prior.sinceMs ?? now }\n      }\n      const next: OperatorHealthStateDoc = {\n        checkId: id,\n        label,\n        status,\n        sinceMs: now,\n        detail: summary,\n        updatedAtMs: now,\n      }\n      transaction.set(ref, next)\n      // A check first seen healthy is the ordinary case and says nothing; a\n      // check first seen degraded is an install that was broken before\n      // anybody watched, which is exactly the one to say out loud.\n      const transition: OperatorHealthTransition =\n        status === 'degraded' ? 'degraded' : prior ? 'recovered' : null\n      return { transition, priorSinceMs: prior?.sinceMs ?? now }\n    })\n    lastWritten.set(id, status)\n    if (edge.transition) {\n      const { raiseOperatorAlert } = await import('./operator-alerts')\n      if (edge.transition === 'degraded') {\n        await raiseOperatorAlert('system.healthDegraded', {\n          dedupeKey: id,\n          context: { check: label, detail: summary },\n        })\n      } else {\n        await raiseOperatorAlert('system.healthRecovered', {\n          dedupeKey: id,\n          context: {\n            check: label,\n            duration: describeHealthDuration(now - edge.priorSinceMs),\n          },\n        })\n      }\n    }\n    return edge.transition\n  } catch (error) {\n    // Once per verdict per instance: a store that is down must not turn\n    // every poll of a public endpoint into another log line. The next\n    // change of verdict tries again.\n    lastWritten.set(id, status)\n    console.error(`[operator-health] ${id} could not be recorded`, error)\n    return null\n  }\n}\n\n/** Every recorded check, for the staff page and the sweep's report. */\nexport async function listHealthStates(): Promise<OperatorHealthStateDoc[]> {\n  try {\n    const snapshot = await (await firestore()).collection(OPERATOR_HEALTH_COLLECTION).get()\n    return snapshot.docs.map((doc: any) => doc.data() as OperatorHealthStateDoc)\n  } catch (error) {\n    console.error('[operator-health] states unreadable', error)\n    return []\n  }\n}\n\n/**\n * One health endpoint's body as a state: `status` is the endpoint's own\n * verdict, and the detail names each failing check and its code.\n */\nexport function healthStateOfBody(\n  body: Record<string, unknown> | null,\n  httpStatus: number | null,\n): { status: OperatorHealthStatus; detail: string } {\n  if (!body) {\n    return {\n      status: 'degraded',\n      detail:\n        httpStatus === null\n          ? 'The endpoint did not answer.'\n          : `The endpoint answered ${httpStatus} with no readable body.`,\n    }\n  }\n  const checks = (body['checks'] ?? {}) as Record<string, Record<string, unknown> | undefined>\n  const failing = Object.entries(checks)\n    .filter(([, check]) => check && check['ok'] !== true)\n    .map(([name, check]) =>\n      typeof check?.['code'] === 'string' ? `${name} (${check['code']})` : name,\n    )\n  const degraded =\n    body['status'] === 'degraded' ||\n    (body['status'] !== 'ok' && (httpStatus === 503 || failing.length > 0))\n  return {\n    status: degraded ? 'degraded' : 'ok',\n    detail: degraded\n      ? failing.length\n        ? `Failing: ${failing.join(', ')}.`\n        : `The endpoint answered ${httpStatus ?? 'nothing'}.`\n      : 'All checks pass.',\n  }\n}\n\n/**\n * What each health endpoint is called on the board and in an alert, keyed by\n * the `service` its body names. An unlisted service is called by its name.\n */\nexport const HEALTH_SERVICE_LABELS: Readonly<Record<string, string>> = {\n  console: 'Console serving',\n  'console-auth-doors': 'Ways in',\n  'console-backups': 'Backups & exports',\n  'console-billing': 'Billing webhook',\n  'console-crons': 'Scheduled jobs',\n  'console-error-beacon': 'Console error beacon',\n  'console-journeys': 'Console journeys',\n  'console-rate-limits': 'Rate limiters',\n  'console-server-errors': 'Server errors',\n  'console-signup-volume': 'Signup volume',\n  'console-email': 'Email delivery',\n  tenant: 'Site serving',\n  'tenant-error-beacon': 'Site error beacon',\n  'tenant-funnel': 'Site capture funnel',\n  'tenant-render-site': 'Site rendering',\n  'tenant-render-marketing': 'Marketing site rendering',\n}\n\n/**\n * Records a health endpoint's own response and hands it back unchanged\n * (AGL-3377): `return recordHealthResponse(Response.json(healthBody(…)))`.\n * The check id is the body's `service`, so the endpoint, the scheduled sweep\n * and any other poller all write the same state. Costs nothing while the\n * verdict is the one this instance last wrote. Never throws.\n */\nexport async function recordHealthResponse(response: Response): Promise<Response> {\n  try {\n    const body = (await response.clone().json()) as Record<string, unknown> | null\n    const service = String(body?.['service'] ?? '').trim()\n    if (service) {\n      const { status, detail } = healthStateOfBody(body, response.status)\n      await recordHealthState(service, status, detail, {\n        label: HEALTH_SERVICE_LABELS[service] ?? service,\n      })\n    }\n  } catch (error) {\n    console.error('[operator-health] a health response could not be recorded', error)\n  }\n  return response\n}\n"],"names":["firestore","default","firebaseAdmin","app","OPERATOR_HEALTH_COLLECTION","lastWritten","Map","resetOperatorHealthCacheForTests","clear","describeHealthDuration","ms","minutes","Math","max","round","hours","floor","rest","recordHealthState","checkId","status","detail","options","id","String","trim","get","now","Date","label","summary","slice","db","ref","collection","doc","stored","exists","set","edge","runTransaction","transaction","snapshot","prior","data","transition","priorSinceMs","sinceMs","next","updatedAtMs","raiseOperatorAlert","dedupeKey","context","check","duration","error","console","listHealthStates","docs","map","healthStateOfBody","body","httpStatus","checks","failing","Object","entries","filter","name","degraded","length","join","HEALTH_SERVICE_LABELS","tenant","recordHealthResponse","response","clone","json","service"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;CAuBC,GAED,eAAeA;IACb,MAAM,EAAEC,SAASC,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,OAAOA,cAAcC,GAAG,GAAGH,SAAS;AACtC;AAEA,OAAO,MAAMI,6BAA6B,sBAAqB;AAgB/D,mDAAmD,GACnD,MAAMC,cAAc,IAAIC;AAExB,eAAe,GACf,OAAO,SAASC;IACdF,YAAYG,KAAK;AACnB;AAEA,4CAA4C,GAC5C,OAAO,SAASC,uBAAuBC,EAAU;IAC/C,MAAMC,UAAUC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACJ,KAAK;IAC5C,IAAIC,UAAU,IAAI,OAAO,GAAGA,QAAQ,IAAI,CAAC;IACzC,MAAMI,QAAQH,KAAKI,KAAK,CAACL,UAAU;IACnC,MAAMM,OAAON,UAAU;IACvB,IAAII,QAAQ,IAAI,OAAOE,OAAO,GAAGF,MAAM,GAAG,EAAEE,KAAK,IAAI,CAAC,GAAG,GAAGF,MAAM,EAAE,CAAC;IACrE,OAAO,GAAGH,KAAKE,KAAK,CAACC,QAAQ,IAAI,KAAK,CAAC;AACzC;AAEA,OAAO,eAAeG,kBACpBC,OAAe,EACfC,MAA4B,EAC5BC,MAAc,EACdC,UAA4C,CAAC,CAAC;QAKlCA,cACSA;IAJrB,MAAMC,KAAKC,OAAOL,kBAAAA,UAAW,IAAIM,IAAI;IACrC,IAAI,CAACF,IAAI,OAAO;IAChB,IAAIlB,YAAYqB,GAAG,CAACH,QAAQH,QAAQ,OAAO;IAC3C,MAAMO,OAAML,eAAAA,QAAQK,GAAG,YAAXL,eAAeM,KAAKD,GAAG;IACnC,MAAME,QAAQL,QAAOF,iBAAAA,QAAQO,KAAK,YAAbP,iBAAiB,IAAIG,IAAI,MAAMF;IACpD,MAAMO,UAAUN,OAAOH,iBAAAA,SAAU,IAAII,IAAI,GAAGM,KAAK,CAAC,GAAG;IACrD,IAAI;QACF,MAAMC,KAAK,MAAMhC;QACjB,MAAMiC,MAAMD,GAAGE,UAAU,CAAC9B,4BAA4B+B,GAAG,CAACZ;QAC1D,MAAMa,SAAS,MAAMH,IAAIP,GAAG;QAC5B,IAAIU,OAAOC,MAAM,IAAID,OAAOV,GAAG,CAAC,cAAcN,QAAQ;YACpDf,YAAYiC,GAAG,CAACf,IAAIH;YACpB,OAAO;QACT;QACA,MAAMmB,OAAO,MAAMP,GAAGQ,cAAc,CAAC,OAAOC;;YAC1C,MAAMC,WAAW,MAAMD,YAAYf,GAAG,CAACO;YACvC,MAAMU,QAASD,SAASL,MAAM,GAAGK,SAASE,IAAI,KAAK;YAGnD,IAAID,CAAAA,yBAAAA,MAAOvB,MAAM,MAAKA,QAAQ;oBACauB;gBAAzC,OAAO;oBAAEE,YAAY;oBAAMC,YAAY,GAAEH,iBAAAA,MAAMI,OAAO,YAAbJ,iBAAiBhB;gBAAI;YAChE;YACA,MAAMqB,OAA+B;gBACnC7B,SAASI;gBACTM;gBACAT;gBACA2B,SAASpB;gBACTN,QAAQS;gBACRmB,aAAatB;YACf;YACAc,YAAYH,GAAG,CAACL,KAAKe;YACrB,sEAAsE;YACtE,iEAAiE;YACjE,6DAA6D;YAC7D,MAAMH,aACJzB,WAAW,aAAa,aAAauB,QAAQ,cAAc;YAC7D,OAAO;gBAAEE;gBAAYC,YAAY,UAAEH,yBAAAA,MAAOI,OAAO,mBAAIpB;YAAI;QAC3D;QACAtB,YAAYiC,GAAG,CAACf,IAAIH;QACpB,IAAImB,KAAKM,UAAU,EAAE;YACnB,MAAM,EAAEK,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;YAC5C,IAAIX,KAAKM,UAAU,KAAK,YAAY;gBAClC,MAAMK,mBAAmB,yBAAyB;oBAChDC,WAAW5B;oBACX6B,SAAS;wBAAEC,OAAOxB;wBAAOR,QAAQS;oBAAQ;gBAC3C;YACF,OAAO;gBACL,MAAMoB,mBAAmB,0BAA0B;oBACjDC,WAAW5B;oBACX6B,SAAS;wBACPC,OAAOxB;wBACPyB,UAAU7C,uBAAuBkB,MAAMY,KAAKO,YAAY;oBAC1D;gBACF;YACF;QACF;QACA,OAAOP,KAAKM,UAAU;IACxB,EAAE,OAAOU,OAAO;QACd,oEAAoE;QACpE,kEAAkE;QAClE,iCAAiC;QACjClD,YAAYiC,GAAG,CAACf,IAAIH;QACpBoC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAEhC,GAAG,sBAAsB,CAAC,EAAEgC;QAC/D,OAAO;IACT;AACF;AAEA,qEAAqE,GACrE,OAAO,eAAeE;IACpB,IAAI;QACF,MAAMf,WAAW,MAAM,AAAC,CAAA,MAAM1C,WAAU,EAAGkC,UAAU,CAAC9B,4BAA4BsB,GAAG;QACrF,OAAOgB,SAASgB,IAAI,CAACC,GAAG,CAAC,CAACxB,MAAaA,IAAIS,IAAI;IACjD,EAAE,OAAOW,OAAO;QACdC,QAAQD,KAAK,CAAC,uCAAuCA;QACrD,OAAO,EAAE;IACX;AACF;AAEA;;;CAGC,GACD,OAAO,SAASK,kBACdC,IAAoC,EACpCC,UAAyB;QAWTD;IAThB,IAAI,CAACA,MAAM;QACT,OAAO;YACLzC,QAAQ;YACRC,QACEyC,eAAe,OACX,iCACA,CAAC,sBAAsB,EAAEA,WAAW,uBAAuB,CAAC;QACpE;IACF;IACA,MAAMC,UAAUF,eAAAA,IAAI,CAAC,SAAS,YAAdA,eAAkB,CAAC;IACnC,MAAMG,UAAUC,OAAOC,OAAO,CAACH,QAC5BI,MAAM,CAAC,CAAC,GAAGd,MAAM,GAAKA,SAASA,KAAK,CAAC,KAAK,KAAK,MAC/CM,GAAG,CAAC,CAAC,CAACS,MAAMf,MAAM,GACjB,QAAOA,yBAAAA,KAAO,CAAC,OAAO,MAAK,WAAW,GAAGe,KAAK,EAAE,EAAEf,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAGe;IAEzE,MAAMC,WACJR,IAAI,CAAC,SAAS,KAAK,cAClBA,IAAI,CAAC,SAAS,KAAK,QAASC,CAAAA,eAAe,OAAOE,QAAQM,MAAM,GAAG,CAAA;IACtE,OAAO;QACLlD,QAAQiD,WAAW,aAAa;QAChChD,QAAQgD,WACJL,QAAQM,MAAM,GACZ,CAAC,SAAS,EAAEN,QAAQO,IAAI,CAAC,MAAM,CAAC,CAAC,GACjC,CAAC,sBAAsB,EAAET,qBAAAA,aAAc,UAAU,CAAC,CAAC,GACrD;IACN;AACF;AAEA;;;CAGC,GACD,OAAO,MAAMU,wBAA0D;IACrEhB,SAAS;IACT,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,iBAAiB;IACjB,wBAAwB;IACxB,oBAAoB;IACpB,uBAAuB;IACvB,yBAAyB;IACzB,yBAAyB;IACzB,iBAAiB;IACjBiB,QAAQ;IACR,uBAAuB;IACvB,iBAAiB;IACjB,sBAAsB;IACtB,2BAA2B;AAC7B,EAAC;AAED;;;;;;CAMC,GACD,OAAO,eAAeC,qBAAqBC,QAAkB;IAC3D,IAAI;;QACF,MAAMd,OAAQ,MAAMc,SAASC,KAAK,GAAGC,IAAI;QACzC,MAAMC,UAAUtD,eAAOqC,wBAAAA,IAAM,CAAC,UAAU,mBAAI,IAAIpC,IAAI;QACpD,IAAIqD,SAAS;gBAGFN;YAFT,MAAM,EAAEpD,MAAM,EAAEC,MAAM,EAAE,GAAGuC,kBAAkBC,MAAMc,SAASvD,MAAM;YAClE,MAAMF,kBAAkB4D,SAAS1D,QAAQC,QAAQ;gBAC/CQ,KAAK,GAAE2C,iCAAAA,qBAAqB,CAACM,QAAQ,YAA9BN,iCAAkCM;YAC3C;QACF;IACF,EAAE,OAAOvB,OAAO;QACdC,QAAQD,KAAK,CAAC,6DAA6DA;IAC7E;IACA,OAAOoB;AACT"}
+{"version":3,"sources":["../../../../../../../../libs/tenant/data/admin/src/lib/server/operator-health.ts"],"sourcesContent":["/**\n * @license\n * Copyright 2026 Aglyn LLC\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n *   http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n/**\n * HEALTH TRANSITIONS (AGL-3377): the health checks, remembered.\n *\n * Every `/api/health/*` endpoint answers 200 or 503 to whoever asks and\n * forgets the answer. An install with an external uptime monitor hears the\n * 503; an install without one — most self-hosted ones — hears nothing, and\n * a dead cron or a stale backup is found on the day it is needed.\n *\n * `recordHealthState` keeps the last verdict per check in\n * `operatorHealthState/{checkId}` and raises an operator alert on the EDGE:\n * `system.healthDegraded` when a healthy (or never-seen) check goes\n * degraded, `system.healthRecovered` when it comes back. Once each, however\n * many instances and pollers see the same state, because the edge is decided\n * inside a transaction on the check's document.\n *\n * Cheap on the hot path: each instance remembers the last state it wrote, so\n * a poll that finds nothing changed costs no Firestore round trip at all. An\n * instance's FIRST poll of a check reads the document outside a transaction\n * and writes nothing when the stored status already agrees: serverless\n * instances come and go all day, and rewriting `detail` on every cold start\n * was a write per check per instance that nothing reads.\n * The Admin SDK is loaded on the first write rather than at import, so every\n * health route can import this without initializing anything. Never throws.\n *\n * The edge is told once, and a check that STAYS degraded must not be\n * forgotten after it (AGL-3860). `escalateStillDegradedHealth`, run by the\n * operator alerts tick, raises `system.healthStillDegraded` at 1 h and every\n * 2 h after, as Must know from 2 h on.\n */\n\nimport type { OperatorAlertOutcome } from './operator-alerts'\n\nasync function firestore(): Promise<any> {\n  const { default: firebaseAdmin } = await import('./firebase-admin')\n  return firebaseAdmin.app().firestore()\n}\n\nexport const OPERATOR_HEALTH_COLLECTION = 'operatorHealthState'\n\nexport type OperatorHealthStatus = 'ok' | 'degraded'\n\nexport interface OperatorHealthStateDoc {\n  checkId: string\n  label: string\n  status: OperatorHealthStatus\n  /** When the check entered `status`. */\n  sinceMs: number\n  detail: string\n  updatedAtMs: number\n  /**\n   * The last still-degraded reminder told for THIS incident (AGL-3860), as\n   * its step: 1 at 1 h, 2 at 3 h, 3 at 5 h. A transition rewrites the whole\n   * document, so a new incident starts again from none.\n   */\n  escalatedStep?: number\n  lastEscalatedMs?: number\n}\n\nexport type OperatorHealthTransition = 'degraded' | 'recovered' | null\n\n/** The last status this instance wrote per check. */\nconst lastWritten = new Map<string, OperatorHealthStatus>()\n\n/** Test seam. */\nexport function resetOperatorHealthCacheForTests(): void {\n  lastWritten.clear()\n}\n\n/** \"3 h 5 min\", for a recovery's duration. */\nexport function describeHealthDuration(ms: number): string {\n  const minutes = Math.max(1, Math.round(ms / 60_000))\n  if (minutes < 60) return `${minutes} min`\n  const hours = Math.floor(minutes / 60)\n  const rest = minutes % 60\n  if (hours < 48) return rest ? `${hours} h ${rest} min` : `${hours} h`\n  return `${Math.round(hours / 24)} days`\n}\n\nexport async function recordHealthState(\n  checkId: string,\n  status: OperatorHealthStatus,\n  detail: string,\n  options: { label?: string; now?: number } = {},\n): Promise<OperatorHealthTransition> {\n  const id = String(checkId ?? '').trim()\n  if (!id) return null\n  if (lastWritten.get(id) === status) return null\n  const now = options.now ?? Date.now()\n  const label = String(options.label ?? '').trim() || id\n  const summary = String(detail ?? '').trim().slice(0, 1_000)\n  try {\n    const db = await firestore()\n    const ref = db.collection(OPERATOR_HEALTH_COLLECTION).doc(id)\n    const stored = await ref.get()\n    if (stored.exists && stored.get('status') === status) {\n      lastWritten.set(id, status)\n      return null\n    }\n    const edge = await db.runTransaction(async (transaction: any) => {\n      const snapshot = await transaction.get(ref)\n      const prior = (snapshot.exists ? snapshot.data() : null) as\n        | Partial<OperatorHealthStateDoc>\n        | null\n      if (prior?.status === status) {\n        return { transition: null, priorSinceMs: prior.sinceMs ?? now }\n      }\n      const next: OperatorHealthStateDoc = {\n        checkId: id,\n        label,\n        status,\n        sinceMs: now,\n        detail: summary,\n        updatedAtMs: now,\n      }\n      transaction.set(ref, next)\n      // A check first seen healthy is the ordinary case and says nothing; a\n      // check first seen degraded is an install that was broken before\n      // anybody watched, which is exactly the one to say out loud.\n      const transition: OperatorHealthTransition =\n        status === 'degraded' ? 'degraded' : prior ? 'recovered' : null\n      return { transition, priorSinceMs: prior?.sinceMs ?? now }\n    })\n    lastWritten.set(id, status)\n    if (edge.transition) {\n      const { raiseOperatorAlert } = await import('./operator-alerts')\n      if (edge.transition === 'degraded') {\n        await raiseOperatorAlert('system.healthDegraded', {\n          dedupeKey: id,\n          context: { check: label, detail: summary },\n        })\n      } else {\n        await raiseOperatorAlert('system.healthRecovered', {\n          dedupeKey: id,\n          context: {\n            check: label,\n            duration: describeHealthDuration(now - edge.priorSinceMs),\n          },\n        })\n      }\n    }\n    return edge.transition\n  } catch (error) {\n    // Once per verdict per instance: a store that is down must not turn\n    // every poll of a public endpoint into another log line. The next\n    // change of verdict tries again.\n    lastWritten.set(id, status)\n    console.error(`[operator-health] ${id} could not be recorded`, error)\n    return null\n  }\n}\n\n/** Every recorded check, for the staff page and the sweep's report. */\nexport async function listHealthStates(): Promise<OperatorHealthStateDoc[]> {\n  try {\n    const snapshot = await (await firestore()).collection(OPERATOR_HEALTH_COLLECTION).get()\n    return snapshot.docs.map((doc: any) => doc.data() as OperatorHealthStateDoc)\n  } catch (error) {\n    console.error('[operator-health] states unreadable', error)\n    return []\n  }\n}\n\n/** The first still-degraded reminder, after this long degraded. */\nexport const HEALTH_ESCALATION_FIRST_MS = 60 * 60_000\n/** Each reminder after the first, this long after the one before. */\nexport const HEALTH_ESCALATION_EVERY_MS = 2 * 60 * 60_000\n/** From this long degraded on, a reminder is Must know. */\nexport const HEALTH_ESCALATION_MUST_AFTER_MS = 2 * 60 * 60_000\n\n/**\n * The reminder a check degraded this long is owed: 0 before 1 h, 1 from\n * 1 h, 2 from 3 h, 3 from 5 h. A tick that missed a step owes only the\n * latest, so a sweep that was down for hours tells it once, not N times.\n */\nexport function healthEscalationStep(degradedForMs: number): number {\n  if (!(degradedForMs >= HEALTH_ESCALATION_FIRST_MS)) return 0\n  return 1 + Math.floor((degradedForMs - HEALTH_ESCALATION_FIRST_MS) / HEALTH_ESCALATION_EVERY_MS)\n}\n\nexport interface HealthEscalation {\n  checkId: string\n  step: number\n  tier: 'should' | 'must'\n  /** What raising it did; `dry-run` when nothing was raised on purpose. */\n  outcome: OperatorAlertOutcome | 'dry-run'\n}\n\n/**\n * Raises `system.healthStillDegraded` for every recorded check that has\n * been degraded long enough to be owed its next reminder (AGL-3860). The\n * step is claimed in a transaction on the check's own state document, and\n * only while the document still holds the same incident (`sinceMs`), so any\n * number of ticks on any number of instances tell each step once, and a\n * check that recovered in between is not told at all. Never throws.\n */\nexport async function escalateStillDegradedHealth(\n  options: { now?: number; dryRun?: boolean; states?: readonly OperatorHealthStateDoc[] } = {},\n): Promise<HealthEscalation[]> {\n  const now = options.now ?? Date.now()\n  const states = options.states ?? (await listHealthStates())\n  const told: HealthEscalation[] = []\n  for (const state of states) {\n    if (state?.status !== 'degraded' || !state.checkId) continue\n    const sinceMs = Number(state.sinceMs)\n    if (!Number.isFinite(sinceMs)) continue\n    const degradedForMs = now - sinceMs\n    const step = healthEscalationStep(degradedForMs)\n    if (!step || (state.escalatedStep ?? 0) >= step) continue\n    const tier = degradedForMs >= HEALTH_ESCALATION_MUST_AFTER_MS ? 'must' : 'should'\n    if (options.dryRun) {\n      told.push({ checkId: state.checkId, step, tier, outcome: 'dry-run' })\n      continue\n    }\n    try {\n      const db = await firestore()\n      const ref = db.collection(OPERATOR_HEALTH_COLLECTION).doc(state.checkId)\n      const claimed = await db.runTransaction(async (transaction: any) => {\n        const snapshot = await transaction.get(ref)\n        const current = (snapshot.exists ? snapshot.data() : null) as\n          | Partial<OperatorHealthStateDoc>\n          | null\n        if (\n          current?.status !== 'degraded' ||\n          current.sinceMs !== sinceMs ||\n          (current.escalatedStep ?? 0) >= step\n        ) {\n          return false\n        }\n        transaction.update(ref, { escalatedStep: step, lastEscalatedMs: now })\n        return true\n      })\n      if (!claimed) continue\n      const { raiseOperatorAlert } = await import('./operator-alerts')\n      const result = await raiseOperatorAlert('system.healthStillDegraded', {\n        dedupeKey: `${state.checkId}:${step}`,\n        tier,\n        context: {\n          check: String(state.label ?? '').trim() || state.checkId,\n          duration: describeHealthDuration(degradedForMs),\n          detail: String(state.detail ?? '').trim(),\n        },\n      })\n      told.push({ checkId: state.checkId, step, tier, outcome: result.outcome })\n    } catch (error) {\n      // The next tick tries the same step again: it was never claimed.\n      console.error(`[operator-health] ${state.checkId} could not be escalated`, error)\n    }\n  }\n  return told\n}\n\n/**\n * One health endpoint's body as a state: `status` is the endpoint's own\n * verdict, and the detail names each failing check and its code.\n */\nexport function healthStateOfBody(\n  body: Record<string, unknown> | null,\n  httpStatus: number | null,\n): { status: OperatorHealthStatus; detail: string } {\n  if (!body) {\n    return {\n      status: 'degraded',\n      detail:\n        httpStatus === null\n          ? 'The endpoint did not answer.'\n          : `The endpoint answered ${httpStatus} with no readable body.`,\n    }\n  }\n  const checks = (body['checks'] ?? {}) as Record<string, Record<string, unknown> | undefined>\n  const failing = Object.entries(checks)\n    .filter(([, check]) => check && check['ok'] !== true)\n    .map(([name, check]) =>\n      typeof check?.['code'] === 'string' ? `${name} (${check['code']})` : name,\n    )\n  const degraded =\n    body['status'] === 'degraded' ||\n    (body['status'] !== 'ok' && (httpStatus === 503 || failing.length > 0))\n  return {\n    status: degraded ? 'degraded' : 'ok',\n    detail: degraded\n      ? failing.length\n        ? `Failing: ${failing.join(', ')}.`\n        : `The endpoint answered ${httpStatus ?? 'nothing'}.`\n      : 'All checks pass.',\n  }\n}\n\n/**\n * What each health endpoint is called on the board and in an alert, keyed by\n * the `service` its body names. An unlisted service is called by its name.\n */\nexport const HEALTH_SERVICE_LABELS: Readonly<Record<string, string>> = {\n  console: 'Console serving',\n  'console-auth-doors': 'Ways in',\n  'console-backups': 'Backups & exports',\n  'console-billing': 'Billing webhook',\n  'console-crons': 'Scheduled jobs',\n  'console-error-beacon': 'Console error beacon',\n  'console-journeys': 'Console journeys',\n  'console-rate-limits': 'Rate limiters',\n  'console-server-errors': 'Server errors',\n  'console-signup-volume': 'Signup volume',\n  'console-email': 'Email delivery',\n  tenant: 'Site serving',\n  'tenant-error-beacon': 'Site error beacon',\n  'tenant-funnel': 'Site capture funnel',\n  'tenant-render-site': 'Site rendering',\n  'tenant-render-marketing': 'Marketing site rendering',\n}\n\n/**\n * Records a health endpoint's own response and hands it back unchanged\n * (AGL-3377): `return recordHealthResponse(Response.json(healthBody(…)))`.\n * The check id is the body's `service`, so the endpoint, the scheduled sweep\n * and any other poller all write the same state. Costs nothing while the\n * verdict is the one this instance last wrote. Never throws.\n */\nexport async function recordHealthResponse(response: Response): Promise<Response> {\n  try {\n    const body = (await response.clone().json()) as Record<string, unknown> | null\n    const service = String(body?.['service'] ?? '').trim()\n    if (service) {\n      const { status, detail } = healthStateOfBody(body, response.status)\n      await recordHealthState(service, status, detail, {\n        label: HEALTH_SERVICE_LABELS[service] ?? service,\n      })\n    }\n  } catch (error) {\n    console.error('[operator-health] a health response could not be recorded', error)\n  }\n  return response\n}\n"],"names":["firestore","default","firebaseAdmin","app","OPERATOR_HEALTH_COLLECTION","lastWritten","Map","resetOperatorHealthCacheForTests","clear","describeHealthDuration","ms","minutes","Math","max","round","hours","floor","rest","recordHealthState","checkId","status","detail","options","id","String","trim","get","now","Date","label","summary","slice","db","ref","collection","doc","stored","exists","set","edge","runTransaction","transaction","snapshot","prior","data","transition","priorSinceMs","sinceMs","next","updatedAtMs","raiseOperatorAlert","dedupeKey","context","check","duration","error","console","listHealthStates","docs","map","HEALTH_ESCALATION_FIRST_MS","HEALTH_ESCALATION_EVERY_MS","HEALTH_ESCALATION_MUST_AFTER_MS","healthEscalationStep","degradedForMs","escalateStillDegradedHealth","states","told","state","Number","isFinite","step","escalatedStep","tier","dryRun","push","outcome","claimed","current","update","lastEscalatedMs","result","healthStateOfBody","body","httpStatus","checks","failing","Object","entries","filter","name","degraded","length","join","HEALTH_SERVICE_LABELS","tenant","recordHealthResponse","response","clone","json","service"],"mappings":"AAAA;;;;;;;;;;;;;;;CAeC,GAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4BC,GAID,eAAeA;IACb,MAAM,EAAEC,SAASC,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC;IAChD,OAAOA,cAAcC,GAAG,GAAGH,SAAS;AACtC;AAEA,OAAO,MAAMI,6BAA6B,sBAAqB;AAuB/D,mDAAmD,GACnD,MAAMC,cAAc,IAAIC;AAExB,eAAe,GACf,OAAO,SAASC;IACdF,YAAYG,KAAK;AACnB;AAEA,4CAA4C,GAC5C,OAAO,SAASC,uBAAuBC,EAAU;IAC/C,MAAMC,UAAUC,KAAKC,GAAG,CAAC,GAAGD,KAAKE,KAAK,CAACJ,KAAK;IAC5C,IAAIC,UAAU,IAAI,OAAO,GAAGA,QAAQ,IAAI,CAAC;IACzC,MAAMI,QAAQH,KAAKI,KAAK,CAACL,UAAU;IACnC,MAAMM,OAAON,UAAU;IACvB,IAAII,QAAQ,IAAI,OAAOE,OAAO,GAAGF,MAAM,GAAG,EAAEE,KAAK,IAAI,CAAC,GAAG,GAAGF,MAAM,EAAE,CAAC;IACrE,OAAO,GAAGH,KAAKE,KAAK,CAACC,QAAQ,IAAI,KAAK,CAAC;AACzC;AAEA,OAAO,eAAeG,kBACpBC,OAAe,EACfC,MAA4B,EAC5BC,MAAc,EACdC,UAA4C,CAAC,CAAC;QAKlCA,cACSA;IAJrB,MAAMC,KAAKC,OAAOL,kBAAAA,UAAW,IAAIM,IAAI;IACrC,IAAI,CAACF,IAAI,OAAO;IAChB,IAAIlB,YAAYqB,GAAG,CAACH,QAAQH,QAAQ,OAAO;IAC3C,MAAMO,OAAML,eAAAA,QAAQK,GAAG,YAAXL,eAAeM,KAAKD,GAAG;IACnC,MAAME,QAAQL,QAAOF,iBAAAA,QAAQO,KAAK,YAAbP,iBAAiB,IAAIG,IAAI,MAAMF;IACpD,MAAMO,UAAUN,OAAOH,iBAAAA,SAAU,IAAII,IAAI,GAAGM,KAAK,CAAC,GAAG;IACrD,IAAI;QACF,MAAMC,KAAK,MAAMhC;QACjB,MAAMiC,MAAMD,GAAGE,UAAU,CAAC9B,4BAA4B+B,GAAG,CAACZ;QAC1D,MAAMa,SAAS,MAAMH,IAAIP,GAAG;QAC5B,IAAIU,OAAOC,MAAM,IAAID,OAAOV,GAAG,CAAC,cAAcN,QAAQ;YACpDf,YAAYiC,GAAG,CAACf,IAAIH;YACpB,OAAO;QACT;QACA,MAAMmB,OAAO,MAAMP,GAAGQ,cAAc,CAAC,OAAOC;;YAC1C,MAAMC,WAAW,MAAMD,YAAYf,GAAG,CAACO;YACvC,MAAMU,QAASD,SAASL,MAAM,GAAGK,SAASE,IAAI,KAAK;YAGnD,IAAID,CAAAA,yBAAAA,MAAOvB,MAAM,MAAKA,QAAQ;oBACauB;gBAAzC,OAAO;oBAAEE,YAAY;oBAAMC,YAAY,GAAEH,iBAAAA,MAAMI,OAAO,YAAbJ,iBAAiBhB;gBAAI;YAChE;YACA,MAAMqB,OAA+B;gBACnC7B,SAASI;gBACTM;gBACAT;gBACA2B,SAASpB;gBACTN,QAAQS;gBACRmB,aAAatB;YACf;YACAc,YAAYH,GAAG,CAACL,KAAKe;YACrB,sEAAsE;YACtE,iEAAiE;YACjE,6DAA6D;YAC7D,MAAMH,aACJzB,WAAW,aAAa,aAAauB,QAAQ,cAAc;YAC7D,OAAO;gBAAEE;gBAAYC,YAAY,UAAEH,yBAAAA,MAAOI,OAAO,mBAAIpB;YAAI;QAC3D;QACAtB,YAAYiC,GAAG,CAACf,IAAIH;QACpB,IAAImB,KAAKM,UAAU,EAAE;YACnB,MAAM,EAAEK,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;YAC5C,IAAIX,KAAKM,UAAU,KAAK,YAAY;gBAClC,MAAMK,mBAAmB,yBAAyB;oBAChDC,WAAW5B;oBACX6B,SAAS;wBAAEC,OAAOxB;wBAAOR,QAAQS;oBAAQ;gBAC3C;YACF,OAAO;gBACL,MAAMoB,mBAAmB,0BAA0B;oBACjDC,WAAW5B;oBACX6B,SAAS;wBACPC,OAAOxB;wBACPyB,UAAU7C,uBAAuBkB,MAAMY,KAAKO,YAAY;oBAC1D;gBACF;YACF;QACF;QACA,OAAOP,KAAKM,UAAU;IACxB,EAAE,OAAOU,OAAO;QACd,oEAAoE;QACpE,kEAAkE;QAClE,iCAAiC;QACjClD,YAAYiC,GAAG,CAACf,IAAIH;QACpBoC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAEhC,GAAG,sBAAsB,CAAC,EAAEgC;QAC/D,OAAO;IACT;AACF;AAEA,qEAAqE,GACrE,OAAO,eAAeE;IACpB,IAAI;QACF,MAAMf,WAAW,MAAM,AAAC,CAAA,MAAM1C,WAAU,EAAGkC,UAAU,CAAC9B,4BAA4BsB,GAAG;QACrF,OAAOgB,SAASgB,IAAI,CAACC,GAAG,CAAC,CAACxB,MAAaA,IAAIS,IAAI;IACjD,EAAE,OAAOW,OAAO;QACdC,QAAQD,KAAK,CAAC,uCAAuCA;QACrD,OAAO,EAAE;IACX;AACF;AAEA,iEAAiE,GACjE,OAAO,MAAMK,6BAA6B,KAAK,MAAM;AACrD,mEAAmE,GACnE,OAAO,MAAMC,6BAA6B,IAAI,KAAK,MAAM;AACzD,yDAAyD,GACzD,OAAO,MAAMC,kCAAkC,IAAI,KAAK,MAAM;AAE9D;;;;CAIC,GACD,OAAO,SAASC,qBAAqBC,aAAqB;IACxD,IAAI,CAAEA,CAAAA,iBAAiBJ,0BAAyB,GAAI,OAAO;IAC3D,OAAO,IAAIhD,KAAKI,KAAK,CAAC,AAACgD,CAAAA,gBAAgBJ,0BAAyB,IAAKC;AACvE;AAUA;;;;;;;CAOC,GACD,OAAO,eAAeI,4BACpB3C,UAA0F,CAAC,CAAC;QAEhFA,cACGA;IADf,MAAMK,OAAML,eAAAA,QAAQK,GAAG,YAAXL,eAAeM,KAAKD,GAAG;IACnC,MAAMuC,UAAS5C,kBAAAA,QAAQ4C,MAAM,YAAd5C,kBAAmB,MAAMmC;IACxC,MAAMU,OAA2B,EAAE;IACnC,KAAK,MAAMC,SAASF,OAAQ;YAMZE;QALd,IAAIA,CAAAA,yBAAAA,MAAOhD,MAAM,MAAK,cAAc,CAACgD,MAAMjD,OAAO,EAAE;QACpD,MAAM4B,UAAUsB,OAAOD,MAAMrB,OAAO;QACpC,IAAI,CAACsB,OAAOC,QAAQ,CAACvB,UAAU;QAC/B,MAAMiB,gBAAgBrC,MAAMoB;QAC5B,MAAMwB,OAAOR,qBAAqBC;QAClC,IAAI,CAACO,QAAQ,EAACH,uBAAAA,MAAMI,aAAa,YAAnBJ,uBAAuB,MAAMG,MAAM;QACjD,MAAME,OAAOT,iBAAiBF,kCAAkC,SAAS;QACzE,IAAIxC,QAAQoD,MAAM,EAAE;YAClBP,KAAKQ,IAAI,CAAC;gBAAExD,SAASiD,MAAMjD,OAAO;gBAAEoD;gBAAME;gBAAMG,SAAS;YAAU;YACnE;QACF;QACA,IAAI;gBAwBgBR,cAECA;YAzBnB,MAAMpC,KAAK,MAAMhC;YACjB,MAAMiC,MAAMD,GAAGE,UAAU,CAAC9B,4BAA4B+B,GAAG,CAACiC,MAAMjD,OAAO;YACvE,MAAM0D,UAAU,MAAM7C,GAAGQ,cAAc,CAAC,OAAOC;oBAQ1CqC;gBAPH,MAAMpC,WAAW,MAAMD,YAAYf,GAAG,CAACO;gBACvC,MAAM6C,UAAWpC,SAASL,MAAM,GAAGK,SAASE,IAAI,KAAK;gBAGrD,IACEkC,CAAAA,2BAAAA,QAAS1D,MAAM,MAAK,cACpB0D,QAAQ/B,OAAO,KAAKA,WACpB,EAAC+B,yBAAAA,QAAQN,aAAa,YAArBM,yBAAyB,MAAMP,MAChC;oBACA,OAAO;gBACT;gBACA9B,YAAYsC,MAAM,CAAC9C,KAAK;oBAAEuC,eAAeD;oBAAMS,iBAAiBrD;gBAAI;gBACpE,OAAO;YACT;YACA,IAAI,CAACkD,SAAS;YACd,MAAM,EAAE3B,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC;YAC5C,MAAM+B,SAAS,MAAM/B,mBAAmB,8BAA8B;gBACpEC,WAAW,GAAGiB,MAAMjD,OAAO,CAAC,CAAC,EAAEoD,MAAM;gBACrCE;gBACArB,SAAS;oBACPC,OAAO7B,QAAO4C,eAAAA,MAAMvC,KAAK,YAAXuC,eAAe,IAAI3C,IAAI,MAAM2C,MAAMjD,OAAO;oBACxDmC,UAAU7C,uBAAuBuD;oBACjC3C,QAAQG,QAAO4C,gBAAAA,MAAM/C,MAAM,YAAZ+C,gBAAgB,IAAI3C,IAAI;gBACzC;YACF;YACA0C,KAAKQ,IAAI,CAAC;gBAAExD,SAASiD,MAAMjD,OAAO;gBAAEoD;gBAAME;gBAAMG,SAASK,OAAOL,OAAO;YAAC;QAC1E,EAAE,OAAOrB,OAAO;YACd,iEAAiE;YACjEC,QAAQD,KAAK,CAAC,CAAC,kBAAkB,EAAEa,MAAMjD,OAAO,CAAC,uBAAuB,CAAC,EAAEoC;QAC7E;IACF;IACA,OAAOY;AACT;AAEA;;;CAGC,GACD,OAAO,SAASe,kBACdC,IAAoC,EACpCC,UAAyB;QAWTD;IAThB,IAAI,CAACA,MAAM;QACT,OAAO;YACL/D,QAAQ;YACRC,QACE+D,eAAe,OACX,iCACA,CAAC,sBAAsB,EAAEA,WAAW,uBAAuB,CAAC;QACpE;IACF;IACA,MAAMC,UAAUF,eAAAA,IAAI,CAAC,SAAS,YAAdA,eAAkB,CAAC;IACnC,MAAMG,UAAUC,OAAOC,OAAO,CAACH,QAC5BI,MAAM,CAAC,CAAC,GAAGpC,MAAM,GAAKA,SAASA,KAAK,CAAC,KAAK,KAAK,MAC/CM,GAAG,CAAC,CAAC,CAAC+B,MAAMrC,MAAM,GACjB,QAAOA,yBAAAA,KAAO,CAAC,OAAO,MAAK,WAAW,GAAGqC,KAAK,EAAE,EAAErC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAGqC;IAEzE,MAAMC,WACJR,IAAI,CAAC,SAAS,KAAK,cAClBA,IAAI,CAAC,SAAS,KAAK,QAASC,CAAAA,eAAe,OAAOE,QAAQM,MAAM,GAAG,CAAA;IACtE,OAAO;QACLxE,QAAQuE,WAAW,aAAa;QAChCtE,QAAQsE,WACJL,QAAQM,MAAM,GACZ,CAAC,SAAS,EAAEN,QAAQO,IAAI,CAAC,MAAM,CAAC,CAAC,GACjC,CAAC,sBAAsB,EAAET,qBAAAA,aAAc,UAAU,CAAC,CAAC,GACrD;IACN;AACF;AAEA;;;CAGC,GACD,OAAO,MAAMU,wBAA0D;IACrEtC,SAAS;IACT,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,iBAAiB;IACjB,wBAAwB;IACxB,oBAAoB;IACpB,uBAAuB;IACvB,yBAAyB;IACzB,yBAAyB;IACzB,iBAAiB;IACjBuC,QAAQ;IACR,uBAAuB;IACvB,iBAAiB;IACjB,sBAAsB;IACtB,2BAA2B;AAC7B,EAAC;AAED;;;;;;CAMC,GACD,OAAO,eAAeC,qBAAqBC,QAAkB;IAC3D,IAAI;;QACF,MAAMd,OAAQ,MAAMc,SAASC,KAAK,GAAGC,IAAI;QACzC,MAAMC,UAAU5E,eAAO2D,wBAAAA,IAAM,CAAC,UAAU,mBAAI,IAAI1D,IAAI;QACpD,IAAI2E,SAAS;gBAGFN;YAFT,MAAM,EAAE1E,MAAM,EAAEC,MAAM,EAAE,GAAG6D,kBAAkBC,MAAMc,SAAS7E,MAAM;YAClE,MAAMF,kBAAkBkF,SAAShF,QAAQC,QAAQ;gBAC/CQ,KAAK,GAAEiE,iCAAAA,qBAAqB,CAACM,QAAQ,YAA9BN,iCAAkCM;YAC3C;QACF;IACF,EAAE,OAAO7C,OAAO;QACdC,QAAQD,KAAK,CAAC,6DAA6DA;IAC7E;IACA,OAAO0C;AACT"}