@smthrs/notifications 0.0.0-stage → 1.0.0-rc.13

package/src/AlertPolicy.ts

@@ -0,0 +1,377 @@
+/**
+ * Alert policy: what a condition is, when a journal leaves one open, and
+ * which open conditions have outlived their delay.
+ *
+ * Everything here is pure and a function of journal time, so the same entries
+ * decide the same alerts in any process at any instant past the delay. The
+ * transport that pages about them is `./layerWebhook.ts` and the journal-backed
+ * tick that drives them is `./AlertRuntime.ts`; `./Alerts.ts` is the module a
+ * consumer imports.
+ *
+ * @since 1.0.0
+ */
+
+import type { JournalEvent } from "@smthrs/journal"
+import { Schema } from "effect"
+
+/**
+ * The journal event type one delivered alert is recorded under.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const deliveredEventType = "flows.alerts.delivered"
+
+/**
+ * The journal event type one failed delivery attempt is recorded under.
+ *
+ * One entry is written per alert per failure code, never one per tick: a
+ * webhook that stays down for an hour leaves one record, not a hundred and
+ * twenty that every later tick has to read past.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const failedEventType = "flows.alerts.failed"
+
+/**
+ * How loud an alert is.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const Severity = Schema.Literals(["info", "warning", "critical"])
+
+/**
+ * How loud an alert is.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Severity = typeof Severity.Type
+
+/**
+ * How a condition is recognized in the journal.
+ *
+ * `field` names a payload key, `value` the value that means the condition
+ * holds, and `eventTypes` narrows which entries are consulted at all. An
+ * `eventTypes` item ending in `.*` names a family: `control.*` consults
+ * `control.run.parked` and `control.monitor.beat` alike. An entry that carries
+ * the field with a different value CLOSES the condition, which is what makes a
+ * resume clear an approval alert without a second vocabulary for "cleared".
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const Detector = Schema.Struct({
+  field: Schema.NonEmptyString,
+  value: Schema.NonEmptyString,
+  eventTypes: Schema.optional(Schema.Array(Schema.NonEmptyString))
+})
+
+/**
+ * How a condition is recognized in the journal.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Detector = typeof Detector.Type
+
+/**
+ * What to do about one condition, and after how long.
+ *
+ * `afterMs` is a whole, non-negative number of milliseconds. The bound is the
+ * schema's job because the delay is also arithmetic on journal time: a `NaN`
+ * delay fires on the first tick and stamps a `firedAt` that JSON writes as
+ * `null`, and a negative one fires with a `firedAt` earlier than the condition
+ * it describes.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const Rule = Schema.Struct({
+  afterMs: Schema.Int.pipe(Schema.check(Schema.isGreaterThanOrEqualTo(0))),
+  severity: Schema.optional(Severity),
+  owner: Schema.optional(Schema.String),
+  runbook: Schema.optional(Schema.String)
+})
+
+/**
+ * What to do about one condition, and after how long.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Rule = typeof Rule.Type
+
+/**
+ * The conditions a policy alerts on and the delays it alerts after.
+ *
+ * `defaults` fills in what a rule leaves out, so a policy states the delay per
+ * condition and the ownership once.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export const Policy = Schema.Struct({
+  defaults: Schema.optional(Schema.Struct({
+    severity: Schema.optional(Severity),
+    owner: Schema.optional(Schema.String),
+    runbook: Schema.optional(Schema.String)
+  })),
+  rules: Schema.Record(Schema.String, Rule),
+  /** Detectors for conditions this deployment names itself. */
+  detectors: Schema.optional(Schema.Record(Schema.String, Detector))
+})
+
+/**
+ * The conditions a policy alerts on and the delays it alerts after.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Policy = typeof Policy.Type
+
+/**
+ * The four conditions a control plane journals out of the box.
+ *
+ * `status` rides on every `control.run.*` entry, and `health` on every
+ * `control.monitor.beat`. A deployment whose supervisor journals a park reason
+ * gets `quota-parked` from the same field the run summary reports it under.
+ *
+ * Each consults the `control.*` family only. A run's journal is shared with
+ * flow steps, agent traces, and anything else that writes to the run, and a
+ * `status` or `health` field on one of those entries would otherwise open or
+ * close a condition the control plane never reported.
+ *
+ * @category constants
+ * @since 0.1.0
+ */
+export const defaultDetectors: Readonly<Record<string, Detector>> = {
+  "waiting-approval": { field: "status", value: "waiting-approval", eventTypes: ["control.*"] },
+  failed: { field: "status", value: "failed", eventTypes: ["control.*"] },
+  stalled: { field: "health", value: "stalled", eventTypes: ["control.*"] },
+  "quota-parked": { field: "waitingReason", value: "quota", eventTypes: ["control.*"] }
+}
+
+/**
+ * Whether a detector's `eventTypes` names an entry's event type, exactly or
+ * through a `.*` family.
+ */
+const names = (eventTypes: ReadonlyArray<string>, eventType: string): boolean =>
+  eventTypes.some((named) => named.endsWith(".*") ? eventType.startsWith(named.slice(0, -1)) : named === eventType)
+
+/**
+ * A condition that is open on a run, and when it opened.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Open {
+  readonly runId: string
+  readonly condition: string
+  /** The journal time of the entry that opened the condition. */
+  readonly since: number
+}
+
+/**
+ * One alert a policy decided to raise.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Alert {
+  readonly runId: string
+  readonly condition: string
+  readonly since: number
+  /**
+   * The journal instant the condition outlived its delay: `since` plus the
+   * rule's `afterMs`, never the wall clock of whichever tick noticed.
+   */
+  readonly firedAt: number
+  readonly severity: Severity
+  readonly coalescingKey: string
+  readonly owner?: string | undefined
+  readonly runbook?: string | undefined
+}
+
+/**
+ * The journal source the alerter writes one outcome of one alert under.
+ *
+ * A delivery record counts as evidence that an alert went out only under this
+ * source, so another producer's `flows.alerts.delivered` record with a
+ * matching `alertId` does not suppress a page. This scopes records; it does
+ * not authenticate the writer, because the journal does not bind a source to
+ * a writer.
+ *
+ * @param id the alert's {@link alertId}
+ * @param outcome `delivered`, `refused`, or `failed/<code>`
+ * @private
+ * @since 1.0.0
+ */
+export const recordSourceId = (id: string, outcome: string): string => `/notifications/alerts/${id}/${outcome}`
+
+/**
+ * The key an alert coalesces on: one open condition on one run.
+ *
+ * Each component is percent-encoded before it is joined, so a run id or a
+ * condition name containing the separator cannot forge another pair's key and
+ * suppress a page that belongs to somebody else. Both components are values a
+ * deployment chooses, which is why the encoding is not optional.
+ *
+ * @param runId the run the condition is open on
+ * @param condition the condition name the policy uses
+ * @category getters
+ * @since 1.0.0
+ */
+export const coalescingKey = (runId: string, condition: string): string =>
+  `${encodeURIComponent(runId)}:${encodeURIComponent(condition)}`
+
+/**
+ * The identity a delivery is recorded under.
+ *
+ * The opening time is part of it on purpose. A condition that clears and
+ * re-opens is a NEW alert, because the second approval wait is not the first
+ * one, and a key without the time would suppress it forever.
+ *
+ * @param alert the alert
+ * @category getters
+ * @since 0.1.0
+ */
+export const alertId = (alert: Pick<Alert, "coalescingKey" | "since">): string =>
+  `alert:${alert.coalescingKey}:${alert.since}`
+
+/**
+ * The detectors one policy decides with: the four this package ships, with
+ * the deployment's own overriding them by name.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const detectorsOf = (policy: Policy): Readonly<Record<string, Detector>> => ({
+  ...defaultDetectors,
+  ...policy.detectors
+})
+
+/**
+ * One entry's payload when it is a record a detector can read a field off,
+ * and `undefined` for the arrays and primitives that carry no fields at all.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const payloadRecord = (payload: unknown): Readonly<Record<string, unknown>> | undefined =>
+  typeof payload === "object" && payload !== null && !Array.isArray(payload)
+    ? payload as Readonly<Record<string, unknown>>
+    : undefined
+
+/**
+ * Folds one entry into the open-condition map. Shared by {@link conditions}
+ * and the runtime's incremental read, so both decide a condition the same
+ * way.
+ *
+ * The alerter's own records are never evidence about a condition. They are
+ * written into the same journal they are read from, and they carry the alert's
+ * own vocabulary: a refusal reports the answering HTTP `status`, which a
+ * detector watching the run's `status` would read as the condition clearing.
+ * A page would then close the condition it paged about and the next tick would
+ * re-open it, so a webhook that answered 503 once would alert forever.
+ *
+ * @private
+ * @since 1.0.0
+ */
+export const observe = (
+  policy: Policy,
+  detectors: Readonly<Record<string, Detector>>,
+  entry: JournalEvent.Entry,
+  payload: Readonly<Record<string, unknown>>,
+  open: Map<string, number>
+): void => {
+  if (entry.eventType === deliveredEventType || entry.eventType === failedEventType) return
+  for (const condition of Object.keys(policy.rules)) {
+    const detector = detectors[condition]
+    if (detector === undefined) continue
+    if (detector.eventTypes !== undefined && !names(detector.eventTypes, entry.eventType)) continue
+    // Own properties only. `in` walks `Object.prototype`, so a detector named
+    // `toString` or `constructor` would read every record-shaped entry in the
+    // run as evidence and close the condition on all of them.
+    if (!Object.hasOwn(payload, detector.field)) continue
+    if (payload[detector.field] === detector.value) {
+      if (!open.has(condition)) open.set(condition, entry.emittedAtMs)
+    } else {
+      open.delete(condition)
+    }
+  }
+}
+
+/**
+ * Every condition a run's journal leaves open, with the time each opened.
+ *
+ * An entry that carries a detector's field opens the condition when the value
+ * matches and closes it when it does not. Entries the detector does not name
+ * are ignored entirely, so a monitor beat cannot close an approval wait and an
+ * approval cannot close a stall.
+ *
+ * @param policy the policy whose detectors decide what counts
+ * @param runId the run the entries belong to
+ * @param entries the run's journal, oldest first
+ * @category projections
+ * @since 0.1.0
+ */
+export const conditions = (
+  policy: Policy,
+  runId: string,
+  entries: ReadonlyArray<JournalEvent.Entry>
+): ReadonlyArray<Open> => {
+  const detectors = detectorsOf(policy)
+  const open = new Map<string, number>()
+  for (const entry of entries) {
+    const payload = payloadRecord(entry.payload)
+    if (payload === undefined) continue
+    observe(policy, detectors, entry, payload, open)
+  }
+  return Array.from(open, ([condition, since]) => ({ runId, condition, since }))
+}
+
+/**
+ * The alerts a policy raises for the conditions open at `now`.
+ *
+ * Pure, and a function of journal time: the same journal raises the same
+ * alerts in any process, at any instant past the delay. `now` decides WHETHER
+ * an alert is raised; it never appears in one. A condition that has been open
+ * for less than its rule's delay raises nothing, which is the whole point of
+ * the delay, because most stalls clear themselves.
+ *
+ * @param policy the policy
+ * @param open the conditions the journal left open
+ * @param now the instant to judge them at
+ * @category projections
+ * @since 0.1.0
+ */
+export const decide = (
+  policy: Policy,
+  open: ReadonlyArray<Open>,
+  now: number
+): ReadonlyArray<Alert> =>
+  open.flatMap((condition) => {
+    const rule = policy.rules[condition.condition]
+    if (rule === undefined) return []
+    if (now - condition.since < rule.afterMs) return []
+    const severity = rule.severity ?? policy.defaults?.severity ?? "warning"
+    const owner = rule.owner ?? policy.defaults?.owner
+    const runbook = rule.runbook ?? policy.defaults?.runbook
+    return [{
+      runId: condition.runId,
+      condition: condition.condition,
+      since: condition.since,
+      // Derived from the journal, not read off the clock. The alert id is
+      // stable across ticks, and the queue refuses a reused id whose content
+      // changed, so an alert stamped with the reading time would become
+      // permanently undeliverable the moment a tick refused and time moved on.
+      firedAt: condition.since + rule.afterMs,
+      severity,
+      coalescingKey: coalescingKey(condition.runId, condition.condition),
+      ...(owner === undefined ? {} : { owner }),
+      ...(runbook === undefined ? {} : { runbook })
+    }]
+  })

package/src/AlertRuntime.ts

@@ -0,0 +1,280 @@
+/**
+ * The journal-backed alert runtime: one tick per run.
+ *
+ * A tick folds the run's journal into the conditions it leaves open, decides
+ * which have outlived their delay, admits each one as a coalesced system
+ * event, pages the sink, and journals what the sink did with it. The fold is
+ * retained per run, so a tick costs the entries committed since the previous
+ * one rather than the run's whole journal.
+ *
+ * The rules it enforces are `./AlertPolicy.ts` and the channel it pages over
+ * is `./AlertSink.ts`; neither knows this module exists.
+ *
+ * @since 1.0.0
+ */
+
+import { Journal, JournalEvent } from "@smthrs/journal"
+import { Clock, Context, Effect, HashSet, Layer, Result, Schema } from "effect"
+import {
+  type Alert,
+  alertId,
+  decide,
+  deliveredEventType,
+  detectorsOf,
+  failedEventType,
+  observe,
+  payloadRecord,
+  Policy,
+  recordSourceId
+} from "./AlertPolicy.ts"
+import { Sink } from "./AlertSink.ts"
+import * as FoldCache from "./internal/foldCache.ts"
+import type * as NotificationModel from "./Notification.ts"
+import { type NotificationError, NotificationQueue } from "./NotificationQueue.ts"
+
+/**
+ * What one tick decided.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export interface Tick {
+  /** Alerts raised and delivered on this tick. */
+  readonly delivered: ReadonlyArray<Alert>
+  /** Alerts the sink refused on this tick. They are retried on the next one. */
+  readonly failed: ReadonlyArray<Alert>
+  /**
+   * Alerts the notification queue refused because the run is at capacity. The
+   * sink was not called for them, and they are retried on the next tick.
+   */
+  readonly refused: ReadonlyArray<Alert>
+  /** Alerts that had already been delivered, and were not delivered again. */
+  readonly suppressed: ReadonlyArray<Alert>
+}
+
+/**
+ * The alert runtime: one tick per run.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export interface RuntimeService {
+  /**
+   * Reads one run's journal and pages about whatever has waited too long.
+   *
+   * The two failures stay apart because they mean different things to an
+   * operator. A `JournalError` is the alerter's own record channel failing: it
+   * read no entries, or it could not write the delivery record. A
+   * `NotificationError` is the notification queue rejecting the alert outright,
+   * which is a producer or storage fault rather than a full queue. Neither is
+   * a sink failure: a refused page is journaled as `flows.alerts.failed` and
+   * retried on the next tick, and it comes back in {@link Tick.failed}. A queue
+   * at capacity is not a failure either; it comes back in {@link Tick.refused}.
+   */
+  readonly tick: (runId: string) => Effect.Effect<Tick, Journal.JournalError | NotificationError>
+}
+
+/**
+ * The {@link RuntimeService} tag.
+ *
+ * @category services
+ * @since 0.1.0
+ */
+export class AlertRuntime extends Context.Service<AlertRuntime, RuntimeService>()(
+  "/notifications/AlertRuntime"
+) {}
+
+/**
+ * One run's folded alert history: the conditions still open, the alerts
+ * already paged, and the sequence the fold stopped at.
+ */
+interface Watched {
+  readonly open: ReadonlyMap<string, number>
+  /**
+   * The alerts already paged, as a persistent set: a tick adds a node and
+   * leaves the fold the previous tick published untouched, where copying the
+   * set would cost the run's whole delivery history on every tick.
+   */
+  readonly delivered: HashSet.HashSet<string>
+  readonly cursor: number | undefined
+}
+
+/**
+ * How many runs one alert runtime keeps folded at a time. The retention policy
+ * is scan-resistant, so a sweep of more runs than this reads journal tails for
+ * all but a fixed handful of them.
+ */
+const maximumWatchedRuns = 64
+
+const alertNotification = (alert: Alert): NotificationModel.Notification => ({
+  _tag: "system-event",
+  id: alertId(alert),
+  targetLineageId: alert.runId,
+  delivery: "queue",
+  coalescingKey: alert.coalescingKey,
+  provenance: {
+    sourceRunId: alert.runId,
+    sourceLineageId: alert.runId,
+    sourceTurn: 0,
+    sourceActor: "alerts"
+  },
+  payload: {
+    condition: alert.condition,
+    severity: alert.severity,
+    since: alert.since,
+    firedAt: alert.firedAt,
+    ...(alert.owner === undefined ? {} : { owner: alert.owner }),
+    ...(alert.runbook === undefined ? {} : { runbook: alert.runbook })
+  }
+})
+
+/**
+ * The alert runtime over one policy.
+ *
+ * A tick reads the run's journal, derives the open conditions, decides which
+ * have outlived their delay, and for each one that has not already been
+ * delivered: admits a coalesced system event, sends it to the sink, and
+ * journals the delivery. The admission comes first because it is idempotent
+ * and the send is not: a crash between them costs a duplicate admission, which
+ * the queue drops, while a crash between an accepted send and the delivery
+ * record costs a duplicate PAGE, which is why {@link SinkService.deliver} is
+ * required to dedupe on {@link alertId}.
+ *
+ * The policy is decoded when the layer is built, so a rule with an impossible
+ * delay fails the composition by name instead of mis-paging at 3am.
+ *
+ * @param policy the rules to enforce
+ * @category layers
+ * @since 0.1.0
+ */
+export const layer = (
+  policy: Policy
+): Layer.Layer<AlertRuntime, never, Journal.Journal | NotificationQueue | Sink> =>
+  Layer.effect(
+    AlertRuntime,
+    Effect.gen(function*() {
+      const checked = yield* Effect.orDie(Schema.decodeUnknownEffect(Policy)(policy))
+      const detectors = detectorsOf(checked)
+      const journal = yield* Journal.Journal
+      const queue = yield* NotificationQueue
+      const sink = yield* Sink
+      // Folded history per run, so a tick costs the entries committed since
+      // the previous one rather than the run's whole journal.
+      const watched = FoldCache.make<Watched>(maximumWatchedRuns)
+
+      const observed = (runId: JournalEvent.RunId): Effect.Effect<Watched, Journal.JournalError> =>
+        Effect.gen(function*() {
+          const base: Watched = watched.get(runId) ??
+            { open: new Map(), delivered: HashSet.empty(), cursor: undefined }
+          const fresh: Array<JournalEvent.Entry> = []
+          let after = base.cursor === undefined ? undefined : JournalEvent.Seq.make(base.cursor)
+          while (true) {
+            const page = yield* journal.entries({ runId, ...(after === undefined ? {} : { after }), limit: 512 })
+            fresh.push(...page.entries)
+            if (!page.hasMore || page.entries.length === 0) break
+            after = page.entries.at(-1)!.seq
+          }
+          if (fresh.length === 0) return base
+
+          // `open` is bounded by the policy's rules, so it is copied; the
+          // delivery history is not, so it grows by one persistent node.
+          const open = new Map(base.open)
+          let delivered = base.delivered
+          let cursor = base.cursor
+          for (const entry of fresh) {
+            cursor = entry.seq
+            const payload = payloadRecord(entry.payload)
+            if (payload === undefined) continue
+            if (entry.eventType === deliveredEventType) {
+              const id = payload["alertId"]
+              // Only a record under the source the alerter writes counts, so
+              // another producer's delivered record cannot suppress a page.
+              // The journal does not bind a source to a writer.
+              if (typeof id === "string" && entry.sourceId === recordSourceId(id, "delivered")) {
+                delivered = HashSet.add(delivered, id)
+              }
+            }
+            observe(checked, detectors, entry, payload, open)
+          }
+          const next: Watched = { open, delivered, cursor }
+          // Published at commit, exactly as `NotificationQueue.load` publishes
+          // its fold. A tick inside an enclosing transaction reads that
+          // transaction's uncommitted rows, and caching them would outlive the
+          // rollback that erased them: the run would page about a condition
+          // durable history never held.
+          yield* journal.whenCommitted(Effect.sync(() => watched.put(runId, next)))
+          return next
+        })
+
+      const record = (
+        eventType: string,
+        alert: Alert,
+        source: string,
+        outcome: { readonly code?: string | undefined; readonly status?: number | undefined }
+      ): Effect.Effect<void, Journal.JournalError> =>
+        journal.emitDurableUnfenced(
+          new JournalEvent.Input({
+            runId: JournalEvent.RunId.make(alert.runId),
+            sourceId: JournalEvent.SourceId.make(recordSourceId(alertId(alert), source)),
+            sourceSeq: JournalEvent.SourceSeq.make(0),
+            // One record per alert per outcome. Without an explicit identity
+            // the journal allocates a new sequence on every attempt, and a
+            // webhook that stays down appends a row per tick forever.
+            dedupe: "identity",
+            eventType,
+            payload: {
+              runId: alert.runId,
+              condition: alert.condition,
+              since: alert.since,
+              severity: alert.severity,
+              alertId: alertId(alert),
+              ...(outcome.code === undefined ? {} : { code: outcome.code }),
+              ...(outcome.status === undefined ? {} : { status: outcome.status })
+            }
+          })
+        ).pipe(Effect.asVoid)
+
+      return AlertRuntime.of({
+        tick: Effect.fn("AlertRuntime.tick")(function*(runId: string) {
+          const journalRunId = JournalEvent.RunId.make(runId)
+          const history = yield* observed(journalRunId)
+          const now = yield* Clock.currentTimeMillis
+          const open = Array.from(history.open, ([condition, since]) => ({ runId, condition, since }))
+          const raised = decide(checked, open, now)
+          const delivered: Array<Alert> = []
+          const failed: Array<Alert> = []
+          const refused: Array<Alert> = []
+          const suppressed: Array<Alert> = []
+          for (const alert of raised) {
+            if (HashSet.has(history.delivered, alertId(alert))) {
+              suppressed.push(alert)
+              continue
+            }
+            // Raised unchanged: a queue that rejects the admission outright is
+            // not the journal failing, and calling it `sink_failed` would send
+            // an operator looking at the webhook.
+            const receipt = yield* queue.admit(runId, alertNotification(alert))
+            if (receipt.decision === "rejected-full") {
+              // The alert has nowhere durable to sit, so paging about it would
+              // tell an operator about something the run will never read.
+              yield* record(failedEventType, alert, "refused", { code: "notification_full" })
+              refused.push(alert)
+              continue
+            }
+            const outcome = yield* Effect.result(sink.deliver(alert))
+            if (Result.isFailure(outcome)) {
+              yield* record(failedEventType, alert, `failed/${outcome.failure.code}`, {
+                code: outcome.failure.code,
+                status: outcome.failure.status
+              })
+              failed.push(alert)
+            } else {
+              yield* record(deliveredEventType, alert, "delivered", {})
+              delivered.push(alert)
+            }
+          }
+          return { delivered, failed, refused, suppressed }
+        })
+      })
+    })
+  )