@openlup/platform 0.28.0 → 0.30.0

package/src/operations/server/controlPlaneHandler.ts

@@ -0,0 +1,56 @@
+import { createBffResponder, type BffResponderOptions, type BffResponder } from "../../bff/index.js";
+import type { HttpRequest, HttpResponse } from "../../http/index.js";
+import { platformControlPlaneMutationResponseSchema, platformControlPlaneMutationSchema, platformControlPlaneReadResponseSchema } from "../index.js";
+import { PlatformControlPlaneConflictError } from "../index.js";
+import type { PlatformControlPlanePort } from "../contracts/index.js";
+
+export function createPlatformControlPlaneHandler(deps: {
+  port: PlatformControlPlanePort;
+  authorizeAdmin: () => Promise<boolean>;
+} & BffResponderOptions) {
+  const { sendBffError, sendBffSuccess, sendMethodNotAllowed } = createBffResponder(deps);
+  return async (req: HttpRequest, res: HttpResponse): Promise<void> => {
+    if (!(await deps.authorizeAdmin())) {
+      sendBffError(res, "UNAUTHORIZED", "Admin session required");
+      return;
+    }
+    try {
+      if (req.method === "GET") {
+        const parsed = platformControlPlaneReadResponseSchema.safeParse(
+          await deps.port.readControlPlane({}),
+        );
+        if (!parsed.success) return invalidResponse(res, sendBffError);
+        sendBffSuccess(res, parsed.data);
+        return;
+      }
+      if (req.method === "POST") {
+        const parsed = platformControlPlaneMutationSchema.safeParse(body(req.body));
+        if (!parsed.success) {
+          sendBffError(res, "BAD_REQUEST", "Invalid platform control-plane request");
+          return;
+        }
+        const response = platformControlPlaneMutationResponseSchema.safeParse(
+          await deps.port.mutateControlPlane(parsed.data),
+        );
+        if (!response.success) return invalidResponse(res, sendBffError);
+        sendBffSuccess(res, response.data);
+        return;
+      }
+      sendMethodNotAllowed(res, ["GET", "POST"]);
+    } catch (error) {
+      if (error instanceof PlatformControlPlaneConflictError) {
+        sendBffError(res, "CONFLICT", error.message);
+        return;
+      }
+      sendBffError(res, "UPSTREAM_UNAVAILABLE", "Platform control plane unavailable");
+    }
+  };
+}
+
+function body(value: unknown): unknown {
+  if (typeof value !== "string") return value;
+  try { return JSON.parse(value); } catch { return null; }
+}
+function invalidResponse(res: HttpResponse, sendBffError: BffResponder["sendBffError"]): void {
+  sendBffError(res, "INVALID_RESPONSE", "Platform control plane returned invalid response");
+}

package/src/operations/server/controlPlaneStore.ts

@@ -0,0 +1,150 @@
+import { PlatformControlPlaneConflictError, PlatformControlPlaneUnavailableError } from "../index.js";
+import { createHash } from "node:crypto";
+
+import type {
+  PlatformControlPlaneMutation,
+  PlatformControlPlaneMutationResponse,
+  PlatformControlPlaneReadResponse,
+} from "../contracts/index.js";
+import {
+  type PlatformControlPlanePort,
+} from "../contracts/index.js";
+import type { PgQueryExecutor } from "../../data/index.js";
+
+const rpc = (name: string, types: string[]) =>
+  `SELECT * FROM public.${name}(${types.map((type, index) => `$${index + 1}::${type}`).join(", ")})`;
+
+export const platformControlPlaneSql = {
+  active: rpc("platform_control_operator_is_active", ["uuid"]),
+  setControl: rpc("platform_control_set_control", ["uuid", "text", "text", "text", "boolean"]),
+  heartbeat: rpc("platform_control_record_heartbeat", ["uuid", "text", "text", "text", "integer", "text", "boolean"]),
+  controls: rpc("platform_control_list_controls", ["uuid"]),
+  jobs: rpc("platform_control_list_jobs", ["uuid"]),
+  alerts: rpc("platform_control_list_alerts", ["uuid"]),
+} as const;
+
+export function createPlatformControlPlanePort(
+  client: PgQueryExecutor,
+  operatorId: string,
+): PlatformControlPlanePort {
+  return {
+    async readControlPlane() {
+      try {
+        const [controls, jobs, alerts] = await Promise.all([
+          client.query(platformControlPlaneSql.controls, [operatorId]),
+          client.query(platformControlPlaneSql.jobs, [operatorId]),
+          client.query(platformControlPlaneSql.alerts, [operatorId]),
+        ]);
+        return {
+          controls: controls.rows.map((row) => ({
+            controlKey: text(row.control_key), enabled: bool(row.enabled),
+            revision: positive(row.revision), updatedAt: timestamp(row.updated_at),
+          })),
+          jobs: jobs.rows.map((row) => ({
+            jobName: text(row.job_name), enabled: bool(row.enabled),
+            activeDriver: driver(row.active_driver), lastStatus: jobStatus(row.last_status),
+            lastStartedAt: nullableTimestamp(row.last_started_at),
+            lastFinishedAt: nullableTimestamp(row.last_finished_at),
+            lastSuccessAt: nullableTimestamp(row.last_success_at),
+          })),
+          alerts: alerts.rows.map((row) => ({
+            dedupeKey: text(row.dedupe_key), severity: severity(row.severity),
+            status: alertStatus(row.status), owner: text(row.owner), title: text(row.title),
+            firstSeenAt: timestamp(row.first_seen_at), lastSeenAt: timestamp(row.last_seen_at),
+            resolvedAt: nullableTimestamp(row.resolved_at),
+          })),
+        } satisfies PlatformControlPlaneReadResponse;
+      } catch (error) {
+        throw mapError(error);
+      }
+    },
+
+    async mutateControlPlane(request) {
+      try {
+        const fingerprint = digest(JSON.stringify(request));
+        const result = request.action === "set-control"
+          ? await client.query(platformControlPlaneSql.setControl, [
+              operatorId, request.idempotencyKey, fingerprint, request.controlKey, request.enabled,
+            ])
+          : await client.query(platformControlPlaneSql.heartbeat, [
+              operatorId, request.idempotencyKey, fingerprint, request.health,
+              request.firingCount, request.maxSeverity, request.promotionReady,
+            ]);
+        return mutation(first(result.rows), request.action);
+      } catch (error) {
+        throw mapError(error);
+      }
+    },
+  };
+}
+
+export async function isPlatformControlOperatorActive(
+  client: PgQueryExecutor,
+  principalId: string,
+): Promise<boolean> {
+  try {
+    return first((await client.query(platformControlPlaneSql.active, [principalId])).rows)
+      ?.platform_control_operator_is_active === true;
+  } catch (error) {
+    throw mapError(error);
+  }
+}
+
+function mutation(
+  row: Record<string, unknown> | undefined,
+  expected: PlatformControlPlaneMutation["action"],
+): PlatformControlPlaneMutationResponse {
+  if (!row || row.action !== expected) throw new PlatformControlPlaneUnavailableError();
+  return {
+    action: expected,
+    replayed: bool(row.replayed),
+    revision: row.revision == null ? null : positive(row.revision),
+    recordedAt: timestamp(row.recorded_at),
+  };
+}
+
+function digest(value: string): string {
+  return createHash("sha256").update(value).digest("hex");
+}
+function first(rows: Record<string, unknown>[]): Record<string, unknown> | undefined { return rows[0]; }
+function text(value: unknown): string {
+  if (typeof value !== "string" || value.length === 0) throw new PlatformControlPlaneUnavailableError();
+  return value;
+}
+function bool(value: unknown): boolean {
+  if (typeof value !== "boolean") throw new PlatformControlPlaneUnavailableError();
+  return value;
+}
+function positive(value: unknown): number {
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 1) throw new PlatformControlPlaneUnavailableError();
+  return value;
+}
+function timestamp(value: unknown): string {
+  const result = text(value);
+  if (!Number.isFinite(Date.parse(result))) throw new PlatformControlPlaneUnavailableError();
+  return result;
+}
+function nullableTimestamp(value: unknown): string | null { return value == null ? null : timestamp(value); }
+function driver(value: unknown): "worker" | "scheduler" | "operator" {
+  if (value === "worker" || value === "scheduler" || value === "operator") return value;
+  throw new PlatformControlPlaneUnavailableError();
+}
+function jobStatus(value: unknown): "running" | "success" | "failed" | "skipped" | null {
+  if (value == null) return null;
+  if (value === "running" || value === "success" || value === "failed" || value === "skipped") return value;
+  throw new PlatformControlPlaneUnavailableError();
+}
+function severity(value: unknown): "p0" | "p1" | "p2" | "p3" {
+  if (value === "p0" || value === "p1" || value === "p2" || value === "p3") return value;
+  throw new PlatformControlPlaneUnavailableError();
+}
+function alertStatus(value: unknown): "open" | "acknowledged" | "resolved" {
+  if (value === "open" || value === "acknowledged" || value === "resolved") return value;
+  throw new PlatformControlPlaneUnavailableError();
+}
+function mapError(error: unknown): Error {
+  if (error instanceof PlatformControlPlaneUnavailableError || error instanceof PlatformControlPlaneConflictError) return error;
+  const code = typeof error === "object" && error && "code" in error ? (error as { code?: unknown }).code : undefined;
+  if (code === "23505") return new PlatformControlPlaneConflictError("platform_control_idempotency_conflict");
+  return new PlatformControlPlaneUnavailableError("platform_control_plane_unavailable");
+}

package/src/operations/server/controlPlaneWatchdog.ts

@@ -0,0 +1,28 @@
+import type { PlatformControlPlanePort } from "../contracts/index.js";
+
+export type DirectPlatformWatchdogResult = { status: number; body: Record<string, unknown> };
+
+/** Existing public-bundle evaluation of persisted neutral job facts. No provider calls. */
+export async function runControlPlaneWatchdogTick(
+  port: PlatformControlPlanePort,
+  now: Date,
+  checkOnly: boolean,
+): Promise<DirectPlatformWatchdogResult> {
+  const snapshot = await port.readControlPlane({});
+  const failedJobs = snapshot.jobs.filter((job) => job.enabled && job.lastStatus === "failed");
+  const health = failedJobs.length > 0 ? "degraded" as const : "healthy" as const;
+  const recorded = checkOnly ? null : await port.mutateControlPlane({
+    action: "record-heartbeat",
+    idempotencyKey: `platform-watchdog:${now.toISOString()}`,
+    health,
+    firingCount: failedJobs.length,
+    maxSeverity: failedJobs.length > 0 ? "p2" : null,
+    promotionReady: failedJobs.length === 0,
+  });
+  return { status: 200, body: {
+    ok: true, health, firingCount: failedJobs.length,
+    failedJobs: failedJobs.map((job) => job.jobName),
+    checkedAt: now.toISOString(), heartbeatRecorded: recorded != null,
+    replayed: recorded?.replayed ?? false,
+  } };
+}

package/src/operations/server/index.ts

@@ -0,0 +1,12 @@
+/** Server mechanisms; authorization, persistence and provider transports are injected. */
+export * from "./adminSettingsHandlers.js";
+export * from "./adminPipelineHandlers.js";
+export * from "./adminAlertsHandlers.js";
+export * from "./controlPlaneHandler.js";
+export * from "./controlPlaneStore.js";
+export * from "./platformWatchdogService.js";
+export * from "./platformWatchdogDecision.js";
+export * from "./platformWatchdogDeliveryPolicy.js";
+export * from "./platformWatchdogQueue.js";
+export * from "./controlPlaneWatchdog.js";
+export * from "./migrations.js";

package/src/operations/server/migrations.ts

@@ -0,0 +1,7 @@
+/** Sparse projection only: install the complete canonical native prefix, never this unit alone. */
+export const OPERATIONS_MIGRATIONS = {
+  native: { units: [{ position: 28, catalogPath: "db/platform/migrations/20260814010000_platform_control_plane.sql",
+    sha256: "b0c8ef6424ebc6f1a84cb132c37908f4831fa40484a729c3713cfb35936559c9",
+    url: new URL("../../../sql/20260814010000_platform_control_plane.sql", import.meta.url) }] },
+  managed: { verified: false },
+} as const;

package/src/operations/server/platformWatchdogDecision.ts

@@ -0,0 +1,93 @@
+import type {
+  AlertDecision,
+  AlertNotificationOutcome,
+  AlertSeverity,
+} from "../contracts/index.js";
+import type { AlertLedgerPort, AlertSinkPort } from "../contracts/index.js";
+import { shouldPage } from "../index.js";
+import { throttleSecondsForSeverity } from "../index.js";
+import {
+  isActionabilitySuppressed,
+  isDeliveryBackoff,
+  isPagingSuppressed,
+  notificationSchedule,
+  shouldAttemptNotification,
+} from "./platformWatchdogDeliveryPolicy.js";
+
+const BELOW_PAGING_SEVERITY_OUTCOME: AlertNotificationOutcome<"webhook"> = {
+  channel: "webhook", status: "skipped", provider: "webhook", error: "below_paging_severity",
+};
+const NON_PAGEABLE_DIAGNOSTIC_OUTCOME: AlertNotificationOutcome<"webhook"> = {
+  channel: "webhook", status: "skipped", provider: "webhook", error: "non_pageable_diagnostic",
+};
+
+export function emptyWatchdogDecisionCounts() {
+  return {
+    notified: 0,
+    notificationFailures: 0,
+    deliveryBackoffCount: 0,
+    throttledNotifications: 0,
+    skippedNotifications: 0,
+    belowThreshold: 0,
+    suppressed: 0,
+    actionableCriticalCount: 0,
+    suppressedCriticalCount: 0,
+    actionablePageableCount: 0,
+  };
+}
+
+/** One complete ledger transition; callers must serialize the same dedupe key. */
+export async function processWatchdogDecision<Channel extends string>(
+  decision: AlertDecision<Channel>,
+  {
+    ledgerPort, sinkPort, now, pagingMinSeverity, notificationThrottleSeconds,
+  }: {
+    ledgerPort: AlertLedgerPort<Channel>;
+    sinkPort: AlertSinkPort<Channel>;
+    now: Date;
+    pagingMinSeverity: AlertSeverity;
+    notificationThrottleSeconds?: number;
+  },
+) {
+  const counts = emptyWatchdogDecisionCounts();
+  const alert = await ledgerPort.upsertOpenAlert(decision, now);
+  const critical = isCriticalDecision(decision);
+  const actionableSuppressed = isActionabilitySuppressed(alert, now);
+  const pagingSuppressed = isPagingSuppressed(alert, now);
+  const pageable = decision.paging !== "never" && shouldPage(decision.severity, pagingMinSeverity);
+
+  // Acknowledgement silences pages, but only a snooze suppresses actionability.
+  if (critical) {
+    if (actionableSuppressed) counts.suppressedCriticalCount += 1;
+    else counts.actionableCriticalCount += 1;
+  }
+  if (pageable && !pagingSuppressed) counts.actionablePageableCount += 1;
+  if (pageable && !pagingSuppressed && isDeliveryBackoff(alert, now)) counts.deliveryBackoffCount += 1;
+  if (pagingSuppressed) {
+    counts.suppressed += 1;
+    return counts;
+  }
+
+  const throttleSeconds = notificationThrottleSeconds ?? throttleSecondsForSeverity(decision.severity);
+  if (!shouldAttemptNotification(alert, now, throttleSeconds)) {
+    counts.throttledNotifications += 1;
+    return counts;
+  }
+
+  // Non-pageable and below-threshold alerts retain their skipped-attempt audit.
+  const nonPageable = decision.paging === "never";
+  const paging = !nonPageable && shouldPage(decision.severity, pagingMinSeverity);
+  const outcome = paging
+    ? await sinkPort.send(decision, alert)
+    : nonPageable ? NON_PAGEABLE_DIAGNOSTIC_OUTCOME : BELOW_PAGING_SEVERITY_OUTCOME;
+  if (outcome.status === "sent") counts.notified += 1;
+  else if (outcome.status === "failed") counts.notificationFailures += 1;
+  else counts.skippedNotifications += 1;
+  if (!paging) counts.belowThreshold += 1;
+  await ledgerPort.recordNotification(alert.id, outcome, now, notificationSchedule(alert, outcome, now, throttleSeconds));
+  return counts;
+}
+
+export function isCriticalDecision(decision: AlertDecision): boolean {
+  return decision.paging !== "never" && (decision.severity === "p0" || decision.severity === "p1");
+}

package/src/operations/server/platformWatchdogDeliveryPolicy.ts

@@ -0,0 +1,92 @@
+import type {
+  AlertDecision,
+  AlertNotificationOutcome,
+  OpenAlert,
+} from "../contracts/index.js";
+
+const FAILURE_RETRY_INITIAL_SECONDS = 10 * 60;
+const FAILURE_RETRY_MAX_SECONDS = 60 * 60;
+
+export function shouldAttemptNotification(
+  alert: Pick<
+    OpenAlert,
+    "lastNotifiedAt" | "lastNotificationStatus" | "nextNotificationAttemptAt" | "snoozedUntil"
+  >,
+  now: Date,
+  throttleSeconds: number,
+): boolean {
+  const snoozeExpiry = alert.snoozedUntil ? Date.parse(alert.snoozedUntil) : Number.NaN;
+  const lastSentAt = alert.lastNotifiedAt ? Date.parse(alert.lastNotifiedAt) : Number.NaN;
+  if (
+    alert.lastNotificationStatus !== "failed"
+    && Number.isFinite(snoozeExpiry)
+    && snoozeExpiry <= now.getTime()
+    && (!Number.isFinite(lastSentAt) || snoozeExpiry > lastSentAt)
+  ) {
+    return true;
+  }
+  const nextAttemptAt = alert.nextNotificationAttemptAt
+    ? Date.parse(alert.nextNotificationAttemptAt)
+    : Number.NaN;
+  if (Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime()) return false;
+  if (!alert.lastNotifiedAt) return true;
+  return now.getTime() - new Date(alert.lastNotifiedAt).getTime() >= throttleSeconds * 1000;
+}
+
+export function notificationSchedule(
+  alert: Pick<OpenAlert, "notificationFailureCount">,
+  outcome: AlertNotificationOutcome,
+  now: Date,
+  throttleSeconds: number,
+): { nextAttemptAt: Date; failureCount: number } {
+  if (outcome.status === "failed") {
+    const failureCount = (alert.notificationFailureCount ?? 0) + 1;
+    const exponent = Math.max(0, Math.min(failureCount - 1, 16));
+    const retrySeconds = Math.min(
+      FAILURE_RETRY_MAX_SECONDS,
+      FAILURE_RETRY_INITIAL_SECONDS * 2 ** exponent,
+    );
+    return {
+      nextAttemptAt: new Date(now.getTime() + retrySeconds * 1000),
+      failureCount,
+    };
+  }
+
+  return {
+    // Sent uses the severity cadence. Skipped uses the same due field only to
+    // avoid duplicate audit rows; last_notified_at still advances on sent only.
+    nextAttemptAt: new Date(now.getTime() + throttleSeconds * 1000),
+    failureCount: 0,
+  };
+}
+
+export function isActionabilitySuppressed(
+  alert: Pick<OpenAlert, "snoozedUntil">,
+  now: Date,
+): boolean {
+  return alert.snoozedUntil != null && new Date(alert.snoozedUntil).getTime() > now.getTime();
+}
+
+export function isPagingSuppressed(
+  alert: Pick<OpenAlert, "status" | "snoozedUntil">,
+  now: Date,
+): boolean {
+  return alert.status === "acknowledged" || isActionabilitySuppressed(alert, now);
+}
+
+export function isDeliveryBackoff(
+  alert: Pick<OpenAlert, "lastNotificationStatus" | "nextNotificationAttemptAt">,
+  now: Date,
+): boolean {
+  if (alert.lastNotificationStatus !== "failed" || !alert.nextNotificationAttemptAt) return false;
+  const nextAttemptAt = Date.parse(alert.nextNotificationAttemptAt);
+  return Number.isFinite(nextAttemptAt) && nextAttemptAt > now.getTime();
+}
+
+export function isSeverityEscalation(
+  decision: Pick<AlertDecision, "severity">,
+  alert: Pick<OpenAlert, "severity">,
+): boolean {
+  const rank = { p0: 0, p1: 1, p2: 2, p3: 3 };
+  return rank[decision.severity] < rank[alert.severity];
+}

package/src/operations/server/platformWatchdogQueue.ts

@@ -0,0 +1,96 @@
+import type { AlertDecision } from "../contracts/index.js";
+import type { AlertSinkPort } from "../contracts/index.js";
+
+type WatchdogQueueItem = Pick<AlertDecision, "dedupeKey" | "severity" | "paging">;
+const WATCHDOG_CONCURRENCY = 4;
+const SEVERITY_RANK = { p0: 0, p1: 1, p2: 2, p3: 3 };
+
+/**
+ * Only each key's first unfinished item is eligible. A head heap prioritizes
+ * normalized severity without reordering the caller's decisions or duplicates.
+ */
+export async function runWatchdogQueue(
+  items: readonly WatchdogQueueItem[],
+  process: (index: number) => Promise<void>,
+): Promise<void> {
+  const next = new Array<number>(items.length).fill(-1);
+  const lastByKey = new Map<string, number>();
+  const ready: number[] = [];
+  for (let index = 0; index < items.length; index += 1) {
+    const previous = lastByKey.get(items[index].dedupeKey);
+    if (previous === undefined) push(index);
+    else next[previous] = index;
+    lastByKey.set(items[index].dedupeKey, index);
+  }
+
+  let failed = false;
+  let firstError: unknown;
+  async function worker() {
+    while (!failed) {
+      const index = pop();
+      if (index === undefined) return;
+      try {
+        await process(index);
+      } catch (error) {
+        if (!failed) firstError = error;
+        failed = true;
+        return;
+      }
+      if (!failed && next[index] !== -1) push(next[index]);
+    }
+  }
+
+  // Workers catch their own errors so every admitted chain settles before reject.
+  await Promise.all(Array.from({ length: WATCHDOG_CONCURRENCY }, () => worker()));
+  if (failed) throw firstError;
+
+  function compare(left: number, right: number): number {
+    const a = items[left];
+    const b = items[right];
+    return SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity]
+      || Number(a.paging === "never") - Number(b.paging === "never")
+      || a.dedupeKey.localeCompare(b.dedupeKey)
+      || left - right;
+  }
+
+  function push(index: number) {
+    let position = ready.length;
+    ready.push(index);
+    while (position > 0) {
+      const parent = Math.floor((position - 1) / 2);
+      if (compare(ready[parent], index) <= 0) break;
+      ready[position] = ready[parent];
+      position = parent;
+    }
+    ready[position] = index;
+  }
+
+  function pop(): number | undefined {
+    if (ready.length === 0) return undefined;
+    const first = ready[0];
+    const last = ready.pop()!;
+    if (ready.length === 0) return first;
+    let position = 0;
+    while (position * 2 + 1 < ready.length) {
+      let child = position * 2 + 1;
+      if (child + 1 < ready.length && compare(ready[child + 1], ready[child]) < 0) child += 1;
+      if (compare(last, ready[child]) <= 0) break;
+      ready[position] = ready[child];
+      position = child;
+    }
+    ready[position] = last;
+    return first;
+  }
+}
+
+/** Per-tick transport limit; admitted waiters still run if an earlier send throws. */
+export function createSerialWatchdogSink<Channel extends string>(sinkPort: AlertSinkPort<Channel>): AlertSinkPort<Channel> {
+  let tail = Promise.resolve();
+  return {
+    send(decision, alert) {
+      const attempt = tail.then(() => sinkPort.send(decision, alert));
+      tail = attempt.then(() => undefined, () => undefined);
+      return attempt;
+    },
+  };
+}