@zerotal/monitor 1.0.0

package/src/panel.ts

@@ -0,0 +1,159 @@
+/**
+ * The monitor panel's contribution surface — how other packages add what *they*
+ * want monitored.
+ *
+ * The monitor owns the shell, the time-range selector, the storage and the
+ * retention policy; a package owns the knowledge of what is worth watching about
+ * itself. So the monitor is a host: it publishes the write surface below, binds
+ * it into the container as `monitor.panel`, and a package pushes a section into
+ * it at boot:
+ *
+ *   // packages/scheduler/src/monitor.ts
+ *   interface MonitorHost {                    // declared locally — no monitor dependency
+ *     enabled(id: string): boolean;
+ *     section(s: { id: string; label: string; resolve(range: string): unknown }): void;
+ *   }
+ *
+ *   export function installSchedulerMonitor(app: Application): void {
+ *     const monitor = app.container.tryMake("monitor.panel") as MonitorHost | undefined;
+ *     if (!monitor?.enabled("scheduler")) return;
+ *     monitor.section({ id: "scheduler", label: "Scheduled tasks", resolve });
+ *   }
+ *
+ * A section is *described*, not rendered: the contributing package returns stats
+ * and tables, and the monitor draws them. That keeps the panel coherent no matter
+ * who contributed a section, and means a package needs no JSX and no dependency
+ * on this one.
+ *
+ * This mirrors how the recorders already work. A satellite writes its measurements
+ * into `monitor.store` through its own `observability.ts`; this is the other half —
+ * saying what those measurements should look like on screen.
+ */
+import type { MonitorRange } from "./store/types.ts";
+
+/** Semantic tone for a stat or cell — the panel maps it to its own palette. */
+export type MonitorTone = "default" | "good" | "warn" | "bad";
+
+/** A row in a contributed table. */
+export type MonitorRow = Record<string, unknown>;
+
+/** A headline figure at the top of a section. */
+export interface MonitorStat {
+  label: string;
+  value: string | number;
+  /** Secondary line under the value — a total, a rate, a comparison. */
+  detail?: string;
+  tone?: MonitorTone;
+  /** Draw a 0–100 progress bar under the value. */
+  percent?: number;
+}
+
+/** One column of a contributed table. */
+export interface MonitorTableColumn {
+  key: string;
+  label: string;
+  align?: "start" | "end";
+  /** Render in a monospace face — ids, keys, class names. */
+  mono?: boolean;
+  /** Turn the raw value into display text. Defaults to `String(value)`. */
+  format?: (value: unknown, row: MonitorRow) => string;
+  /** Tint the cell. */
+  tone?: (value: unknown, row: MonitorRow) => MonitorTone | null;
+}
+
+/** A table beneath a section's stats. */
+export interface MonitorTable {
+  title: string;
+  columns: MonitorTableColumn[];
+  rows: MonitorRow[];
+  empty?: string;
+}
+
+/** What a section shows for the selected range. */
+export interface MonitorSectionData {
+  stats?: MonitorStat[];
+  tables?: MonitorTable[];
+}
+
+/** A section contributed to the monitor's navigation. */
+export interface MonitorSection {
+  /** Stable id — the URL segment (`/monitor/<id>`) and the nav key. */
+  id: string;
+  label: string;
+  /** Sidebar group heading. Contributed groups sort after the built-in ones. */
+  group?: string;
+  /** Raw SVG markup for the nav icon. Falls back to a generic glyph. */
+  icon?: string;
+  sort?: number;
+  /**
+   * Resolve the section's content for the selected time range.
+   *
+   * Called on every render and every auto-refresh, so it should read from
+   * whatever the package already records rather than doing expensive work.
+   */
+  resolve(range: MonitorRange): Promise<MonitorSectionData> | MonitorSectionData;
+}
+
+/**
+ * The monitor's write surface, bound into the container as `monitor.panel`.
+ *
+ * Contributors should declare their own minimal copy of the members they use
+ * rather than importing this type, so they depend on this package not at all.
+ */
+export interface MonitorPanelHost {
+  /**
+   * Whether the app has left this contributor switched on. Check it first and
+   * return early — `sections: { scheduler: false }` in `config/monitor.ts` turns
+   * a contributor off without uninstalling its provider.
+   */
+  enabled(id: string): boolean;
+  section(section: MonitorSection): void;
+}
+
+/** Registered contributed sections, in registration order. */
+const _sections: MonitorSection[] = [];
+
+/** Contributors the app has switched off, from `config/monitor.ts`. */
+let _disabled: Record<string, boolean> = {};
+
+/**
+ * The monitor's contribution registry.
+ *
+ * Module-level rather than instance state because the panel is a single
+ * process-wide product, matching how the store and config are published.
+ */
+export const MonitorPanel = {
+  /** Record which contributors the app disabled. Called by the provider on boot. */
+  configure(sections: Record<string, boolean> | undefined): void {
+    _disabled = sections ?? {};
+  },
+
+  /** The write surface handed to contributors through the container. */
+  host(): MonitorPanelHost {
+    return {
+      enabled: (id) => _disabled[id] !== false,
+      section: (section) => {
+        if (_sections.some((s) => s.id === section.id)) return;
+        _sections.push(section);
+      },
+    };
+  },
+
+  /** Every contributed section, sorted for display. */
+  sections(): MonitorSection[] {
+    return [..._sections].sort(
+      (a, b) => (a.sort ?? 0) - (b.sort ?? 0) || a.label.localeCompare(b.label),
+    );
+  },
+
+  /** Resolve a contributed section by id. */
+  find(id: string): MonitorSection | undefined {
+    return _sections.find((s) => s.id === id);
+  },
+
+  /** Reset the registry (tests). */
+  reset(): void {
+    _sections.length = 0;
+    _disabled = {};
+  },
+};

package/src/prometheus.ts

@@ -0,0 +1,128 @@
+/**
+ * Prometheus text-exposition exporter. Renders a {@link MonitorSnapshot} (plus
+ * core `HttpMetrics`) as `text/plain; version=0.0.4` so any Prometheus scraper —
+ * and the whole Grafana/Alertmanager ecosystem — can read Zerotal's metrics.
+ *
+ * HTTP totals are real counters (cumulative since boot, from `HttpMetrics`);
+ * everything else is a gauge over the current window.
+ */
+import { httpMetrics } from "@zerotal/core/metrics";
+import type { MonitorSnapshot } from "./store/types.ts";
+
+function _sanitize(s: string): string {
+  return s.toLowerCase().replace(/[^a-z0-9_]/g, "_");
+}
+
+function _escapeLabel(s: string): string {
+  return s.replace(/\\/g, "\\\\").replace(/"/g, '\\"').replace(/\n/g, " ");
+}
+
+export function renderPrometheus(snap: MonitorSnapshot): string {
+  const m = httpMetrics();
+  const out: string[] = [];
+  const metric = (name: string, type: "gauge" | "counter", help: string, value: number): void => {
+    out.push(`# HELP ${name} ${help}`, `# TYPE ${name} ${type}`, `${name} ${value}`);
+  };
+
+  // ── HTTP ──────────────────────────────────────────────────────────────────────
+  metric("zerotal_http_requests_total", "counter", "HTTP requests since boot", m.total);
+  metric("zerotal_http_in_flight", "gauge", "HTTP requests currently being processed", m.inFlight);
+  metric("zerotal_http_server_errors_total", "counter", "5xx responses since boot", m.serverErrors);
+  metric("zerotal_http_client_errors_total", "counter", "4xx responses since boot", m.clientErrors);
+  metric("zerotal_http_request_duration_ms_avg", "gauge", "Mean response time (ms)", m.avgMs);
+  metric(
+    "zerotal_http_request_duration_ms_p95",
+    "gauge",
+    "p95 response time over 5m (ms)",
+    m.p95Ms,
+  );
+  metric("zerotal_http_requests_per_minute", "gauge", "Requests per minute over 5m", m.perMinute);
+  metric("zerotal_http_success_rate", "gauge", "Success rate (%)", m.successRate);
+  metric("zerotal_apdex", "gauge", "Apdex satisfaction score", snap.apdex);
+
+  // ── Cache ─────────────────────────────────────────────────────────────────────
+  metric("zerotal_cache_hit_rate", "gauge", "Cache hit rate (%)", snap.cache.hitRate);
+  metric("zerotal_cache_evictions", "gauge", "Cache evictions in window", snap.cache.evictions);
+
+  // ── Queues ────────────────────────────────────────────────────────────────────
+  metric(
+    "zerotal_queue_pending",
+    "gauge",
+    "Pending jobs across queues",
+    snap.queues.reduce((a, q) => a + q.pending, 0),
+  );
+  metric("zerotal_queue_failed", "gauge", "Failed jobs", snap.failedJobs.length);
+
+  // ── Database ──────────────────────────────────────────────────────────────────
+  metric(
+    "zerotal_db_transactions_committed",
+    "gauge",
+    "Committed transactions in window",
+    snap.transactions.committed,
+  );
+  metric(
+    "zerotal_db_transactions_rolledback",
+    "gauge",
+    "Rolled-back transactions in window",
+    snap.transactions.rolledBack,
+  );
+  metric(
+    "zerotal_db_nplus_offenders",
+    "gauge",
+    "Distinct N+1 offenders in window",
+    snap.nplusOnes.length,
+  );
+
+  // ── Exceptions ────────────────────────────────────────────────────────────────
+  metric(
+    "zerotal_exceptions_total",
+    "gauge",
+    "Exception occurrences in window",
+    snap.exceptions.reduce((a, e) => a + e.count, 0),
+  );
+
+  // ── Realtime / WebSocket ──────────────────────────────────────────────────────
+  metric(
+    "zerotal_ws_connections",
+    "gauge",
+    "Active WebSocket connections",
+    snap.realtime.activeConnections,
+  );
+  metric(
+    "zerotal_ws_actions_per_minute",
+    "gauge",
+    "WebSocket actions per minute",
+    snap.realtime.actionsPerMin,
+  );
+  metric(
+    "zerotal_ws_action_duration_ms_avg",
+    "gauge",
+    "Average WS action time (ms)",
+    snap.realtime.avgActionMs,
+  );
+
+  // ── System gauges (CPU / Memory / Heap) ───────────────────────────────────────
+  for (const g of snap.gauges) {
+    metric(
+      `zerotal_system_${_sanitize(g.label)}_percent`,
+      "gauge",
+      `${g.label} usage (%)`,
+      g.value,
+    );
+  }
+
+  // ── Per-route latency (labelled — HELP/TYPE once) ─────────────────────────────
+  if (snap.slowRoutes.length) {
+    out.push(
+      "# HELP zerotal_route_duration_ms_avg Average response time per route (ms)",
+      "# TYPE zerotal_route_duration_ms_avg gauge",
+    );
+    for (const r of snap.slowRoutes) {
+      out.push(
+        `zerotal_route_duration_ms_avg{method="${_escapeLabel(r.method)}",route="${_escapeLabel(r.path)}"} ${r.ms}`,
+      );
+    }
+  }
+
+  return out.join("\n") + "\n";
+}

package/src/provider/MonitorProvider.ts

@@ -0,0 +1,318 @@
+/**
+ * Wires `@zerotal/monitor` into a Zerotal app:
+ *
+ *   1. Resolves `config/monitor.ts` (with defaults) and binds it as `monitor`.
+ *   2. Creates the {@link MonitorStore}, binds it to the IoC container and the
+ *      live app (so it can read queue/scheduler/health), and publishes it via
+ *      the process handle used by the recorder and the `Monitor` facade.
+ *   3. Installs the event-driven recorder (when `record: true`).
+ *   4. Registers the Flow panel route at `config.path` behind the auth guard.
+ *
+ * The panel is a Flow component, so this provider depends on `FlowProvider`
+ * (declared via `dependsOn`) — it is registered and booted automatically, in the
+ * right order, even in apps that don't otherwise use Flow:
+ *
+ *   import { MonitorProvider } from "@zerotal/monitor";
+ *   const providers = [MonitorProvider]; // FlowProvider comes along via dependsOn
+ */
+import { ServiceProvider, Router } from "@zerotal/core";
+import { FlowProvider } from "@zerotal/flow";
+import type { AppEnvironment, HttpContext } from "@zerotal/core";
+import type { ConfigManager } from "@zerotal/core/config";
+import type { MonitorRange } from "../store/types.ts";
+import { LogManager, frameworkLog } from "@zerotal/core/logger";
+import { MonitorConfig, type ResolvedMonitorConfig } from "../config.ts";
+import { MonitorStore } from "../MonitorStore.ts";
+import { _setStore } from "../instance.ts";
+import { installMonitorEventBridge } from "../recorder/MonitorEventBridge.ts";
+import { renderPrometheus } from "../prometheus.ts";
+import { evaluateAlerts, alertsToFire, _dispatchAlert, onAlert } from "../alerting.ts";
+import { detectDeploy } from "../sources/system.ts";
+import { MonitorAuthMiddleware } from "../middleware/MonitorAuthMiddleware.ts";
+import { MonitorPanel } from "../panel.ts";
+import type { MonitorPanelHost } from "../panel.ts";
+import { MonitorPayloadMiddleware } from "../middleware/MonitorPayloadMiddleware.ts";
+import { MonitorPage } from "../ui/MonitorPage.tsx";
+
+declare module "@zerotal/core" {
+  interface ContainerBindings {
+    monitor: ResolvedMonitorConfig;
+    "monitor.store": MonitorStore;
+    /** The panel's contribution surface — see `panel.ts`. */
+    "monitor.panel": MonitorPanelHost;
+  }
+}
+
+export class MonitorProvider extends ServiceProvider {
+  static override provides = ["monitor", "monitor.store", "monitor.panel"] as const;
+  // The panel mounts via Flow's `Router.flow()` macro, so FlowProvider must
+  // register/boot first to install it. Declaring the dependency wires it in
+  // automatically (and in order) even in non-Flow apps.
+  static override dependsOn = [FlowProvider];
+  // The panel + metrics endpoints mount via Flow's `Router.flow()` macro and the
+  // HTTP router, so this only runs where Flow/the server do — not in `console`
+  // (CLI) boots, where the macro is absent and there's nothing to serve.
+  static override environments: AppEnvironment[] = ["web", "test"];
+
+  private _disposeBridge: (() => void) | undefined = undefined;
+  private _disposeLogTap: (() => void) | undefined = undefined;
+  private _pruneTimer: ReturnType<typeof setInterval> | undefined = undefined;
+  private _alertTimer: ReturnType<typeof setInterval> | undefined = undefined;
+  private _disposeAlertHook: (() => void) | undefined = undefined;
+  private readonly _firingAlerts = new Set<string>();
+  private readonly _lastFiredAt = new Map<string, number>();
+
+  override onRegister(): void {
+    this.app.container.singleton("monitor", () => {
+      const config = this.app.container.makeSync("config") as ConfigManager;
+      const raw = config.get<ResolvedMonitorConfig>("monitor");
+      // Apply defaults even if the app didn't author config/monitor.ts.
+      return raw ? MonitorConfig(raw) : MonitorConfig();
+    });
+
+    // Publish the contribution surface during registration, so any provider can
+    // add a section from its own `onBooting` regardless of boot order —
+    // contributors deliberately do not (and must not) depend on this package.
+    //
+    // The factory body runs on first resolution, not now: reading config during
+    // the registration phase would force it earlier than the rest of the provider
+    // needs it, and isolated tests bind no config at all.
+    this.app.container.singleton("monitor.panel", () => {
+      try {
+        const cfg = this.app.container.makeSync("monitor") as ResolvedMonitorConfig;
+        MonitorPanel.configure(cfg.sections);
+      } catch {
+        // No config bound — every contributor stays enabled.
+      }
+      return MonitorPanel.host();
+    });
+
+    this.app.container.singleton("monitor.store", () => {
+      const cfg = this.app.container.makeSync("monitor") as ResolvedMonitorConfig;
+      const deploy = cfg.deploy ?? detectDeploy();
+      const store = new MonitorStore({
+        apdexTargetMs: cfg.apdexTargetMs,
+        slowQueryMs: cfg.slowQueryMs,
+        slowRequestMs: cfg.slowRequestMs,
+        snapshotCacheMs: cfg.snapshotCacheMs,
+        storage: cfg.storage,
+        retentionDays: cfg.retentionDays,
+        retentionMode: cfg.retentionMode,
+        ...(cfg.zerotalVersion ? { zerotalVersion: cfg.zerotalVersion } : {}),
+        ...(cfg.region ? { region: cfg.region } : {}),
+        ...(deploy ? { deploy } : {}),
+      });
+      return store;
+    });
+  }
+
+  override async onBooting(): Promise<void> {
+    const cfg = (await this.app.container.make("monitor")) as ResolvedMonitorConfig;
+    const store = (await this.app.container.make("monitor.store")) as MonitorStore;
+
+    // Let the store reach the live subsystems, and publish it process-wide.
+    store.bindApp(this.app as unknown as { container: { tryMake(name: string): unknown } });
+    _setStore(store);
+
+    // Expose branding/refresh/path/retention to the page without a container round-trip.
+    (globalThis as { __monitorConfig?: unknown }).__monitorConfig = {
+      title: cfg.title,
+      subtitle: cfg.subtitle,
+      refreshMs: cfg.refreshMs,
+      path: cfg.path.replace(/\/$/, "") || "/monitor",
+      retentionDays: cfg.retentionDays,
+      retentionMode: cfg.retentionMode,
+    };
+
+    // Opt-in: capture request/response headers + bodies for the request trace.
+    if (cfg.record && cfg.capturePayloads) {
+      this.app.use(MonitorPayloadMiddleware);
+    }
+
+    // Install the recorder: one FrameworkEvents bridge feeds every panel
+    // (requests + queries + cache + mail + exceptions + outgoing HTTP + jobs).
+    if (cfg.record) {
+      this._disposeBridge = installMonitorEventBridge(store);
+
+      // Surface application logs in the panel via a LogManager tap.
+      this._disposeLogTap = LogManager.tap((entry) => {
+        const status =
+          entry.level === "error" || entry.level === "fatal"
+            ? "bad"
+            : entry.level === "warn"
+              ? "warn"
+              : "info";
+        store.recordEvent({
+          kind: "log",
+          label: entry.level,
+          status,
+          route: entry.requestId ?? null,
+          data: {
+            detail: entry.message,
+            ...(entry.error ? { error: entry.error } : {}),
+            ...entry.context,
+          },
+        });
+      });
+    }
+
+    // Enforce retention: prune past-window data now and hourly thereafter.
+    store.prune();
+    this._pruneTimer = setInterval(() => store.prune(), 60 * 60 * 1000);
+    (this._pruneTimer as { unref?: () => void }).unref?.();
+
+    // Evaluate threshold alerts every 15s (edge-triggered, deduped).
+    if (cfg.alerts) {
+      this._alertTimer = setInterval(() => void this._runAlerts(store, cfg), 15_000);
+      (this._alertTimer as { unref?: () => void }).unref?.();
+
+      // Ship newly-firing alerts to a webhook so they page someone, not just the panel.
+      if (cfg.alertWebhook) {
+        const webhook = cfg.alertWebhook;
+        this._disposeAlertHook = onAlert(async (a) => {
+          try {
+            await fetch(webhook, {
+              method: "POST",
+              headers: { "Content-Type": "application/json" },
+              body: JSON.stringify({
+                text: `[${a.level.toUpperCase()}] ${a.title} — ${a.detail}`,
+                level: a.level,
+                title: a.title,
+                detail: a.detail,
+              }),
+            });
+          } catch {
+            /* delivery is best-effort — never throw back into the alert loop */
+          }
+        });
+      }
+    }
+
+    // Mount the panel. The base path renders the Overview; `/:section` deep-links
+    // each page (e.g. /monitor/requests, /monitor/exceptions) to the same shell.
+    const path = cfg.path.replace(/\/$/, "") || "/monitor";
+    Router.flow(path, MonitorPage, [MonitorAuthMiddleware]);
+    Router.flow(`${path}/:section`, MonitorPage, [MonitorAuthMiddleware]);
+
+    // Snapshot export — the full derived snapshot as a JSON download, behind the
+    // same auth gate as the panel. `?range=live|1h|24h|7d` selects the window.
+    Router.get(
+      `${path}/export.json`,
+      async (http: HttpContext) => {
+        const q = http.query("range", "live") ?? "live";
+        const range: MonitorRange = (["live", "1h", "24h", "7d"] as const).includes(
+          q as MonitorRange,
+        )
+          ? (q as MonitorRange)
+          : "live";
+        const snap = await store.snapshot(range);
+        return new Response(JSON.stringify(snap, null, 2), {
+          headers: {
+            "Content-Type": "application/json; charset=utf-8",
+            "Content-Disposition": `attachment; filename="zerotal-monitor-${range}.json"`,
+          },
+        });
+      },
+      [MonitorAuthMiddleware],
+    );
+
+    // Prometheus text-exposition endpoint (protect at the network layer).
+    if (cfg.metrics) {
+      Router.get(cfg.metricsPath, async () => {
+        const snap = await store.snapshot("live");
+        return new Response(renderPrometheus(snap), {
+          headers: { "Content-Type": "text/plain; version=0.0.4; charset=utf-8" },
+        });
+      });
+    }
+  }
+
+  /** Evaluate alerts against a live snapshot; fire newly-breaching ones, reset resolved. */
+  private async _runAlerts(store: MonitorStore, cfg: ResolvedMonitorConfig): Promise<void> {
+    try {
+      const snap = await store.snapshot("live");
+      const active = evaluateAlerts(snap, cfg.alertThresholds);
+      const activeIds = new Set(active.map((a) => a.id));
+
+      // Edge-triggered + cooldown: fire each newly-breaching alert once, and don't
+      // re-fire within the cooldown even if it recovers and re-breaches.
+      const toFire = alertsToFire(
+        active,
+        this._firingAlerts,
+        this._lastFiredAt,
+        Date.now(),
+        cfg.alertCooldownMs,
+      );
+      for (const a of toFire) {
+        frameworkLog("monitor").warn(`ALERT [${a.level}] ${a.title} — ${a.detail}`, {
+          level: a.level,
+        });
+        store.recordEvent({
+          kind: "alert",
+          label: a.title,
+          status: a.level === "critical" ? "bad" : "warn",
+          route: a.id,
+          // Capture the metric + a snapshot of the surrounding state so the panel can
+          // explain *why* it fired without the operator hunting across tabs.
+          data: {
+            id: a.id,
+            level: a.level,
+            detail: a.detail,
+            metric: a.metric,
+            value: a.value,
+            threshold: a.threshold,
+            unit: a.unit,
+            context: this._alertContext(snap),
+          },
+        });
+        _dispatchAlert(a);
+      }
+      // Reset alerts that have recovered so they can fire again next time.
+      for (const id of [...this._firingAlerts]) {
+        if (!activeIds.has(id)) this._firingAlerts.delete(id);
+      }
+    } catch {
+      /* alerting must never break the app */
+    }
+  }
+
+  /** A compact snapshot of the surrounding state, recorded with each alert firing. */
+  private _alertContext(
+    snap: import("../store/types.ts").MonitorSnapshot,
+  ): Record<string, unknown> {
+    const errCard = snap.statCards.find((c) => c.label === "Error rate");
+    const rpmCard = snap.statCards.find((c) => c.label === "Requests / min");
+    const pct = (label: string): number =>
+      snap.percentiles.find((p) => p.label === label)?.value ?? 0;
+    return {
+      errorRate: errCard ? parseFloat(errCard.value) || 0 : 0,
+      rpm: rpmCard ? parseFloat(rpmCard.value.replace(/,/g, "")) || 0 : 0,
+      p50: pct("p50"),
+      p95: pct("p95"),
+      p99: pct("p99"),
+      apdex: snap.apdex,
+      pending: snap.queues.reduce((acc, q) => acc + q.pending, 0),
+      rolledBack: snap.transactions.rolledBack,
+      slowRoutes: snap.slowRoutes
+        .slice(0, 3)
+        .map((r) => ({ method: r.method, path: r.path, ms: r.ms })),
+      topException: snap.exceptions[0]?.type ?? null,
+    };
+  }
+
+  override async onStopping(): Promise<void> {
+    if (this._pruneTimer) clearInterval(this._pruneTimer);
+    this._pruneTimer = undefined;
+    if (this._alertTimer) clearInterval(this._alertTimer);
+    this._alertTimer = undefined;
+    this._disposeAlertHook?.();
+    this._disposeAlertHook = undefined;
+    this._disposeBridge?.();
+    this._disposeBridge = undefined;
+    this._disposeLogTap?.();
+    this._disposeLogTap = undefined;
+    const store = this.app.container.tryMake("monitor.store") as MonitorStore | null;
+    store?.dispose();
+    _setStore(undefined);
+  }
+}