@zerotal/monitor 1.0.0

package/src/recorder/MonitorEventBridge.ts

@@ -0,0 +1,157 @@
+/**
+ * The monitor's core-signal bridge: subscribes to the framework's request
+ * lifecycle (`RequestHandled` / `RequestFailed`), outgoing HTTP calls, and console
+ * commands, and records each request together with the query / N+1 / custom-context
+ * / payload that feature packages buffered against it while it was in flight (see
+ * recorder/ctxBuffers.ts).
+ *
+ * Feature packages contribute their own events (DB, cache, mail, jobs, auth,
+ * realtime) through their own monitor bridge, which resolves the {@link MonitorStore}
+ * from the container. The monitor therefore imports none of them, and adding a
+ * feature package requires no change here.
+ */
+import { FrameworkEvents } from "@zerotal/core";
+import type {
+  RequestHandled,
+  RequestFailed,
+  OutgoingRequestCompleted,
+  CommandRan,
+} from "@zerotal/core";
+import type { MonitorStore } from "../MonitorStore.ts";
+import { collectRequestState, markRecorded } from "./ctxBuffers.ts";
+
+// Framework/asset noise the panel shouldn't chart as application traffic.
+const IGNORE_PREFIXES = [
+  "/monitor",
+  "/metrics",
+  "/__flow",
+  "/__zerotal",
+  "/__dev",
+  "/health",
+  "/favicon",
+  "/assets",
+  "/build",
+];
+
+interface HttpCtxShape {
+  url?: { pathname?: string };
+  request?: { method?: string };
+  response?: { status?: number };
+  _routeDef?: { pattern?: string };
+  user?: unknown;
+  ip?: () => string | null;
+}
+
+/** Method, matched-route template, raw path, and status from an HttpContext. */
+function _ctxInfo(raw: object): { method: string; path: string; rawPath: string; status: number } {
+  const c = raw as HttpCtxShape;
+  const rawPath = c.url?.pathname ?? "/";
+  return {
+    method: (c.request?.method ?? "GET").toUpperCase(),
+    path: c._routeDef?.pattern ?? rawPath, // prefer /posts/:id over /posts/42
+    rawPath,
+    status: c.response?.status ?? 0,
+  };
+}
+
+/** Authenticated user identity (email → id → name), or null when unauthenticated. */
+function _ctxUser(raw: object): string | null {
+  const u = (raw as HttpCtxShape).user;
+  if (u == null) return null;
+  if (typeof u === "string" || typeof u === "number") return String(u);
+  if (typeof u === "object") {
+    const o = u as Record<string, unknown>;
+    const v = o.email ?? o.id ?? o.name;
+    return v != null ? String(v) : null;
+  }
+  return null;
+}
+
+/** Client IP from the HttpContext (best-effort — null when unavailable). */
+function _ctxIp(raw: object): string | null {
+  try {
+    return (raw as HttpCtxShape).ip?.() ?? null;
+  } catch {
+    return null;
+  }
+}
+
+/** Recover a "XxxError" class name from a framework error message, else "Error". */
+function _errorType(message: string): string {
+  const match = /^([A-Za-z]*Error)\b/.exec(message);
+  return match?.[1] ?? "Error";
+}
+
+export function installMonitorEventBridge(store: MonitorStore): () => void {
+  /** Finalise one request: drain its buffered correlation state, record it once. */
+  const recordRequest = (
+    ctx: object,
+    statusOverride: number,
+    durationMs: number,
+    error: string | null = null,
+  ): void => {
+    if (!markRecorded(ctx)) return; // success + failure can both fire; record once
+
+    const info = _ctxInfo(ctx);
+    // Drain buffers even for ignored paths so they don't leak into a later request
+    // that reuses the context object.
+    const { queries, nplus, context, payload } = store.collectRequestState(ctx);
+    if (IGNORE_PREFIXES.some((p) => info.rawPath.startsWith(p))) return;
+
+    store.recordRequest({
+      method: info.method,
+      path: info.path,
+      status: statusOverride || info.status,
+      ms: durationMs,
+      queries,
+      nplus,
+      user: _ctxUser(ctx),
+      ip: _ctxIp(ctx),
+      // Process heap at completion — a per-request memory proxy (the runtime shares
+      // one heap, so this is indicative, not isolated like PHP-FPM peak memory).
+      memKb: Math.round(process.memoryUsage().heapUsed / 1024),
+      context,
+      payload,
+      error,
+    });
+  };
+
+  const unsubs: Array<() => void> = [
+    // ── Requests — full per-request trace ──────────────────────────────────────
+    FrameworkEvents.on<RequestHandled>("RequestHandled", (e) =>
+      recordRequest(e.ctx, 0, e.durationMs),
+    ),
+    FrameworkEvents.on<RequestFailed>("RequestFailed", (e) => {
+      store.recordException(
+        { type: _errorType(e.error), message: e.error },
+        _ctxInfo(e.ctx).path,
+        _ctxUser(e.ctx),
+      );
+      recordRequest(e.ctx, e.status, e.durationMs, e.error);
+    }),
+
+    // ── Outgoing HTTP — per-host calls, p95, error rate ────────────────────────
+    FrameworkEvents.on<OutgoingRequestCompleted>("OutgoingRequestCompleted", (e) => {
+      store.recordHttp({ host: e.host, ms: e.durationMs, error: !e.ok });
+    }),
+
+    // ── Console / Artisan command runs ─────────────────────────────────────────
+    FrameworkEvents.on<CommandRan>("CommandRan", (e) =>
+      store.recordEvent({
+        kind: "command",
+        label: e.name,
+        status: e.ok ? "ok" : "bad",
+        route: null,
+        data: {
+          ms: Math.round(e.durationMs),
+          code: e.exitCode,
+          detail: e.error ?? `exit ${e.exitCode}`,
+        },
+      }),
+    ),
+  ];
+
+  return () => {
+    for (const unsub of unsubs) unsub();
+  };
+}

package/src/recorder/ctxBuffers.ts

@@ -0,0 +1,91 @@
+/**
+ * Per-request correlation buffers, keyed by the HttpContext object and GC'd with
+ * it via WeakMap/WeakSet. Feature packages buffer their per-request signal here
+ * (through {@link MonitorStore}'s thin delegating methods) while a request is in
+ * flight; the request-lifecycle handler drains it when the request finalises so
+ * the recorded row carries the real queries, N+1 flag, custom context, and payload
+ * that occurred during it.
+ *
+ * This is the shared substrate that lets the monitor correlate cross-package
+ * activity to a request without any package importing another: the ORM buffers
+ * queries, the payload middleware attaches bodies, and the request/Flow-action
+ * handlers collect the lot.
+ */
+import type { RequestQuery, RequestPayload } from "../store/types.ts";
+
+const _ctxQueries = new WeakMap<object, RequestQuery[]>();
+const _ctxNPlus = new WeakSet<object>();
+const _ctxRecorded = new WeakSet<object>();
+const _ctxContext = new WeakMap<object, Record<string, unknown>>();
+const _ctxPayload = new WeakMap<object, RequestPayload>();
+
+/** A request is an N+1 offender once its buffered query count crosses this. */
+const NPLUS_QUERY_THRESHOLD = 25;
+
+/** Buffer one query span against the request context it ran under. */
+export function bufferQuery(ctx: object, q: RequestQuery): void {
+  let arr = _ctxQueries.get(ctx);
+  if (!arr) {
+    arr = [];
+    _ctxQueries.set(ctx, arr);
+  }
+  arr.push(q);
+}
+
+/** Flag the request context as having triggered an N+1 pattern. */
+export function markNPlus(ctx: object): void {
+  _ctxNPlus.add(ctx);
+}
+
+/** Merge custom metadata onto the request context (used by `Monitor.context`). */
+export function addContext(ctx: object, data: Record<string, unknown>): void {
+  _ctxContext.set(ctx, { ...(_ctxContext.get(ctx) ?? {}), ...data });
+}
+
+/** Attach captured headers/bodies to the request (used by MonitorPayloadMiddleware). */
+export function addPayload(ctx: object, payload: RequestPayload): void {
+  _ctxPayload.set(ctx, payload);
+}
+
+/**
+ * Record-once guard for a context: returns `true` the first time it is called for
+ * a given context and `false` thereafter, so a request that fires both success
+ * and failure events is recorded a single time.
+ */
+export function markRecorded(ctx: object): boolean {
+  if (_ctxRecorded.has(ctx)) return false;
+  _ctxRecorded.add(ctx);
+  return true;
+}
+
+/** The correlation state drained for a finalised request or Flow action. */
+export interface RequestState {
+  queries: RequestQuery[];
+  nplus: boolean;
+  context: Record<string, unknown>;
+  payload: RequestPayload | null;
+}
+
+/**
+ * Read and clear all buffered correlation state for a context. Called once, when
+ * the request (or Flow action) that owns the context finalises.
+ */
+export function collectRequestState(ctx: object): RequestState {
+  const queries = _ctxQueries.get(ctx) ?? [];
+  const nplus = _ctxNPlus.has(ctx) || queries.length > NPLUS_QUERY_THRESHOLD;
+  const context = _ctxContext.get(ctx) ?? {};
+  const payload = _ctxPayload.get(ctx) ?? null;
+  _ctxQueries.delete(ctx);
+  _ctxNPlus.delete(ctx);
+  _ctxContext.delete(ctx);
+  _ctxPayload.delete(ctx);
+  return { queries, nplus, context, payload };
+}
+
+/** Drop any buffered state for a context without recording it (ignored paths). */
+export function discardRequestState(ctx: object): void {
+  _ctxQueries.delete(ctx);
+  _ctxNPlus.delete(ctx);
+  _ctxContext.delete(ctx);
+  _ctxPayload.delete(ctx);
+}

package/src/sources/live.ts

@@ -0,0 +1,249 @@
+/**
+ * Adapters that read real data from the queue, scheduler, and health
+ * subsystems when they are installed. Each reader is defensive: if the
+ * subsystem isn't bound in the container (the package is an optional peer
+ * dependency), it returns `null` and the store falls back to representative
+ * sample data so the panel stays populated and useful.
+ */
+import { Health } from "@zerotal/core/health";
+import type {
+  CheckIn,
+  DeadJob,
+  FailedJob,
+  HealthEntry,
+  QueueRow,
+  ScheduledJob,
+  Worker,
+} from "../store/types.ts";
+import { ago } from "../support/time.ts";
+
+// ── Structural interfaces (avoid hard type-coupling to optional peers) ────────
+
+interface QueueStatsShape {
+  processedTotal: number;
+  failedTotal: number;
+  processedLast5m: number;
+  failedLast5m: number;
+}
+
+interface FailedRecordShape {
+  id: number;
+  queue: string;
+  className: string;
+  attempts: number;
+  maxAttempts?: number;
+  error: string;
+  failedAt: string;
+}
+
+interface QueueManagerShape {
+  queues(): Promise<{ queue: string; pending: number }[]>;
+  failed(queue?: string): Promise<FailedRecordShape[]>;
+  retryFailed(id: number): Promise<boolean>;
+  forgetFailed(id: number): Promise<void>;
+  stats(): QueueStatsShape;
+  readonly activeJobCount?: number;
+}
+
+interface ScheduledTaskShape {
+  readonly _name: string;
+  readonly _schedule: string;
+  readonly _running: boolean;
+  readonly _lastRunAt?: Date;
+  readonly _lastOk?: boolean;
+  readonly _lastDurationMs?: number;
+}
+
+interface SchedulerManagerShape {
+  readonly tasks: ReadonlyMap<string, ScheduledTaskShape>;
+}
+
+interface ContainerShape {
+  tryMake(name: string): unknown;
+}
+
+interface AppShape {
+  container: ContainerShape;
+}
+
+function resolve<T>(app: AppShape | undefined, binding: string): T | null {
+  if (!app) return null;
+  try {
+    const v = app.container.tryMake(binding);
+    return (v ?? null) as T | null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Queues ────────────────────────────────────────────────────────────────────
+
+export function queueManager(app?: AppShape): QueueManagerShape | null {
+  return resolve<QueueManagerShape>(app, "queue");
+}
+
+export async function liveQueues(app?: AppShape): Promise<QueueRow[] | null> {
+  const mgr = queueManager(app);
+  if (!mgr) return null;
+  try {
+    const rows = await mgr.queues();
+    return rows.map((r) => ({
+      name: r.queue,
+      pending: r.pending,
+      wait: 0,
+      throughput: 0,
+      paused: false,
+      series: [],
+    }));
+  } catch {
+    return null;
+  }
+}
+
+export async function liveFailedJobs(app?: AppShape): Promise<FailedJob[] | null> {
+  const mgr = queueManager(app);
+  if (!mgr) return null;
+  try {
+    const rows = await mgr.failed();
+    return rows.map((r) => ({
+      id: r.id,
+      name: r.className,
+      error: r.error,
+      failedAt: r.failedAt,
+      attempts: r.attempts,
+      tags: [r.queue],
+    }));
+  } catch {
+    return null;
+  }
+}
+
+// The dead-letter view: jobs that exhausted every retry (attempts ≥ maxAttempts).
+// In the SQLite driver the failed store only holds exhausted jobs, so this is the
+// authoritative "won't run again without intervention" list backing the Requeue action.
+export async function liveDeadLetter(app?: AppShape): Promise<DeadJob[] | null> {
+  const mgr = queueManager(app);
+  if (!mgr) return null;
+  try {
+    const rows = await mgr.failed();
+    return rows
+      .filter((r) => r.attempts >= (r.maxAttempts ?? r.attempts))
+      .map((r) => ({
+        id: r.id,
+        name: r.className,
+        error: r.error,
+        attempts: r.attempts,
+        deadAt: r.failedAt,
+      }));
+  } catch {
+    return null;
+  }
+}
+
+export function liveQueueStats(app?: AppShape): QueueStatsShape | null {
+  const mgr = queueManager(app);
+  if (!mgr) return null;
+  try {
+    return mgr.stats();
+  } catch {
+    return null;
+  }
+}
+
+export async function retryFailedJob(app: AppShape | undefined, id: number): Promise<boolean> {
+  const mgr = queueManager(app);
+  if (!mgr) return false;
+  try {
+    return await mgr.retryFailed(id);
+  } catch {
+    return false;
+  }
+}
+
+export async function forgetFailedJob(app: AppShape | undefined, id: number): Promise<boolean> {
+  const mgr = queueManager(app);
+  if (!mgr) return false;
+  try {
+    await mgr.forgetFailed(id);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// Workers are not modelled by the SQLite driver; surface active job count as a
+// single logical worker when the manager exposes it.
+export function liveWorkers(app?: AppShape): Worker[] | null {
+  const mgr = queueManager(app);
+  if (!mgr || typeof mgr.activeJobCount !== "number") return null;
+  return [
+    {
+      name: "worker-1",
+      status: "running",
+      processes: 1,
+      jobsPerMin: mgr.activeJobCount,
+      series: [],
+    },
+  ];
+}
+
+// ── Scheduler ──────────────────────────────────────────────────────────────────
+
+export function liveCheckins(app?: AppShape): CheckIn[] | null {
+  const mgr = resolve<SchedulerManagerShape>(app, "scheduler");
+  if (!mgr) return null;
+  try {
+    const out: CheckIn[] = [];
+    for (const task of mgr.tasks.values()) {
+      const last = task._lastRunAt ? ago(task._lastRunAt.getTime()) : "never";
+      const status: CheckIn["status"] =
+        task._lastRunAt == null ? "missed" : task._lastOk === false ? "missed" : "ok";
+      out.push({ name: task._name, cron: task._schedule, last, status });
+    }
+    return out;
+  } catch {
+    return null;
+  }
+}
+
+export function liveScheduled(app?: AppShape): ScheduledJob[] | null {
+  const mgr = resolve<SchedulerManagerShape>(app, "scheduler");
+  if (!mgr) return null;
+  try {
+    const out: ScheduledJob[] = [];
+    for (const task of mgr.tasks.values()) {
+      out.push({
+        name: task._name,
+        runsIn: task._running ? "running" : "—",
+        queue: "scheduler",
+        tag: "cron",
+      });
+    }
+    return out;
+  } catch {
+    return null;
+  }
+}
+
+// ── Health ─────────────────────────────────────────────────────────────────────
+
+export async function liveHealth(_app?: AppShape): Promise<HealthEntry[] | null> {
+  if (Health.names.length === 0) return null;
+  try {
+    const report = await Health.run({
+      name: "app",
+      version: "1.1.0",
+      environment: process.env.NODE_ENV ?? "production",
+      uptime: Math.floor(process.uptime()),
+    });
+    return Object.entries(report.checks).map(([name, check]) => ({
+      name: name.charAt(0).toUpperCase() + name.slice(1),
+      ok: check.status === "ok",
+      detail: check.message ?? `${check.status} · ${check.durationMs}ms`,
+    }));
+  } catch {
+    return null;
+  }
+}
+
+export type { AppShape };

package/src/sources/realtime.ts

@@ -0,0 +1,24 @@
+/**
+ * Live realtime/WebSocket readings from `@zerotal/flow`. Defensive: if Flow
+ * changes or isn't active, falls back to empty/0 rather than throwing.
+ */
+import { flowActiveConnections, flowConnections } from "@zerotal/flow";
+import type { FlowConnection } from "@zerotal/flow";
+
+/** Number of currently-open Flow WebSocket connections. */
+export function activeConnections(): number {
+  try {
+    return flowActiveConnections();
+  } catch {
+    return 0;
+  }
+}
+
+/** The currently-connected Flow clients (who, not just how many). */
+export function connectedClients(): FlowConnection[] {
+  try {
+    return flowConnections();
+  } catch {
+    return [];
+  }
+}

package/src/sources/system.ts

@@ -0,0 +1,151 @@
+/**
+ * System-level readings (memory, uptime, runtime metadata) derived from the
+ * running process. CPU and disk are best-effort: Bun does not expose a portable
+ * cross-platform CPU% without sampling, so we approximate from load average
+ * where available and fall back to a derived value otherwise.
+ */
+import { cpus, loadavg, totalmem, freemem } from "node:os";
+import { readFileSync } from "node:fs";
+import { currentApp } from "@zerotal/core";
+import { httpMetrics } from "@zerotal/core/metrics";
+import type { Gauge, SystemMeta } from "../store/types.ts";
+
+/** Bun's native in-flight HTTP count, or 0 when no server is bound (console/test). */
+function bunPendingRequests(): number {
+  try {
+    return currentApp().serverMetrics().pendingRequests;
+  } catch {
+    return 0;
+  }
+}
+
+/**
+ * Live HTTP pulse — in-flight concurrency plus the recent rate and error rate.
+ * These are "right now" readings, independent of the dashboard's range tab, for the
+ * Overview pulse row. Active requests come from Bun's authoritative
+ * `server.pendingRequests` (https://bun.com/docs/runtime/http/metrics), falling back
+ * to the in-process gauge when no server is bound.
+ */
+export function httpPulse(): {
+  activeRequests: number;
+  requestsPerSec: number;
+  errorRatePct: number;
+} {
+  try {
+    const m = httpMetrics();
+    const recent = m.last5m.total;
+    return {
+      activeRequests: bunPendingRequests() || m.inFlight,
+      requestsPerSec: Math.round((m.perMinute / 60) * 10) / 10,
+      errorRatePct: recent ? +((m.last5m.errors / recent) * 100).toFixed(1) : 0,
+    };
+  } catch {
+    return { activeRequests: 0, requestsPerSec: 0, errorRatePct: 0 };
+  }
+}
+
+const BOOTED_AT = Date.now();
+
+/**
+ * Detect the current deploy SHA from the common platform env vars, falling back
+ * to reading `.git/HEAD`. Returns a 7-char short SHA, or undefined when unknown.
+ */
+export function detectDeploy(): string | undefined {
+  const env = process.env;
+  const fromEnv =
+    env.DEPLOY_SHA ??
+    env.GIT_COMMIT ??
+    env.SOURCE_VERSION ?? // Heroku
+    env.RENDER_GIT_COMMIT ??
+    env.VERCEL_GIT_COMMIT_SHA ??
+    env.RAILWAY_GIT_COMMIT_SHA ??
+    env.GITHUB_SHA;
+  if (fromEnv) return fromEnv.slice(0, 7);
+
+  try {
+    const head = readFileSync(`${process.cwd()}/.git/HEAD`, "utf8").trim();
+    const ref = head.startsWith("ref:") ? head.slice(4).trim() : null;
+    const sha = ref ? readFileSync(`${process.cwd()}/.git/${ref}`, "utf8").trim() : head;
+    return sha.slice(0, 7);
+  } catch {
+    return undefined;
+  }
+}
+
+/** Format a millisecond duration as `Nd Nh` / `Nh Nm` / `Nm`. */
+export function formatUptime(ms: number): string {
+  const s = Math.floor(ms / 1000);
+  const d = Math.floor(s / 86400);
+  const h = Math.floor((s % 86400) / 3600);
+  const m = Math.floor((s % 3600) / 60);
+  if (d > 0) return `${d}d ${h}h`;
+  if (h > 0) return `${h}h ${m}m`;
+  return `${m}m`;
+}
+
+/** Memory / CPU / disk gauges as 0..100 percentages. */
+export function systemGauges(): Gauge[] {
+  const total = totalmem();
+  const free = freemem();
+  const usedBytes = total - free;
+  const memPct = total > 0 ? Math.round((usedBytes / total) * 100) : 0;
+
+  const cores = cpus().length || 1;
+  // loadavg() is 0 on Windows; clamp to a sane 0..100 against core count.
+  const load1 = loadavg()[0] ?? 0;
+  const cpuPct = Math.min(100, Math.round((load1 / cores) * 100));
+
+  const gb = (b: number) => (b / 1024 ** 3).toFixed(1);
+
+  return [
+    { label: "CPU", value: cpuPct, sub: `${cores} vCPU` },
+    {
+      label: "Memory",
+      value: memPct,
+      sub: `${gb(usedBytes)} / ${gb(total)} GB`,
+    },
+    // Disk is not portably available without a syscall; surface process RSS
+    // share of total memory as a proxy so the gauge is meaningful, not faked.
+    {
+      label: "Heap",
+      value: heapPct(),
+      sub: heapSub(),
+    },
+  ];
+}
+
+function heapPct(): number {
+  const mem = process.memoryUsage();
+  if (!mem.heapTotal) return 0;
+  return Math.min(100, Math.round((mem.heapUsed / mem.heapTotal) * 100));
+}
+
+function heapSub(): string {
+  const mem = process.memoryUsage();
+  const mb = (b: number) => Math.round(b / 1024 ** 2);
+  return `${mb(mem.heapUsed)} / ${mb(mem.heapTotal)} MB`;
+}
+
+/** Runtime metadata for the System footer. */
+export function systemMeta(opts: {
+  zerotal?: string;
+  region?: string;
+  deploy?: string;
+}): SystemMeta {
+  return {
+    uptime: formatUptime(Date.now() - BOOTED_AT),
+    bun:
+      typeof Bun !== "undefined" && Bun.version
+        ? Bun.version
+        : (process.versions.bun ?? process.version.replace(/^v/, "")),
+    zerotal: opts.zerotal ?? "v1.1.0",
+    region: opts.region ?? process.env.FLY_REGION ?? process.env.AWS_REGION ?? "local",
+    deploy: opts.deploy ?? process.env.DEPLOY_SHA?.slice(0, 7) ?? "dev",
+    environment: process.env.APP_ENV ?? process.env.NODE_ENV ?? "development",
+  };
+}
+
+/** Process uptime in milliseconds since the monitor booted. */
+export function uptimeMs(): number {
+  return Date.now() - BOOTED_AT;
+}