@ultimat3/admin 29.4.0 → 30.1.0

package/src/dev/data.ts

@@ -22,6 +22,7 @@ import type {
   LiveQueryFact,
   LiveSubscriberFact,
   MailFact,
+  MailTemplateFact,
   ManifestFact,
   PolicyFact,
   QueueFact,
@@ -54,6 +55,7 @@ export function staticDevSources(facts: Partial<DevSources> = {}): DevSources {
     runSql:
       facts.runSql ?? ((): Promise<SqlResult> => empty({ columns: [], rows: [], elapsedMs: 0 })),
     mail: facts.mail ?? ((): Promise<readonly MailFact[]> => empty([])),
+    mailTemplates: facts.mailTemplates ?? ((): Promise<readonly MailTemplateFact[]> => empty([])),
     cacheGraph: facts.cacheGraph ?? ((): Promise<readonly CacheEdgeFact[]> => empty([])),
     invalidations: facts.invalidations ?? ((): Promise<readonly InvalidationFact[]> => empty([])),
     policyMatrix: facts.policyMatrix ?? ((): Promise<readonly PolicyFact[]> => empty([])),
@@ -122,6 +124,10 @@ const STEP_STATUS: Readonly<Record<StepStatus, JobStepFact['status']>> = {
   sleeping: 'sleeping',
   waiting: 'pending',
   failed: 'failed',
+  // A `once` step whose `fn` began and recorded no outcome: in flight, or unknown until an operator
+  // says so with `x jobs resolve-step`. `cleared` is that operator's "it did not run": it runs again.
+  started: 'running',
+  cleared: 'pending',
 };
 
 /**
@@ -335,6 +341,9 @@ export function defaultDevSources(opts: DevSourceOptions = {}): DevSources {
 
     runSql: unwired<SqlResult>('runSql', 'db'),
     mail: unwired<readonly MailFact[]>('mail', 'mail'),
+    // The mail registry and the app's locales are the host's: `x dev` renders each declared
+    // `preview` through `@ultimat3/mail`, which this package does not depend on.
+    mailTemplates: unwired<readonly MailTemplateFact[]>('mailTemplates', 'mail'),
 
     /**
      * The tag graph is read through `dependentsOf`, one entity tag at a time: the cache owns

package/src/dev/facts.ts

@@ -225,6 +225,30 @@ export interface MailFact {
   readonly sentAt: string;
 }
 
+/** One locale's render of a declared mail's `preview`, or the instruction its schema refused with. */
+export type MailTemplatePreviewFact =
+  | {
+      readonly locale: string;
+      readonly ok: true;
+      readonly subject: string;
+      readonly html: string;
+      readonly text: string;
+      /** Catalog keys this locale could not resolve: `send` would refuse these. */
+      readonly missing: readonly string[];
+    }
+  | {
+      readonly locale: string;
+      readonly ok: false;
+      readonly error: { readonly code: string; readonly cause: string; readonly fix: string };
+    };
+
+/** A declared mail (`defineMail`) and its preview in every locale; none when it declares none. */
+export interface MailTemplateFact {
+  readonly id: string;
+  readonly hasPreview: boolean;
+  readonly previews: readonly MailTemplatePreviewFact[];
+}
+
 export interface CacheEdgeFact {
   readonly tag: string;
   readonly dependents: readonly { readonly kind: string; readonly id: string }[];
@@ -270,6 +294,7 @@ export interface DevSources {
   drift(): Promise<readonly DriftFact[]>;
   runSql(sql: string): Promise<SqlResult>;
   mail(): Promise<readonly MailFact[]>;
+  mailTemplates(): Promise<readonly MailTemplateFact[]>;
   cacheGraph(): Promise<readonly CacheEdgeFact[]>;
   invalidations(): Promise<readonly InvalidationFact[]>;
   policyMatrix(): Promise<readonly PolicyFact[]>;

package/src/dev/index.ts

@@ -17,6 +17,8 @@ export type {
   LiveQueryFact,
   LiveSubscriberFact,
   MailFact,
+  MailTemplateFact,
+  MailTemplatePreviewFact,
   ManifestFact,
   PolicyFact,
   QueueFact,

package/src/dev/panel-mail.ts

@@ -1,8 +1,8 @@
 // Panel: Mail.
-// Kills: "what did that email actually look like, in that locale?" — every caught message,
-// rendered, grouped per locale so a missing translation is visible instead of inferred.
+// Kills: "what did that email actually look like, in that locale?" — every caught message, and
+// every declared template rendered from its `preview` in every locale with no flow triggered.
 
-import type { MailFact } from './facts';
+import type { MailFact, MailTemplateFact } from './facts';
 import type { DevPanel } from './panel';
 
 export interface MailPanelData {
@@ -14,14 +14,30 @@ export interface MailPanelData {
     readonly subject: string;
     readonly missing: readonly string[];
   }[];
+  /** Every declared mail, each with its preview rendered per locale (`defineMail({ preview })`). */
+  readonly templates: readonly MailTemplateFact[];
+  readonly selectedTemplate: MailTemplateFact | null;
+  /** A template's catalog keys a locale could not resolve — the gap before anything is sent. */
+  readonly templateGaps: readonly {
+    readonly id: string;
+    readonly locale: string;
+    readonly missing: readonly string[];
+  }[];
 }
 
+/** `?template=` names one; none named is the first; an unknown name is nothing, never the first. */
+const selectedOf = (
+  templates: readonly MailTemplateFact[],
+  wanted: string | null,
+): MailTemplateFact | null =>
+  (wanted === null ? templates[0] : templates.find((one) => one.id === wanted)) ?? null;
+
 export const mailPanel: DevPanel<MailPanelData> = {
   key: 'mail',
   titleKey: 'dev.panel.mail.title',
   questionKey: 'dev.panel.mail.question',
   async data(sources, params): Promise<MailPanelData> {
-    const all = await sources.mail();
+    const [all, templates] = await Promise.all([sources.mail(), sources.mailTemplates()]);
     const locale = params.get('locale');
     const messages = locale === null ? all : all.filter((mail) => mail.locale === locale);
     const wanted = params.get('id');
@@ -46,6 +62,15 @@ export const mailPanel: DevPanel<MailPanelData> = {
           missing: locales.filter((candidate) => !seen.has(candidate)),
         }))
         .filter((entry) => entry.missing.length > 0),
+      templates,
+      selectedTemplate: selectedOf(templates, params.get('template')),
+      templateGaps: templates.flatMap((template) =>
+        template.previews.flatMap((preview) =>
+          preview.ok && preview.missing.length > 0
+            ? [{ id: template.id, locale: preview.locale, missing: preview.missing }]
+            : [],
+        ),
+      ),
     };
   },
 };

package/src/error-groups-pages.ts

@@ -0,0 +1,50 @@
+// Single responsibility: `/admin/errors` as `pages:` entries — the list and one group's detail,
+// guarded like every custom page (`admin:read` plus `errors:read`). Data, not a new screen kind:
+// the route table, the nav and the audit of a refusal are the ones `pages.ts` already builds.
+
+import type { ErrorGroupsStore } from './error-groups-store';
+import { ErrorsDetail } from './errors-detail';
+import { ErrorsList } from './errors-list';
+import type { AdminCustomPage } from './pages';
+
+/** The permission the two pages compose after `admin:read`. */
+export const ERRORS_READ = 'errors:read';
+
+export interface ErrorGroupsPagesOptions {
+  readonly store: ErrorGroupsStore;
+  /** The nav group the list is linked under; omitted, reachable by URL only. */
+  readonly navGroup?: string | undefined;
+  /** Rooted at the admin's `basePath`. Default `/errors`. */
+  readonly path?: string | undefined;
+}
+
+export function errorGroupsPages(options: ErrorGroupsPagesOptions): readonly AdminCustomPage[] {
+  const path = options.path ?? '/errors';
+  return [
+    {
+      path,
+      titleKey: 'admin.errors.title',
+      permissions: [ERRORS_READ],
+      ...(options.navGroup === undefined ? {} : { navGroup: options.navGroup }),
+      component: (props) =>
+        ErrorsList({
+          store: options.store,
+          basePath: new URL(props.url).pathname,
+          url: props.url,
+        }),
+    },
+    {
+      path: `${path}/:fingerprint`,
+      titleKey: 'admin.errors.detail.title',
+      permissions: [ERRORS_READ],
+      component: (props) => {
+        const pathname = new URL(props.url).pathname;
+        return ErrorsDetail({
+          store: options.store,
+          listPath: pathname.slice(0, pathname.lastIndexOf('/')),
+          fingerprint: props.params['fingerprint'] ?? '',
+        });
+      },
+    },
+  ];
+}

package/src/error-groups-pg.ts

@@ -0,0 +1,163 @@
+// The durable error-group store: one upsert per occurrence on `x_error_groups`, read back newest
+// first. What `errorGroupsSink({ store: postgresErrorGroupsStore() })` names in a deployment with
+// more than one process — the memory twin forgets at every restart and counts per replica.
+
+import { isJsonObject } from '@ultimat3/core';
+import { type DbClient, db } from '@ultimat3/db';
+import { ERROR_GROUPS_TABLE } from './error-groups-schema';
+import {
+  ERROR_GROUP_STATUSES,
+  type ErrorGroup,
+  type ErrorGroupSample,
+  type ErrorGroupStatus,
+  type ErrorGroupsStore,
+  errorGroupsLimit,
+} from './error-groups-store';
+
+const COLUMNS =
+  'fingerprint, code, source, operation, message, status, first_seen, last_seen, count, sample';
+
+/**
+ * `xmax = 0` is true only on the row version an INSERT wrote: under a concurrent conflict one
+ * session inserts and every other one updates, so exactly one caller reads `inserted`. A resolved
+ * group re-opens here, in the same statement, so a regression can never be counted and stay shut.
+ */
+export const SQL_ERROR_GROUP_UPSERT = `
+insert into x_error_groups as g (
+  fingerprint, code, source, operation, message, status, first_seen, last_seen, count, sample
+) values ($1, $2, $3, $4, $5, 'open', $6::timestamptz, $6::timestamptz, $7, $8::jsonb)
+on conflict (fingerprint) do update set
+  message   = excluded.message,
+  status    = case when g.status = 'resolved' then 'open' else g.status end,
+  last_seen = greatest(g.last_seen, excluded.last_seen),
+  count     = g.count + excluded.count,
+  sample    = excluded.sample
+returning (xmax = 0) as inserted
+`;
+
+interface GroupRow {
+  readonly fingerprint: string;
+  readonly code: string;
+  readonly source: string;
+  readonly operation: string | null;
+  readonly message: string;
+  readonly status: string;
+  readonly first_seen: Date | string;
+  readonly last_seen: Date | string;
+  readonly count: number | string | bigint;
+  readonly sample: unknown;
+}
+
+const iso = (value: Date | string): string =>
+  (value instanceof Date ? value : new Date(value)).toISOString();
+
+/** A `jsonb` column, whichever way the driver hands it back: parsed, or as its text. */
+const parsed = (value: unknown): unknown => {
+  if (typeof value !== 'string') return value;
+  try {
+    return JSON.parse(value);
+  } catch {
+    return null;
+  }
+};
+
+/** An own string field, or `null`: a stored row is data, never a prototype to read through. */
+const text = (record: Readonly<Record<string, unknown>>, key: string): string | null => {
+  const value = Object.hasOwn(record, key) ? record[key] : undefined;
+  return typeof value === 'string' ? value : null;
+};
+
+/** Written by this file from `ErrorGroupSample`; a hand-edited row reads as what it holds. */
+function sampleOfRow(value: unknown): ErrorGroupSample {
+  const record = isJsonObject(value) ? value : {};
+  const meta = record['meta'];
+  // Spelled apart: the stored instruction is data read back, not a `fix:` this file writes.
+  const fix = text(record, 'fix') ?? '';
+  return {
+    at: text(record, 'at') ?? '',
+    severity: text(record, 'severity') ?? 'error',
+    title: text(record, 'title') ?? '',
+    cause: text(record, 'cause') ?? '',
+    fix,
+    release: text(record, 'release'),
+    environment: text(record, 'environment') ?? '',
+    requestId: text(record, 'requestId'),
+    traceId: text(record, 'traceId'),
+    meta: isJsonObject(meta) ? meta : null,
+    stack: text(record, 'stack'),
+  };
+}
+
+const groupOf = (row: GroupRow): ErrorGroup => ({
+  fingerprint: row.fingerprint,
+  code: row.code,
+  source: row.source,
+  operation: row.operation,
+  message: row.message,
+  status: ERROR_GROUP_STATUSES.find((known) => known === row.status) ?? 'open',
+  firstSeen: iso(row.first_seen),
+  lastSeen: iso(row.last_seen),
+  count: Number(row.count),
+  sample: sampleOfRow(parsed(row.sample)),
+});
+
+export interface PostgresErrorGroupsStoreOptions {
+  /** Omitted, the process's own `db()`. The sink already left any open transaction. */
+  readonly client?: DbClient | undefined;
+}
+
+export function postgresErrorGroupsStore(
+  options: PostgresErrorGroupsStoreOptions = {},
+): ErrorGroupsStore {
+  const client = (): DbClient => options.client ?? db();
+  return {
+    kind: 'postgres',
+    async record(occurrence) {
+      const rows = await client().query<{ inserted: boolean }>({
+        text: SQL_ERROR_GROUP_UPSERT,
+        values: [
+          occurrence.fingerprint,
+          occurrence.code,
+          occurrence.source,
+          occurrence.operation,
+          occurrence.message,
+          occurrence.at.toISOString(),
+          occurrence.count,
+          JSON.stringify(occurrence.sample),
+        ],
+      });
+      return { isNew: rows[0]?.inserted === true };
+    },
+    async list(query = {}) {
+      const limit = errorGroupsLimit('postgresErrorGroupsStore', query.limit);
+      const filtered = query.status !== undefined;
+      return (
+        await client().query<GroupRow>({
+          text: `select ${COLUMNS} from ${ERROR_GROUPS_TABLE}${filtered ? ' where status = $2' : ''} order by last_seen desc, fingerprint limit $1`,
+          values: filtered ? [limit, query.status] : [limit],
+        })
+      ).map(groupOf);
+    },
+    async get(fingerprint) {
+      const rows = await client().query<GroupRow>({
+        text: `select ${COLUMNS} from ${ERROR_GROUPS_TABLE} where fingerprint = $1`,
+        values: [fingerprint],
+      });
+      return rows[0] === undefined ? undefined : groupOf(rows[0]);
+    },
+    async setStatus(fingerprint, status: ErrorGroupStatus) {
+      const rows = await client().query<GroupRow>({
+        text: `update ${ERROR_GROUPS_TABLE} set status = $2 where fingerprint = $1 returning ${COLUMNS}`,
+        values: [fingerprint, status],
+      });
+      return rows[0] === undefined ? undefined : groupOf(rows[0]);
+    },
+    async purge(beforeMs) {
+      const rows = await client().query<{ removed: number | string }>({
+        text: `with gone as (delete from ${ERROR_GROUPS_TABLE} where last_seen < $1::timestamptz returning 1) select count(*)::int as removed from gone`,
+        values: [new Date(beforeMs).toISOString()],
+      });
+      return Number(rows[0]?.removed ?? 0);
+    },
+  };
+}

package/src/error-groups-schema.ts

@@ -0,0 +1,36 @@
+// The DDL of the error-groups table, and nothing else: a LEAF module, so the boot's
+// `FRAMEWORK_SCHEMA` (`@ultimat3/cli`) can apply it in every role without pulling one screen into
+// a worker's module graph — `audit-schema.ts`'s rule, for the same reason.
+
+/** The table. One name, read by this DDL and by `error-groups-pg.ts`'s statements. */
+export const ERROR_GROUPS_TABLE = 'x_error_groups';
+
+/**
+ * Applied by the boot, never by an app migration — the rule every framework table follows.
+ *
+ * **One row per fingerprint** (`errorFingerprint`, core): the primary key IS the group, so the
+ * upsert's `xmax = 0` answers "this insert made it" for exactly one of N racing replicas.
+ *
+ * **`sample` is what the sink already redacted** — credentials by key, the stack cut — and never
+ * the thrown value. A replayable bundle is `x_failures`', not this table's.
+ *
+ * **One index**, `(last_seen desc, fingerprint)`: the list screen's order and the purge's range.
+ * Retention is the app's: `errorGroupsPurgeTarget(store, { olderThanMs })` has no default.
+ */
+export const SQL_ERROR_GROUPS_TABLE = `
+create table if not exists x_error_groups (
+  fingerprint text        primary key,
+  code        text        not null,
+  source      text        not null,
+  operation   text,
+  message     text        not null,
+  status      text        not null default 'open',
+  first_seen  timestamptz not null,
+  last_seen   timestamptz not null,
+  count       bigint      not null,
+  sample      jsonb       not null
+);
+
+create index if not exists x_error_groups_seen_idx
+  on x_error_groups (last_seen desc, fingerprint);
+`;

package/src/error-groups-sink.ts

@@ -0,0 +1,181 @@
+// Single responsibility: `errorGroupsSink` — an `ErrorReporter` that folds every report into its
+// `errorFingerprint` group in a store, outside the request's transaction, bounded per minute, and
+// enqueues the app's `onNewGroup` job exactly once per new group. It never re-enters the reporter.
+
+import type { Clock, ErrorReport, ErrorReporter } from '@ultimat3/core';
+import {
+  errorFingerprint,
+  finiteCount,
+  isRedactedKey,
+  logger,
+  REDACTED,
+  renderThrowable,
+  runToCompletion,
+  systemClock,
+} from '@ultimat3/core';
+import { outsideTransaction } from '@ultimat3/db';
+import type { JobHandle } from '@ultimat3/jobs';
+import type {
+  ErrorGroupOccurrence,
+  ErrorGroupSample,
+  ErrorGroupsStore,
+} from './error-groups-store';
+
+/** What `onNewGroup` is enqueued with: enough to title an issue, nothing a report did not group on. */
+export interface ErrorGroupNotice {
+  readonly fingerprint: string;
+  readonly code: string;
+  readonly source: string;
+  readonly operation: string | null;
+  readonly message: string;
+  readonly firstSeen: string;
+}
+
+export interface ErrorGroupsSinkOptions {
+  readonly store: ErrorGroupsStore;
+  /** The app's job: which tracker, which title, which assignee are its own (axiom 8). */
+  readonly onNewGroup?: Pick<JobHandle<ErrorGroupNotice>, 'name' | 'enqueue'> | undefined;
+  /** A report this answers `true` for is not grouped. */
+  readonly ignore?: ((report: ErrorReport) => boolean) | undefined;
+  /** Store writes per process per minute. Past it, reports are counted and written as one later. */
+  readonly perMinute?: number | undefined;
+  readonly clock?: Clock | undefined;
+}
+
+export interface ErrorGroupsSink extends ErrorReporter {
+  /** Resolves when every write started so far has settled. For tests and a drain hook. */
+  settled(): Promise<void>;
+}
+
+export const ERROR_GROUPS_PER_MINUTE = 120;
+/** The cause's head: a list row, never a page of text. */
+const MESSAGE_HEAD = 200;
+const STACK_MAX = 8_000;
+/** Groups held back while over the cap. Past this many, a report is dropped and that is logged. */
+const DEFERRED_MAX = 1_000;
+
+/** Credentials by key, as the log line and the monitor envelope redact them: one rule. */
+function redactedMeta(meta: ErrorReport['meta']): Readonly<Record<string, unknown>> | null {
+  if (meta === undefined) return null;
+  const out = new Map<string, unknown>();
+  for (const [key, value] of Object.entries(meta))
+    out.set(key, isRedactedKey(key) ? REDACTED : value);
+  return Object.fromEntries(out);
+}
+
+export function sampleOf(report: ErrorReport): ErrorGroupSample {
+  return {
+    at: new Date(report.at).toISOString(),
+    severity: report.severity,
+    title: report.title,
+    cause: report.cause,
+    fix: report.fix,
+    release: report.release,
+    environment: report.environment,
+    requestId: report.scope.requestId ?? null,
+    traceId: report.scope.traceId ?? null,
+    meta: redactedMeta(report.meta),
+    stack: report.stack === undefined ? null : report.stack.slice(0, STACK_MAX),
+  };
+}
+
+function occurrenceOf(report: ErrorReport, count: number): ErrorGroupOccurrence {
+  return {
+    fingerprint: errorFingerprint(report),
+    code: report.code,
+    source: report.source,
+    operation: report.scope.operation ?? null,
+    message: report.cause.split('\n')[0]?.slice(0, MESSAGE_HEAD) ?? '',
+    at: new Date(report.at),
+    count,
+    sample: sampleOf(report),
+  };
+}
+
+export function errorGroupsSink(options: ErrorGroupsSinkOptions): ErrorGroupsSink {
+  const clock = options.clock ?? systemClock;
+  const cap = finiteCount(
+    'errorGroupsSink',
+    'perMinute',
+    options.perMinute ?? ERROR_GROUPS_PER_MINUTE,
+    1,
+  );
+  const deferred = new Map<string, ErrorGroupOccurrence>();
+  const pending = new Set<Promise<void>>();
+  let window = -1;
+  let written = 0;
+
+  /** Its own failure is logged, never reported: a sink that reports itself loops on an outage. */
+  const write = (occurrence: ErrorGroupOccurrence): void => {
+    const run = outsideTransaction(() =>
+      runToCompletion(async () => {
+        const { isNew } = await options.store.record(occurrence);
+        if (!isNew || options.onNewGroup === undefined) return;
+        await options.onNewGroup.enqueue({
+          fingerprint: occurrence.fingerprint,
+          code: occurrence.code,
+          source: occurrence.source,
+          operation: occurrence.operation,
+          message: occurrence.message,
+          firstSeen: occurrence.at.toISOString(),
+        });
+      }),
+    ).catch((failure: unknown) => {
+      logger.warn('error group write failed', {
+        fingerprint: occurrence.fingerprint,
+        code: occurrence.code,
+        error: renderThrowable(failure),
+      });
+    });
+    pending.add(run);
+    void run.finally(() => pending.delete(run));
+  };
+
+  /** A new minute writes what the last one held back, each group once, before the new report. */
+  const turn = (minute: number): void => {
+    if (minute === window) return;
+    window = minute;
+    written = 0;
+    const held = [...deferred.values()];
+    deferred.clear();
+    for (const occurrence of held) {
+      written += 1;
+      write(occurrence);
+    }
+  };
+
+  const hold = (occurrence: ErrorGroupOccurrence): void => {
+    const seen = deferred.get(occurrence.fingerprint);
+    if (seen !== undefined) {
+      deferred.set(occurrence.fingerprint, { ...occurrence, count: seen.count + occurrence.count });
+    } else if (deferred.size < DEFERRED_MAX) {
+      deferred.set(occurrence.fingerprint, occurrence);
+    } else {
+      logger.warn('error group dropped over the per-minute cap', {
+        fingerprint: occurrence.fingerprint,
+        perMinute: cap,
+      });
+    }
+  };
+
+  return {
+    report(report: ErrorReport): void {
+      try {
+        if (options.ignore?.(report) === true) return;
+        turn(Math.floor(clock.now().getTime() / 60_000));
+        const occurrence = occurrenceOf(report, 1);
+        if (written >= cap) {
+          hold(occurrence);
+          return;
+        }
+        written += 1;
+        write(occurrence);
+      } catch (failure) {
+        logger.warn('error group sink failed', { error: renderThrowable(failure) });
+      }
+    },
+    async settled(): Promise<void> {
+      while (pending.size > 0) await Promise.all([...pending]);
+    },
+  };
+}