@ultimat3/core 25.1.0 → 26.0.0

package/README.md

@@ -55,6 +55,7 @@ Zero dependencies, zero `@ultimat3/*` imports.
 | `Clock` — the only source of "now" | `clock.ts` |
 | UUIDv7, nanoid, branded ids | `ids.ts` |
 | structured JSON logging + redaction; `setLogSink(sink)` — the test seam that sends every default-writer line to a sink instead of the process's streams (a test preload drops them; a test asserting on the process logger collects them) | `logger.ts` |
+| `addLogSink(sink)` — the supported tee: every default-writer line, after redaction, to `sink` **beside** the streams (or a `setLogSink` seam), never instead; returns the unsubscribe. A sink that throws is skipped and reported once (`log.sink_failed`); a line a sink logs is not teed back | `log-tee.ts` |
 | OTel-shaped spans, always on, no-op by default | `telemetry.ts` |
 | the sampling decision, and `OTEL_TRACES_SAMPLER*` | `sampler.ts` |
 | OTLP/HTTP JSON: endpoint, headers, value encoding | `otlp.ts` |
@@ -70,7 +71,7 @@ Zero dependencies, zero `@ultimat3/*` imports.
 | graceful drain, `/healthz`, `/readyz` | `lifecycle.ts` |
 | what a health endpoint tells whom — `healthBody(report, role, detailed)`, `healthPeerListed(peers, address)`, `DEFAULT_HEALTH_DETAIL_PEERS`; the one rule `@ultimat3/http` and the sync node's own listener both call | `health-disclosure.ts` |
 | the readiness grace between `/readyz` → 503 and the listener closing (`drain.readinessGraceMs`) | `lifecycle-grace.ts` |
-| the drain budget's default and domain (`drain.deadlineMs`, 25 s, 1–3600000 ms) — `DRAIN_DEADLINE_DEFAULT_MS`, `DRAIN_DEADLINE_MAX_MS` | `drain-deadline.ts` |
+| the drain budget's default and domain (`drain.deadlineMs`, 25 s, 1–3600000 ms; the worker's optional `drain.workerDeadlineMs`, 1–86400000 ms) — `DRAIN_DEADLINE_DEFAULT_MS`, `DRAIN_DEADLINE_MAX_MS`, `WORKER_DRAIN_DEADLINE_MAX_MS` | `drain-deadline.ts` |
 | `jobs.concurrency`'s default and domain — one slot count for every queue a worker serves, or a table per queue (`{ banks: 4, 'banks-long': 2 }`; a queue the table does not name runs at the default). `JOBS_CONCURRENCY_DEFAULT` (8), `type JobsConcurrency` | `config-jobs.ts` |
 | SIGTERM/SIGINT → the one drain; on Windows also SIGHUP (console close) and SIGBREAK (Ctrl-Break) — `drainSignals(platform)` | `lifecycle-signals.ts` |
 | is this directory inside a `bun build --compile` binary? `isCompiledBundle(import.meta.dir)` — `/$bunfs/` and Windows' `B:\~BUN\` | `bunfs.ts` |
@@ -485,6 +486,10 @@ match with `sealAll()`; uniqueness cannot be held across keys.
   once has to keep it: a discarded one is a hook per `start()`, each retaining the resource it
   was going to drain, and the next drain runs every one of them against a torn-down copy.
   `shutdownHookCount()` is the test-only probe that makes the leak assertable.
+- `isRetiring()` is true from the moment a worker's retire begins (SIGUSR2 in production, `x dev`'s
+  restart) for the rest of the process — the retire finishes every held job BEFORE the drain, so
+  `isDraining()` is still false the whole time. Read it where the app reports its own state (a
+  heartbeat). `markRetiring()` is the retire's own call; there is no un-retire.
 - `registerReadinessCheck(name, check)` is what makes `/readyz` mean **usable** rather than
   **bound**. `ReadinessCheck` is `() => boolean` and must stay synchronous — a probe that awaits its
   dependency turns a slow dependency into a wedged endpoint and then a restart loop; keep a boolean
@@ -496,6 +501,16 @@ match with `sealAll()`; uniqueness cannot be held across keys.
   before trusting an empty `checks`.
 - Anything that opens a socket calls `markListening(server.url.origin)` and releases it on close.
   That is what tells the sealed test network a loopback request is this process, not egress.
+- **A live suite's throwaway database is named by `probeDatabaseName(prefix)`** — test support,
+  here because tier 0 is the one tier every package with a live suite can import. Prefix, pid,
+  creation second (base 36), eight random hex characters, at most `PROBE_DATABASE_NAME_MAX` (63) bytes, valid unquoted: a
+  fixed name let two runs against one server drop each other's database.
+  `bun run probe-databases` refuses `create database` on any other name (`X_PROBE_DATABASE_FIXED`).
+  A killed run never reaches its `afterAll`, so call `sweepProbeDatabases(executor, PROBE_DB)`
+  before the `create`: it drops the same prefix's probe databases whose pid is dead on this host
+  AND which no backend is connected to AND whose creation second, carried in the name, is older than
+  `PROBE_DATABASE_MIN_AGE_MS` (10 minutes) — a concurrent run on any host is left alone, even between
+  its `create` and its first connection.
 
 ## Metrics: same seam as tracing, one signal over
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/core",
-  "version": "25.1.0",
+  "version": "26.0.0",
   "description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
   "license": "MIT",
   "type": "module",
@@ -37,6 +37,6 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/schema": "25.1.0"
+    "@ultimat3/schema": "26.0.0"
   }
 }

package/src/config-defaults.ts

@@ -47,7 +47,12 @@ export function configDefaults(name: string): Omit<AppConfig, Sectioned> {
     notify: { inboxReadRetentionMs: undefined, inboxUnreadRetentionMs: undefined },
     ai: { mcp: { expose: true } },
     // Read from the process env when the config is DEFINED — the same env the drain will run in.
-    drain: { readinessGraceMs: defaultReadinessGraceMs(), deadlineMs: DRAIN_DEADLINE_DEFAULT_MS },
+    drain: {
+      readinessGraceMs: defaultReadinessGraceMs(),
+      deadlineMs: DRAIN_DEADLINE_DEFAULT_MS,
+      // Unset: the worker drains on `deadlineMs`, like every role.
+      workerDeadlineMs: undefined,
+    },
     health: { readiness: 'dependencies' },
   };
 }

package/src/config-health.ts

@@ -40,9 +40,17 @@ export interface DrainConfig {
    * The drain budget: how long a SIGTERM'd process has to finish what it holds — in-flight requests,
    * a running job — after the grace, before the lifecycle abandons the rest. Applied to EVERY role,
    * so this is the knob that gives a long job room to finish on a deploy. Default 25000. A whole
-   * number, 1–3600000. The ONE drain budget, the web role's included.
+   * number, 1–3600000. The ONE drain budget, the web role's included — except the worker's when
+   * `workerDeadlineMs` is set.
    */
   readonly deadlineMs: number;
+  /**
+   * `ROLE=worker`'s drain budget, in place of `deadlineMs`, for a job that must finish rather than
+   * be cut off and replayed and may run past the hour `deadlineMs` is capped at. It also bounds a
+   * retire (SIGUSR2) that a SIGTERM lands in. Unset (the default): the worker drains on
+   * `deadlineMs`. A whole number, 1–86400000. The chart's worker grace period is derived from it.
+   */
+  readonly workerDeadlineMs: number | undefined;
 }
 
 /** Why a value is not a readiness mode, or `undefined` when it is one. */

package/src/drain-deadline.ts

@@ -1,5 +1,5 @@
-// Single responsibility: the drain budget's default and its domain — `drain.deadlineMs` in
-// `app.config.ts`. One module because the config validator, the lifecycle's own default and the
+// Single responsibility: the drain budget's default and its domain — `drain.deadlineMs` and the
+// worker's `drain.workerDeadlineMs` in `app.config.ts`. One module because the config validator, the lifecycle's own default and the
 // chart's grace period must all mean the same number.
 
 import { countIssue } from './config-count';
@@ -19,7 +19,15 @@ export const DRAIN_DEADLINE_DEFAULT_MS = 25_000;
  */
 export const DRAIN_DEADLINE_MAX_MS = 3_600_000;
 
+/**
+ * A day: `drain.workerDeadlineMs`'s ceiling. The worker is the one role whose in-flight unit — a
+ * job that must not run twice, a bank login — can outlast the hour, and a retired worker
+ * (`SIGUSR2`) is bound by this budget when a SIGTERM lands mid-retire.
+ */
+export const WORKER_DRAIN_DEADLINE_MAX_MS = 86_400_000;
+
 const DEADLINE_KEY = 'drain.deadlineMs';
+const WORKER_DEADLINE_KEY = 'drain.workerDeadlineMs';
 
 /**
  * Why a value is not a drain budget, or `undefined` when it is one. A whole number of milliseconds
@@ -34,10 +42,28 @@ export function drainDeadlineIssue(value: unknown): string | undefined {
     : undefined;
 }
 
+/**
+ * Why a value is not the worker's drain budget, or `undefined` when it is one (or is unset — the
+ * worker then drains on `drain.deadlineMs`). A whole number in `1 ≤ v ≤ 86400000`.
+ */
+export function workerDrainDeadlineIssue(value: unknown): string | undefined {
+  if (value === undefined) return undefined;
+  const count = countIssue(WORKER_DEADLINE_KEY, value, 1);
+  if (count !== undefined) return count;
+  return (value as number) > WORKER_DRAIN_DEADLINE_MAX_MS
+    ? `${WORKER_DEADLINE_KEY} must be at most ${WORKER_DRAIN_DEADLINE_MAX_MS} milliseconds (a day) — a job longer than that belongs in steps, which replay instead of re-running`
+    : undefined;
+}
+
 /** The `drain` section's issues, one per key, for `defineConfig`'s validator. */
 export function drainIssues(drain: {
   readonly readinessGraceMs: unknown;
   readonly deadlineMs: unknown;
+  readonly workerDeadlineMs?: unknown;
 }): readonly (string | undefined)[] {
-  return [readinessGraceIssue(drain.readinessGraceMs), drainDeadlineIssue(drain.deadlineMs)];
+  return [
+    readinessGraceIssue(drain.readinessGraceMs),
+    drainDeadlineIssue(drain.deadlineMs),
+    workerDrainDeadlineIssue(drain.workerDeadlineMs),
+  ];
 }

package/src/index.ts

@@ -225,7 +225,11 @@ export {
   CursorSecretDevError,
   devSecretsRefused,
 } from './dev-secrets';
-export { DRAIN_DEADLINE_DEFAULT_MS, DRAIN_DEADLINE_MAX_MS } from './drain-deadline';
+export {
+  DRAIN_DEADLINE_DEFAULT_MS,
+  DRAIN_DEADLINE_MAX_MS,
+  WORKER_DRAIN_DEADLINE_MAX_MS,
+} from './drain-deadline';
 export type {
   Env,
   EnvBooleanVar,
@@ -613,8 +617,10 @@ export {
   idleWaiterCount,
   inflightCount,
   isDraining,
+  isRetiring,
   lifecycleState,
   markReady,
+  markRetiring,
   onShutdown,
   readinessCheckCount,
   readinessChecks,
@@ -638,6 +644,8 @@ export type { Direction } from './locale-direction';
 export { directionOf, isRtl } from './locale-direction';
 export type { LocalePathSplit } from './locale-path';
 export { localeSegment, localizePath, splitLocalePath } from './locale-path';
+// The supported tee: every default-writer line, after redaction, beside the streams.
+export { addLogSink } from './log-tee';
 // The process logger's test seam, beside nothing it groups with: where a default-writer line goes.
 export type { LogSink } from './logger';
 export { setLogSink } from './logger';
@@ -676,6 +684,15 @@ export {
 } from './page-meta';
 /** The structural Postgres seam http, auth, action and jobs share without a `@ultimat3/db` edge. */
 export type { PgExecutor } from './pg-executor';
+/** Test support: the one name a live suite gives the database it creates (`probe-databases`). */
+export type { ProbeDatabaseEntropy } from './probe-database';
+export { PROBE_DATABASE_NAME_MAX, probeDatabaseName } from './probe-database';
+export type { ProbeDatabaseSweepOptions } from './probe-database-sweep';
+export {
+  PROBE_DATABASE_MIN_AGE_MS,
+  probeDatabaseAlive,
+  sweepProbeDatabases,
+} from './probe-database-sweep';
 export type { ProcessMetricsOptions, ProcessReading } from './process-metrics';
 export { readProcess, resetProcessMetrics, startProcessMetrics } from './process-metrics';
 export { hasPublicCause, registerPublicCause, resetPublicCauses } from './public-cause';

package/src/lifecycle.ts

@@ -61,6 +61,8 @@ let readinessMode: ReadinessMode = 'dependencies';
 let clock: Clock = systemClock;
 let log: Logger = rootLogger;
 let state: HealthState = 'starting';
+/** One-way: set by a worker's retire, never cleared but by `resetLifecycle()`. */
+let retiring = false;
 let startedAtMono = clock.monotonic();
 let inflight = 0;
 /** Bumped by `resetLifecycle()`: a `beginWork()` finisher only ever counts down its own lifetime. */
@@ -215,6 +217,22 @@ export function isDraining(): boolean {
   return state === 'draining' || state === 'stopped';
 }
 
+/**
+ * True from the moment a worker's retire begins (SIGUSR2, `@ultimat3/cli`'s `serve-retire.ts`) for
+ * the rest of the process. The retire stops claiming and finishes every held job BEFORE the drain,
+ * so `isDraining()` stays false the whole time — this is how the app's own code (a heartbeat, a
+ * status row) tells "finishing up, about to exit" from "serving". Not a drain: `/readyz` and new
+ * work are untouched.
+ */
+export function isRetiring(): boolean {
+  return retiring;
+}
+
+/** The retire's own call, made before it stops the worker. Idempotent; there is no un-retire. */
+export function markRetiring(): void {
+  retiring = true;
+}
+
 function waitForIdle(timeoutMs: number): Promise<boolean> {
   if (inflight === 0) return Promise.resolve(true);
   return new Promise<boolean>((resolve) => {
@@ -401,6 +419,7 @@ export function resetLifecycle(): void {
   clock = systemClock;
   log = rootLogger;
   state = 'starting';
+  retiring = false;
   startedAtMono = clock.monotonic();
   inflight = 0;
   lifetime += 1;

package/src/log-tee.ts

@@ -0,0 +1,73 @@
+// Single responsibility: `addLogSink` — the supported tee of every default-writer log line, beside
+// the process's streams and never instead of them (`setLogSink` is the test seam that replaces).
+// Total: a sink that throws or logs cannot cost the line, the other sinks, or the process.
+
+import { systemClock } from './clock';
+import { renderThrowable } from './error-render';
+import type { LogLevel, LogSink } from './logger';
+
+let sinks: readonly LogSink[] = [];
+/** Set while sinks run: a line a sink logs reaches the streams, never the tee it came from. */
+let teeing = false;
+/** Each sink says it failed ONCE — a sink that throws on every line would double the log. */
+const reported = new WeakSet<LogSink>();
+
+/**
+ * Tee every line the process logger writes — the complete JSON line, AFTER redaction, and its
+ * level — to `sink`, in addition to stdout/stderr (or a `setLogSink` test sink). Returns the
+ * unsubscribe; calling it twice is a no-op.
+ *
+ * The population is `setLogSink`'s: every logger without its own `writer` — `logger`, every
+ * `child()` of it, `ctx.logger`. Sinks run synchronously, in the order added, after the line is
+ * written; one added while a line is being teed sees the next line, not that one. A sink that
+ * throws is skipped for that line and reported once as `log.sink_failed` on the streams; a line a
+ * sink logs itself is written but not teed back into the sinks.
+ */
+export function addLogSink(sink: LogSink): () => void {
+  sinks = [...sinks, sink];
+  return () => {
+    sinks = sinks.filter((candidate) => candidate !== sink);
+  };
+}
+
+/** How many sinks are teeing now — a count that climbs across a test is an unsubscribe missed. */
+export function logTeeCount(): number {
+  return sinks.length;
+}
+
+/** Called by the logger's default writer after `write` has put the line on the streams. */
+export function teeLogLine(line: string, level: LogLevel, write: LogSink): void {
+  if (teeing || sinks.length === 0) return;
+  teeing = true;
+  try {
+    for (const sink of sinks) {
+      try {
+        sink(line, level);
+      } catch (thrown) {
+        reportOnce(sink, thrown, write);
+      }
+    }
+  } finally {
+    teeing = false;
+  }
+}
+
+/** Straight to the streams, never through the logger: the logger is what called the sink. */
+function reportOnce(sink: LogSink, thrown: unknown, write: LogSink): void {
+  if (reported.has(sink)) return;
+  reported.add(sink);
+  try {
+    write(
+      JSON.stringify({
+        ts: systemClock.now().toISOString(),
+        level: 'warn',
+        msg: 'log.sink_failed',
+        cause: `a log sink added with addLogSink threw ${renderThrowable(thrown)}; it is skipped for that line and still called for the next — this is reported once per sink`,
+        fix: 'addLogSink((line, level) => { try { store(line, level); } catch { /* drop it: a sink must not throw */ } })',
+      }),
+      'warn',
+    );
+  } catch {
+    // The streams themselves are gone; there is nowhere left to say it.
+  }
+}

package/src/logger.ts

@@ -5,6 +5,7 @@ import { assert } from './assert';
 import { type Clock, systemClock } from './clock';
 import { renderCauseValue } from './error-render';
 import { isUltimateError } from './errors';
+import { teeLogLine } from './log-tee';
 import { isSecret, REDACTED } from './secret';
 
 // Re-exported, not redefined: `secret.ts` owns the placeholder because a `Secret` has to render
@@ -219,6 +220,13 @@ export function setLogSink(sink: LogSink | undefined): LogSink | undefined {
  * its log stream, and where there is a `process` this writes to the fd as it always did.
  */
 function defaultWriter(line: string, level: LogLevel): void {
+  streamWriter(line, level);
+  // After the streams, so a sink can never cost a line its stdout (`addLogSink`, `log-tee.ts`).
+  teeLogLine(line, level, streamWriter);
+}
+
+/** The process's streams, or the `setLogSink` seam standing in for them. */
+function streamWriter(line: string, level: LogLevel): void {
   if (logSink !== undefined) {
     logSink(line, level);
     return;

package/src/probe-database-sweep.ts

@@ -0,0 +1,68 @@
+// Test support: drop the probe databases a KILLED run of the same suite left behind. A per-run name
+// (`probe-database.ts`) is never reused, so nothing else would ever drop it — the fixed name it
+// replaced was dropped by the next run. Only a database whose run is provably over goes: its pid
+// dead on this host, no backend connected, AND older than `PROBE_DATABASE_MIN_AGE_MS` — a run on
+// another host that has created its database and not yet connected reads dead and idle here.
+
+import { stringField } from './error-render';
+import type { PgExecutor } from './pg-executor';
+
+/** `<prefix>_<pid>_<seconds, base 36>_<8 hex>` — exactly what `probeDatabaseName` returns. */
+const PROBE_SHAPE = /^([a-z_][a-z0-9_]*)_(\d+)_([0-9a-z]+)_[0-9a-f]{8}$/;
+
+/** A leftover younger than this is left alone: no suite takes ten minutes to make its first query. */
+export const PROBE_DATABASE_MIN_AGE_MS = 10 * 60 * 1000;
+
+export interface ProbeDatabaseSweepOptions {
+  /** Whether a pid is a live process on this host. Defaults to `probeDatabaseAlive`. */
+  readonly isAlive?: ((pid: number) => boolean) | undefined;
+  /** The clock a leftover's age is read against. Defaults to `Date.now`. */
+  readonly now?: (() => number) | undefined;
+}
+
+/**
+ * `kill(pid, 0)` delivers nothing and answers whether the process exists: `EPERM` is a process
+ * someone else owns, which is alive. Another host's pid reads as dead here, which is why the sweep
+ * also demands that nothing is connected.
+ */
+export function probeDatabaseAlive(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (thrown) {
+    return stringField(thrown, 'code') === 'EPERM';
+  }
+}
+
+/**
+ * Drop every database sharing `probeName`'s prefix whose run is over, never `probeName` itself.
+ * Call it beside the `create database`; it answers the names it dropped. A name not shaped like
+ * `probeDatabaseName`'s answer has no siblings to find, and nothing is sent.
+ */
+export async function sweepProbeDatabases(
+  executor: PgExecutor,
+  probeName: string,
+  options: ProbeDatabaseSweepOptions = {},
+): Promise<readonly string[]> {
+  const prefix = PROBE_SHAPE.exec(probeName)?.[1];
+  if (prefix === undefined) return [];
+  const isAlive = options.isAlive ?? probeDatabaseAlive;
+  const now = (options.now ?? Date.now)();
+  const rows = await executor.query<{ datname: string; backends: number | string }>(
+    `select d.datname, (select count(*) from pg_stat_activity a where a.datname = d.datname) as backends
+       from pg_database d where d.datname like $1`,
+    [`${prefix.replaceAll('_', '\\_')}\\_%`],
+  );
+  const swept: string[] = [];
+  for (const row of rows) {
+    const shape = PROBE_SHAPE.exec(row.datname);
+    if (row.datname === probeName || shape?.[1] !== prefix) continue;
+    if (Number(row.backends) > 0 || isAlive(Number(shape[2]))) continue;
+    const createdMs = Number.parseInt(shape[3] ?? '', 36) * 1000;
+    if (!Number.isFinite(createdMs) || now - createdMs < PROBE_DATABASE_MIN_AGE_MS) continue;
+    // The name matched `PROBE_SHAPE`, so it is `[a-z0-9_]` only — quoting it needs no escape.
+    await executor.query(`drop database if exists "${row.datname}" with (force)`, []);
+    swept.push(row.datname);
+  }
+  return swept;
+}

package/src/probe-database.ts

@@ -0,0 +1,44 @@
+// Test support: the ONE name a live or contract suite gives the database it creates and drops —
+// prefix, pid, creation second (base 36), eight random hex characters. A fixed name let two runs against one server drop each
+// other's database mid-suite (#705). Here, at tier 0, because every package with such a suite can
+// import core and `@ultimat3/testing` (tier 5) is out of reach below it. `bun run probe-databases`.
+
+/** Postgres's NAMEDATALEN - 1: a longer identifier is silently truncated, so two names could meet. */
+export const PROBE_DATABASE_NAME_MAX = 63;
+
+const RANDOM_HEX = 8;
+
+/** Where a probe name's uniqueness comes from — injectable so a test can pin the answer. */
+export interface ProbeDatabaseEntropy {
+  /** Defaults to `process.pid`: two concurrent runs are two processes. */
+  readonly pid?: number | undefined;
+  /** Defaults to a fresh `crypto.randomUUID()`: two suites in one process differ by this. */
+  readonly random?: string | undefined;
+  /** Defaults to `Date.now()`: the creation second the sweep ages a leftover by. */
+  readonly now?: number | undefined;
+}
+
+/** Lowercase `[a-z0-9_]`, starting with a letter or `_` — an identifier no statement must quote. */
+function identifierPrefix(prefix: string): string {
+  const folded = prefix.toLowerCase().replace(/[^a-z0-9_]/g, '_');
+  if (folded === '') return 'x_probe';
+  return /^[a-z_]/.test(folded) ? folded : `x_${folded}`;
+}
+
+/**
+ * A database name unique to this call, valid unquoted, at most 63 bytes — truncation takes the
+ * prefix, never the suffix. The creation second is in the name because `pg_database` records none,
+ * and `sweepProbeDatabases` must not drop a run's database in the moment between its `create` and
+ * its first connection (`<prefix>_<pid>_<seconds, base 36>_<8 hex>`).
+ */
+export function probeDatabaseName(prefix: string, entropy: ProbeDatabaseEntropy = {}): string {
+  const pid = Math.trunc(Math.abs(entropy.pid ?? process.pid));
+  const hex = (entropy.random ?? crypto.randomUUID())
+    .toLowerCase()
+    .replace(/[^0-9a-f]/g, '')
+    .slice(0, RANDOM_HEX)
+    .padEnd(RANDOM_HEX, '0');
+  const seconds = Math.floor(Math.max(0, entropy.now ?? Date.now()) / 1000).toString(36);
+  const suffix = `_${String(pid)}_${seconds}_${hex}`;
+  return `${identifierPrefix(prefix).slice(0, PROBE_DATABASE_NAME_MAX - suffix.length)}${suffix}`;
+}