@ultimat3/core 25.2.0 → 26.0.0

package/README.md

@@ -55,6 +55,7 @@ Zero dependencies, zero `@ultimat3/*` imports.
 | `Clock` — the only source of "now" | `clock.ts` |
 | UUIDv7, nanoid, branded ids | `ids.ts` |
 | structured JSON logging + redaction; `setLogSink(sink)` — the test seam that sends every default-writer line to a sink instead of the process's streams (a test preload drops them; a test asserting on the process logger collects them) | `logger.ts` |
+| `addLogSink(sink)` — the supported tee: every default-writer line, after redaction, to `sink` **beside** the streams (or a `setLogSink` seam), never instead; returns the unsubscribe. A sink that throws is skipped and reported once (`log.sink_failed`); a line a sink logs is not teed back | `log-tee.ts` |
 | OTel-shaped spans, always on, no-op by default | `telemetry.ts` |
 | the sampling decision, and `OTEL_TRACES_SAMPLER*` | `sampler.ts` |
 | OTLP/HTTP JSON: endpoint, headers, value encoding | `otlp.ts` |
@@ -70,7 +71,7 @@ Zero dependencies, zero `@ultimat3/*` imports.
 | graceful drain, `/healthz`, `/readyz` | `lifecycle.ts` |
 | what a health endpoint tells whom — `healthBody(report, role, detailed)`, `healthPeerListed(peers, address)`, `DEFAULT_HEALTH_DETAIL_PEERS`; the one rule `@ultimat3/http` and the sync node's own listener both call | `health-disclosure.ts` |
 | the readiness grace between `/readyz` → 503 and the listener closing (`drain.readinessGraceMs`) | `lifecycle-grace.ts` |
-| the drain budget's default and domain (`drain.deadlineMs`, 25 s, 1–3600000 ms) — `DRAIN_DEADLINE_DEFAULT_MS`, `DRAIN_DEADLINE_MAX_MS` | `drain-deadline.ts` |
+| the drain budget's default and domain (`drain.deadlineMs`, 25 s, 1–3600000 ms; the worker's optional `drain.workerDeadlineMs`, 1–86400000 ms) — `DRAIN_DEADLINE_DEFAULT_MS`, `DRAIN_DEADLINE_MAX_MS`, `WORKER_DRAIN_DEADLINE_MAX_MS` | `drain-deadline.ts` |
 | `jobs.concurrency`'s default and domain — one slot count for every queue a worker serves, or a table per queue (`{ banks: 4, 'banks-long': 2 }`; a queue the table does not name runs at the default). `JOBS_CONCURRENCY_DEFAULT` (8), `type JobsConcurrency` | `config-jobs.ts` |
 | SIGTERM/SIGINT → the one drain; on Windows also SIGHUP (console close) and SIGBREAK (Ctrl-Break) — `drainSignals(platform)` | `lifecycle-signals.ts` |
 | is this directory inside a `bun build --compile` binary? `isCompiledBundle(import.meta.dir)` — `/$bunfs/` and Windows' `B:\~BUN\` | `bunfs.ts` |
@@ -485,6 +486,10 @@ match with `sealAll()`; uniqueness cannot be held across keys.
   once has to keep it: a discarded one is a hook per `start()`, each retaining the resource it
   was going to drain, and the next drain runs every one of them against a torn-down copy.
   `shutdownHookCount()` is the test-only probe that makes the leak assertable.
+- `isRetiring()` is true from the moment a worker's retire begins (SIGUSR2 in production, `x dev`'s
+  restart) for the rest of the process — the retire finishes every held job BEFORE the drain, so
+  `isDraining()` is still false the whole time. Read it where the app reports its own state (a
+  heartbeat). `markRetiring()` is the retire's own call; there is no un-retire.
 - `registerReadinessCheck(name, check)` is what makes `/readyz` mean **usable** rather than
   **bound**. `ReadinessCheck` is `() => boolean` and must stay synchronous — a probe that awaits its
   dependency turns a slow dependency into a wedged endpoint and then a restart loop; keep a boolean

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ultimat3/core",
-  "version": "25.2.0",
+  "version": "26.0.0",
   "description": "Ultimate's foundation: errors, context, env, config, clock, ids, logging, telemetry, lifecycle",
   "license": "MIT",
   "type": "module",
@@ -37,6 +37,6 @@
     "test": "bun test"
   },
   "dependencies": {
-    "@ultimat3/schema": "25.2.0"
+    "@ultimat3/schema": "26.0.0"
   }
 }

package/src/config-defaults.ts

@@ -47,7 +47,12 @@ export function configDefaults(name: string): Omit<AppConfig, Sectioned> {
     notify: { inboxReadRetentionMs: undefined, inboxUnreadRetentionMs: undefined },
     ai: { mcp: { expose: true } },
     // Read from the process env when the config is DEFINED — the same env the drain will run in.
-    drain: { readinessGraceMs: defaultReadinessGraceMs(), deadlineMs: DRAIN_DEADLINE_DEFAULT_MS },
+    drain: {
+      readinessGraceMs: defaultReadinessGraceMs(),
+      deadlineMs: DRAIN_DEADLINE_DEFAULT_MS,
+      // Unset: the worker drains on `deadlineMs`, like every role.
+      workerDeadlineMs: undefined,
+    },
     health: { readiness: 'dependencies' },
   };
 }

package/src/config-health.ts

@@ -40,9 +40,17 @@ export interface DrainConfig {
    * The drain budget: how long a SIGTERM'd process has to finish what it holds — in-flight requests,
    * a running job — after the grace, before the lifecycle abandons the rest. Applied to EVERY role,
    * so this is the knob that gives a long job room to finish on a deploy. Default 25000. A whole
-   * number, 1–3600000. The ONE drain budget, the web role's included.
+   * number, 1–3600000. The ONE drain budget, the web role's included — except the worker's when
+   * `workerDeadlineMs` is set.
    */
   readonly deadlineMs: number;
+  /**
+   * `ROLE=worker`'s drain budget, in place of `deadlineMs`, for a job that must finish rather than
+   * be cut off and replayed and may run past the hour `deadlineMs` is capped at. It also bounds a
+   * retire (SIGUSR2) that a SIGTERM lands in. Unset (the default): the worker drains on
+   * `deadlineMs`. A whole number, 1–86400000. The chart's worker grace period is derived from it.
+   */
+  readonly workerDeadlineMs: number | undefined;
 }
 
 /** Why a value is not a readiness mode, or `undefined` when it is one. */

package/src/drain-deadline.ts

@@ -1,5 +1,5 @@
-// Single responsibility: the drain budget's default and its domain — `drain.deadlineMs` in
-// `app.config.ts`. One module because the config validator, the lifecycle's own default and the
+// Single responsibility: the drain budget's default and its domain — `drain.deadlineMs` and the
+// worker's `drain.workerDeadlineMs` in `app.config.ts`. One module because the config validator, the lifecycle's own default and the
 // chart's grace period must all mean the same number.
 
 import { countIssue } from './config-count';
@@ -19,7 +19,15 @@ export const DRAIN_DEADLINE_DEFAULT_MS = 25_000;
  */
 export const DRAIN_DEADLINE_MAX_MS = 3_600_000;
 
+/**
+ * A day: `drain.workerDeadlineMs`'s ceiling. The worker is the one role whose in-flight unit — a
+ * job that must not run twice, a bank login — can outlast the hour, and a retired worker
+ * (`SIGUSR2`) is bound by this budget when a SIGTERM lands mid-retire.
+ */
+export const WORKER_DRAIN_DEADLINE_MAX_MS = 86_400_000;
+
 const DEADLINE_KEY = 'drain.deadlineMs';
+const WORKER_DEADLINE_KEY = 'drain.workerDeadlineMs';
 
 /**
  * Why a value is not a drain budget, or `undefined` when it is one. A whole number of milliseconds
@@ -34,10 +42,28 @@ export function drainDeadlineIssue(value: unknown): string | undefined {
     : undefined;
 }
 
+/**
+ * Why a value is not the worker's drain budget, or `undefined` when it is one (or is unset — the
+ * worker then drains on `drain.deadlineMs`). A whole number in `1 ≤ v ≤ 86400000`.
+ */
+export function workerDrainDeadlineIssue(value: unknown): string | undefined {
+  if (value === undefined) return undefined;
+  const count = countIssue(WORKER_DEADLINE_KEY, value, 1);
+  if (count !== undefined) return count;
+  return (value as number) > WORKER_DRAIN_DEADLINE_MAX_MS
+    ? `${WORKER_DEADLINE_KEY} must be at most ${WORKER_DRAIN_DEADLINE_MAX_MS} milliseconds (a day) — a job longer than that belongs in steps, which replay instead of re-running`
+    : undefined;
+}
+
 /** The `drain` section's issues, one per key, for `defineConfig`'s validator. */
 export function drainIssues(drain: {
   readonly readinessGraceMs: unknown;
   readonly deadlineMs: unknown;
+  readonly workerDeadlineMs?: unknown;
 }): readonly (string | undefined)[] {
-  return [readinessGraceIssue(drain.readinessGraceMs), drainDeadlineIssue(drain.deadlineMs)];
+  return [
+    readinessGraceIssue(drain.readinessGraceMs),
+    drainDeadlineIssue(drain.deadlineMs),
+    workerDrainDeadlineIssue(drain.workerDeadlineMs),
+  ];
 }

package/src/index.ts

@@ -225,7 +225,11 @@ export {
   CursorSecretDevError,
   devSecretsRefused,
 } from './dev-secrets';
-export { DRAIN_DEADLINE_DEFAULT_MS, DRAIN_DEADLINE_MAX_MS } from './drain-deadline';
+export {
+  DRAIN_DEADLINE_DEFAULT_MS,
+  DRAIN_DEADLINE_MAX_MS,
+  WORKER_DRAIN_DEADLINE_MAX_MS,
+} from './drain-deadline';
 export type {
   Env,
   EnvBooleanVar,
@@ -613,8 +617,10 @@ export {
   idleWaiterCount,
   inflightCount,
   isDraining,
+  isRetiring,
   lifecycleState,
   markReady,
+  markRetiring,
   onShutdown,
   readinessCheckCount,
   readinessChecks,
@@ -638,6 +644,8 @@ export type { Direction } from './locale-direction';
 export { directionOf, isRtl } from './locale-direction';
 export type { LocalePathSplit } from './locale-path';
 export { localeSegment, localizePath, splitLocalePath } from './locale-path';
+// The supported tee: every default-writer line, after redaction, beside the streams.
+export { addLogSink } from './log-tee';
 // The process logger's test seam, beside nothing it groups with: where a default-writer line goes.
 export type { LogSink } from './logger';
 export { setLogSink } from './logger';

package/src/lifecycle.ts

@@ -61,6 +61,8 @@ let readinessMode: ReadinessMode = 'dependencies';
 let clock: Clock = systemClock;
 let log: Logger = rootLogger;
 let state: HealthState = 'starting';
+/** One-way: set by a worker's retire, never cleared but by `resetLifecycle()`. */
+let retiring = false;
 let startedAtMono = clock.monotonic();
 let inflight = 0;
 /** Bumped by `resetLifecycle()`: a `beginWork()` finisher only ever counts down its own lifetime. */
@@ -215,6 +217,22 @@ export function isDraining(): boolean {
   return state === 'draining' || state === 'stopped';
 }
 
+/**
+ * True from the moment a worker's retire begins (SIGUSR2, `@ultimat3/cli`'s `serve-retire.ts`) for
+ * the rest of the process. The retire stops claiming and finishes every held job BEFORE the drain,
+ * so `isDraining()` stays false the whole time — this is how the app's own code (a heartbeat, a
+ * status row) tells "finishing up, about to exit" from "serving". Not a drain: `/readyz` and new
+ * work are untouched.
+ */
+export function isRetiring(): boolean {
+  return retiring;
+}
+
+/** The retire's own call, made before it stops the worker. Idempotent; there is no un-retire. */
+export function markRetiring(): void {
+  retiring = true;
+}
+
 function waitForIdle(timeoutMs: number): Promise<boolean> {
   if (inflight === 0) return Promise.resolve(true);
   return new Promise<boolean>((resolve) => {
@@ -401,6 +419,7 @@ export function resetLifecycle(): void {
   clock = systemClock;
   log = rootLogger;
   state = 'starting';
+  retiring = false;
   startedAtMono = clock.monotonic();
   inflight = 0;
   lifetime += 1;

package/src/log-tee.ts

@@ -0,0 +1,73 @@
+// Single responsibility: `addLogSink` — the supported tee of every default-writer log line, beside
+// the process's streams and never instead of them (`setLogSink` is the test seam that replaces).
+// Total: a sink that throws or logs cannot cost the line, the other sinks, or the process.
+
+import { systemClock } from './clock';
+import { renderThrowable } from './error-render';
+import type { LogLevel, LogSink } from './logger';
+
+let sinks: readonly LogSink[] = [];
+/** Set while sinks run: a line a sink logs reaches the streams, never the tee it came from. */
+let teeing = false;
+/** Each sink says it failed ONCE — a sink that throws on every line would double the log. */
+const reported = new WeakSet<LogSink>();
+
+/**
+ * Tee every line the process logger writes — the complete JSON line, AFTER redaction, and its
+ * level — to `sink`, in addition to stdout/stderr (or a `setLogSink` test sink). Returns the
+ * unsubscribe; calling it twice is a no-op.
+ *
+ * The population is `setLogSink`'s: every logger without its own `writer` — `logger`, every
+ * `child()` of it, `ctx.logger`. Sinks run synchronously, in the order added, after the line is
+ * written; one added while a line is being teed sees the next line, not that one. A sink that
+ * throws is skipped for that line and reported once as `log.sink_failed` on the streams; a line a
+ * sink logs itself is written but not teed back into the sinks.
+ */
+export function addLogSink(sink: LogSink): () => void {
+  sinks = [...sinks, sink];
+  return () => {
+    sinks = sinks.filter((candidate) => candidate !== sink);
+  };
+}
+
+/** How many sinks are teeing now — a count that climbs across a test is an unsubscribe missed. */
+export function logTeeCount(): number {
+  return sinks.length;
+}
+
+/** Called by the logger's default writer after `write` has put the line on the streams. */
+export function teeLogLine(line: string, level: LogLevel, write: LogSink): void {
+  if (teeing || sinks.length === 0) return;
+  teeing = true;
+  try {
+    for (const sink of sinks) {
+      try {
+        sink(line, level);
+      } catch (thrown) {
+        reportOnce(sink, thrown, write);
+      }
+    }
+  } finally {
+    teeing = false;
+  }
+}
+
+/** Straight to the streams, never through the logger: the logger is what called the sink. */
+function reportOnce(sink: LogSink, thrown: unknown, write: LogSink): void {
+  if (reported.has(sink)) return;
+  reported.add(sink);
+  try {
+    write(
+      JSON.stringify({
+        ts: systemClock.now().toISOString(),
+        level: 'warn',
+        msg: 'log.sink_failed',
+        cause: `a log sink added with addLogSink threw ${renderThrowable(thrown)}; it is skipped for that line and still called for the next — this is reported once per sink`,
+        fix: 'addLogSink((line, level) => { try { store(line, level); } catch { /* drop it: a sink must not throw */ } })',
+      }),
+      'warn',
+    );
+  } catch {
+    // The streams themselves are gone; there is nowhere left to say it.
+  }
+}

package/src/logger.ts

@@ -5,6 +5,7 @@ import { assert } from './assert';
 import { type Clock, systemClock } from './clock';
 import { renderCauseValue } from './error-render';
 import { isUltimateError } from './errors';
+import { teeLogLine } from './log-tee';
 import { isSecret, REDACTED } from './secret';
 
 // Re-exported, not redefined: `secret.ts` owns the placeholder because a `Secret` has to render
@@ -219,6 +220,13 @@ export function setLogSink(sink: LogSink | undefined): LogSink | undefined {
  * its log stream, and where there is a `process` this writes to the fd as it always did.
  */
 function defaultWriter(line: string, level: LogLevel): void {
+  streamWriter(line, level);
+  // After the streams, so a sink can never cost a line its stdout (`addLogSink`, `log-tee.ts`).
+  teeLogLine(line, level, streamWriter);
+}
+
+/** The process's streams, or the `setLogSink` seam standing in for them. */
+function streamWriter(line: string, level: LogLevel): void {
   if (logSink !== undefined) {
     logSink(line, level);
     return;