@ultimat3/cli 29.4.0 → 30.1.0

package/src/repro-write.ts

@@ -0,0 +1,100 @@
+// Single responsibility: where a repro test lands and what running it says. `<root>/repro/`, one
+// file per subject — outside every surface, so it may import the app's API index for its
+// registrations without crossing a boundary — and `--verify` is one `bun test` of that file.
+
+import { existsSync } from 'node:fs'; // why: a synchronous probe of two optional app files.
+// why: Bun exposes no path-join primitive.
+import { join } from 'node:path';
+import { isJsonObject } from '@ultimat3/core';
+import { API_INDEX } from './app-root';
+import type { Runner } from './exec';
+import { formatWritten } from './generate-format-written';
+import type { ReproSubject } from './repro-bundle';
+import { jobTestSource } from './repro-job';
+import { requestTestSource } from './repro-request';
+import { testEnvOverrides } from './test-dotenv';
+
+/** The directory every written repro lands in, relative to the app root. */
+export const REPRO_DIR = 'repro';
+
+/** `Received: "X_CODE: cause"` — what `expect(outcome.failure).toBeNull()` prints on a red run. */
+const RECEIVED = /Received:\s*"(X_[A-Z0-9_]+)/;
+
+/** Lower-case alphanumerics only: the id came from a person or a URL, the name is a file name. */
+const shortOf = (id: string): string =>
+  id
+    .toLowerCase()
+    .replace(/[^a-z0-9]/g, '')
+    .slice(0, 12) || 'unnamed';
+
+export function reproFile(subject: ReproSubject): string {
+  return subject.kind === 'job'
+    ? `${REPRO_DIR}/repro-${shortOf(subject.failure.jobId)}.job.test.ts`
+    : `${REPRO_DIR}/repro-${shortOf(subject.bundle.requestId ?? subject.bundle.id)}.contract.test.ts`;
+}
+
+/** The app's `@<app>/db` package name, when it has one that exports the driver the code reads. */
+async function dbPackageOf(root: string): Promise<string | null> {
+  const path = join(root, 'packages', 'db', 'package.json');
+  if (!existsSync(path)) return null;
+  const parsed: unknown = await Bun.file(path).json();
+  return isJsonObject(parsed) && typeof parsed['name'] === 'string' ? parsed['name'] : null;
+}
+
+/** The source a subject replays as; the API index is imported for what it registers. */
+export async function reproSource(
+  root: string,
+  subject: ReproSubject,
+  id: string,
+  build: string | null,
+): Promise<string> {
+  const imports = existsSync(join(root, API_INDEX))
+    ? [`../${API_INDEX.replace(/\/index\.ts$/, '')}`]
+    : [];
+  if (subject.kind === 'job') return jobTestSource(subject.failure, { id, imports });
+  return requestTestSource(subject.bundle, {
+    id,
+    imports,
+    build,
+    dbPackage: await dbPackageOf(root),
+  });
+}
+
+export async function writeRepro(
+  root: string,
+  file: string,
+  source: string,
+  runner: Runner,
+): Promise<void> {
+  await Bun.write(join(root, file), source);
+  await formatWritten(root, [file], runner);
+}
+
+export interface ReproVerdict {
+  readonly reproduced: boolean;
+  /** The code the written test ended in, read off its output; `null` for a pass. */
+  readonly got: string | null;
+}
+
+/**
+ * Reproduced = the written test is RED and its output carries the captured code. A green run, or a
+ * red one for another reason, is the answer `X_REPRO_NOT_REPRODUCED` reports — never a crash.
+ */
+export async function verifyRepro(
+  root: string,
+  file: string,
+  expected: string | null,
+  runner: Runner,
+  env: Readonly<Record<string, string | undefined>>,
+): Promise<ReproVerdict> {
+  // A test process, never the shell's deploy: no `DATABASE_URL` for the app's driver to follow
+  // and no `ULTIMATE_ENV` to outrank `bun test`'s own — the replay runs where the app's tests run.
+  const run = await runner(['bun', 'test', `./${file}`], {
+    cwd: root,
+    env: { ...testEnvOverrides(root, env), DATABASE_URL: undefined, ULTIMATE_ENV: undefined },
+  });
+  if (run.ok) return { reproduced: false, got: null };
+  // The ANSWER the assertion printed — never the test's name or its source, which name the code too.
+  const got = RECEIVED.exec(`${run.stdout}\n${run.stderr}`)?.[1] ?? 'a failure with no code';
+  return { reproduced: expected !== null && got === expected, got };
+}

package/src/retain-sweep.ts

@@ -0,0 +1,27 @@
+// The `manifest` step's retention half: an entity declaring `retain` that no `purge()` covers is a
+// promise nothing keeps — rows past the window are never deleted, and nobody finds out until an
+// audit. `X_ENTITY_RETAIN_UNSWEPT` makes it a build error (axiom 3).
+
+import { unsweptEntities } from '@ultimat3/jobs';
+import { loadApp } from './app-load';
+import type { AppLoader } from './app-permissions';
+import type { Finding } from './output';
+
+export function retainUnsweptFinding(entity: string): Finding {
+  return {
+    code: 'X_ENTITY_RETAIN_UNSWEPT',
+    cause: `entity "${entity}" declares retain, and no purge() registered in this app sweeps it — rows past its window are kept forever`,
+    fix: "export const retention = purge({ entities: 'declared' }) — in a jobs module named in jobs: [...] in apps/web/api/index.ts, then x verify",
+    at: 'apps/web/api/index.ts',
+  };
+}
+
+/** The step's answer, after the app loaded — a short registry is the load's finding, not this one. */
+export async function retainUnsweptFindings(
+  root: string,
+  load: AppLoader = loadApp,
+  unswept: () => readonly string[] = unsweptEntities,
+): Promise<readonly Finding[]> {
+  if ((await load(root)).findings.length > 0) return [];
+  return unswept().map(retainUnsweptFinding);
+}

package/src/retained-assets-boot.ts

@@ -0,0 +1,113 @@
+// The web role's boot half of `assets.retain`: read the section the `*Of(config)` way, refuse a
+// disk the app never declared, publish every hashed file this build serves — before the role
+// binds, so before `/readyz` can say ready — and hand back the reader the asset routes miss into.
+
+import type { AppConfig, AssetsRetainConfig } from '@ultimat3/core';
+import { ConfigInvalidError, renderFixLiteral } from '@ultimat3/core';
+import type { Storage } from '@ultimat3/storage';
+import type { IslandBundle } from './island-bundle';
+import type { BuildAsset, PublishReport, RetainLog } from './retained-assets';
+import { publishBuildAssets } from './retained-assets';
+import { retainedPrefix } from './retained-assets-layout';
+import type { RetainedAssets } from './retained-assets-read';
+import { retainedAssets } from './retained-assets-read';
+import { siteAssetTable } from './site-assets';
+import type { StyleBundle } from './style-bundle';
+import type { FrameworkScript } from './worker-bundle';
+
+/** `assets.retain`, or `undefined` — absent is today's behaviour: each pod serves what it built. */
+export function retainOf(config: AppConfig | undefined): AssetsRetainConfig | undefined {
+  return config?.assets.retain;
+}
+
+/** What this build serves at a content-addressed URL: the whole of what an old tab can ask for. */
+export interface BuiltSurface {
+  readonly root: string;
+  readonly islands: IslandBundle;
+  readonly styles: StyleBundle;
+  readonly scripts: readonly FrameworkScript[];
+}
+
+const encode = (text: string): Uint8Array => new TextEncoder().encode(text);
+
+/** Every hashed file, once per URL, in route order: styles, islands, scripts, site assets. */
+export function buildAssetsOf(surface: BuiltSurface): readonly BuildAsset[] {
+  const assets: BuildAsset[] = [
+    ...surface.styles.chunks.map((chunk) => ({
+      url: chunk.url,
+      contentType: 'text/css; charset=utf-8',
+      body: encode(chunk.css),
+    })),
+    ...[...surface.islands.chunks, ...surface.islands.shared].map((chunk) => ({
+      url: chunk.url,
+      contentType: 'text/javascript',
+      body: encode(chunk.code),
+    })),
+    ...surface.scripts.map((script) => ({
+      url: script.url,
+      contentType: 'text/javascript',
+      body: encode(script.code),
+    })),
+    // A `Bun.file`, so a disk refuses one past its ceiling by size without reading it.
+    ...siteAssetTable(surface.root)
+      .all()
+      .map((asset) => ({
+        url: asset.url,
+        contentType: asset.contentType,
+        body: Bun.file(asset.file),
+      })),
+  ];
+  return [...new Map(assets.map((asset) => [asset.url, asset] as const)).values()];
+}
+
+export interface RetainBootInput extends BuiltSurface {
+  readonly config: AppConfig | undefined;
+  /** The disks the app's routes read (`servedStorage`) — `retain.disk` must be one of them. */
+  readonly storage: Storage;
+  readonly buildId: string;
+  /** `resolveEnvironment` of the boot's env: half of the scope, with `config.name`. */
+  readonly environment: string;
+  readonly log?: RetainLog | undefined;
+  /** Test seam; `RETAIN_PUBLISH_DEADLINE_MS` otherwise. */
+  readonly deadlineMs?: number | undefined;
+}
+
+export interface RetainedBoot {
+  /** What the asset routes read a miss through — `withRetainedAssets(routes, reader)`. */
+  readonly reader: RetainedAssets;
+  readonly report: PublishReport;
+}
+
+/**
+ * `undefined` when the app declared no window. A misnamed disk refuses the boot: it is the app's
+ * config, wrong, and every pod of the deploy would otherwise run without the window it asked for.
+ * A disk that refuses a write does NOT: `publishBuildAssets` logs it and the pod serves, reading
+ * earlier builds' files all the same. The scope is `config.name` and the environment, so staging
+ * and production on one bucket never sweep or serve each other.
+ */
+export async function retainBuildAssets(input: RetainBootInput): Promise<RetainedBoot | undefined> {
+  const retain = retainOf(input.config);
+  if (input.config === undefined || retain === undefined) return undefined;
+  const names = input.storage.diskNames;
+  if (!names.includes(retain.disk)) {
+    throw new ConfigInvalidError({
+      cause: `assets.retain.disk is "${retain.disk}" but the app's disks are: ${names.join(', ')}`,
+      fix: `assets: { retain: { disk: ${renderFixLiteral(names[0], "'<disk>'")}, builds: ${String(retain.builds)} } }   # in app.config.ts; the declared disks: ${names.join(', ')}`,
+      meta: { key: 'assets.retain.disk' },
+    });
+  }
+  const scope = {
+    disk: input.storage.disk(retain.disk),
+    prefix: retainedPrefix(input.config.name, input.environment),
+  };
+  const report = await publishBuildAssets({
+    ...scope,
+    diskName: retain.disk,
+    buildId: input.buildId,
+    builds: retain.builds,
+    assets: buildAssetsOf(input),
+    ...(input.log === undefined ? {} : { log: input.log }),
+    ...(input.deadlineMs === undefined ? {} : { deadlineMs: input.deadlineMs }),
+  });
+  return { reader: retainedAssets(scope), report };
+}

package/src/retained-assets-layout.ts

@@ -0,0 +1,110 @@
+// Where `assets.retain` keeps things on its disk, and the reads every half shares: the key of a
+// file and of a manifest, one app-and-environment prefix, and the manifests as written. The
+// publish, the sweep and the read each import this; none imports another's file for it.
+
+import { isUltimateError } from '@ultimat3/core';
+import type { StorageDriver, StorageListEntry, StorageRead } from '@ultimat3/storage';
+
+/** Outside `org/`, so `/_storage` and `/media` refuse it (`assertReadableKey`): served only here. */
+export const RETAINED_ASSET_PREFIX = 'x-build-assets';
+/** Disk calls in flight at once: a boot over S3 is round trips, not bytes. */
+const CONCURRENCY = 8;
+
+/**
+ * One app in one environment on the disk: `x-build-assets/<app>/<environment>`. Staging and
+ * production sharing a bucket must neither sweep nor serve each other's builds.
+ */
+export interface RetainedScope {
+  readonly disk: StorageDriver;
+  readonly prefix: string;
+}
+
+/** `config.name` is already a slug (`defineConfig`); the environment is one of `ENVIRONMENTS`. */
+export const retainedPrefix = (app: string, environment: string): string =>
+  `${RETAINED_ASSET_PREFIX}/${encodeURIComponent(app)}/${encodeURIComponent(environment)}`;
+
+/** `/styles/ab12cd34.css` → `<prefix>/files/styles/ab12cd34.css`. Content-addressed: shared. */
+export const retainedFileKey = (prefix: string, url: string): string => `${prefix}/files${url}`;
+
+/** The listing prefix every file of the scope sits under. */
+export const filesPrefix = (prefix: string): string => `${prefix}/files/`;
+
+/** The listing prefix every manifest of the scope sits under. */
+export const manifestsPrefix = (prefix: string): string => `${prefix}/builds/`;
+
+/** Where a build says it is complete — written after its files. */
+export const manifestKey = (prefix: string, buildId: string): string =>
+  `${manifestsPrefix(prefix)}${encodeURIComponent(buildId)}.json`;
+
+/** What a manifest holds: the build, when it last went live, and every URL it serves. */
+export interface BuildManifest {
+  readonly buildId: string;
+  readonly publishedAt: number;
+  readonly urls: readonly string[];
+}
+
+export async function inBatches<T>(
+  items: readonly T[],
+  each: (item: T) => Promise<void>,
+): Promise<void> {
+  for (let at = 0; at < items.length; at += CONCURRENCY) {
+    await Promise.all(items.slice(at, at + CONCURRENCY).map(each));
+  }
+}
+
+export async function listAll(
+  disk: StorageDriver,
+  prefix: string,
+): Promise<readonly StorageListEntry[]> {
+  const all: StorageListEntry[] = [];
+  let cursor: string | null = null;
+  for (;;) {
+    const page = await disk.list({ prefix, cursor });
+    all.push(...page.rows);
+    if (!page.hasMore) return all;
+    cursor = page.nextCursor;
+  }
+}
+
+function parseManifest(bytes: Uint8Array): BuildManifest | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(new TextDecoder().decode(bytes));
+  } catch {
+    return undefined;
+  }
+  if (typeof parsed !== 'object' || parsed === null) return undefined;
+  const { buildId, publishedAt, urls } = parsed as Record<string, unknown>;
+  if (typeof buildId !== 'string' || typeof publishedAt !== 'number') return undefined;
+  if (!Array.isArray(urls) || !urls.every((url) => typeof url === 'string')) return undefined;
+  return { buildId, publishedAt, urls };
+}
+
+export interface StoredManifest {
+  readonly key: string;
+  readonly manifest: BuildManifest;
+}
+
+/**
+ * Every manifest of the scope, or `undefined` when one does not parse: what it names is unknown,
+ * so a caller deciding what to delete or to serve must not guess.
+ */
+export async function readManifests(
+  scope: RetainedScope,
+): Promise<readonly StoredManifest[] | undefined> {
+  const manifests: StoredManifest[] = [];
+  for (const entry of await listAll(scope.disk, manifestsPrefix(scope.prefix))) {
+    let read: StorageRead;
+    try {
+      read = await scope.disk.get(entry.key);
+    } catch (error) {
+      // Deleted by another pod's sweep between the listing and the read: no longer a manifest.
+      if (isUltimateError(error) && error.code === 'X_STORAGE_NOT_FOUND') continue;
+      throw error;
+    }
+    const manifest = parseManifest(read.bytes);
+    if (manifest === undefined) return undefined;
+    manifests.push({ key: entry.key, manifest });
+  }
+  return manifests;
+}

package/src/retained-assets-read.ts

@@ -0,0 +1,142 @@
+// The read half of `assets.retain`: an asset route that missed asks the retained disk, and the disk
+// answers only a URL a retained build's manifest names — the manifest is the one gate, so no route
+// restates a URL shape. Composed over the web boot's routes by `serve-web.ts`; `x dev` never is.
+
+import type { Clock } from '@ultimat3/core';
+import { finiteCount, logger, renderThrowable, systemClock } from '@ultimat3/core';
+import type { Route, UltimateRequest } from '@ultimat3/http';
+import { objectResponse } from '@ultimat3/http';
+import type { StorageObject } from '@ultimat3/storage';
+import type { RetainedScope } from './retained-assets-layout';
+import { readManifests, retainedFileKey } from './retained-assets-layout';
+
+/**
+ * How often an unnamed URL may make the reader list the manifests again. A new build's pods
+ * publish after this one booted, and its URLs must become readable here (a new document reaching
+ * an old pod); a URL nobody ever published must not cost a listing per request.
+ */
+export const RETAINED_REFRESH_MS = 5_000;
+
+/** What `serve-web.ts` hands the routes: the retained answer for a request, or none. */
+export interface RetainedAssets {
+  read(request: UltimateRequest): Promise<Response | undefined>;
+}
+
+export interface RetainedAssetsOptions extends RetainedScope {
+  readonly clock?: Clock | undefined;
+  readonly refreshMs?: number | undefined;
+}
+
+/** A body whose first chunk has arrived, or a throw: what was swept since `stat` is a miss. */
+async function startedBody(
+  body: ReadableStream<Uint8Array>,
+): Promise<{ readonly stream: ReadableStream<Uint8Array>; readonly cancel: () => void }> {
+  const reader = body.getReader();
+  const first = await reader.read();
+  const stream = new ReadableStream<Uint8Array>({
+    start(controller) {
+      if (first.done) controller.close();
+      else controller.enqueue(first.value);
+    },
+    async pull(controller) {
+      const next = await reader.read();
+      if (next.done) controller.close();
+      else controller.enqueue(next.value);
+    },
+    cancel: (reason) => reader.cancel(reason),
+  });
+  return { stream, cancel: () => void reader.cancel().catch(() => undefined) };
+}
+
+export function retainedAssets(options: RetainedAssetsOptions): RetainedAssets {
+  const clock = options.clock ?? systemClock;
+  const refreshMs = finiteCount(
+    'retainedAssets',
+    'refreshMs',
+    options.refreshMs ?? RETAINED_REFRESH_MS,
+  );
+  let named: ReadonlySet<string> = new Set();
+  let refreshedAt = Number.NEGATIVE_INFINITY;
+  let refreshing: Promise<void> | undefined;
+
+  const refresh = (): Promise<void> => {
+    refreshing ??= (async () => {
+      refreshedAt = clock.now().getTime();
+      const manifests = await readManifests(options);
+      // A manifest that does not parse leaves the last good set: never serve on a guess.
+      if (manifests !== undefined) {
+        named = new Set(manifests.flatMap(({ manifest }) => manifest.urls));
+      }
+    })().finally(() => {
+      refreshing = undefined;
+    });
+    return refreshing;
+  };
+
+  const isNamed = async (pathname: string): Promise<boolean> => {
+    if (named.has(pathname)) return true;
+    if (clock.now().getTime() - refreshedAt < refreshMs) return false;
+    await refresh();
+    return named.has(pathname);
+  };
+
+  return {
+    async read(request: UltimateRequest): Promise<Response | undefined> {
+      const { pathname } = request;
+      try {
+        if (!(await isNamed(pathname))) return undefined;
+        const key = retainedFileKey(options.prefix, pathname);
+        const object: StorageObject | undefined = await options.disk.stat(key);
+        if (object === undefined) return undefined;
+        const body = await startedBody(await options.disk.stream(key));
+        let sent = false;
+        const response = await objectResponse(
+          request,
+          {
+            object,
+            open: (range) => {
+              if (range === undefined) {
+                sent = true;
+                return body.stream;
+              }
+              return options.disk.streamRange(key, [range.start, range.end]);
+            },
+          },
+          // `{}`, not the stored-object sandbox: these are the app's own scripts and stylesheets,
+          // and an `attachment` disposition would make a browser refuse to run them.
+          { cache: { mode: 'immutable' }, headers: {} },
+        );
+        if (!sent) body.cancel();
+        return response;
+      } catch (error) {
+        // A disk that cannot answer is a miss: the route's own 404 is still true for this process.
+        logger.warn('assets.retain_read_failed', { pathname, cause: renderThrowable(error) });
+        return undefined;
+      }
+    },
+  };
+}
+
+/**
+ * Every `GET` route tagged `assets` asks `retained` when it would answer 404. Over the composed
+ * table rather than inside each factory, so the routes `x dev` mounts are byte-for-byte the
+ * container's and no factory gains a second way to answer.
+ */
+export function withRetainedAssets(
+  routes: readonly Route[],
+  retained: RetainedAssets | undefined,
+): readonly Route[] {
+  if (retained === undefined) return routes;
+  return routes.map((route) =>
+    route.method !== 'GET' || !(route.meta?.tags ?? []).includes('assets')
+      ? route
+      : {
+          ...route,
+          handler: async (request, ctx) => {
+            const answer = await route.handler(request, ctx);
+            if (answer.status !== 404) return answer;
+            return (await retained.read(request)) ?? answer;
+          },
+        },
+  );
+}

package/src/retained-assets-sweep.ts

@@ -0,0 +1,70 @@
+// The `assets.retain` sweep: keep the newest `builds` manifests of one app-and-environment scope
+// and delete the files none of them names — without ever deleting what another pod is publishing
+// at the same moment. Run at the end of a publish.
+
+import type { RetainedScope, StoredManifest } from './retained-assets-layout';
+import {
+  filesPrefix,
+  inBatches,
+  listAll,
+  readManifests,
+  retainedFileKey,
+} from './retained-assets-layout';
+
+/**
+ * A file put this recently is spared whatever the manifests say: it may belong to a publish still
+ * in flight on another pod, whose manifest is not written yet. Far longer than a publish may take
+ * (`RETAIN_PUBLISH_DEADLINE_MS`), so clock skew between a pod and the disk does not eat it.
+ */
+export const RETAIN_SWEEP_GRACE_MS = 10 * 60_000;
+
+const urlsOf = (manifests: readonly StoredManifest[]): readonly string[] =>
+  manifests.flatMap(({ manifest }) => manifest.urls);
+
+/**
+ * Manifests, then files, then the dropped manifests: a pod killed anywhere leaves extra objects,
+ * never a missing one. Three guards against a concurrent publish: a file younger than the grace
+ * stays; the manifests are read AGAIN just before deleting, and any that appeared keeps what it
+ * names; a manifest that does not parse stops the sweep. Returns the files deleted.
+ */
+export async function sweepRetained(
+  scope: RetainedScope,
+  builds: number,
+  startedAt: number,
+): Promise<number> {
+  const manifests = await readManifests(scope);
+  if (manifests === undefined) return 0;
+  const ordered = [...manifests].sort(
+    (a, b) =>
+      b.manifest.publishedAt - a.manifest.publishedAt ||
+      a.manifest.buildId.localeCompare(b.manifest.buildId),
+  );
+  const dropped = ordered.slice(builds);
+  if (dropped.length === 0) return 0;
+  const droppedAt = new Map(dropped.map(({ key, manifest }) => [key, manifest.publishedAt]));
+  const files = await listAll(scope.disk, filesPrefix(scope.prefix));
+  const again = await readManifests(scope);
+  if (again === undefined) return 0;
+  // Still dropped: read the first time as dropped and not re-dated since by a pod of that build
+  // booting again (a rollback), which makes it the newest and not ours to drop.
+  const stillDropped = (one: StoredManifest): boolean =>
+    droppedAt.get(one.key) === one.manifest.publishedAt;
+  const named = new Set(
+    urlsOf(again.filter((one) => !stillDropped(one))).map((url) =>
+      retainedFileKey(scope.prefix, url),
+    ),
+  );
+  const redated = new Set(
+    again.filter((one) => droppedAt.has(one.key) && !stillDropped(one)).map(({ key }) => key),
+  );
+  const cutoff = startedAt - RETAIN_SWEEP_GRACE_MS;
+  const doomed = files
+    .filter((entry) => !named.has(entry.key))
+    // Unknown age is spared, as `sweepOrphans` spares it: never epoch 0, never "old enough".
+    .filter((entry) => entry.lastModified !== undefined && entry.lastModified.getTime() < cutoff)
+    .map((entry) => entry.key);
+  await inBatches(doomed, (key) => scope.disk.delete(key));
+  const drop = dropped.filter(({ key }) => !redated.has(key));
+  await inBatches(drop, ({ key }) => scope.disk.delete(key));
+  return doomed.length;
+}