@ultimat3/cli 20.2.0 → 21.0.0

package/src/cdp-offline-script.ts

@@ -0,0 +1,73 @@
+// One responsibility: `navigator.onLine` reads `false` from a document's FIRST script while the
+// session's offline switch is thrown, and reads `true` again — with the `online` event a page
+// reconnects on — when it goes back. `Network.emulateNetworkConditions` cuts the NETWORK for a
+// document created under the switch but, measured on Chrome 150, never tells it so: a reload under
+// the switch read `navigator.onLine === true` at its first script every time, COOP or not, still
+// read `true` a second later, and got no `online` event when the switch went back.
+// `Network.overrideNetworkState` changed nothing. The dummy's page boot asks at its first script
+// and replayed its outbox with a real POST under the cut (`offline-like.e2e.test.ts`, two attempts).
+//
+// The override is an OWN property on `navigator`, so the real getter on `Navigator.prototype` is one
+// `delete` away. A document Chrome DID tell (it saw `offline`) gets Chrome's own `online` event on
+// restore; one it never told gets one from `RESTORE_ONLINE`, so a page reconnects exactly once.
+
+import type { CdpResult } from './cdp-connection';
+
+type Send = (
+  method: string,
+  params: Record<string, unknown>,
+  session: string,
+) => Promise<CdpResult>;
+
+/** Runs before the page's own scripts, in every document a page session creates while cut. */
+export const OFFLINE_FIRST_SCRIPT = `(() => {
+  let told = false;
+  Object.defineProperty(navigator, 'onLine', { configurable: true, get: () => false });
+  addEventListener('offline', () => { told = true; });
+  addEventListener('online', () => { delete navigator.onLine; }, { once: true });
+  Object.defineProperty(window, '__xRestoreOnLine', { configurable: true, value: () => {
+    if (!Object.getOwnPropertyDescriptor(navigator, 'onLine')) return;
+    delete navigator.onLine;
+    if (!told) dispatchEvent(new Event('online'));
+  } });
+})();`;
+
+/** Evaluated in every open page when the switch goes back: a no-op in a document never cut. */
+export const RESTORE_ONLINE = `window.__xRestoreOnLine?.()`;
+
+export interface OfflineScripts {
+  /** Register the script on one page session; answers once the browser has it. */
+  add(session: string): Promise<unknown>;
+  /** Unregister it from one page session, if it holds it, and restore the open document. */
+  remove(session: string): Promise<unknown>;
+}
+
+const field = (from: unknown, key: string): string | undefined => {
+  const value =
+    typeof from === 'object' && from !== null ? (from as Record<string, unknown>)[key] : undefined;
+  return typeof value === 'string' ? value : undefined;
+};
+
+/** Per-session registration, keyed by the identifier the browser hands back. */
+export function offlineScripts(send: Send): OfflineScripts {
+  const held = new Map<string, string>();
+  return {
+    async add(session) {
+      const answer = await send(
+        'Page.addScriptToEvaluateOnNewDocument',
+        { source: OFFLINE_FIRST_SCRIPT },
+        session,
+      );
+      const identifier = field(answer.result, 'identifier');
+      if (identifier !== undefined) held.set(session, identifier);
+      return answer;
+    },
+    async remove(session) {
+      const identifier = held.get(session);
+      if (identifier === undefined) return undefined;
+      held.delete(session);
+      await send('Page.removeScriptToEvaluateOnNewDocument', { identifier }, session);
+      return send('Runtime.evaluate', { expression: RESTORE_ONLINE }, session);
+    },
+  };
+}

package/src/cdp-pipe.ts

@@ -0,0 +1,77 @@
+// One responsibility: CDP over the pipe `--remote-debugging-pipe` opens — Chrome reads commands on
+// its fd 3 and writes replies and events on its fd 4, each message one JSON text ended by a NUL
+// byte. This is the e2e driver's wire; the WebSocket in `cdp-connection.ts` is for a remote browser.
+//
+// Why a pipe rather than the WebSocket Chrome also offers: Bun 1.4.0's WebSocket client handed
+// `onmessage` text spliced from several frames under the dummy's `offline-feed` load — 64
+// unparseable frames in one run, one of them a `Runtime.evaluate` reply that then waited out its
+// 30 s deadline. A frame nobody can parse has no `id`, so no layer above can even tell which call
+// it lost. A pipe is bytes and a delimiter, read here, and nothing in between.
+
+import type { CdpTransport } from './cdp-connection';
+
+/** The two ends a transport needs: a sink for whole messages, and the byte stream Chrome writes. */
+export interface PipeEnds {
+  /** Write these bytes to Chrome's fd 3, in order. */
+  readonly write: (bytes: Uint8Array) => void;
+  /** Chrome's fd 4. */
+  readonly read: ReadableStream<Uint8Array>;
+  /** Release the write end — Chrome treats its fd 3 closing as the client going away. */
+  readonly end: () => void;
+}
+
+const NUL = 0;
+
+/** Concatenate two byte arrays; the reader only ever holds the unterminated tail. */
+const join = (a: Uint8Array, b: Uint8Array): Uint8Array => {
+  if (a.length === 0) return b;
+  const out = new Uint8Array(a.length + b.length);
+  out.set(a);
+  out.set(b, a.length);
+  return out;
+};
+
+export function pipeTransport(ends: PipeEnds): CdpTransport {
+  const encoder = new TextEncoder();
+  let closed = false;
+  const reader = ends.read.getReader();
+  return {
+    send(text: string): void {
+      if (closed) return;
+      const body = encoder.encode(text);
+      const framed = new Uint8Array(body.length + 1);
+      framed.set(body);
+      framed[body.length] = NUL;
+      ends.write(framed);
+    },
+    close(): void {
+      if (closed) return;
+      closed = true;
+      ends.end();
+      void reader.cancel().catch(() => undefined);
+    },
+    listen(handlers): void {
+      void (async () => {
+        // Split on BYTES, decoded per message: a multi-byte character may straddle two reads, and
+        // decoding each read on its own would corrupt it — the failure this file exists to end.
+        const decoder = new TextDecoder();
+        let tail: Uint8Array = new Uint8Array(0);
+        try {
+          for (;;) {
+            const { value, done } = await reader.read();
+            if (done) break;
+            tail = join(tail, value);
+            for (let at = tail.indexOf(NUL); at !== -1; at = tail.indexOf(NUL)) {
+              handlers.message(decoder.decode(tail.subarray(0, at)));
+              tail = tail.subarray(at + 1);
+            }
+          }
+        } catch {
+          // A read that fails is a pipe that is gone — reported as the close it is, below.
+        }
+        closed = true;
+        handlers.closed('the browser closed the CDP pipe');
+      })();
+    },
+  };
+}

package/src/cmd-deploy.ts

@@ -11,6 +11,7 @@ import { msg } from './messages';
 import type { CommandResult, JsonValue } from './output';
 import { flagBool, flagString } from './parse';
 import { quoteArg } from './shell-quote';
+import { PROD_ENV_FILE } from './templates/scaffold-container';
 
 /**
  * Ordered, and the order is the design. `migrate` GATES — it runs to completion before anything
@@ -146,6 +147,12 @@ export function planDeploy(image: string, method: DeployMethod, root: string): D
       command: [
         'docker',
         'compose',
+        // Compose interpolates `${SYNC_URL:?…}` and `${POSTGRES_PASSWORD:?…}` from the shell and
+        // `--env-file` only — never from a service's `env_file:`. Without this an operator who put
+        // them in `.env.production`, the one file the compose file tells them to fill, had every
+        // step die on a parse error. Global flag, so it precedes `-f`; the shell still wins over it.
+        '--env-file',
+        join(root, PROD_ENV_FILE),
         '-f',
         join(root, 'docker', 'docker-compose.prod.yml'),
         ONE_SHOT_ROLES.includes(role) ? 'run' : 'up',

package/src/cmd-dev.ts

@@ -9,58 +9,43 @@ import { devShellStyle } from '@ultimat3/admin/dev';
 import type { Role } from '@ultimat3/core';
 import { configureTelemetry, METRICS_PATH, noopExporter } from '@ultimat3/core';
 import { setStatementObserver } from '@ultimat3/db';
-import type { OverlayNotice, RequestContext, Route } from '@ultimat3/http';
+import type { OverlayNotice, RequestContext } from '@ultimat3/http';
 import { asCtx } from '@ultimat3/http';
 import type { Manifest } from '@ultimat3/manifest';
 import { MANIFEST_FILENAME } from '@ultimat3/manifest';
-import { describeRoutes } from '@ultimat3/render';
-import { apiRoutes } from './api-routes';
 import { loadSignInPath } from './app-auth';
 import { appManifest } from './app-manifest';
-import { mountAppMcp } from './app-mcp';
 import { requireAppRoot } from './app-root';
 import { loadAppRuntime } from './app-runtime';
 import type { CliCommand, CommandContext } from './command';
-import { assetRoutes } from './dev-assets';
 import type { DevDashboardInput, DevStatus } from './dev-dashboard';
-import { devDashboardRoutes, devPanels } from './dev-dashboard';
+import { devPanels } from './dev-dashboard';
 import { declareDevEnvironment } from './dev-environment';
 import { liveFeedLabel } from './dev-live-feed';
 import { clearLock, preflight, writeLock } from './dev-lock';
 import { createStatementLedger } from './dev-n-plus-one';
 import { coalesceReloads } from './dev-reload';
-import { appRoutes } from './dev-render';
 import { replicaOverrides } from './dev-replica';
 import type { RunningRoles } from './dev-roles';
 import { DEV_BINDING, DEV_ROLES, selectRoles, startRoles } from './dev-roles';
+import { devRouteTable } from './dev-route-table';
 import type { RunningServices } from './dev-runtime';
 import { cdnLabel, describeCdn, describeMail, mailLabel, startServices } from './dev-runtime';
 import type { DevServices } from './dev-services';
 import { describeServices, reportedUrls, resolveServices } from './dev-services';
-import { storageRoutes } from './dev-storage';
 import { createTraceRecorder } from './dev-traces';
 import { watchTree } from './dev-watch-tree';
-import { errorPageStyleSources } from './error-page-csp';
 import { intFlagOr, PORT_RANGE } from './flag-number';
 import { holdUntilShutdown } from './hold';
 import type { IslandBundle } from './island-bundle';
 import { buildIslands } from './island-bundle';
 import { FRAME_STYLE } from './island-harness';
-import { islandHarnessRoutes } from './island-harness-route';
-import { islandRoutes } from './island-routes';
-import { loadIslandStates } from './island-states-load';
 import { msg } from './messages';
 import type { CommandResult, Finding } from './output';
 import { findingFrom } from './output';
 import { flagString } from './parse';
-import { loadPwaArtifacts } from './pwa-artifacts';
 import { metricsPortFor } from './serve';
 import { loopFacts, loopFinding, loopNotice } from './statement-loop';
-import { styleBundle } from './style-bundle';
-import { styleRoutes } from './style-routes';
-import { serviceWorkerArtifacts } from './sw-artifacts';
-import { serviceWorkerRoutes } from './sw-routes';
-import { loadThemeMode, themeBoot } from './theme-boot';
 
 const DEFAULT_PORT = 3000;
 
@@ -68,7 +53,10 @@ export interface DevServer {
   readonly url: string;
   readonly services: DevServices;
   readonly roles: readonly Role[];
-  /** The manifest as it stands now — a reload that registers a new route moves it. */
+  /**
+   * `BUILD_ID` when stamped — the id every response carries. Otherwise the manifest as it stands
+   * now, so a reload that registers a new route moves it.
+   */
   readonly buildId: string;
   /**
    * Modules that would not import, primitives that would not register, reloads that would not
@@ -164,7 +152,12 @@ export async function startDev(options: StartDevOptions): Promise<DevServer> {
   // boot on purpose: the header is handed to the HTTP config and the render modes once, and a
   // reload cannot re-pin it — `state.manifest.buildId` is what `/_x` and `--json` report, so a
   // divergence between the two is visible rather than silent, and a restart closes it.
-  const buildId = state.manifest.buildId;
+  // `BUILD_ID` wins when set — `serve.ts`'s rule, so an e2e `deploy.newBuild()` can restart `x dev`
+  // as a new build with the same sources.
+  // Stamped, it is ALSO what `server.buildId` answers below: the process serves no other build.
+  const rawStamp = options.env['BUILD_ID'];
+  const stamped = rawStamp !== undefined && rawStamp !== '' ? rawStamp : undefined;
+  const buildId = stamped ?? state.manifest.buildId;
 
   let server: DevServer;
   // Read at request time, never captured at boot: `/_x/services` must report the reload counter
@@ -185,70 +178,14 @@ export async function startDev(options: StartDevOptions): Promise<DevServer> {
   };
   const panels = devPanels(dashboard).map((panel) => panel.key);
 
-  // Resolved once, before the first route: the manifest's bytes and the three head elements that
-  // name it. `undefined` for an app that is not installable, and then nothing is mounted and no
-  // document changes — the 0kb baseline is not spent on a `<link>` to a file that does not exist.
-  const pwa = await loadPwaArtifacts(options.root);
-  const theme = themeBoot(await loadThemeMode(options.root));
-  const errorStyles = await errorPageStyleSources(options.root);
-  // Built once at boot, from this process's own route table and island bundle. `x dev` rebuilds
-  // islands on the watcher tick and the worker is NOT rebuilt with them, deliberately: a service
-  // worker that changes under a page it already controls is the update path, and re-emitting one
-  // per keystroke would exercise it on every save.
-  const serviceWorker =
-    pwa === undefined
-      ? undefined
-      : serviceWorkerArtifacts({
-          pwa,
-          buildId,
-          routes: describeRoutes(),
-          islands: state.islands,
-          styles: styleBundle(),
-        });
-
-  // The app's own MCP endpoint, discovered from `apps/<app>/mcp.ts` and mounted through the SAME
-  // call `runRole` makes — `POST /mcp` answered 404 in every process the framework booted until
-  // one of them asked. Warned once here when `expose` is true and nothing can be mounted.
-  const mcpMount = await mountAppMcp(options.root);
-  const routes: readonly Route[] = [
-    ...devDashboardRoutes(dashboard),
-    // The same API table the container serves: a read that answers here and 404s in production
-    // is exactly the drift one composition exists to prevent.
-    ...apiRoutes(),
-    ...mcpMount.routes,
-    // The image pipeline's only HTTP surface: the icons the web manifest declares, and the
-    // variants every `srcset` promises. Mounted before the app's own routes so a page route can
-    // never shadow `/icons` or `/media`.
-    ...assetRoutes({
-      root: options.root,
-      storage: runtime.storage,
-      ...(pwa === undefined ? {} : { pwa }),
-    }),
-    ...storageRoutes({ storage: runtime.storage }),
-    // The chunks the documents below name. Mounted before the app's routes for the reason
-    // `/icons` and `/media` are: a page route must not be able to shadow an asset URL.
-    ...islandRoutes(() => state.islands),
-    // And the stylesheet every one of those documents links. Read through the getter for the
-    // reason the islands are: a rebuilt island registers CSS, which mints a new URL, and a table
-    // captured at boot would answer 404 for the href the document now carries.
-    ...styleRoutes(() => styleBundle()),
-    // `x shot --island`'s harness, in the `/_x` dev namespace so no app route can shadow it. It
-    // lives here rather than in a second server because everything it needs is in THIS process:
-    // the built chunks, the app's stylesheet registry, and the one embedded Postgres a checkout
-    // may have. The states are read per REQUEST — an author editing a state and re-running the
-    // command must not need a restart to see it.
-    ...islandHarnessRoutes({
-      islands: () => state.islands,
-      states: () => loadIslandStates(options.root),
-    }),
-    ...(serviceWorker === undefined ? [] : serviceWorkerRoutes(serviceWorker)),
-    ...appRoutes({
-      buildId,
-      resolveIsland: (file) => state.islands.resolverFor(file),
-      themeHead: theme.head,
-      ...(pwa === undefined ? {} : { pwaHead: pwa.head + (serviceWorker?.head ?? '') }),
-    }),
-  ];
+  const { routes, theme, errorStyles, mcpPath } = await devRouteTable({
+    root: options.root,
+    env: options.env,
+    buildId,
+    storage: runtime.storage,
+    dashboard,
+    islands: () => state.islands,
+  });
 
   // The app's `apps/<app>/runtime.ts`, composed exactly as `runRole` composes a caller's
   // `runtime`: the replica scope in front, the app's own middleware behind it. Before this the
@@ -320,9 +257,9 @@ export async function startDev(options: StartDevOptions): Promise<DevServer> {
     url: running.url ?? `http://localhost:${options.port}`,
     services,
     roles: running.roles,
-    mcp: mcpMount.path,
+    mcp: mcpPath,
     get buildId(): string {
-      return state.manifest.buildId;
+      return stamped ?? state.manifest.buildId;
     },
     // A getter, not a snapshot: `/_x` and `--json` must show the reload that just failed and the
     // loop the last request tripped, not the findings as they were when the route table was built.

package/src/cmd-shot.ts

@@ -32,6 +32,7 @@ import { shotBrowserChoice } from './shot-browser';
 import type { BootDevServer, ShotServer } from './shot-server';
 import { allowHostsFrom, devServerFor, SHOT_DIR } from './shot-server';
 import { SETTLE_POLL_MS, settleIslands } from './shot-settle';
+import { readThemeFlag, themeChoiceExpression } from './shot-theme';
 import type { IslandCount, ShotArtifacts } from './shot-verdict';
 import {
   buildVerdict,
@@ -191,9 +192,13 @@ export interface ShotRun {
    */
   readonly extraHosts?: string | undefined;
   /**
-   * What `prefers-color-scheme` the page sees, emulated BEFORE navigation so the boot script's
-   * "system" branch answers the same on every box. Absent means the box's own preference — what
-   * `x shot` has always done — and `ui.shot` names one explicitly for exactly that reason.
+   * The theme the picture is of. Two things happen BEFORE navigation, because the boot script runs
+   * inline and nothing after `goto` can reach it: `prefers-color-scheme` is emulated so the boot's
+   * "system" branch answers the same on every box, AND the scheme is stored as the visitor's
+   * choice under `THEME_STORAGE_KEY` (`shot-theme.ts`), because an app with `theme.defaultMode`
+   * set answers that before the OS and only a stored choice beats it (issue #489). Absent means
+   * neither: the box's own preference and the app's own default — what `x shot` has always done,
+   * and the point of `defaultMode` — and `ui.shot` names one explicitly for exactly that reason.
    */
   readonly colorScheme?: ColorScheme | undefined;
   readonly now?: (() => Date) | undefined;
@@ -234,7 +239,11 @@ export async function runShot(options: ShotRun): Promise<ShotArtifacts> {
       timeoutMs: options.timeoutMs,
     });
     const page = session.page;
-    if (options.colorScheme !== undefined) await page.colorScheme(options.colorScheme);
+    if (options.colorScheme !== undefined) {
+      await page.colorScheme(options.colorScheme);
+      const choice = themeChoiceExpression(options.colorScheme);
+      if (choice !== undefined) await page.prepare(choice);
+    }
     await page.goto(requestedUrl, { timeout: options.timeoutMs });
     if (options.settleMs > 0) await Bun.sleep(options.settleMs);
     // The probe may legitimately answer nothing — a page that refuses evaluation, a driver with no
@@ -322,6 +331,11 @@ export const shotCommand: CliCommand = {
         summary: 'attach to a browser somebody else is running (a provider session, a sidecar)',
       },
       { name: 'allow-hosts', type: 'string', summary: 'extra hosts the page may request' },
+      {
+        name: 'theme',
+        type: 'string',
+        summary: "light or dark, stored as the visitor's choice; absent is the app's own default",
+      },
       // A FLAG on `x shot` and never a second command: photographing a route and photographing a
       // component are one job with two subjects, and a parallel command would be the second path
       // axiom 1 refuses.
@@ -352,6 +366,7 @@ export const shotCommand: CliCommand = {
     // report, which is the rule `x routes` and `x mcp` already follow.
     const island = flagString(ctx.args, 'island');
     const state = flagString(ctx.args, 'state');
+    const theme = readThemeFlag(flagString(ctx.args, 'theme'));
     const positional = ctx.args.positionals[0];
     const sweep = flagBool(ctx.args, 'all-islands');
     // Every ambiguous pair refused BY NAME, before a value is read: a reader who typed two
@@ -364,6 +379,16 @@ export const shotCommand: CliCommand = {
       refuseRouteWithIsland(positional, island);
     }
     const component = sweep || (island !== undefined && island !== '');
+    // An island is photographed in BOTH themes by the harness, which owns its `data-theme` and
+    // carries no boot script — so a theme asked for beside one is a request nothing could honour.
+    if (component && theme !== undefined) {
+      throw new BadFlagError({
+        flag: 'theme',
+        command: 'shot',
+        reason: 'photographs a route; an island is photographed in both themes',
+        fix: 'x shot / --theme light --json',
+      });
+    }
     const route = component ? '' : readRoute(positional);
     const port = intFlag(ctx.args, 'port', PORT_RANGE.min, DEFAULT_PORT, PORT_RANGE.max);
     const settleMs = intFlag(ctx.args, 'settle', 0, DEFAULT_SETTLE_MS);
@@ -415,6 +440,7 @@ export const shotCommand: CliCommand = {
         timeoutMs,
         fullPage: flagBool(ctx.args, 'full'),
         extraHosts: flagString(ctx.args, 'allow-hosts'),
+        ...(theme === undefined ? {} : { colorScheme: theme }),
       }),
     );
   },

package/src/dev-live-feed.ts

@@ -49,6 +49,8 @@ export async function startLiveFeed(input: LiveFeedInput): Promise<RunningLiveFe
   }
   const bridge = await startLiveReplicator({
     registry: input.sync.registry,
+    // The same changes the channels read — a real node's change subscription feeds both.
+    channels: input.sync.hub,
     // Logged, never thrown: one change nobody could fan out must not take the dev server down.
     onError: (error) =>
       logger.warn('live.bridge_delivery_failed', { error: renderThrowable(error) }),