@mostra/sdk 0.0.0-stage → 0.2.0

package/src/bundle/setup.ts

@@ -0,0 +1,278 @@
+/**
+ * Making an extracted bundle usable (RT-01, D142): its files checked against `files.json`, which
+ * `bundle.json` pins by SHA-256; the verify browser fetched through the bundle's own Playwright
+ * and checked against the bundle's pin; then `.mostra-setup.json`, the one marker every reader
+ * trusts. The SDK runs it after a download; `bin/mostra-setup` runs it for a bundle extracted by
+ * hand, and for the Rust crate (WP-12), so there is one implementation of what "ready" means.
+ */
+import { spawn } from "node:child_process";
+import { readFile, rename, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import {
+  BUNDLE_SETUP_MARKER,
+  BundleFiles,
+  BundleInfo,
+  BundleSetup,
+  createDiagnostic,
+  RpcError,
+} from "@mostra/protocol";
+import { scrubUrls } from "./download.ts";
+import { checkTree, hashFile, safeRelative } from "./tree.ts";
+
+/** The environment setup reads. */
+export type Environment = Readonly<Record<string, string | undefined>>;
+
+/** An extracted bundle and what its `bundle.json` says. */
+export interface Bundle {
+  /** Its root directory, holding `bundle.json`. */
+  readonly root: string;
+  /** Its parsed `bundle.json`. */
+  readonly info: BundleInfo;
+}
+
+/** One step of setup, for progress reports. */
+export type SetupStep = "files" | "browser";
+
+/** Options of {@link setupBundle}. */
+export interface SetupOptions {
+  /** The environment: `MOSTRA_OFFLINE`, proxies, Playwright's download mirror. */
+  readonly env?: Environment | undefined;
+  /** Gives up the browser's download. */
+  readonly signal?: AbortSignal | undefined;
+  /** Each step as it starts. */
+  readonly onStep?: ((step: SetupStep) => void) | undefined;
+  /** Skip checking the files, for a tree the caller checked a moment ago. */
+  readonly filesChecked?: boolean | undefined;
+}
+
+/** The directory Playwright's browsers live in, inside a bundle. */
+export function browsersDir(root: string): string {
+  return join(root, "browsers");
+}
+
+/** Whether `MOSTRA_OFFLINE` says Mostra is offline: any value but empty, `0`, `false` or `no`. */
+export function offline(env: Environment): boolean {
+  const value = env["MOSTRA_OFFLINE"]?.trim().toLowerCase();
+  return (
+    value !== undefined && value !== "" && value !== "0" && value !== "false" && value !== "no"
+  );
+}
+
+function invalid(message: string): RpcError {
+  return new RpcError(createDiagnostic("MOSTRA_ENV_BUNDLE_INVALID", { message }));
+}
+
+/** Reads `bundle.json` at `root`. Throws `MOSTRA_ENV_BUNDLE_INVALID` when it isn't a bundle's. */
+export async function readBundle(root: string): Promise<Bundle> {
+  let text: string;
+  try {
+    text = await readFile(join(root, "bundle.json"), "utf8");
+  } catch (error) {
+    throw invalid(`${root} has no readable bundle.json (${(error as Error).message}).`);
+  }
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(text);
+  } catch {
+    throw invalid(`${join(root, "bundle.json")} isn't JSON.`);
+  }
+  const result = BundleInfo.safeParse(parsed);
+  if (!result.success) throw invalid(`${join(root, "bundle.json")} isn't a runtime bundle's.`);
+  const info = result.data;
+  const paths = [
+    info.node.path,
+    ...Object.values(info.entries),
+    info.files.path,
+    info.browser.installer,
+  ];
+  if (!paths.every(safeRelative))
+    throw invalid(`${join(root, "bundle.json")} names a path outside the bundle.`);
+  return { root, info };
+}
+
+/** The setup marker, when it describes exactly this bundle. */
+export async function readSetup(bundle: Bundle): Promise<BundleSetup | undefined> {
+  try {
+    const marker = BundleSetup.parse(
+      JSON.parse(await readFile(join(bundle.root, BUNDLE_SETUP_MARKER), "utf8")),
+    );
+    const { info } = bundle;
+    return marker.files === info.files.sha256 &&
+      marker.version === info.version &&
+      marker.platform === info.platform &&
+      marker.browser.revision === info.browser.revision
+      ? marker
+      : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+/** What `checkTree` may find besides the listed files: the bundle's own records and browsers. */
+export function setupExtras(path: string): boolean {
+  return (
+    path === "bundle.json" ||
+    path === "files.json" ||
+    path === "browsers" ||
+    path.startsWith("browsers/") ||
+    path === BUNDLE_SETUP_MARKER ||
+    path.startsWith(`${BUNDLE_SETUP_MARKER}.tmp-`)
+  );
+}
+
+/** Checks the bundle's files; throws `MOSTRA_ENV_BUNDLE_INVALID` naming what differs. */
+export async function checkFiles(bundle: Bundle): Promise<void> {
+  const { root, info } = bundle;
+  const listing = join(root, info.files.path);
+  const { sha256 } = await hashFile(listing).catch(() => ({ sha256: "" }));
+  if (sha256 !== info.files.sha256) {
+    throw invalid(`${info.files.path} doesn't match the SHA-256 that bundle.json pins.`);
+  }
+  const manifest = BundleFiles.safeParse(JSON.parse(await readFile(listing, "utf8")));
+  if (!manifest.success) throw invalid(`${info.files.path} isn't a bundle's file list.`);
+  const problems = await checkTree(root, manifest.data, { allowExtra: setupExtras });
+  if (problems.length > 0) {
+    throw invalid(
+      `Mostra ${info.version}'s runtime bundle for ${info.platform} differs from its file list: ${problems.join("; ")}.`,
+    );
+  }
+}
+
+/** The browser files that differ from the pin, relative to `browsers/`. */
+export async function checkBrowser(bundle: Bundle): Promise<readonly string[]> {
+  const problems = await checkTree(
+    browsersDir(bundle.root),
+    { schemaVersion: 1, files: bundle.info.browser.files, links: [] },
+    // Playwright's own markers, its `.links` registry and any other browser it fetched.
+    { allowExtra: () => true },
+  ).catch((error: unknown) => [`browsers: ${(error as Error).message}`]);
+  return problems;
+}
+
+/** The environment the browser install runs in: proxies and Playwright's mirror pass through. */
+function installEnvironment(bundle: Bundle, env: Environment): NodeJS.ProcessEnv {
+  const out: NodeJS.ProcessEnv = {};
+  for (const [name, value] of Object.entries(env)) {
+    if (value !== undefined && !name.startsWith("MOSTRA_") && name !== "PLAYWRIGHT_BROWSERS_PATH")
+      out[name] = value;
+  }
+  out["PLAYWRIGHT_BROWSERS_PATH"] = browsersDir(bundle.root);
+  return out;
+}
+
+/** The variables whose URLs may carry credentials the browser's fetch uses. */
+const CREDENTIAL_URLS = [
+  "PLAYWRIGHT_DOWNLOAD_HOST",
+  "HTTPS_PROXY",
+  "https_proxy",
+  "HTTP_PROXY",
+  "http_proxy",
+  "ALL_PROXY",
+  "all_proxy",
+];
+
+/** `text` without URLs' credentials or queries, and without any password `env`'s URLs carry. */
+export function scrubSecrets(text: string, env: Environment): string {
+  let out = scrubUrls(text);
+  for (const name of CREDENTIAL_URLS) {
+    const value = env[name];
+    if (!value) continue;
+    // A proxy is often written without a scheme (`user:pass@host:3128`), which `URL` would read
+    // as the scheme `user:`.
+    const spelled = value.includes("://") ? value : `http://${value}`;
+    const parsed = URL.canParse(spelled) ? new URL(spelled) : undefined;
+    for (const secret of [parsed?.password, parsed?.username]) {
+      if (!secret) continue;
+      for (const form of new Set([secret, decodeURIComponent(secret)]))
+        if (form.length >= 3) out = out.split(form).join("***");
+    }
+  }
+  return out;
+}
+
+/** Runs the bundle's Playwright to fetch the pinned browser; at most 15 minutes. */
+async function installBrowser(bundle: Bundle, options: SetupOptions): Promise<void> {
+  const { root, info } = bundle;
+  const output: string[] = [];
+  const code = await new Promise<number | string>((resolve) => {
+    const child = spawn(
+      join(root, info.node.path),
+      [join(root, info.browser.installer), "install", "chromium-headless-shell"],
+      {
+        cwd: root,
+        env: installEnvironment(bundle, options.env ?? process.env),
+        stdio: ["ignore", "pipe", "pipe"],
+        ...(options.signal ? { signal: options.signal } : {}),
+        timeout: 15 * 60_000,
+      },
+    );
+    const keep = (chunk: Buffer): void => {
+      output.push(chunk.toString("utf8"));
+      if (output.length > 40) output.shift();
+    };
+    child.stdout.on("data", keep);
+    child.stderr.on("data", keep);
+    child.on("error", (error) => resolve(error.message));
+    child.on("close", (exit, signal) => resolve(exit ?? signal ?? "unknown"));
+  });
+  if (code === 0) return;
+  // Playwright prints the URLs it fetches, and a mirror's or proxy's may carry credentials
+  // (Opus review finding 3): every URL is reduced to its printable form, and any password the
+  // environment names is removed wherever it appears.
+  const tail = scrubSecrets(
+    output.join("").trim().split("\n").slice(-6).join(" | "),
+    options.env ?? process.env,
+  );
+  throw new RpcError(
+    createDiagnostic("MOSTRA_ENV_BUNDLE_DOWNLOAD_FAILED", {
+      message: `Fetching the verify browser (${info.browser.name} ${info.browser.browserVersion}) failed (${code}): ${tail}`,
+    }),
+  );
+}
+
+/**
+ * Makes the bundle at `root` ready: checks its files, fetches and checks the browser when it is
+ * missing or differs, and writes `.mostra-setup.json`. Idempotent: a ready bundle returns its
+ * marker after the checks. Throws `MOSTRA_ENV_BUNDLE_INVALID`, `MOSTRA_ENV_BUNDLE_UNAVAILABLE`
+ * (offline with no browser) or `MOSTRA_ENV_BUNDLE_DOWNLOAD_FAILED`.
+ */
+export async function setupBundle(root: string, options: SetupOptions = {}): Promise<BundleSetup> {
+  const env = options.env ?? process.env;
+  const bundle = await readBundle(root);
+  if (!options.filesChecked) {
+    options.onStep?.("files");
+    await checkFiles(bundle);
+  }
+  options.onStep?.("browser");
+  if ((await checkBrowser(bundle)).length > 0) {
+    if (offline(env)) {
+      throw new RpcError(
+        createDiagnostic("MOSTRA_ENV_BUNDLE_UNAVAILABLE", {
+          message: `The runtime bundle at ${root} has no verify browser yet, and MOSTRA_OFFLINE keeps Mostra from fetching it.`,
+        }),
+      );
+    }
+    // Fresh, never over what's there: a directory that differs from the pin may still carry
+    // Playwright's "installation complete" marker, and Playwright would then fetch nothing.
+    await rm(browsersDir(root), { recursive: true, force: true });
+    await installBrowser(bundle, options);
+    const problems = await checkBrowser(bundle);
+    if (problems.length > 0) {
+      throw invalid(
+        `The verify browser Playwright fetched differs from the bundle's pin: ${problems.join("; ")}.`,
+      );
+    }
+  }
+  const marker: BundleSetup = {
+    schemaVersion: 1,
+    version: bundle.info.version,
+    platform: bundle.info.platform,
+    files: bundle.info.files.sha256,
+    browser: { revision: bundle.info.browser.revision },
+    at: new Date().toISOString(),
+  };
+  const temporary = join(root, `${BUNDLE_SETUP_MARKER}.tmp-${process.pid}`);
+  await writeFile(temporary, `${JSON.stringify(marker, null, 2)}\n`);
+  await rename(temporary, join(root, BUNDLE_SETUP_MARKER));
+  return marker;
+}

package/src/bundle/tree.ts

@@ -0,0 +1,176 @@
+/**
+ * A bundle's files against its manifest (RT-01): one walk that both writes `files.json` (the
+ * assembler) and checks an extracted bundle against it (setup), so the two can't disagree about
+ * what counts. Links must stay inside the bundle; nothing may be missing, changed or extra.
+ */
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { lstat, readdir, readlink } from "node:fs/promises";
+import { dirname, isAbsolute, join, resolve, sep } from "node:path";
+import type { BundleFileEntry, BundleFiles, BundleLinkEntry } from "@mostra/protocol";
+
+/** The SHA-256 and size of one file, read as a stream. */
+export async function hashFile(path: string): Promise<{ sha256: string; size: number }> {
+  const hash = createHash("sha256");
+  let size = 0;
+  for await (const chunk of createReadStream(path)) {
+    hash.update(chunk as Buffer);
+    size += (chunk as Buffer).length;
+  }
+  return { sha256: hash.digest("hex"), size };
+}
+
+/** One entry of a walk: a regular file or a link, by `/`-separated path from the root. */
+type Walked =
+  | { readonly kind: "file"; readonly path: string; readonly executable: boolean }
+  | { readonly kind: "link"; readonly path: string; readonly target: string }
+  | { readonly kind: "other"; readonly path: string };
+
+/** Every file and link under `root`, sorted by path, skipping `exclude`d paths (and below). */
+async function walk(root: string, exclude: (path: string) => boolean): Promise<Walked[]> {
+  const out: Walked[] = [];
+  const visit = async (dir: string): Promise<void> => {
+    const entries = await readdir(join(root, dir), { withFileTypes: true });
+    for (const entry of entries) {
+      const path = dir === "" ? entry.name : `${dir}/${entry.name}`;
+      if (exclude(path)) continue;
+      if (entry.isDirectory()) await visit(path);
+      else if (entry.isSymbolicLink()) {
+        out.push({ kind: "link", path, target: await readlink(join(root, path)) });
+      } else if (entry.isFile()) {
+        const stats = await lstat(join(root, path));
+        out.push({ kind: "file", path, executable: (stats.mode & 0o100) !== 0 });
+      } else out.push({ kind: "other", path });
+    }
+  };
+  await visit("");
+  return out.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+}
+
+/** Runs `task` over `items`, at most `limit` at a time, keeping the order of results. */
+async function mapLimit<T, R>(
+  items: readonly T[],
+  limit: number,
+  task: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results = new Array<R>(items.length);
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    while (next < items.length) {
+      const index = next++;
+      results[index] = await task(items[index] as T);
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+  return results;
+}
+
+/** Whether a link's target is relative and resolves inside `root`. */
+export function linkStaysInside(root: string, path: string, target: string): boolean {
+  if (isAbsolute(target)) return false;
+  const base = resolve(root);
+  const resolved = resolve(base, dirname(path), target);
+  return resolved === base || resolved.startsWith(base + sep);
+}
+
+/** Whether a manifest path is a plain relative path: no absolute start, no `.` or `..` segment. */
+export function safeRelative(path: string): boolean {
+  if (path === "" || isAbsolute(path) || path.includes("\\") || path.includes("\0")) return false;
+  return path.split("/").every((segment) => segment !== "" && segment !== "." && segment !== "..");
+}
+
+/**
+ * The manifest of the tree under `root`: every regular file with its SHA-256, size and execute
+ * bit, every link with its target. Throws on anything else (a device, a socket, a link leaving
+ * the tree), which a bundle must never contain.
+ */
+export async function describeTree(
+  root: string,
+  exclude: (path: string) => boolean = () => false,
+): Promise<BundleFiles> {
+  const walked = await walk(root, exclude);
+  const files: BundleFileEntry[] = [];
+  const links: BundleLinkEntry[] = [];
+  const regular = walked.filter((entry) => entry.kind === "file");
+  const hashed = await mapLimit(regular, 32, (entry) => hashFile(join(root, entry.path)));
+  regular.forEach((entry, index) => {
+    const { sha256, size } = hashed[index] as { sha256: string; size: number };
+    files.push({ path: entry.path, size, sha256, executable: entry.executable });
+  });
+  for (const entry of walked) {
+    if (entry.kind === "other") throw new Error(`${entry.path} is neither a file nor a link.`);
+    if (entry.kind !== "link") continue;
+    if (!linkStaysInside(root, entry.path, entry.target))
+      throw new Error(`${entry.path} links outside the tree (${entry.target}).`);
+    links.push({ path: entry.path, target: entry.target });
+  }
+  return { schemaVersion: 1, files, links };
+}
+
+/** What a check found: an empty list means the tree matches. Paths are relative to the root. */
+export type TreeProblems = readonly string[];
+
+/**
+ * Checks the tree under `root` against `manifest`: each listed file present, regular, with its
+ * size, SHA-256 and execute bit; each link present with its target, staying inside; and nothing
+ * else besides what `allowExtra` accepts. Reports at most `limit` problems.
+ */
+export async function checkTree(
+  root: string,
+  manifest: BundleFiles,
+  options: { readonly allowExtra?: (path: string) => boolean; readonly limit?: number } = {},
+): Promise<TreeProblems> {
+  const limit = options.limit ?? 20;
+  const allowExtra = options.allowExtra ?? (() => false);
+  const problems: string[] = [];
+  const add = (problem: string): void => {
+    if (problems.length < limit) problems.push(problem);
+  };
+  const expectedFiles = new Map(manifest.files.map((entry) => [entry.path, entry]));
+  const expectedLinks = new Map(manifest.links.map((entry) => [entry.path, entry]));
+  for (const path of [...expectedFiles.keys(), ...expectedLinks.keys()]) {
+    if (!safeRelative(path)) add(`${path}: not a plain relative path`);
+  }
+  if (problems.length > 0) return problems;
+  // An allowed extra is skipped whole unless something listed lies at or below it.
+  const listedDirs = new Set<string>();
+  for (const path of [...expectedFiles.keys(), ...expectedLinks.keys()]) {
+    const segments = path.split("/");
+    for (let index = 1; index < segments.length; index++)
+      listedDirs.add(segments.slice(0, index).join("/"));
+  }
+  const walked = await walk(
+    root,
+    (path) =>
+      allowExtra(path) &&
+      !expectedFiles.has(path) &&
+      !expectedLinks.has(path) &&
+      !listedDirs.has(path),
+  );
+  const found = new Map(walked.map((entry) => [entry.path, entry]));
+  for (const entry of walked) {
+    if (!expectedFiles.has(entry.path) && !expectedLinks.has(entry.path)) {
+      if (!allowExtra(entry.path)) add(`${entry.path}: not in the bundle's file list`);
+    }
+  }
+  for (const [path, link] of expectedLinks) {
+    const actual = found.get(path);
+    if (actual?.kind !== "link") add(`${path}: missing link`);
+    else if (actual.target !== link.target)
+      add(`${path}: links to ${actual.target}, not ${link.target}`);
+    else if (!linkStaysInside(root, path, link.target)) add(`${path}: links outside the bundle`);
+  }
+  const files = [...expectedFiles.values()];
+  const results = await mapLimit(files, 32, async (entry) => {
+    const actual = found.get(entry.path);
+    if (actual?.kind !== "file") return `${entry.path}: missing file`;
+    if (actual.executable !== entry.executable)
+      return `${entry.path}: ${entry.executable ? "not executable" : "executable"}`;
+    const { sha256, size } = await hashFile(join(root, entry.path));
+    if (size !== entry.size) return `${entry.path}: ${size} bytes, not ${entry.size}`;
+    if (sha256 !== entry.sha256) return `${entry.path}: its SHA-256 differs`;
+    return undefined;
+  });
+  for (const result of results) if (result !== undefined) add(result);
+  return problems;
+}