@hajimism/dek 0.4.0 → 0.5.0

package/src/core/sandbox.ts

@@ -0,0 +1,217 @@
+import { DekError } from "./error.ts";
+import { runJsonWorker } from "./spawn.ts";
+
+/**
+ * Set to 1, the user accepts that Chromium draws without its sandbox where the sandbox cannot
+ * start, as in a container running as root: a ref's slides too, which are refused otherwise, and
+ * the project's own with no warning. Read only from the environment dek starts in (ADR 12).
+ */
+const ALLOW_UNSANDBOXED = "DEK_ALLOW_UNSANDBOXED";
+
+/**
+ * What a Playwright worker is told of Chromium's sandbox: draw only inside it, or inside it where
+ * it starts and without it where it does not. A worker told anything else requires it.
+ */
+export type SandboxMode = "required" | "preferred";
+
+/** Chromium drew without its sandbox: why, in Chromium's words, and how to have it back. */
+type Unsandboxed = { reason: string; hint: string };
+
+function accepted(): boolean {
+  return process.env[ALLOW_UNSANDBOXED] === "1";
+}
+
+/**
+ * How a worker may draw: a ref's slide scripts are someone else's, so only in the sandbox unless
+ * the user accepted otherwise; the project's own wherever Chromium starts (ADR 29).
+ */
+export function sandboxMode(ref: boolean): SandboxMode {
+  return ref && !accepted() ? "required" : "preferred";
+}
+
+/**
+ * What Chromium says when its sandbox cannot start: as root, without user namespaces, or with its
+ * SUID helper set up wrong; then what Playwright 1.64 puts in place of Chromium's log when it holds
+ * one of those, which the call log after it still holds.
+ */
+const SANDBOX_SAYS = [
+  /No usable sandbox!|Running as root without --no-sandbox|SUID sandbox helper|Failed to move to new namespace|Received invalid process ID for zygote/,
+  /Chromium sandboxing failed!/,
+];
+
+/** A line Chromium wrote to stderr, as Playwright's call log marks it, or one it logged as FATAL. */
+const CHROMIUM_SAYS = /\[err\]|FATAL/;
+
+/** The most of a line a reason keeps, in whole sentences; it keeps the first however long. */
+const REASON_CHARS = 280;
+
+/** A sentence: up to a `.`, `!`, or `?` that ends a word, outside parentheses, as `(0 vs. 1)` is. */
+const SENTENCE = /(?:\([^)]*\)|\((?!\)|$)|[^()!?\.]|[.!?](?!\s|$))+(?:[.!?]+|$)/g;
+
+/**
+ * What Chromium said when it failed: its words for a sandbox that cannot start, else its first
+ * line of error, past the call log's `- [pid=…][err]` and its own `[…:FATAL:…]`. Undefined when
+ * it said nothing, as when Playwright found no Chromium to start.
+ */
+export function chromiumFailure(error: unknown): string | undefined {
+  const lines = (error instanceof Error ? error.message : String(error)).split("\n");
+  for (const pattern of [...SANDBOX_SAYS, CHROMIUM_SAYS]) {
+    const line = lines.find((text) => pattern.test(text));
+    if (line !== undefined) {
+      return sentences(line.replace(/^[\s-]*(?:\[[^\]]*\]\s*)*/, ""));
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Why a launch failed, in a line: what Chromium said, else the first line of Playwright's error.
+ * Never the command line Playwright's call log opens with, or its advice to launch without the
+ * sandbox.
+ */
+export function launchFailure(error: unknown): string {
+  const first = (error instanceof Error ? error.message : String(error)).split("\n")[0] ?? "";
+  return chromiumFailure(error) ?? sentences(first);
+}
+
+function sentences(text: string): string {
+  let kept = "";
+  for (const [found] of text.trim().matchAll(SENTENCE)) {
+    const sentence = found.trim();
+    const next = kept ? `${kept} ${sentence}` : sentence;
+    if (kept && next.length > REASON_CHARS) {
+      break;
+    }
+    kept = next;
+  }
+  return kept || text.trim();
+}
+
+const GENERAL_HELP =
+  "run dekc where Chromium's sandbox can start: as a user other than root, with user namespaces allowed (on Ubuntu 23.10 and later, `sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0`)";
+
+/** What lets the sandbox start, for what Chromium said. */
+function sandboxHelp(reason: string): string {
+  if (/Running as root/.test(reason)) {
+    return "run dekc as a user other than root, where Chromium's sandbox can start";
+  }
+  if (/No usable sandbox|namespace|zygote|AppArmor/i.test(reason)) {
+    return "let Chromium make user namespaces: on Ubuntu 23.10 and later, `sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0`, or an AppArmor profile for it; in a container, a seccomp profile that allows them";
+  }
+  return GENERAL_HELP;
+}
+
+/** How to have the sandbox back, and while the user has not, how to accept going without it. */
+function unsandboxedHint(reason: string): string {
+  const help = sandboxHelp(reason);
+  return accepted() ? help : `${help}; or set ${ALLOW_UNSANDBOXED}=1 to accept drawing without it`;
+}
+
+/** Why Chromium drew without its sandbox in this run, once it has; see `takeUnsandboxed`. */
+let drewUnsandboxed: string | undefined;
+
+/**
+ * Says on stderr that Chromium drew without its sandbox, once a run, as lint --visual starts a
+ * worker or two per deck and the dev server one per pass; and keeps why, for the result. The
+ * project's own pages go unsaid once the user accepted it, a ref's never: it is someone else's code.
+ */
+function noteUnsandboxed(reason: string, ref: boolean): void {
+  if (drewUnsandboxed === undefined && (ref || !accepted())) {
+    process.stderr.write(
+      ref
+        ? `warning: Chromium drew a ref without its sandbox, as ${ALLOW_UNSANDBOXED}=1 lets it: ${reason}\n  help: ${sandboxHelp(reason)}\n`
+        : `warning: Chromium drew without its sandbox, which did not start here: ${reason}\n  help: ${unsandboxedHint(reason)}\n`,
+    );
+  }
+  drewUnsandboxed = reason;
+}
+
+/**
+ * `{ unsandboxed }` when Chromium drew without its sandbox since the last call, for the result's
+ * `--json`, and nothing otherwise, as `skipped` is absent when every check ran. Each call starts
+ * the next run afresh.
+ */
+export function takeUnsandboxed(): { unsandboxed?: Unsandboxed } {
+  const reason = drewUnsandboxed;
+  drewUnsandboxed = undefined;
+  return reason === undefined ? {} : { unsandboxed: { reason, hint: unsandboxedHint(reason) } };
+}
+
+/**
+ * A worker's answer and why it drew without the sandbox; or, having drawn nothing, why the
+ * sandbox did not start, and why Chromium did not start without it either when it tried.
+ */
+type Drawn<T> =
+  | { answer: T; unsandboxed?: string }
+  | { refused: string }
+  | { failed: string; unsandboxed: string };
+
+/** The worker's JSON as `parse` reads it, with what it says of the sandbox. */
+function readDrawn<T>(
+  text: string,
+  sandbox: SandboxMode,
+  parse: (text: string) => T | null,
+): Drawn<T> | null {
+  let said: { unsandboxed?: unknown; failed?: unknown } | null;
+  try {
+    said = JSON.parse(text);
+  } catch {
+    return null;
+  }
+  const { unsandboxed, failed } = said ?? {};
+  if (unsandboxed !== undefined && typeof unsandboxed !== "string") {
+    return null;
+  }
+  if (typeof unsandboxed === "string" && typeof failed === "string") {
+    return { failed, unsandboxed };
+  }
+  if (unsandboxed !== undefined && sandbox === "required") {
+    return { refused: unsandboxed };
+  }
+  const answer = parse(text);
+  return answer === null ? null : { answer, ...(unsandboxed === undefined ? {} : { unsandboxed }) };
+}
+
+/**
+ * Runs a Playwright worker as `runJsonWorker` does, telling it how it may draw: `ref` pages only in
+ * Chromium's sandbox unless the user accepted otherwise. A draw without the sandbox is said and
+ * kept for the result; a ref's pages the sandbox could not hold are refused, undrawn.
+ */
+export async function runDrawingWorker<T>(
+  cmd: string[],
+  request: object,
+  parse: (text: string) => T | null,
+  options: Parameters<typeof runJsonWorker>[3] & { ref?: boolean },
+): Promise<T> {
+  const { ref = false, ...worker } = options;
+  const sandbox = sandboxMode(ref);
+  const drawn = await runJsonWorker(
+    cmd,
+    { ...request, sandbox },
+    (text) => readDrawn(text, sandbox, parse),
+    worker,
+  );
+  if ("failed" in drawn) {
+    const first =
+      drawn.failed === drawn.unsandboxed ? "" : `; with it, Chromium said: ${drawn.unsandboxed}`;
+    throw new DekError(`Chromium did not start, with its sandbox or without: ${drawn.failed}`, {
+      hint: `${worker.hint}${first}`,
+    });
+  }
+  if ("refused" in drawn) {
+    // A failure that names no sandbox may be Chromium missing as well.
+    const missing = /sandbox|namespace|zygote|as root/i.test(drawn.refused)
+      ? ""
+      : `${worker.hint}; or `;
+    throw new DekError(
+      `Chromium did not start with its sandbox here, and a ref is drawn only inside it: ${drawn.refused}`,
+      {
+        hint: `${missing}${sandboxHelp(drawn.refused)}; if you accept running this ref's slide scripts without the sandbox, set ${ALLOW_UNSANDBOXED}=1 and run again`,
+      },
+    );
+  }
+  if (drawn.unsandboxed !== undefined) {
+    noteUnsandboxed(drawn.unsandboxed, ref);
+  }
+  return drawn.answer;
+}

package/src/core/shot-cache.ts

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+// biome-ignore lint/style/noRestrictedImports: rmSync, which prunes old entries by name in the folder outputDir returned, and takes a link, not its target.
 import { readdirSync, rmSync } from "node:fs";
 import { join } from "node:path";
 import { escapeRegExp } from "./escape.ts";

package/src/core/slide-eval-worker.ts

@@ -6,6 +6,7 @@
  * that never ends is cut off; the parent's timeout covers anything that
  * escapes into the microtask queue.
  */
+// biome-ignore lint/style/noRestrictedImports: writeSync, to stdout (fd 1), so each answer is out before the worker is killed or exits.
 import { readFileSync, writeSync } from "node:fs";
 import vm from "node:vm";
 

package/src/core/slide-script-eval.ts

@@ -28,6 +28,7 @@ export function slideScriptsProblems(scripts: ScriptInput[]): SlideScriptProblem
   const { results, pending } = prepareScripts(scripts);
   const codes = uncached(pending);
   if (codes.length > 0) {
+    // biome-ignore lint/nursery/noJsRestrictedProperties: lint's path is synchronous, the scripts get an empty environment, and a killed worker's lines still count; runPiped does none of these.
     const stdout = Bun.spawnSync(evaluatorCommand(), {
       ...EVALUATOR_OPTIONS,
       stdin: Buffer.from(JSON.stringify(codes)),
@@ -48,6 +49,7 @@ export async function evaluateSlideScripts(
   const { results, pending } = prepareScripts(scripts);
   const codes = uncached(pending);
   if (codes.length > 0) {
+    // biome-ignore lint/nursery/noJsRestrictedProperties: as for spawnSync above; SIGTERM ends the worker, as no host object reaches its scripts.
     const child = Bun.spawn(evaluatorCommand(), {
       ...EVALUATOR_OPTIONS,
       stdin: Buffer.from(JSON.stringify(codes)),

package/src/core/spawn.ts

@@ -1,18 +1,5 @@
 import { DekError } from "./error.ts";
 
-export async function awaitPiped(proc: {
-  stdout: ReadableStream<Uint8Array> | number;
-  stderr: ReadableStream<Uint8Array> | number;
-  exited: Promise<number>;
-}): Promise<{ stdout: string; stderr: string; exitCode: number }> {
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(proc.stdout as ReadableStream<Uint8Array>).text(),
-    new Response(proc.stderr as ReadableStream<Uint8Array>).text(),
-    proc.exited,
-  ]);
-  return { stdout, stderr, exitCode };
-}
-
 /**
  * How dek runs bun: with no auto install, and without the working directory's `.env` or
  * `bunfig.toml`, which a repository someone else wrote can commit to set variables or preload code.
@@ -28,6 +15,15 @@ export function workerCommand(bin: string, args: string[] = []): string[] {
 /** A browser that hangs must not hang dek; a whole visual run or capture fits well inside this. */
 const DEFAULT_WORKER_TIMEOUT_MS = 10 * 60 * 1000;
 
+/** How long a command stopped at its timeout has to exit on SIGTERM before it gets SIGKILL. */
+const KILL_GRACE_MS = 2000;
+
+/**
+ * How long the output is still read once the command has exited. A process it started and left
+ * running holds the pipes open as long as it lives, so what came by then is the output.
+ */
+const DRAIN_MS = 1000;
+
 type PipedOptions = {
   /** The error message when the command cannot start or overruns, such as "ffmpeg failed". */
   label: string;
@@ -37,9 +33,20 @@ type PipedOptions = {
   stdin?: string;
   /** Covers the whole run, from spawn to exit. */
   timeoutMs: number;
+  /** Where the command runs; dek's working directory when left out. */
+  cwd?: string;
 };
 
-type Piped = { stdout: string; stderr: string; exitCode: number };
+type Piped = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  /** Set when the output was let go before it ended, `DRAIN_MS` after the command exited. */
+  cutShort?: true;
+};
+
+/** What `runPiped` throws for a command it stopped at its timeout, unlike one that cannot start. */
+export class OverrunError extends DekError {}
 
 /**
  * Runs `cmd` to its exit with its output piped back. A command that cannot start, or runs past
@@ -48,36 +55,123 @@ type Piped = { stdout: string; stderr: string; exitCode: number };
 export async function runPiped(cmd: string[], options: PipedOptions): Promise<Piped> {
   let proc: Bun.Subprocess<"pipe", "pipe", "pipe">;
   try {
-    proc = Bun.spawn(cmd, { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
+    proc = Bun.spawn(cmd, {
+      ...(options.cwd !== undefined ? { cwd: options.cwd } : {}),
+      // The environment as it is now: left out, Bun passes the one dek started with, which holds
+      // what a `.env` set before dek took it out (see dropEnvFileVariables).
+      env: process.env,
+      stdin: "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+    });
   } catch (error) {
     throw new DekError(options.label, { hint: options.hint, cause: error });
   }
-  let timedOut = false;
-  const timer = setTimeout(() => {
-    timedOut = true;
-    proc.kill();
-  }, options.timeoutMs);
+  // Read before writing: a command that answers while still reading must not block on a full pipe.
+  const output = readOutput(proc);
+  const stop = stopAfter(proc, options.timeoutMs);
+  const overrun = () =>
+    new OverrunError(options.label, {
+      hint: `it did not finish within ${Math.round(options.timeoutMs / 1000)}s and was stopped`,
+    });
   try {
-    // Read before writing: a command that answers while still reading must not block on a full pipe.
-    const piped = awaitPiped(proc);
     if (options.stdin !== undefined) {
       proc.stdin.write(options.stdin);
     }
     await proc.stdin.end();
-    const result = await piped;
-    if (timedOut) {
-      throw new DekError(options.label, {
-        hint: `it did not finish within ${Math.round(options.timeoutMs / 1000)}s and was stopped`,
-      });
+    const exitCode = await proc.exited;
+    // It finished in time, however long what it left behind keeps the pipes open.
+    stop.clear();
+    const { stdout, stderr, cutShort } = await output.drain();
+    if (stop.fired()) {
+      throw overrun();
     }
-    return result;
+    return { stdout, stderr, exitCode, ...(cutShort ? { cutShort } : {}) };
   } catch (error) {
     if (error instanceof DekError) {
       throw error;
     }
+    // Killing the command breaks the pipe a write to its stdin was waiting on.
+    if (stop.fired()) {
+      throw overrun();
+    }
     throw new DekError(options.label, { hint: options.hint, cause: error });
   } finally {
-    clearTimeout(timer);
+    stop.clear();
+    output.cancel();
+  }
+}
+
+/**
+ * Stops `proc` once `timeoutMs` passes: SIGTERM, then SIGKILL if it is still running
+ * `KILL_GRACE_MS` later. `clear` calls off the timeout, never a SIGKILL already on its way.
+ *
+ * Only the command is signalled, not what it started: a command that forks a child and is killed
+ * leaves the child running. `bun add -d rumdl` puts a Node wrapper in front of rumdl that runs the
+ * binary with spawnSync, so a rumdl that hangs is stopped and its binary lives on as an orphan;
+ * the drain keeps it from holding dek. Killing the whole group would need the command in a session
+ * of its own (`detached`), where the terminal's Ctrl-C no longer reaches it.
+ */
+function stopAfter(
+  proc: Bun.Subprocess,
+  timeoutMs: number,
+): { fired: () => boolean; clear: () => void } {
+  let fired = false;
+  const timer = setTimeout(() => {
+    fired = true;
+    proc.kill();
+    const kill = setTimeout(() => proc.kill("SIGKILL"), KILL_GRACE_MS);
+    void proc.exited.finally(() => clearTimeout(kill));
+  }, timeoutMs);
+  return { fired: () => fired, clear: () => clearTimeout(timer) };
+}
+
+/**
+ * The command's stdout and stderr, read as they come. `drain`, once it has exited, waits
+ * `DRAIN_MS` at most for both to end, and says when it had to let go first (`cutShort`); `cancel`
+ * lets go of both, which an open pipe needs before dek can exit.
+ */
+function readOutput(proc: Bun.Subprocess<"pipe", "pipe", "pipe">): {
+  drain: () => Promise<{ stdout: string; stderr: string; cutShort: boolean }>;
+  cancel: () => void;
+} {
+  const readers = [proc.stdout.getReader(), proc.stderr.getReader()];
+  const texts = Promise.all(readers.map(readToEnd));
+  // Awaited by `drain`; a read that fails before then must not go unhandled.
+  texts.catch(() => {});
+  const cancel = () => {
+    for (const reader of readers) {
+      reader.cancel().catch(() => {});
+    }
+  };
+  return {
+    drain: async () => {
+      let cutShort = false;
+      const timer = setTimeout(() => {
+        cutShort = true;
+        cancel();
+      }, DRAIN_MS);
+      try {
+        const [stdout = "", stderr = ""] = await texts;
+        return { stdout, stderr, cutShort };
+      } finally {
+        clearTimeout(timer);
+      }
+    },
+    cancel,
+  };
+}
+
+/** The text a reader reads until its stream ends or is cancelled. */
+async function readToEnd(reader: ReadableStreamDefaultReader<Uint8Array>): Promise<string> {
+  const decoder = new TextDecoder();
+  let text = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) {
+      return text + decoder.decode();
+    }
+    text += decoder.decode(value, { stream: true });
   }
 }
 
@@ -94,7 +188,8 @@ type JsonWorkerOptions = {
  * Runs a worker that takes one JSON request on stdin and answers with JSON on
  * stdout. Every failure is a DekError named `label`: a worker that exits
  * non-zero contributes its stderr as the hint, one that overruns the timeout
- * is killed and says so, and output `parse` rejects is invalid JSON.
+ * is killed and says so, and output `parse` rejects is invalid JSON, or cut
+ * short when a process the worker left behind held it open.
  */
 export async function runJsonWorker<T>(
   cmd: string[],
@@ -102,7 +197,7 @@ export async function runJsonWorker<T>(
   parse: (stdout: string) => T | null,
   options: JsonWorkerOptions,
 ): Promise<T> {
-  const { stdout, stderr, exitCode } = await runPiped(cmd, {
+  const { stdout, stderr, exitCode, cutShort } = await runPiped(cmd, {
     label: options.label,
     hint: options.hint,
     stdin: JSON.stringify(request),
@@ -113,7 +208,11 @@ export async function runJsonWorker<T>(
   }
   const parsed = parse(stdout);
   if (parsed === null) {
-    throw new DekError(options.label, { hint: "worker returned invalid JSON" });
+    throw new DekError(options.label, {
+      hint: cutShort
+        ? "the worker exited while a process it started still held its output, so its answer was cut short; run the command again"
+        : "worker returned invalid JSON",
+    });
   }
   return parsed;
 }

package/src/core/sync.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadConfig } from "./config.ts";
 import { type DeckPaths, deckPaths } from "./deck-paths.ts";
+import { foreignLine, TITLE_MAX } from "./foreign-text.ts";
 import { parseRefSource, refDir, refTitle } from "./ref.ts";
 import { asResolvedDeck, listSlides, type ResolvedDeck, SLIDE_SIDECARS } from "./resolve.ts";
 import { outputDir, readSourceIfExists, removeInside, writeInside } from "./safe-fs.ts";
@@ -293,23 +294,13 @@ For commands, run \`dekc help --agent\`.
 `;
 }
 
-/** The longest ref title AGENTS.md shows: a title names a deck, it does not describe one. */
-const REF_TITLE_MAX = 80;
-
 /**
  * A ref's title as it goes into AGENTS.md. Someone else wrote it, and agents read dek's block as
  * dek's own guidance, so a title must not start a line of its own there, such as a heading or an
- * instruction: line breaks and control characters become one space, and it is cut to
- * REF_TITLE_MAX characters.
+ * instruction, nor hold what a person cannot see: it is one foreignLine of TITLE_MAX.
  */
 function refTitleLine(title: string | undefined): string | undefined {
-  // biome-ignore lint/suspicious/noControlCharactersInRegex: matching them is the point.
-  const line = title?.replace(/[\s\x00-\x1f\x7f-\x9f]+/gu, " ").trim();
-  if (!line) {
-    return undefined;
-  }
-  const chars = [...line];
-  return chars.length > REF_TITLE_MAX ? `${chars.slice(0, REF_TITLE_MAX - 1).join("")}…` : line;
+  return title === undefined ? undefined : foreignLine(title, TITLE_MAX) || undefined;
 }
 
 /**