@hajimism/dek 0.3.3 → 0.5.0

package/src/core/slide-script-eval.ts

@@ -6,6 +6,7 @@ import {
   lineMatching,
   type SlideScriptProblem,
   staticProblems,
+  withoutImports,
 } from "./slide-script.ts";
 import { BUN_FLAGS } from "./spawn.ts";
 import { suggest } from "./suggest.ts";
@@ -27,6 +28,7 @@ export function slideScriptsProblems(scripts: ScriptInput[]): SlideScriptProblem
   const { results, pending } = prepareScripts(scripts);
   const codes = uncached(pending);
   if (codes.length > 0) {
+    // biome-ignore lint/nursery/noJsRestrictedProperties: lint's path is synchronous, the scripts get an empty environment, and a killed worker's lines still count; runPiped does none of these.
     const stdout = Bun.spawnSync(evaluatorCommand(), {
       ...EVALUATOR_OPTIONS,
       stdin: Buffer.from(JSON.stringify(codes)),
@@ -47,6 +49,7 @@ export async function evaluateSlideScripts(
   const { results, pending } = prepareScripts(scripts);
   const codes = uncached(pending);
   if (codes.length > 0) {
+    // biome-ignore lint/nursery/noJsRestrictedProperties: as for spawnSync above; SIGTERM ends the worker, as no host object reaches its scripts.
     const child = Bun.spawn(evaluatorCommand(), {
       ...EVALUATOR_OPTIONS,
       stdin: Buffer.from(JSON.stringify(codes)),
@@ -77,13 +80,6 @@ type PendingScript = {
   withoutImports: boolean;
 };
 
-const IMPORT_RE = /^[ \t]*import\s+(?:type\s+)?(?:[\s\S]*?\sfrom\s*)?["'][^"'\n]+["'][ \t]*;?/gm;
-
-/** The code with its import declarations blanked out, lines kept in place. */
-function withoutImports(code: string): string {
-  return code.replace(IMPORT_RE, (match) => match.replace(/[^\n]/g, " "));
-}
-
 function prepareScripts(scripts: ScriptInput[]): {
   results: SlideScriptProblem[][];
   pending: PendingScript[];
@@ -97,7 +93,15 @@ function prepareScripts(scripts: ScriptInput[]): {
     const source = importsOnly ? withoutImports(script.code) : script.code;
     const compiled =
       problems.length > 0 && !importsOnly ? undefined : compileSlideScript(source, "slide");
-    if (compiled && "problem" in compiled) {
+    // The blanked code can fail for the blanking's sake: an import it left is refused again, and
+    // already reported, and a syntax error is its own, since the script as written parsed.
+    const blanking =
+      importsOnly &&
+      compiled &&
+      "problem" in compiled &&
+      (compiled.problem.message === IMPORTS_MESSAGE ||
+        compiled.problem.message.startsWith("syntax error: "));
+    if (compiled && "problem" in compiled && !blanking) {
       problems.push(compiled.problem);
     }
     results.push(problems);

package/src/core/slide-script.ts

@@ -4,8 +4,35 @@ import { escapeRegExp } from "./escape.ts";
 import { listSlideFiles } from "./resolve.ts";
 import { stillDrawScript } from "./slide-draw.ts";
 
-/** Slide scripts are `slides/<slug>.ts`; plain JavaScript is valid TypeScript. */
-const transpiler = new Bun.Transpiler({ loader: "ts" });
+/** Thrown in place of reading a script that imports; see `transpiler`. */
+class ImportsRefused extends Error {}
+
+/** A first `#!` line, a comment that `scanImports` rejects; Bun ends it at any line terminator. */
+const HASHBANG_RE = /^#![^\n\r\u2028\u2029]*/;
+
+/**
+ * Bun's transpiler, which nothing else in dek holds. Bun's own `scan` and `transformSync` run a
+ * macro, an import marked `type: "macro"`, in this process while they read the code, so a ref's
+ * slide script or a cloned project's would run inside dek. `scanImports` runs none: both methods
+ * here ask it first, and throw `ImportsRefused` for code that imports anything.
+ * Slide scripts are `slides/<slug>.ts`; plain JavaScript is valid TypeScript.
+ */
+const transpiler = (() => {
+  const bun = new Bun.Transpiler({ loader: "ts" });
+  /** The code to read, once it imports nothing; a syntax error throws as Bun reports it. */
+  function importFree(code: string): string {
+    const source = code.replace(HASHBANG_RE, "");
+    // A `require(…)` calls a function a page does not have, and Bun reads nothing for it.
+    if (bun.scanImports(source).some((found) => found.kind !== "require-call")) {
+      throw new ImportsRefused();
+    }
+    return source;
+  }
+  return {
+    scan: (code: string) => bun.scan(importFree(code)),
+    transformSync: (code: string) => bun.transformSync(importFree(code)),
+  };
+})();
 
 const DEFAULT_EXPORT_RE = /\bexport\s+default\b/;
 /** In transpiled output, comments are gone and each statement starts a line. */
@@ -35,30 +62,94 @@ function withLine(problem: SlideScriptProblem, line: number | undefined): SlideS
   return line === undefined ? problem : { ...problem, line };
 }
 
+function importsProblem(code: string): SlideScriptProblem {
+  return withLine(
+    {
+      message: IMPORTS_MESSAGE,
+      hint: "remove the import and write what it gave in this file; DekSlide is global, from .dek/slide.d.ts",
+    },
+    lineMatching(code, /^\s*import\b/),
+  );
+}
+
+/** A string literal, which can hold a quote, a brace, or a semicolon. */
+const STRING = String.raw`"(?:[^"\\\n]|\\.)*"|'(?:[^'\\\n]|\\.)*'`;
+/**
+ * A name's character. Each part below takes only what it can hold, and none starts with a space
+ * where the part before it ends with any, so no text is read more than once from each place.
+ */
+const NAME = String.raw`[\p{ID_Continue}$]`;
+/** A specifier, then `with { … }` or the older `assert { … }`, to the end of the statement. */
+const SOURCE = String.raw`(?:${STRING})(?:\s*(?:with|assert)\s*\{(?:${NAME}|[\s,:]|${STRING})*\})?[ \t]*;?`;
+/** Where a statement can start: a line's start, or after a statement or a comment on its line. */
+const STATEMENT = String.raw`(?<=^|[;}]|\*\/)[ \t]*`;
+/** What an import declaration binds, up to its `from`, which no other import comes before. */
+const BINDINGS = String.raw`(?:(?!\bimport\b)(?:${NAME}|[\s,*{}])|${STRING})*?\bfrom\s*`;
+/** An `export { … }` clause, or the name of an `export * as`, captured. */
+const EXPORT_NAMES = String.raw`export\s*(?:\{((?:${NAME}|[\s,]|${STRING})*)\}|\*\s*as\s+(${NAME}+|${STRING}))`;
+
+/**
+ * What `scanImports` counts as an import, as the text shows it: an import declaration, an
+ * `export … from`, and the `import` of an `import(…)`.
+ */
+const IMPORT_RE = new RegExp(
+  String.raw`${STATEMENT}(?:import(?!${NAME})\s*(?:type\b\s*)?(?:(?=\S)${BINDINGS})?|(?:${EXPORT_NAMES}|export\s*\*)\s*from\s*)${SOURCE}|(?<!${NAME}|\.)import(?=\s*\()`,
+  "gmu",
+);
+const EXPORT_NAMES_RE = new RegExp(STATEMENT + EXPORT_NAMES, "gmu");
+
+/** The code with its imports blanked out, attributes included, lines kept in place. */
+export function withoutImports(code: string): string {
+  return code.replace(IMPORT_RE, (match) => match.replace(/[^\n]/g, " "));
+}
+
+/**
+ * The names `export { … }` and `export * as` give the module: what follows `as`, or the name
+ * itself. Bun leaves out of a scan one whose binding, or whose `from`, the blanking took.
+ */
+function exportedNames(code: string): string[] {
+  return [...code.matchAll(EXPORT_NAMES_RE)].flatMap(([, clause = "", star]) =>
+    (star === undefined ? clause.split(",") : [star])
+      .map((item) => item.trim())
+      .filter((item) => item !== "" && !/^type\s+(?!as\b)/.test(item))
+      .map((item) => (item.split(/\s+as\s+/).at(-1) ?? item).replace(/^(["'])(.*)\1$/, "$2")),
+  );
+}
+
 /**
  * What stops a script from being one self-contained module whose only export is the default
  * object, found without running it.
  */
 export function staticProblems(code: string): SlideScriptProblem[] {
-  let scan: ReturnType<Bun.Transpiler["scan"]>;
+  const problems: SlideScriptProblem[] = [];
+  let exports: string[] | undefined;
   try {
-    scan = transpiler.scan(code);
+    const scan = transpiler.scan(code);
+    // `require.resolve(…)` is the import scanImports does not report; nothing runs for it.
+    if (scan.imports.length > 0) {
+      problems.push(importsProblem(code));
+    }
+    exports = scan.exports;
   } catch (error) {
-    return [syntaxProblem(error)];
-  }
-  const problems: SlideScriptProblem[] = [];
-  if (scan.imports.length > 0) {
-    problems.push(
-      withLine(
-        {
-          message: IMPORTS_MESSAGE,
-          hint: "remove the import and write what it gave in this file; DekSlide is global, from .dek/slide.d.ts",
-        },
-        lineMatching(code, /^\s*import\b/),
-      ),
-    );
+    if (!(error instanceof ImportsRefused)) {
+      return [syntaxProblem(error)];
+    }
+    // Imports alone do not stop the rest of the checks, which read the script without them.
+    problems.push(importsProblem(code));
+    const blanked = withoutImports(code);
+    try {
+      // In Bun's order, which sorts the names.
+      exports = [...new Set([...transpiler.scan(blanked).exports, ...exportedNames(code)])].sort();
+    } catch (blankedError) {
+      // An error on a line the blanking left alone is the script's own, and all Bun said before.
+      const own = blankedError instanceof ImportsRefused ? undefined : syntaxProblem(blankedError);
+      const at = own?.line === undefined ? undefined : own.line - 1;
+      if (own && at !== undefined && code.split("\n")[at] === blanked.split("\n")[at]) {
+        return [own];
+      }
+    }
   }
-  for (const name of scan.exports.filter((entry) => entry !== "default")) {
+  for (const name of (exports ?? []).filter((entry) => entry !== "default")) {
     problems.push(
       withLine(
         {
@@ -72,7 +163,8 @@ export function staticProblems(code: string): SlideScriptProblem[] {
       ),
     );
   }
-  if (!scan.exports.includes("default") || !DEFAULT_EXPORT_RE.test(code)) {
+  // Exports the blanking left unreadable leave the source to say whether one is the default.
+  if (!(exports?.includes("default") ?? true) || !DEFAULT_EXPORT_RE.test(code)) {
     problems.push({
       message: "missing export default",
       hint: "end the script with export default { draw(slide, { t }) {} } satisfies DekSlide",
@@ -116,7 +208,9 @@ export function compileSlideScript(
   try {
     js = transpiler.transformSync(code);
   } catch (error) {
-    return { problem: syntaxProblem(error) };
+    return {
+      problem: error instanceof ImportsRefused ? importsProblem(code) : syntaxProblem(error),
+    };
   }
   let firstError: unknown;
   for (const match of js.matchAll(EXPORT_DEFAULT_LINE_RE)) {

package/src/core/spawn.ts

@@ -1,18 +1,5 @@
 import { DekError } from "./error.ts";
 
-export async function awaitPiped(proc: {
-  stdout: ReadableStream<Uint8Array> | number;
-  stderr: ReadableStream<Uint8Array> | number;
-  exited: Promise<number>;
-}): Promise<{ stdout: string; stderr: string; exitCode: number }> {
-  const [stdout, stderr, exitCode] = await Promise.all([
-    new Response(proc.stdout as ReadableStream<Uint8Array>).text(),
-    new Response(proc.stderr as ReadableStream<Uint8Array>).text(),
-    proc.exited,
-  ]);
-  return { stdout, stderr, exitCode };
-}
-
 /**
  * How dek runs bun: with no auto install, and without the working directory's `.env` or
  * `bunfig.toml`, which a repository someone else wrote can commit to set variables or preload code.
@@ -28,6 +15,15 @@ export function workerCommand(bin: string, args: string[] = []): string[] {
 /** A browser that hangs must not hang dek; a whole visual run or capture fits well inside this. */
 const DEFAULT_WORKER_TIMEOUT_MS = 10 * 60 * 1000;
 
+/** How long a command stopped at its timeout has to exit on SIGTERM before it gets SIGKILL. */
+const KILL_GRACE_MS = 2000;
+
+/**
+ * How long the output is still read once the command has exited. A process it started and left
+ * running holds the pipes open as long as it lives, so what came by then is the output.
+ */
+const DRAIN_MS = 1000;
+
 type PipedOptions = {
   /** The error message when the command cannot start or overruns, such as "ffmpeg failed". */
   label: string;
@@ -37,9 +33,20 @@ type PipedOptions = {
   stdin?: string;
   /** Covers the whole run, from spawn to exit. */
   timeoutMs: number;
+  /** Where the command runs; dek's working directory when left out. */
+  cwd?: string;
 };
 
-type Piped = { stdout: string; stderr: string; exitCode: number };
+type Piped = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+  /** Set when the output was let go before it ended, `DRAIN_MS` after the command exited. */
+  cutShort?: true;
+};
+
+/** What `runPiped` throws for a command it stopped at its timeout, unlike one that cannot start. */
+export class OverrunError extends DekError {}
 
 /**
  * Runs `cmd` to its exit with its output piped back. A command that cannot start, or runs past
@@ -48,36 +55,123 @@ type Piped = { stdout: string; stderr: string; exitCode: number };
 export async function runPiped(cmd: string[], options: PipedOptions): Promise<Piped> {
   let proc: Bun.Subprocess<"pipe", "pipe", "pipe">;
   try {
-    proc = Bun.spawn(cmd, { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
+    proc = Bun.spawn(cmd, {
+      ...(options.cwd !== undefined ? { cwd: options.cwd } : {}),
+      // The environment as it is now: left out, Bun passes the one dek started with, which holds
+      // what a `.env` set before dek took it out (see dropEnvFileVariables).
+      env: process.env,
+      stdin: "pipe",
+      stdout: "pipe",
+      stderr: "pipe",
+    });
   } catch (error) {
     throw new DekError(options.label, { hint: options.hint, cause: error });
   }
-  let timedOut = false;
-  const timer = setTimeout(() => {
-    timedOut = true;
-    proc.kill();
-  }, options.timeoutMs);
+  // Read before writing: a command that answers while still reading must not block on a full pipe.
+  const output = readOutput(proc);
+  const stop = stopAfter(proc, options.timeoutMs);
+  const overrun = () =>
+    new OverrunError(options.label, {
+      hint: `it did not finish within ${Math.round(options.timeoutMs / 1000)}s and was stopped`,
+    });
   try {
-    // Read before writing: a command that answers while still reading must not block on a full pipe.
-    const piped = awaitPiped(proc);
     if (options.stdin !== undefined) {
       proc.stdin.write(options.stdin);
     }
     await proc.stdin.end();
-    const result = await piped;
-    if (timedOut) {
-      throw new DekError(options.label, {
-        hint: `it did not finish within ${Math.round(options.timeoutMs / 1000)}s and was stopped`,
-      });
+    const exitCode = await proc.exited;
+    // It finished in time, however long what it left behind keeps the pipes open.
+    stop.clear();
+    const { stdout, stderr, cutShort } = await output.drain();
+    if (stop.fired()) {
+      throw overrun();
     }
-    return result;
+    return { stdout, stderr, exitCode, ...(cutShort ? { cutShort } : {}) };
   } catch (error) {
     if (error instanceof DekError) {
       throw error;
     }
+    // Killing the command breaks the pipe a write to its stdin was waiting on.
+    if (stop.fired()) {
+      throw overrun();
+    }
     throw new DekError(options.label, { hint: options.hint, cause: error });
   } finally {
-    clearTimeout(timer);
+    stop.clear();
+    output.cancel();
+  }
+}
+
+/**
+ * Stops `proc` once `timeoutMs` passes: SIGTERM, then SIGKILL if it is still running
+ * `KILL_GRACE_MS` later. `clear` calls off the timeout, never a SIGKILL already on its way.
+ *
+ * Only the command is signalled, not what it started: a command that forks a child and is killed
+ * leaves the child running. `bun add -d rumdl` puts a Node wrapper in front of rumdl that runs the
+ * binary with spawnSync, so a rumdl that hangs is stopped and its binary lives on as an orphan;
+ * the drain keeps it from holding dek. Killing the whole group would need the command in a session
+ * of its own (`detached`), where the terminal's Ctrl-C no longer reaches it.
+ */
+function stopAfter(
+  proc: Bun.Subprocess,
+  timeoutMs: number,
+): { fired: () => boolean; clear: () => void } {
+  let fired = false;
+  const timer = setTimeout(() => {
+    fired = true;
+    proc.kill();
+    const kill = setTimeout(() => proc.kill("SIGKILL"), KILL_GRACE_MS);
+    void proc.exited.finally(() => clearTimeout(kill));
+  }, timeoutMs);
+  return { fired: () => fired, clear: () => clearTimeout(timer) };
+}
+
+/**
+ * The command's stdout and stderr, read as they come. `drain`, once it has exited, waits
+ * `DRAIN_MS` at most for both to end, and says when it had to let go first (`cutShort`); `cancel`
+ * lets go of both, which an open pipe needs before dek can exit.
+ */
+function readOutput(proc: Bun.Subprocess<"pipe", "pipe", "pipe">): {
+  drain: () => Promise<{ stdout: string; stderr: string; cutShort: boolean }>;
+  cancel: () => void;
+} {
+  const readers = [proc.stdout.getReader(), proc.stderr.getReader()];
+  const texts = Promise.all(readers.map(readToEnd));
+  // Awaited by `drain`; a read that fails before then must not go unhandled.
+  texts.catch(() => {});
+  const cancel = () => {
+    for (const reader of readers) {
+      reader.cancel().catch(() => {});
+    }
+  };
+  return {
+    drain: async () => {
+      let cutShort = false;
+      const timer = setTimeout(() => {
+        cutShort = true;
+        cancel();
+      }, DRAIN_MS);
+      try {
+        const [stdout = "", stderr = ""] = await texts;
+        return { stdout, stderr, cutShort };
+      } finally {
+        clearTimeout(timer);
+      }
+    },
+    cancel,
+  };
+}
+
+/** The text a reader reads until its stream ends or is cancelled. */
+async function readToEnd(reader: ReadableStreamDefaultReader<Uint8Array>): Promise<string> {
+  const decoder = new TextDecoder();
+  let text = "";
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) {
+      return text + decoder.decode();
+    }
+    text += decoder.decode(value, { stream: true });
   }
 }
 
@@ -94,7 +188,8 @@ type JsonWorkerOptions = {
  * Runs a worker that takes one JSON request on stdin and answers with JSON on
  * stdout. Every failure is a DekError named `label`: a worker that exits
  * non-zero contributes its stderr as the hint, one that overruns the timeout
- * is killed and says so, and output `parse` rejects is invalid JSON.
+ * is killed and says so, and output `parse` rejects is invalid JSON, or cut
+ * short when a process the worker left behind held it open.
  */
 export async function runJsonWorker<T>(
   cmd: string[],
@@ -102,7 +197,7 @@ export async function runJsonWorker<T>(
   parse: (stdout: string) => T | null,
   options: JsonWorkerOptions,
 ): Promise<T> {
-  const { stdout, stderr, exitCode } = await runPiped(cmd, {
+  const { stdout, stderr, exitCode, cutShort } = await runPiped(cmd, {
     label: options.label,
     hint: options.hint,
     stdin: JSON.stringify(request),
@@ -113,7 +208,11 @@ export async function runJsonWorker<T>(
   }
   const parsed = parse(stdout);
   if (parsed === null) {
-    throw new DekError(options.label, { hint: "worker returned invalid JSON" });
+    throw new DekError(options.label, {
+      hint: cutShort
+        ? "the worker exited while a process it started still held its output, so its answer was cut short; run the command again"
+        : "worker returned invalid JSON",
+    });
   }
   return parsed;
 }

package/src/core/sync.ts

@@ -2,6 +2,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 import { loadConfig } from "./config.ts";
 import { type DeckPaths, deckPaths } from "./deck-paths.ts";
+import { foreignLine, TITLE_MAX } from "./foreign-text.ts";
 import { parseRefSource, refDir, refTitle } from "./ref.ts";
 import { asResolvedDeck, listSlides, type ResolvedDeck, SLIDE_SIDECARS } from "./resolve.ts";
 import { outputDir, readSourceIfExists, removeInside, writeInside } from "./safe-fs.ts";
@@ -132,7 +133,7 @@ export function syncDeck(input: string | ResolvedDeck): SyncResult {
     const skeleton = sectionSkeleton(deck.deck, index, layouts);
     const current = existing.get(section.slug);
     if (current) {
-      const html = readFileSync(current, "utf8");
+      const html = readSourceIfExists(current, project.root) ?? "";
       if (!record.owns(html, skeleton)) {
         continue;
       }
@@ -170,7 +171,7 @@ function removeOrphanSkeletons(
   const removed: string[] = [];
   const kept: string[] = [];
   for (const [slug, path] of orphans) {
-    const html = readFileSync(path, "utf8");
+    const html = readSourceIfExists(path, root) ?? "";
     const beside = ([...SLIDE_SIDECARS, ".js"] as const).some((ext) =>
       existsSync(paths.slide(slug, ext)),
     );
@@ -293,23 +294,13 @@ For commands, run \`dekc help --agent\`.
 `;
 }
 
-/** The longest ref title AGENTS.md shows: a title names a deck, it does not describe one. */
-const REF_TITLE_MAX = 80;
-
 /**
  * A ref's title as it goes into AGENTS.md. Someone else wrote it, and agents read dek's block as
  * dek's own guidance, so a title must not start a line of its own there, such as a heading or an
- * instruction: line breaks and control characters become one space, and it is cut to
- * REF_TITLE_MAX characters.
+ * instruction, nor hold what a person cannot see: it is one foreignLine of TITLE_MAX.
  */
 function refTitleLine(title: string | undefined): string | undefined {
-  // biome-ignore lint/suspicious/noControlCharactersInRegex: matching them is the point.
-  const line = title?.replace(/[\s\x00-\x1f\x7f-\x9f]+/gu, " ").trim();
-  if (!line) {
-    return undefined;
-  }
-  const chars = [...line];
-  return chars.length > REF_TITLE_MAX ? `${chars.slice(0, REF_TITLE_MAX - 1).join("")}…` : line;
+  return title === undefined ? undefined : foreignLine(title, TITLE_MAX) || undefined;
 }
 
 /**