@hajimism/dek 0.3.3 → 0.5.0

package/src/core/ref.ts

@@ -1,12 +1,23 @@
-import { existsSync, lstatSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync, statSync } from "node:fs";
+// biome-ignore lint/style/noRestrictedImports: mkdir, writeFile, rename, and rm, which write a snapshot into a fresh folder in refs/, which outputDir checked, and swap it in whole.
 import { mkdir, rename, rm, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
 import { loadConfig } from "./config.ts";
 import { DekError } from "./error.ts";
-import { parseRefSource, type RefSource } from "./ref-name.ts";
-import { listSlides, resolveProject } from "./resolve.ts";
-import { outputDir } from "./safe-fs.ts";
+import { shellQuote } from "./escape.ts";
+import { foreignList } from "./foreign-text.ts";
+import {
+  isPlainRefName,
+  isRefDeckName,
+  packFile,
+  parseRefSource,
+  REF_DECK_NAME_RULE,
+  type RefSource,
+} from "./ref-name.ts";
+import { listSlides, type Project, resolveProject } from "./resolve.ts";
+import { followInside, outputDir } from "./safe-fs.ts";
+import { readTar } from "./tar.ts";
 
 export {
   isPinnedRev,
@@ -24,6 +35,49 @@ function isLink(path: string): boolean {
   }
 }
 
+/** The first link on the way from `<root>/<top>` down to `path`, both included, if any. */
+function linkUnder(root: string, top: "refs" | "packs", path: string): string | undefined {
+  let at = join(root, top);
+  for (const segment of ["", ...relative(at, path).split(sep)]) {
+    at = join(at, segment);
+    if (isLink(at)) {
+      return at;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Refuses a link on the way to the snapshot at `dir`. refs/ is dek's own folder, which a cloned
+ * project can still commit: a link there, even one that stays in the project, would aim a
+ * snapshot's swap and its delete at what it points to, such as a deck under decks/.
+ */
+export function refuseLinkUnderRefs(root: string, dir: string): void {
+  const link = linkUnder(root, "refs", dir);
+  if (link !== undefined) {
+    const rel = relative(root, link);
+    throw new DekError(`${rel} is a symlink; dek follows no link under refs/`, {
+      path: link,
+      hint: `remove ${rel}: refs/ holds only the snapshots \`dekc ref\` writes, and the next read of a pinned ref fetches it again`,
+    });
+  }
+}
+
+/**
+ * Refuses a link at packs/, or at the copy of `deck`'s pack kept there. dek writes and removes
+ * that copy by name, so through a link it would overwrite, or delete, what the link points to.
+ */
+export function refuseLinkUnderPacks(root: string, deck: string): void {
+  const link = linkUnder(root, "packs", join(root, packFile(deck)));
+  if (link !== undefined) {
+    const rel = relative(root, link);
+    throw new DekError(`${rel} is a symlink; dek follows no link under packs/`, {
+      path: link,
+      hint: `remove ${rel}, then run \`dekc ref <file>.tgz\` with the pack you were sent: packs/ holds the copies it keeps, as plain files`,
+    });
+  }
+}
+
 /** Written into each snapshot; its presence is what makes a directory a ref. */
 export const REF_MARKER = ".ref.json";
 
@@ -44,8 +98,10 @@ export function refDir(root: string, name: string): string {
 
 /** The marker of a snapshot, or nothing when it is missing or not one dek wrote. */
 export function readRefMeta(dir: string): RefMeta | undefined {
-  const path = join(dir, REF_MARKER);
-  if (!existsSync(path)) {
+  const marker = join(dir, REF_MARKER);
+  // dek writes the marker as a plain file; a link out of the snapshot is none of its.
+  const path = existsSync(marker) ? followInside(marker, dir) : undefined;
+  if (path === undefined) {
     return undefined;
   }
   try {
@@ -56,6 +112,87 @@ export function readRefMeta(dir: string): RefMeta | undefined {
   }
 }
 
+/**
+ * The snapshot `dir` is in, and the ref it holds: the outermost folder from `dir` up that is one
+ * dek wrote. A ref may carry a dek.toml, and a marker, of its own in its deck's assets; neither
+ * hides the snapshot around them or names it. A marker anywhere else makes nothing a ref.
+ */
+function refAround(dir: string): { dir: string; name: string } | undefined {
+  let found: { dir: string; name: string } | undefined;
+  for (let at = resolve(dir); ; at = dirname(at)) {
+    const name = snapshotName(at);
+    if (name !== undefined) {
+      found = { dir: at, name };
+    }
+    if (dirname(at) === at) {
+      return found;
+    }
+  }
+}
+
+/**
+ * The ref `dir` holds, when it is a snapshot as dek writes one: a dek.toml beside a marker that
+ * names a ref, at that ref's own place under a project's refs/.
+ */
+export function snapshotName(dir: string): string | undefined {
+  if (!isThere(join(dir, "dek.toml"))) {
+    return undefined;
+  }
+  const name = readRefMeta(dir)?.name;
+  if (name === undefined || !isPlainRefName(name)) {
+    return undefined;
+  }
+  // The same folder, not the same spelling: a disk that ignores case, or a link at refs/, gives
+  // the path to a snapshot another spelling than its name.
+  const place = refDir("", name);
+  const project = resolve(dir, ...place.split(sep).map(() => ".."));
+  return isSameFolder(join(project, place), dir) ? name : undefined;
+}
+
+function isSameFolder(a: string, b: string): boolean {
+  try {
+    const [one, other] = [statSync(a), statSync(b)];
+    return one.dev === other.dev && one.ino === other.ino;
+  } catch {
+    return false;
+  }
+}
+
+function isThere(path: string): boolean {
+  try {
+    lstatSync(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Refuses `dir` when it is inside a ref: no command, the dev server's included, runs or writes there. */
+export function refuseInsideRef(dir: string): void {
+  const ref = refAround(dir);
+  if (ref !== undefined) {
+    throw new DekError("this directory is inside a ref; refs are read-only", {
+      path: ref.dir,
+      hint: `run dek from your own project instead: \`dekc show ${ref.name} <slug>\``,
+    });
+  }
+}
+
+/** The project around `cwd`, which is the user's own: never one inside a ref. */
+export function requireProject(cwd: string): Project {
+  const project = resolveProject(cwd);
+  refuseInsideRef(project.root);
+  return project;
+}
+
+/** What a command that writes says when it is handed a ref by name. */
+export function refReadOnlyError(name: string): DekError {
+  const ref = shellQuote(name);
+  return new DekError(`"${name}" is a ref; refs are read-only`, {
+    hint: `start a deck on its theme with \`dekc new <name> --theme-from ${ref}\`, or read it with \`dekc show ${ref} <slug>\` and write what you need in your own deck's theme`,
+  });
+}
+
 export const LICENSE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING"];
 
 /** The snapshot's license file, relative to the snapshot, when the source had one. */
@@ -81,7 +218,11 @@ export function refState(project: { root: string; configPath: string }, name: st
     name,
     ...(pinned !== undefined ? { pinned } : {}),
     dir,
-    fetched: pinned !== undefined && readRefMeta(dir)?.rev === pinned,
+    // Never through a link under refs/: one is fetched again, which refuses the link.
+    fetched:
+      pinned !== undefined &&
+      linkUnder(project.root, "refs", dir) === undefined &&
+      readRefMeta(dir)?.rev === pinned,
   };
 }
 
@@ -94,42 +235,84 @@ export const MAX_SNAPSHOT_BYTES = 50 * 1024 * 1024;
 /** A pack's tar, unzipped, may hold its files and their headers, not much more. */
 const MAX_PACK_TAR_BYTES = 2 * MAX_SNAPSHOT_BYTES;
 
-const PACK_HINT = "ask for a pack its author made with `dekc pack <deck>`";
-
 /**
- * Reads a GitHub tarball. Bun leaves out links; a path that is absolute, has
- * an empty segment, or climbs with `..` is dropped here, so nothing written
- * later can land outside the snapshot.
+ * A repository's tar, unzipped. A repository of text inflates some 3.5 times, so about five
+ * times the 200 MB dek downloads of it: 1 GB.
  */
-export async function readTarball(bytes: Uint8Array): Promise<RepoFiles> {
-  const files: RepoFiles = new Map();
-  for (const [path, blob] of await new Bun.Archive(bytes).files()) {
-    const [, ...segments] = path.split("/");
-    const safe =
-      segments.length > 0 &&
-      !path.includes("\\") &&
-      segments.every((segment) => segment !== "" && segment !== "." && segment !== "..");
-    if (safe && !path.startsWith("/")) {
-      files.set(segments.join("/"), blob);
-    }
-  }
-  return files;
+const MAX_TARBALL_TAR_BYTES = 1024 * 1024 * 1024;
+
+/** A deck is a few hundred files; a pack of more is not one `dekc pack` made. */
+const MAX_PACK_ENTRIES = 10_000;
+
+/** A repository's files, folders, and the headers before them, as many as a large one holds. */
+const MAX_TARBALL_ENTRIES = 200_000;
+
+const PACK_HINT = "ask for a pack its author made with `dekc pack <deck>`";
+
+/** Reads a GitHub tarball, as `readGzippedTar` reads any. */
+export async function readTarball(
+  bytes: Uint8Array,
+  limits: { maxTarBytes?: number; maxEntries?: number } = {},
+): Promise<RepoFiles> {
+  const maxTarBytes = limits.maxTarBytes ?? MAX_TARBALL_TAR_BYTES;
+  const maxEntries = limits.maxEntries ?? MAX_TARBALL_ENTRIES;
+  return readGzippedTar(
+    bytes,
+    { maxTarBytes, maxEntries },
+    {
+      tooLarge: () =>
+        new DekError(`the repository unpacks to more than ${formatMb(maxTarBytes)}`, {
+          hint: "dekc ref downloads and unpacks the whole repository, so keep decks in a smaller repository, or ask the deck's author for a pack made with `dekc pack <deck>` and add it with `dekc ref <file>.tgz`",
+        }),
+      notTar: () =>
+        new DekError("GitHub's tarball of the repository is not a gzipped tar", {
+          hint: "run the command again",
+        }),
+      holds: (what) =>
+        new DekError(`the repository's tarball holds ${what}, which dek does not read`, {
+          hint: "the repository holds something dek cannot read; ask the deck's author for a pack made with `dekc pack <deck>` and pin the deck from it with `dekc ref <file>.tgz`",
+        }),
+    },
+  );
 }
 
-/**
- * Reads a pack from someone else. It is gunzipped through a count that stops at `maxTarBytes`
- * before Bun takes any of it apart, so a small file that inflates without end is refused rather
- * than held in memory; the tar is then read as a GitHub tarball is.
- */
+/** Reads a pack from someone else, no larger than `maxBytes`, as `readGzippedTar` reads any. */
 export async function readPack(
   bytes: Uint8Array,
-  limits: { maxBytes?: number; maxTarBytes?: number } = {},
+  limits: { maxBytes?: number; maxTarBytes?: number; maxEntries?: number } = {},
 ): Promise<RepoFiles> {
   const maxBytes = limits.maxBytes ?? MAX_SNAPSHOT_BYTES;
   const maxTarBytes = limits.maxTarBytes ?? MAX_PACK_TAR_BYTES;
+  const maxEntries = limits.maxEntries ?? MAX_PACK_ENTRIES;
   if (bytes.byteLength > maxBytes) {
     throw new DekError(`the pack is larger than ${formatMb(maxBytes)}`, { hint: PACK_HINT });
   }
+  return readGzippedTar(
+    bytes,
+    { maxTarBytes, maxEntries },
+    {
+      tooLarge: () =>
+        new DekError(`the pack unpacks to more than ${formatMb(maxTarBytes)}`, { hint: PACK_HINT }),
+      notTar: () =>
+        new DekError("the file is not a pack: a pack is a gzipped tar", { hint: PACK_HINT }),
+      holds: (what) =>
+        new DekError(`the pack holds ${what}, which dek does not read`, { hint: PACK_HINT }),
+    },
+  );
+}
+
+/**
+ * The files of a gzipped tar from elsewhere. It is gunzipped through a count that stops at
+ * `maxTarBytes`, so a small file that inflates without end is refused rather than held in
+ * memory, and the tar is then read by dek's own reader (`readTar`), whose files are the bytes
+ * the tar stores and no more. The top folder is dropped, and so is a path that is absolute, has
+ * an empty segment, or climbs with `..`, so nothing written later can land outside the snapshot.
+ */
+async function readGzippedTar(
+  bytes: Uint8Array,
+  { maxTarBytes, maxEntries }: { maxTarBytes: number; maxEntries: number },
+  errors: { tooLarge: () => DekError; notTar: () => DekError; holds: (what: string) => DekError },
+): Promise<RepoFiles> {
   const chunks: Uint8Array[] = [];
   let size = 0;
   try {
@@ -145,9 +328,7 @@ export async function readPack(
       size += value.byteLength;
       if (size > maxTarBytes) {
         await reader.cancel();
-        throw new DekError(`the pack unpacks to more than ${formatMb(maxTarBytes)}`, {
-          hint: PACK_HINT,
-        });
+        throw errors.tooLarge();
       }
       chunks.push(value);
     }
@@ -155,7 +336,7 @@ export async function readPack(
     if (error instanceof DekError) {
       throw error;
     }
-    throw new DekError("the file is not a pack: a pack is a gzipped tar", { hint: PACK_HINT });
+    throw errors.notTar();
   }
   const tar = new Uint8Array(size);
   let offset = 0;
@@ -163,15 +344,36 @@ export async function readPack(
     tar.set(chunk, offset);
     offset += chunk.byteLength;
   }
-  try {
-    return await readTarball(tar);
-  } catch {
-    throw new DekError("the file is not a pack: a pack is a gzipped tar", { hint: PACK_HINT });
+  // The tar holds them all now; let them go before its files are made.
+  chunks.length = 0;
+  const entries = readTar(tar, { maxBytes: maxTarBytes, maxEntries });
+  if (!(entries instanceof Map)) {
+    throw entries.kind === "too large"
+      ? errors.tooLarge()
+      : entries.kind === "not a tar"
+        ? errors.notTar()
+        : errors.holds(entries.what);
   }
+  const files: RepoFiles = new Map();
+  for (const [path, blob] of entries) {
+    const [, ...segments] = path.split("/");
+    const safe =
+      segments.length > 0 &&
+      !path.includes("\\") &&
+      segments.every((segment) => segment !== "" && segment !== "." && segment !== "..");
+    if (safe && !path.startsWith("/")) {
+      files.set(segments.join("/"), blob);
+    }
+  }
+  return files;
 }
 
 function formatMb(bytes: number): string {
-  return bytes >= 1024 * 1024 ? `${bytes / 1024 / 1024} MB` : `${bytes} bytes`;
+  const mb = 1024 * 1024;
+  if (bytes >= 1024 * mb && bytes % (1024 * mb) === 0) {
+    return `${bytes / 1024 / mb} GB`;
+  }
+  return bytes >= mb ? `${bytes / mb} MB` : `${bytes} bytes`;
 }
 
 const DECK_SCRIPT_RE = /^(?:(.*)\/)?decks\/([^/]+)\/script\.md$/;
@@ -185,7 +387,37 @@ function repoDecks(files: RepoFiles): Array<{ name: string; prefix: string }> {
   });
 }
 
-/** The name of the one deck a pack holds; a pack of none, or of more, is refused. */
+/**
+ * Names from someone else's repository or pack as a message lists them: those `shows` passes,
+ * as foreignList shows them, then how many others there are, in the words of `others`.
+ */
+function listCarried(
+  names: string[],
+  shows: (name: string) => boolean,
+  others: (count: number) => string,
+): string {
+  const shown = names.filter(shows);
+  const left = names.length - shown.length;
+  if (left === 0) {
+    return foreignList(shown);
+  }
+  return shown.length === 0 ? others(left) : `${foreignList(shown)}, and ${others(left)}`;
+}
+
+/** Deck names as a message lists them: only those a ref can carry. */
+function listDecks(names: string[]): string {
+  return listCarried(
+    names,
+    isRefDeckName,
+    (count) => `${count} whose ${count === 1 ? "name" : "names"} a ref cannot carry`,
+  );
+}
+
+/**
+ * The name of the one deck a pack holds; a pack of none, of more, or of one no ref can name is
+ * refused. The names are the pack author's, so the message lists them as listDecks does, and the
+ * hint is dek's.
+ */
 export function deckOfPack(files: RepoFiles): string {
   const names = [...new Set(repoDecks(files).map((deck) => deck.name))].sort();
   const [name, ...others] = names;
@@ -195,14 +427,23 @@ export function deckOfPack(files: RepoFiles): string {
     });
   }
   if (others.length > 0) {
-    throw new DekError(`the pack holds more than one deck: ${names.join(", ")}`, {
+    throw new DekError(`the pack holds more than one deck: ${listDecks(names)}`, {
       hint: `a pack is one deck; ${PACK_HINT}`,
     });
   }
+  if (!isRefDeckName(name)) {
+    throw new DekError("the pack's deck has a name a ref cannot carry", {
+      hint: `${REF_DECK_NAME_RULE}; ask its author to rename the deck's folder and pack it again`,
+    });
+  }
   return name;
 }
 
-/** The directory of the project that holds the deck, as a prefix: "" or "sample/". */
+/**
+ * The directory of the project that holds the deck, as a prefix: "" or "sample/". The folder
+ * names the source holds are its author's: an error lists them in its message, as foreignList
+ * does, and its hint is dek's words alone.
+ */
 export function findDeckInRepo(files: RepoFiles, source: RefSource): string {
   const repo = source.kind === "file" ? "the pack" : `${source.owner}/${source.repo}`;
   const path = source.kind === "github" ? source.path : undefined;
@@ -215,17 +456,32 @@ export function findDeckInRepo(files: RepoFiles, source: RefSource): string {
   const [match, ...others] = matches;
   if (!match) {
     const names = [...new Set(decks.map((deck) => deck.name))].sort();
-    throw new DekError(`${repo} has no deck "${source.deck}"`, {
+    if (names.length === 0) {
+      throw new DekError(`${repo} has no deck "${source.deck}"`, {
+        hint: `${repo} holds no dek project (decks/<name>/script.md beside a dek.toml)`,
+      });
+    }
+    throw new DekError(`${repo} has no deck "${source.deck}", only ${listDecks(names)}`, {
       hint:
-        names.length > 0
-          ? `decks in ${repo}: ${names.join(", ")}`
-          : `${repo} holds no dek project (decks/<name>/script.md beside a dek.toml)`,
+        source.kind === "file"
+          ? `add the pack you were sent with \`dekc ref <file>.tgz\`, which pins the deck it holds`
+          : names.some(isRefDeckName)
+            ? `pin one of them: \`dekc ref ${repo}/<deck>\``
+            : `${REF_DECK_NAME_RULE}; ask its author to rename the deck's folder`,
     });
   }
   if (others.length > 0) {
     const paths = matches.map((deck) => `${deck.prefix}decks/${deck.name}`).sort();
-    throw new DekError(`${repo} has more than one deck named "${source.deck}"`, {
-      hint: `pass the GitHub link to the one you mean; they are at ${paths.join(", ")}`,
+    const listed = listCarried(
+      paths,
+      (path) => path.split("/").every(isRefDeckName),
+      (count) => `${count} whose ${count === 1 ? "path" : "paths"} dek does not print`,
+    );
+    throw new DekError(`${repo} has more than one deck named "${source.deck}", at ${listed}`, {
+      hint:
+        source.kind === "file"
+          ? `a pack is one deck; ${PACK_HINT}`
+          : `pass the GitHub link to the folder of the one you mean, https://github.com/${repo}/tree/<rev>/<path>`,
     });
   }
   return match.prefix;
@@ -319,13 +575,11 @@ export async function writeSnapshot(
   pin: string,
   snapshot: Snapshot,
 ): Promise<{ dir: string; license: string | null; path: string }> {
-  // refs/ is dek's; a link on the way would aim the swap below, and its delete, elsewhere.
   const refsDir = join(root, "refs");
   const dir = refDir(root, name);
+  // A link on the way would aim the swap below, and its delete, at what it points to.
+  refuseLinkUnderRefs(root, dir);
   outputDir(dirname(dir), root);
-  if (isLink(dir)) {
-    await rm(dir);
-  }
   const suffix = `${process.pid}-${Date.now().toString(36)}`;
   const temp = join(refsDir, `.tmp-${suffix}`);
   const old = join(refsDir, `.old-${suffix}`);

package/src/core/resolve.ts

@@ -1,10 +1,11 @@
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync, realpathSync, statSync } from "node:fs";
 import { join, relative, resolve, sep } from "node:path";
 import { deckPaths } from "./deck-paths.ts";
 import { DekError } from "./error.ts";
 import { walkUp } from "./optional.ts";
+import { compareCodeUnits } from "./order.ts";
 import { parseScript } from "./parse.ts";
-import { deckProjectRoot } from "./path.ts";
+import { deckProjectRoot, isInside } from "./path.ts";
 import { readSourceIfExists, requireInside } from "./safe-fs.ts";
 import type { Deck, Section } from "./schema.ts";
 import { suggest } from "./suggest.ts";
@@ -14,6 +15,11 @@ export type ProjectDeck = {
   dir: string;
   scriptPath: string;
   deck: Deck;
+  /**
+   * Set on a ref's deck: the folder that stands for its `.cache`, in the project that reads it,
+   * and that project's root. A snapshot is never written once `dekc ref` wrote it.
+   */
+  cache?: { dir: string; root: string };
 };
 
 export type Project = {
@@ -28,11 +34,6 @@ export type ResolvedDeck = {
   deck: ProjectDeck;
 };
 
-/** The file's text, or nothing when there is no such file. */
-export function readTextIfExists(path: string): string | undefined {
-  return existsSync(path) ? readFileSync(path, "utf8") : undefined;
-}
-
 /**
  * A file of the deck's, or nothing when there is none. It may be a symlink to elsewhere in the
  * project, but not out of it: what a deck reads, a build publishes.
@@ -115,7 +116,7 @@ export function listSlideFiles(
     readdirSync(dir)
       // A declaration file carries types for the editor, not a slide.
       .filter((name) => name.endsWith(ext) && !name.endsWith(".d.ts"))
-      .sort((a, b) => a.localeCompare(b))
+      .sort(compareCodeUnits)
       .flatMap((name) => {
         const path = join(dir, name);
         if (!statSync(requireInside(path, root)).isFile()) {
@@ -150,16 +151,27 @@ function loadDecks(root: string): {
   if (!existsSync(decksDir)) {
     return { decks: [], failed: [] };
   }
-  if (!statSync(decksDir).isDirectory()) {
+  // A link out of the project would list, serve, and watch the folders of wherever it leads. It
+  // may lead elsewhere in the project, but not into a hidden folder; a folder has no kind to keep.
+  const realRoot = realpathSync(root);
+  const realDecks = realpathSync(decksDir);
+  const rel = relative(realRoot, realDecks);
+  if (!isInside(realDecks, realRoot) || rel.split(sep).some((part) => part.startsWith("."))) {
+    throw new DekError("decks leads outside the project", {
+      path: decksDir,
+      hint: "dek follows decks/ only to a folder inside the project and outside hidden folders; move the decks into the project instead",
+    });
+  }
+  if (!statSync(realDecks).isDirectory()) {
     throw new DekError("decks is not a directory", { path: decksDir });
   }
 
   const decks: ProjectDeck[] = [];
   const failed: Project["failed"] = [];
 
-  for (const entry of readdirSync(decksDir, { withFileTypes: true })
+  for (const entry of readdirSync(realDecks, { withFileTypes: true })
     .filter((item) => item.isDirectory())
-    .sort((a, b) => a.name.localeCompare(b.name))) {
+    .sort((a, b) => compareCodeUnits(a.name, b.name))) {
     const loaded = readDeck(root, entry.name);
     if (loaded.ok) {
       decks.push(loaded.deck);