@hajimism/dek 0.3.2 → 0.4.0

package/src/core/ref.ts

@@ -1,16 +1,26 @@
-import { existsSync, lstatSync, readFileSync } from "node:fs";
+import { existsSync, lstatSync, readFileSync, statSync } from "node:fs";
 import { mkdir, rename, rm, writeFile } from "node:fs/promises";
-import { dirname, join } from "node:path";
+import { dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
 import { loadConfig } from "./config.ts";
 import { DekError } from "./error.ts";
-import { parseRefSource, type RefSource } from "./ref-name.ts";
-import { listSlides, resolveProject } from "./resolve.ts";
-import { outputDir } from "./safe-fs.ts";
+import { shellQuote } from "./escape.ts";
+import { listShown } from "./prose.ts";
+import {
+  isPlainRefName,
+  isRefDeckName,
+  packFile,
+  parseRefSource,
+  REF_DECK_NAME_RULE,
+  type RefSource,
+} from "./ref-name.ts";
+import { listSlides, type Project, resolveProject } from "./resolve.ts";
+import { followInside, outputDir } from "./safe-fs.ts";
 
 export {
   isPinnedRev,
   isRefName,
+  packFile,
   parseRefSource,
   type RefSource,
 } from "./ref-name.ts";
@@ -23,6 +33,49 @@ function isLink(path: string): boolean {
   }
 }
 
+/** The first link on the way from `<root>/<top>` down to `path`, both included, if any. */
+function linkUnder(root: string, top: "refs" | "packs", path: string): string | undefined {
+  let at = join(root, top);
+  for (const segment of ["", ...relative(at, path).split(sep)]) {
+    at = join(at, segment);
+    if (isLink(at)) {
+      return at;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Refuses a link on the way to the snapshot at `dir`. refs/ is dek's own folder, which a cloned
+ * project can still commit: a link there, even one that stays in the project, would aim a
+ * snapshot's swap and its delete at what it points to, such as a deck under decks/.
+ */
+export function refuseLinkUnderRefs(root: string, dir: string): void {
+  const link = linkUnder(root, "refs", dir);
+  if (link !== undefined) {
+    const rel = relative(root, link);
+    throw new DekError(`${rel} is a symlink; dek follows no link under refs/`, {
+      path: link,
+      hint: `remove ${rel}: refs/ holds only the snapshots \`dekc ref\` writes, and the next read of a pinned ref fetches it again`,
+    });
+  }
+}
+
+/**
+ * Refuses a link at packs/, or at the copy of `deck`'s pack kept there. dek writes and removes
+ * that copy by name, so through a link it would overwrite, or delete, what the link points to.
+ */
+export function refuseLinkUnderPacks(root: string, deck: string): void {
+  const link = linkUnder(root, "packs", join(root, packFile(deck)));
+  if (link !== undefined) {
+    const rel = relative(root, link);
+    throw new DekError(`${rel} is a symlink; dek follows no link under packs/`, {
+      path: link,
+      hint: `remove ${rel}, then run \`dekc ref <file>.tgz\` with the pack you were sent: packs/ holds the copies it keeps, as plain files`,
+    });
+  }
+}
+
 /** Written into each snapshot; its presence is what makes a directory a ref. */
 export const REF_MARKER = ".ref.json";
 
@@ -35,14 +88,18 @@ export const RefMeta = z.object({
 export type RefMeta = z.infer<typeof RefMeta>;
 
 export function refDir(root: string, name: string): string {
-  const { owner, repo, deck } = parseRefSource(name);
-  return join(root, "refs", owner, repo, deck);
+  const source = parseRefSource(name);
+  return source.kind === "file"
+    ? join(root, "refs", "file", source.deck)
+    : join(root, "refs", source.owner, source.repo, source.deck);
 }
 
 /** The marker of a snapshot, or nothing when it is missing or not one dek wrote. */
 export function readRefMeta(dir: string): RefMeta | undefined {
-  const path = join(dir, REF_MARKER);
-  if (!existsSync(path)) {
+  const marker = join(dir, REF_MARKER);
+  // dek writes the marker as a plain file; a link out of the snapshot is none of its.
+  const path = existsSync(marker) ? followInside(marker, dir) : undefined;
+  if (path === undefined) {
     return undefined;
   }
   try {
@@ -53,7 +110,88 @@ export function readRefMeta(dir: string): RefMeta | undefined {
   }
 }
 
-const LICENSE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING"];
+/**
+ * The snapshot `dir` is in, and the ref it holds: the outermost folder from `dir` up that is one
+ * dek wrote. A ref may carry a dek.toml, and a marker, of its own in its deck's assets; neither
+ * hides the snapshot around them or names it. A marker anywhere else makes nothing a ref.
+ */
+function refAround(dir: string): { dir: string; name: string } | undefined {
+  let found: { dir: string; name: string } | undefined;
+  for (let at = resolve(dir); ; at = dirname(at)) {
+    const name = snapshotName(at);
+    if (name !== undefined) {
+      found = { dir: at, name };
+    }
+    if (dirname(at) === at) {
+      return found;
+    }
+  }
+}
+
+/**
+ * The ref `dir` holds, when it is a snapshot as dek writes one: a dek.toml beside a marker that
+ * names a ref, at that ref's own place under a project's refs/.
+ */
+function snapshotName(dir: string): string | undefined {
+  if (!isThere(join(dir, "dek.toml"))) {
+    return undefined;
+  }
+  const name = readRefMeta(dir)?.name;
+  if (name === undefined || !isPlainRefName(name)) {
+    return undefined;
+  }
+  // The same folder, not the same spelling: a disk that ignores case, or a link at refs/, gives
+  // the path to a snapshot another spelling than its name.
+  const place = refDir("", name);
+  const project = resolve(dir, ...place.split(sep).map(() => ".."));
+  return isSameFolder(join(project, place), dir) ? name : undefined;
+}
+
+function isSameFolder(a: string, b: string): boolean {
+  try {
+    const [one, other] = [statSync(a), statSync(b)];
+    return one.dev === other.dev && one.ino === other.ino;
+  } catch {
+    return false;
+  }
+}
+
+function isThere(path: string): boolean {
+  try {
+    lstatSync(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** Refuses `dir` when it is inside a ref: no command, the dev server's included, runs or writes there. */
+export function refuseInsideRef(dir: string): void {
+  const ref = refAround(dir);
+  if (ref !== undefined) {
+    throw new DekError("this directory is inside a ref; refs are read-only", {
+      path: ref.dir,
+      hint: `run dek from your own project instead: \`dekc show ${ref.name} <slug>\``,
+    });
+  }
+}
+
+/** The project around `cwd`, which is the user's own: never one inside a ref. */
+export function requireProject(cwd: string): Project {
+  const project = resolveProject(cwd);
+  refuseInsideRef(project.root);
+  return project;
+}
+
+/** What a command that writes says when it is handed a ref by name. */
+export function refReadOnlyError(name: string): DekError {
+  const ref = shellQuote(name);
+  return new DekError(`"${name}" is a ref; refs are read-only`, {
+    hint: `start a deck on its theme with \`dekc new <name> --theme-from ${ref}\`, or read it with \`dekc show ${ref} <slug>\` and write what you need in your own deck's theme`,
+  });
+}
+
+export const LICENSE_NAMES = ["LICENSE", "LICENSE.md", "LICENSE.txt", "COPYING"];
 
 /** The snapshot's license file, relative to the snapshot, when the source had one. */
 export function refLicense(dir: string): string | null {
@@ -78,7 +216,11 @@ export function refState(project: { root: string; configPath: string }, name: st
     name,
     ...(pinned !== undefined ? { pinned } : {}),
     dir,
-    fetched: pinned !== undefined && readRefMeta(dir)?.rev === pinned,
+    // Never through a link under refs/: one is fetched again, which refuses the link.
+    fetched:
+      pinned !== undefined &&
+      linkUnder(project.root, "refs", dir) === undefined &&
+      readRefMeta(dir)?.rev === pinned,
   };
 }
 
@@ -86,7 +228,12 @@ export function refState(project: { root: string; configPath: string }, name: st
 export type RepoFiles = Map<string, Blob>;
 
 /** Beyond this, a snapshot is refused: a deck to read is a few hundred kilobytes. */
-const MAX_SNAPSHOT_BYTES = 50 * 1024 * 1024;
+export const MAX_SNAPSHOT_BYTES = 50 * 1024 * 1024;
+
+/** A pack's tar, unzipped, may hold its files and their headers, not much more. */
+const MAX_PACK_TAR_BYTES = 2 * MAX_SNAPSHOT_BYTES;
+
+const PACK_HINT = "ask for a pack its author made with `dekc pack <deck>`";
 
 /**
  * Reads a GitHub tarball. Bun leaves out links; a path that is absolute, has
@@ -108,6 +255,64 @@ export async function readTarball(bytes: Uint8Array): Promise<RepoFiles> {
   return files;
 }
 
+/**
+ * Reads a pack from someone else. It is gunzipped through a count that stops at `maxTarBytes`
+ * before Bun takes any of it apart, so a small file that inflates without end is refused rather
+ * than held in memory; the tar is then read as a GitHub tarball is.
+ */
+export async function readPack(
+  bytes: Uint8Array,
+  limits: { maxBytes?: number; maxTarBytes?: number } = {},
+): Promise<RepoFiles> {
+  const maxBytes = limits.maxBytes ?? MAX_SNAPSHOT_BYTES;
+  const maxTarBytes = limits.maxTarBytes ?? MAX_PACK_TAR_BYTES;
+  if (bytes.byteLength > maxBytes) {
+    throw new DekError(`the pack is larger than ${formatMb(maxBytes)}`, { hint: PACK_HINT });
+  }
+  const chunks: Uint8Array[] = [];
+  let size = 0;
+  try {
+    const reader = new Blob([new Uint8Array(bytes)])
+      .stream()
+      .pipeThrough(new DecompressionStream("gzip"))
+      .getReader();
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      size += value.byteLength;
+      if (size > maxTarBytes) {
+        await reader.cancel();
+        throw new DekError(`the pack unpacks to more than ${formatMb(maxTarBytes)}`, {
+          hint: PACK_HINT,
+        });
+      }
+      chunks.push(value);
+    }
+  } catch (error) {
+    if (error instanceof DekError) {
+      throw error;
+    }
+    throw new DekError("the file is not a pack: a pack is a gzipped tar", { hint: PACK_HINT });
+  }
+  const tar = new Uint8Array(size);
+  let offset = 0;
+  for (const chunk of chunks) {
+    tar.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  try {
+    return await readTarball(tar);
+  } catch {
+    throw new DekError("the file is not a pack: a pack is a gzipped tar", { hint: PACK_HINT });
+  }
+}
+
+function formatMb(bytes: number): string {
+  return bytes >= 1024 * 1024 ? `${bytes / 1024 / 1024} MB` : `${bytes} bytes`;
+}
+
 const DECK_SCRIPT_RE = /^(?:(.*)\/)?decks\/([^/]+)\/script\.md$/;
 
 /** The decks in a repository: each `decks/<name>/script.md` beside its project's dek.toml. */
@@ -119,14 +324,46 @@ function repoDecks(files: RepoFiles): Array<{ name: string; prefix: string }> {
   });
 }
 
+/** Deck names as a hint lists them: only those a ref can carry. */
+function listDecks(names: string[]): string {
+  return listShown(
+    names,
+    isRefDeckName,
+    (count) => `${count} whose ${count === 1 ? "name" : "names"} a ref cannot carry`,
+  );
+}
+
+/** The name of the one deck a pack holds; a pack of none, of more, or of one no ref can name is refused. */
+export function deckOfPack(files: RepoFiles): string {
+  const names = [...new Set(repoDecks(files).map((deck) => deck.name))].sort();
+  const [name, ...others] = names;
+  if (name === undefined) {
+    throw new DekError("the pack holds no deck", {
+      hint: `a pack is one deck beside a dek.toml; ${PACK_HINT}`,
+    });
+  }
+  if (others.length > 0) {
+    throw new DekError(`the pack holds more than one deck: ${listDecks(names)}`, {
+      hint: `a pack is one deck; ${PACK_HINT}`,
+    });
+  }
+  if (!isRefDeckName(name)) {
+    throw new DekError("the pack's deck has a name a ref cannot carry", {
+      hint: `${REF_DECK_NAME_RULE}; ask its author to rename the deck's folder and pack it again`,
+    });
+  }
+  return name;
+}
+
 /** The directory of the project that holds the deck, as a prefix: "" or "sample/". */
 export function findDeckInRepo(files: RepoFiles, source: RefSource): string {
-  const repo = `${source.owner}/${source.repo}`;
+  const repo = source.kind === "file" ? "the pack" : `${source.owner}/${source.repo}`;
+  const path = source.kind === "github" ? source.path : undefined;
   const decks = repoDecks(files);
   const matches = decks.filter(
     (deck) =>
       deck.name === source.deck &&
-      (source.path === undefined || `${deck.prefix}decks/${deck.name}` === source.path),
+      (path === undefined || `${deck.prefix}decks/${deck.name}` === path),
   );
   const [match, ...others] = matches;
   if (!match) {
@@ -134,21 +371,26 @@ export function findDeckInRepo(files: RepoFiles, source: RefSource): string {
     throw new DekError(`${repo} has no deck "${source.deck}"`, {
       hint:
         names.length > 0
-          ? `decks in ${repo}: ${names.join(", ")}`
+          ? `decks in ${repo}: ${listDecks(names)}`
           : `${repo} holds no dek project (decks/<name>/script.md beside a dek.toml)`,
     });
   }
   if (others.length > 0) {
     const paths = matches.map((deck) => `${deck.prefix}decks/${deck.name}`).sort();
+    const listed = listShown(
+      paths,
+      (path) => path.split("/").every(isRefDeckName),
+      (count) => `${count} whose ${count === 1 ? "path" : "paths"} a hint cannot show`,
+    );
     throw new DekError(`${repo} has more than one deck named "${source.deck}"`, {
-      hint: `pass the GitHub link to the one you mean; they are at ${paths.join(", ")}`,
+      hint: `pass the GitHub link to the one you mean; they are at ${listed}`,
     });
   }
   return match.prefix;
 }
 
 /** Inside the deck, only what reading it needs: not dist/, .cache/, or voice/. */
-function keepsInDeck(path: string): boolean {
+export function keepsInDeck(path: string): boolean {
   return (
     path === "script.md" ||
     path === "theme.css" ||
@@ -157,17 +399,19 @@ function keepsInDeck(path: string): boolean {
   );
 }
 
+/** The files a snapshot keeps, by their paths in it, and where the deck was in its source. */
+export type Snapshot = {
+  files: Map<string, Blob>;
+  license: string | undefined;
+  /** The deck's directory in the source repository or pack. */
+  path: string;
+};
+
 /**
- * Writes the snapshot of one deck at one commit under refs/, laid out as the
- * project it came from so it resolves like any dek project. It is written
- * beside the old one and swapped in, so a failure leaves the old one whole.
+ * What a snapshot keeps of a repository or a pack: its project's dek.toml, the deck's files that
+ * reading needs, and its license, laid out as the project it came from.
  */
-export async function installSnapshot(
-  root: string,
-  source: RefSource,
-  sha: string,
-  files: RepoFiles,
-): Promise<{ dir: string; license: string | null; path: string }> {
+export function pickSnapshot(files: RepoFiles, source: RefSource): Snapshot {
   const prefix = findDeckInRepo(files, source);
   const deckPrefix = `${prefix}decks/${source.deck}/`;
   const license =
@@ -190,20 +434,60 @@ export async function installSnapshot(
       hint: "a deck to read as a model should be far smaller; check its assets",
     });
   }
+  return { files: picked, license, path: `${prefix}decks/${source.deck}` };
+}
+
+/**
+ * What a pack's ref is pinned to: the sha256 of every file the snapshot keeps, in byte order of
+ * their paths, each as its path, a NUL, its size in decimal digits, a NUL, and its bytes. It
+ * follows the files, not the tarball, whose headers carry the time it was made.
+ */
+export async function snapshotDigest(snapshot: Snapshot): Promise<string> {
+  const hasher = new Bun.CryptoHasher("sha256");
+  const encoder = new TextEncoder();
+  const paths = [...snapshot.files.keys()].sort((a, b) =>
+    Buffer.compare(encoder.encode(a), encoder.encode(b)),
+  );
+  for (const path of paths) {
+    const blob = snapshot.files.get(path) ?? new Blob([]);
+    hasher.update(`${path}\0${blob.size}\0`);
+    hasher.update(new Uint8Array(await blob.arrayBuffer()));
+  }
+  return hasher.digest("hex");
+}
+
+/** Writes the snapshot of one deck at one commit under refs/; see writeSnapshot. */
+export async function installSnapshot(
+  root: string,
+  source: RefSource,
+  sha: string,
+  files: RepoFiles,
+): Promise<{ dir: string; license: string | null; path: string }> {
+  return writeSnapshot(root, source.name, sha, pickSnapshot(files, source));
+}
 
-  // refs/ is dek's; a link on the way would aim the swap below, and its delete, elsewhere.
+/**
+ * Writes a snapshot under refs/, laid out as the project it came from so it resolves like any
+ * dek project, and marks it with what it is pinned to. It is written beside the old one and
+ * swapped in, so a failure leaves the old one whole.
+ */
+export async function writeSnapshot(
+  root: string,
+  name: string,
+  pin: string,
+  snapshot: Snapshot,
+): Promise<{ dir: string; license: string | null; path: string }> {
   const refsDir = join(root, "refs");
-  const dir = refDir(root, source.name);
+  const dir = refDir(root, name);
+  // A link on the way would aim the swap below, and its delete, at what it points to.
+  refuseLinkUnderRefs(root, dir);
   outputDir(dirname(dir), root);
-  if (isLink(dir)) {
-    await rm(dir);
-  }
   const suffix = `${process.pid}-${Date.now().toString(36)}`;
   const temp = join(refsDir, `.tmp-${suffix}`);
   const old = join(refsDir, `.old-${suffix}`);
-  const meta: RefMeta = { name: source.name, rev: sha, path: `${prefix}decks/${source.deck}` };
+  const meta: RefMeta = { name, rev: pin, path: snapshot.path };
   try {
-    for (const [path, blob] of picked) {
+    for (const [path, blob] of snapshot.files) {
       await mkdir(dirname(join(temp, path)), { recursive: true });
       await writeFile(join(temp, path), new Uint8Array(await blob.arrayBuffer()));
     }
@@ -224,7 +508,7 @@ export async function installSnapshot(
       await rm(old, { recursive: true, force: true });
     }
   }
-  return { dir, license: license ?? null, path: meta.path };
+  return { dir, license: snapshot.license ?? null, path: meta.path };
 }
 
 /** The snapshot deck's title and slide count, or nothing when it does not parse. */

package/src/core/resolve.ts

@@ -1,10 +1,10 @@
-import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { existsSync, readdirSync, realpathSync, statSync } from "node:fs";
 import { join, relative, resolve, sep } from "node:path";
 import { deckPaths } from "./deck-paths.ts";
 import { DekError } from "./error.ts";
 import { walkUp } from "./optional.ts";
 import { parseScript } from "./parse.ts";
-import { deckProjectRoot } from "./path.ts";
+import { deckProjectRoot, isInside } from "./path.ts";
 import { readSourceIfExists, requireInside } from "./safe-fs.ts";
 import type { Deck, Section } from "./schema.ts";
 import { suggest } from "./suggest.ts";
@@ -14,6 +14,11 @@ export type ProjectDeck = {
   dir: string;
   scriptPath: string;
   deck: Deck;
+  /**
+   * Set on a ref's deck: the folder that stands for its `.cache`, in the project that reads it,
+   * and that project's root. A snapshot is never written once `dekc ref` wrote it.
+   */
+  cache?: { dir: string; root: string };
 };
 
 export type Project = {
@@ -28,11 +33,6 @@ export type ResolvedDeck = {
   deck: ProjectDeck;
 };
 
-/** The file's text, or nothing when there is no such file. */
-export function readTextIfExists(path: string): string | undefined {
-  return existsSync(path) ? readFileSync(path, "utf8") : undefined;
-}
-
 /**
  * A file of the deck's, or nothing when there is none. It may be a symlink to elsewhere in the
  * project, but not out of it: what a deck reads, a build publishes.
@@ -150,14 +150,25 @@ function loadDecks(root: string): {
   if (!existsSync(decksDir)) {
     return { decks: [], failed: [] };
   }
-  if (!statSync(decksDir).isDirectory()) {
+  // A link out of the project would list, serve, and watch the folders of wherever it leads. It
+  // may lead elsewhere in the project, but not into a hidden folder; a folder has no kind to keep.
+  const realRoot = realpathSync(root);
+  const realDecks = realpathSync(decksDir);
+  const rel = relative(realRoot, realDecks);
+  if (!isInside(realDecks, realRoot) || rel.split(sep).some((part) => part.startsWith("."))) {
+    throw new DekError("decks leads outside the project", {
+      path: decksDir,
+      hint: "dek follows decks/ only to a folder inside the project and outside hidden folders; move the decks into the project instead",
+    });
+  }
+  if (!statSync(realDecks).isDirectory()) {
     throw new DekError("decks is not a directory", { path: decksDir });
   }
 
   const decks: ProjectDeck[] = [];
   const failed: Project["failed"] = [];
 
-  for (const entry of readdirSync(decksDir, { withFileTypes: true })
+  for (const entry of readdirSync(realDecks, { withFileTypes: true })
     .filter((item) => item.isDirectory())
     .sort((a, b) => a.name.localeCompare(b.name))) {
     const loaded = readDeck(root, entry.name);

package/src/core/shot/morph.ts

@@ -62,7 +62,7 @@ export async function shotMorph(
   const step = stepKey(fromSection.beats, from.beatIndex);
 
   const html = videoDocument(resolved, options.playerScript);
-  const entry = morphEntry(deck.dir, options.from, options.to, at, html);
+  const entry = morphEntry(deck, options.from, options.to, at, html);
   if (!entry.cached) {
     await requirePlaywright(
       {

package/src/core/shot/motion.ts

@@ -2,7 +2,6 @@ import { readFileSync } from "node:fs";
 import { isAbsolute, join, relative } from "node:path";
 import pkg from "../../../package.json";
 import { DekError } from "../error.ts";
-import { deckProjectRoot } from "../path.ts";
 import { type PlaywrightRunner, requirePlaywright } from "../playwright.ts";
 import { deckStops, retreat } from "../position.ts";
 import { asResolvedDeck, type ResolvedDeck, requireSection } from "../resolve.ts";
@@ -103,7 +102,7 @@ export async function shotMotion(
   const key = `${JSON.stringify([pkg.version, SHEET_LAYOUT_VERSION, MOTION_FRACTIONS, plan.from, beats, title])}\0${html}`;
   // Each step keeps its own entry, so runs of different steps neither evict nor race each other.
   const capture = options.step === undefined ? [section.slug] : [section.slug, String(first)];
-  const bundle = bundleEntry(deck.dir, "motion", capture, key);
+  const bundle = bundleEntry(deck, "motion", capture, key);
   const manifest = join(bundle.dir, "motion.json");
   const cached = readMotionManifest(manifest, bundle.dir);
   if (cached) {
@@ -112,7 +111,7 @@ export async function shotMotion(
   }
   // Chromium writes the frames and sheets by name, so no link may stand where one goes.
   dropLinks(bundle.dir);
-  dropLinks(outputDir(join(bundle.dir, "frames"), deckProjectRoot(deck.dir)));
+  dropLinks(outputDir(join(bundle.dir, "frames"), bundle.root));
 
   const response = await requirePlaywright(
     {

package/src/core/shot/still.ts

@@ -84,7 +84,7 @@ export async function shotSheet(
   }));
   // The tiles' paths hash what they show, so the key changes whenever a tile does.
   const bundle = bundleEntry(
-    deck.dir,
+    deck,
     "sheets",
     [],
     JSON.stringify([SHEET_LAYOUT_VERSION, deck.deck.title, cells]),
@@ -136,7 +136,7 @@ export function still(
   step: string = stepKey(section.beats, beatIndex),
 ): Still {
   const html = renderSlideHtml(sources, section.slug, beatIndex);
-  const entry = stillEntry(deck.dir, section, beatIndex, html);
+  const entry = stillEntry(deck, section, beatIndex, html);
   return { html, slug: section.slug, step, screenshotPath: entry.path, entry };
 }