afterpresets 0.2.0 → 0.4.0

package/runtime/scripts/ae-sdk-docs/index.js

@@ -0,0 +1,314 @@
+"use strict";
+
+// Builds the agent-friendly After Effects SDK topic files from the pinned
+// guide source. See ../ae-sdk-docs.js for the command line and config.js for
+// the pin, page selection and task index.
+
+const fs = require("fs");
+const path = require("path");
+const config = require("./config");
+const { convertPage, stripInline } = require("./convert");
+const { findHeadings, planPage, rewriteLinks } = require("./split");
+const { loadSource } = require("./source");
+
+const MANIFEST = ".ae-sdk-docs.json";
+const GENERATOR_VERSION = 1;
+
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+
+/** Pages in navigation order: [{ section, title, path }] from mkdocs.yml's nav. */
+function parseNav(yml) {
+  const lines = yml.split(/\r?\n/);
+  const start = lines.findIndex((l) => /^nav:\s*$/.test(l));
+  if (start === -1) throw new Error("mkdocs.yml has no nav.");
+  const pages = [];
+  let section = "";
+  for (let i = start + 1; i < lines.length; i++) {
+    const line = lines[i];
+    if (/^\S/.test(line)) break;
+    const page = line.match(/^(\s*)-\s+(.*):\s+([\w./-]+\.md)\s*$/);
+    if (page) {
+      pages.push({ section: page[1].length <= 4 ? "" : section, title: page[2].replace(/^["']|["']$/g, ""), path: page[3] });
+      continue;
+    }
+    const group = line.match(/^\s*-\s+(.*):\s*$/);
+    if (group) section = group[1].replace(/^["']|["']$/g, "");
+  }
+  return pages;
+}
+
+function siteUrl(pin, sourcePath, anchor) {
+  const page = sourcePath === "index.md" ? "" : sourcePath.replace(/(^|\/)index\.md$/, "$1").replace(/\.md$/, "/");
+  return pin.site + "/" + page + (anchor ? "#" + anchor : "");
+}
+
+function firstSentence(lines) {
+  const text = lines.find((l) => l.trim() && !/^\s*(#|>|```|- |\* |\||Source:|\*From )/.test(l));
+  if (!text) return "";
+  const plain = stripInline(text).replace(/\s+/g, " ");
+  const end = plain.search(/[.!?](\s|$)/);
+  const sentence = end === -1 ? plain : plain.slice(0, end + 1);
+  return sentence.length > 110 ? sentence.slice(0, 107).replace(/\s+\S*$/, "") + "..." : sentence;
+}
+
+function summarize(headings, lines) {
+  const h2 = headings.filter((h) => h.level === 2).map((h) => h.text);
+  const list = h2.length ? h2 : headings.filter((h) => h.level === 3).map((h) => h.text);
+  if (!list.length) return firstSentence(lines);
+  const shown = list.slice(0, 6).join(", ");
+  return list.length > 6 ? shown + ", ..." : shown;
+}
+
+function clip(text, max) {
+  return text.length > max ? text.slice(0, max - 3).replace(/\s+\S*$/, "") + "..." : text;
+}
+
+function headingsBetween(page, range) {
+  return page.headings.filter((h) => h.index >= range.start && h.index < range.end && h.index !== range.start);
+}
+
+function tableCell(text) {
+  return String(text).replace(/\|/g, "\\|");
+}
+
+/**
+ * Generates the topic files. Returns { files: [{ name, tokens, source }], totalTokens,
+ * sourceTokens, commit, digest, warnings }. Writes them when options.out is set.
+ */
+async function generate(options) {
+  options = options || {};
+  // options.config overrides parts of config.js (tests use a small fixture guide).
+  const cfg = Object.assign({}, config, options.config || {});
+  const pin = cfg.PIN;
+  const limits = cfg.LIMITS;
+  const source = await loadSource(pin, options);
+  const warnings = [];
+
+  const pages = parseNav(source.files["mkdocs.yml"]).filter((p) => !cfg.EXCLUDE.some((re) => re.test(p.path)));
+  const byPath = {};
+  pages.forEach((page) => {
+    const text = source.files["docs/" + page.path];
+    if (text == null) throw new Error("mkdocs.yml lists " + page.path + " but the guide has no such file.");
+    page.name = cfg.outputName(page.path);
+    page.sourceTokens = estimateTokens(text);
+    const converted = convertPage(text);
+    converted.warnings.forEach((w) => warnings.push(page.path + ": " + w));
+    page.lines = converted.lines;
+    page.headings = findHeadings(page.lines);
+    const h1 = page.headings.find((h) => h.level === 1 && h.index === 0);
+    page.heading = h1 ? h1.text : page.title;
+    page.plan = planPage(page, page.lines, page.headings, limits);
+    page.anchors = {};
+    page.headings.forEach((h) => {
+      const entry = page.plan.find((p) => h.index >= p.start && h.index < p.end) || page.plan[0];
+      page.anchors[h.slug] = entry.name;
+    });
+    byPath[page.path] = page;
+  });
+
+  const seen = {};
+  pages.forEach((page) =>
+    page.plan.forEach((entry) => {
+      if (seen[entry.name]) throw new Error("Two topic files would be named " + entry.name + ".md (" + seen[entry.name] + ", " + page.path + "). Add a RENAME in config.js.");
+      seen[entry.name] = page.path;
+    })
+  );
+
+  // Resolves a source reference ("page.md" or "page.md#anchor", relative to `from`).
+  function resolveRef(ref, from) {
+    const hash = ref.indexOf("#");
+    const filePart = hash === -1 ? ref : ref.slice(0, hash);
+    const anchor = hash === -1 ? "" : ref.slice(hash + 1);
+    const target = filePart ? path.posix.normalize(path.posix.join(path.posix.dirname(from), filePart)) : from;
+    return { target, anchor, page: byPath[target] };
+  }
+
+  function linkTo(ref, from, currentName) {
+    if (/^[a-z][a-z0-9+.-]*:/i.test(ref)) return null;
+    const { target, anchor, page } = resolveRef(ref, from);
+    if (!/\.md$/.test(target)) return null;
+    if (!page) return siteUrl(pin, target, anchor);
+    let name = page.plan[0].name;
+    if (anchor) {
+      if (page.anchors[anchor]) name = page.anchors[anchor];
+      else warnings.push(from + ": link to a missing heading " + ref);
+    }
+    if (name === currentName && anchor) return "#" + anchor;
+    return name + ".md" + (anchor ? "#" + anchor : "");
+  }
+
+  // Assemble the files.
+  const out = [];
+  pages.forEach((page) => {
+    const firstAnchor = (entry) => (page.headings.find((h) => h.index === entry.start) || {}).slug;
+    page.plan.forEach((entry, i) => {
+      let body = page.lines.slice(entry.start, entry.end);
+      const header = [];
+      if (i === 0) {
+        if (/^# /.test(body[0] || "")) body = body.slice(1);
+        header.push("# " + page.heading, "", "*From Adobe's After Effects SDK Guide: " + siteUrl(pin, page.path) + "*", "");
+      } else {
+        header.push(
+          "# " + page.heading + ": " + entry.title,
+          "",
+          "*Part of [" + page.heading + "](" + page.plan[0].name + ".md). From Adobe's After Effects SDK Guide: " + siteUrl(pin, page.path, firstAnchor(entry)) + "*",
+          ""
+        );
+      }
+      body = rewriteLinks(body, (href) => linkTo(href, page.path, entry.name));
+      const footer = [];
+      if (entry.overview) {
+        footer.push("", "## Parts of this page", "");
+        page.plan.slice(1).forEach((part) => {
+          const inside = headingsBetween(page, part)
+            .filter((h) => h.level <= 3 && part.title.indexOf(h.text) === -1)
+            .map((h) => clip(h.text, 60));
+          const more = inside.length ? " (also: " + inside.slice(0, 4).join("; ") + (inside.length > 4 ? "; ..." : "") + ")" : "";
+          footer.push("- [" + part.title + "](" + part.name + ".md)" + more);
+        });
+      }
+      const text = header.concat(body, footer).join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n";
+      const headingsIn = headingsBetween(page, entry).filter((h) => h.level > 1 && (!entry.title || entry.title.indexOf(h.text) === -1));
+      out.push({
+        name: entry.name,
+        text,
+        tokens: estimateTokens(text),
+        source: page.path,
+        section: page.section,
+        about: entry.overview ? "Overview and list of parts" : summarize(headingsIn, i === 0 ? page.lines.slice(entry.start, entry.end) : []),
+        pageTitle: page.title,
+        part: i > 0 ? entry.title : null,
+      });
+    });
+  });
+
+  // README.md: the index.
+  const fileFor = (ref) => {
+    const { target, anchor, page } = resolveRef(ref, "");
+    if (!page) throw new Error("config.js refers to " + ref + ", which is not a generated page.");
+    if (anchor && !page.anchors[anchor]) throw new Error("config.js refers to " + ref + ": no such heading.");
+    return (anchor ? page.anchors[anchor] : page.plan[0].name) + ".md" + (anchor ? "#" + anchor : "");
+  };
+  // One link per file; the first reference's heading goes in the href.
+  const linkList = (refs) => {
+    const files = [];
+    refs.map(fileFor).forEach((f) => {
+      const file = f.split("#")[0];
+      if (!files.some((x) => x.split("#")[0] === file)) files.push(f);
+    });
+    return files.map((f) => "[" + f.split("#")[0] + "](" + f + ")").join(", ");
+  };
+  const total = out.reduce((n, f) => n + f.tokens, 0);
+  const short = pin.commit.slice(0, 7);
+  const rules = options.rulesLink === undefined ? "../rules.md" : options.rulesLink;
+  const readme = [
+    "---",
+    "title: After Effects SDK guide (topic files)",
+    "description: Adobe's After Effects C++ plug-in SDK guide split into small topic files, with an index from tasks to files.",
+    "---",
+    "",
+    "# After Effects SDK guide, by topic",
+    "",
+    "Adobe's After Effects C++ SDK guide (" + pin.sdk + "), cut into small files for",
+    "you, the agent: one topic per file, tables flattened, cross-links pointing at these files.",
+    "Generated on this computer from the community copy at",
+    "https://github.com/" + pin.repo + " (commit " + short + ", " + pin.date + ").",
+    "The content is copyright Adobe. It is not part of AfterPresets: do not copy it",
+    "into the product, its listing or its source files.",
+    "",
+    "**Read the task's files below before writing SDK code.** Do not guess struct",
+    "fields, flag names or call order. Where these files and your memory disagree,",
+    "these files win; where they disagree with the SDK headers in the project, the headers win.",
+    rules ? "AfterPresets' own rules for native effects are in [" + path.posix.basename(rules) + "](" + rules + "); they come first." : "",
+    "",
+    "Look up a name across every file:",
+    "",
+    "```sh",
+    'grep -rn "PF_OutFlag_DEEP_COLOR_AWARE" ' + (options.grepPath || "docs/afterpresets/sdk/"),
+    "```",
+    "",
+    "## Start here (first effect)",
+    "",
+    cfg.START_HERE.map((ref, i) => i + 1 + ". " + linkList([ref])).join("\n"),
+    "",
+    "## Tasks",
+    "",
+    "| Task | Read |",
+    "|---|---|",
+    cfg.TASKS.map((t) => "| " + tableCell(t[0]) + " | " + linkList(t[1]) + " |").join("\n"),
+    "",
+    "Every file, with what it covers and its size: [contents.md](contents.md).",
+  ];
+  const readmeText = readme.join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n";
+
+  // contents.md: every file by section.
+  const contents = [
+    "# After Effects SDK guide: all files",
+    "",
+    "By section of Adobe's guide. Start from [README.md](README.md) for tasks. Tokens are estimates (characters / 4).",
+  ];
+  let section = null;
+  out.forEach((f) => {
+    if (f.section !== section) {
+      section = f.section;
+      contents.push("", "## " + (section || "General"), "", "| File | Covers | Tokens |", "|---|---|---|");
+    }
+    const title = stripInline(f.part ? f.pageTitle + ": " + f.part : f.pageTitle);
+    contents.push("| [" + f.name + ".md](" + f.name + ".md) | **" + tableCell(title) + "**" + (f.about ? ". " + tableCell(f.about) : "") + " | " + f.tokens + " |");
+  });
+  const contentsText = contents.join("\n").trim() + "\n";
+  out.unshift({ name: "contents", text: contentsText, tokens: estimateTokens(contentsText), source: null, section: "", about: "All files" });
+  out.unshift({ name: "README", text: readmeText, tokens: estimateTokens(readmeText), source: null, section: "", about: "Index" });
+
+  const sourceTokens = pages.reduce((n, p) => n + p.sourceTokens, 0);
+  const result = {
+    commit: source.commit,
+    digest: source.digest,
+    from: source.from,
+    files: out.map((f) => ({ name: f.name + ".md", tokens: f.tokens, source: f.source })),
+    totalTokens: out.reduce((n, f) => n + f.tokens, 0),
+    sourceTokens,
+    pages: pages.length,
+    warnings,
+  };
+  if (total <= 0) throw new Error("Nothing generated.");
+  if (options.out) writeOutput(options.out, out, result, pin);
+  return result;
+}
+
+/** Writes into `dir`, replacing a previous run's files. Refuses a folder it did not create. */
+function writeOutput(dir, files, result, pin) {
+  const manifestPath = path.join(dir, MANIFEST);
+  if (fs.existsSync(dir)) {
+    const entries = fs.readdirSync(dir);
+    if (entries.length && !fs.existsSync(manifestPath)) {
+      throw new Error(dir + " is not empty and was not made by this generator. Choose an empty folder.");
+    }
+    if (fs.existsSync(manifestPath)) {
+      let old = { files: [] };
+      try {
+        old = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+      } catch (error) {
+        // Unreadable manifest: only our own names below get replaced.
+      }
+      (old.files || []).forEach((f) => {
+        const name = path.basename(String(f.name || ""));
+        if (/^[a-z0-9-]+\.md$|^README\.md$/.test(name)) fs.rmSync(path.join(dir, name), { force: true });
+      });
+    }
+  }
+  fs.mkdirSync(dir, { recursive: true });
+  files.forEach((f) => fs.writeFileSync(path.join(dir, f.name + ".md"), f.text));
+  const manifest = {
+    generator: GENERATOR_VERSION,
+    repo: pin.repo,
+    commit: result.commit,
+    digest: result.digest,
+    files: result.files,
+  };
+  fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + "\n");
+}
+
+module.exports = { generate, parseNav, MANIFEST, GENERATOR_VERSION, estimateTokens };

package/runtime/scripts/ae-sdk-docs/source.js

@@ -0,0 +1,205 @@
+"use strict";
+
+// Gets the pinned After Effects SDK guide source onto this computer.
+//
+// The guide is copyright Adobe and has no open licence, so AfterPresets does
+// not ship it: each user's machine downloads the public GitHub repository at
+// the pinned commit (a tarball, no git needed), keeps only mkdocs.yml and
+// docs/**/*.md, checks them against the pinned digest and caches them by
+// commit in a folder shared by every project.
+//
+// The download is untrusted input: entry paths are filtered to a strict
+// pattern (nothing else is written), the archive size is capped, the commit in
+// the archive header must be the pinned one, and the content digest must match.
+
+const crypto = require("crypto");
+const fs = require("fs");
+const os = require("os");
+const path = require("path");
+const zlib = require("zlib");
+
+const MAX_ARCHIVE_BYTES = 64 * 1024 * 1024;
+const KEEP = /^(mkdocs\.yml|docs\/(?!_)[A-Za-z0-9_-]+(\/[A-Za-z0-9_-]+)*\.md)$/;
+
+function defaultCacheDir() {
+  return path.join(os.homedir(), ".afterpresets", "cache");
+}
+
+/** sha256 over the kept files, sorted by path: "path\0content\0" each. Independent of tar/gzip bytes. */
+function digestFiles(files) {
+  const hash = crypto.createHash("sha256");
+  Object.keys(files)
+    .sort()
+    .forEach((name) => {
+      hash.update(name + "\0");
+      hash.update(files[name]);
+      hash.update("\0");
+    });
+  return "sha256:" + hash.digest("hex");
+}
+
+function readTree(root) {
+  const files = {};
+  const add = (rel) => {
+    files[rel] = fs.readFileSync(path.join(root, rel.split("/").join(path.sep)), "utf8");
+  };
+  if (fs.existsSync(path.join(root, "mkdocs.yml"))) add("mkdocs.yml");
+  const walk = (dirRel) => {
+    const abs = path.join(root, dirRel.split("/").join(path.sep));
+    if (!fs.existsSync(abs)) return;
+    fs.readdirSync(abs, { withFileTypes: true }).forEach((d) => {
+      const rel = dirRel + "/" + d.name;
+      if (d.isDirectory()) walk(rel);
+      else if (d.isFile() && KEEP.test(rel)) add(rel);
+    });
+  };
+  walk("docs");
+  return files;
+}
+
+function parsePax(buf) {
+  const out = {};
+  let i = 0;
+  while (i < buf.length) {
+    const space = buf.indexOf(0x20, i);
+    if (space === -1) break;
+    const len = parseInt(buf.toString("utf8", i, space), 10);
+    if (!len || len < 0) break;
+    const record = buf.toString("utf8", space + 1, i + len - 1);
+    const eq = record.indexOf("=");
+    if (eq !== -1) out[record.slice(0, eq)] = record.slice(eq + 1);
+    i += len;
+  }
+  return out;
+}
+
+function cString(buf, start, end) {
+  const slice = buf.subarray(start, end);
+  const nul = slice.indexOf(0);
+  return slice.toString("utf8", 0, nul === -1 ? slice.length : nul);
+}
+
+/**
+ * Reads a (decompressed) tar archive. Returns { comment, files } where files
+ * holds only entries whose path, minus the top folder, matches KEEP.
+ */
+function readTar(tar) {
+  const files = {};
+  let comment = null;
+  let nextPath = null;
+  let offset = 0;
+  while (offset + 512 <= tar.length) {
+    const header = tar.subarray(offset, offset + 512);
+    if (header.every((b) => b === 0)) break;
+    const size = parseInt(cString(header, 124, 136).trim() || "0", 8);
+    if (!Number.isFinite(size) || size < 0) throw new Error("Bad tar header.");
+    const type = String.fromCharCode(header[156] || 48);
+    const dataStart = offset + 512;
+    const data = tar.subarray(dataStart, dataStart + size);
+    offset = dataStart + Math.ceil(size / 512) * 512;
+
+    if (type === "g") {
+      const pax = parsePax(data);
+      if (pax.comment) comment = pax.comment.trim();
+      continue;
+    }
+    if (type === "x") {
+      const pax = parsePax(data);
+      if (pax.path) nextPath = pax.path;
+      continue;
+    }
+    if (type === "L") {
+      nextPath = cString(data, 0, data.length);
+      continue;
+    }
+    let name = nextPath;
+    nextPath = null;
+    if (!name) {
+      const prefix = cString(header, 345, 500);
+      name = (prefix ? prefix + "/" : "") + cString(header, 0, 100);
+    }
+    if (type !== "0" && type !== "\0") continue;
+    const rel = name.split("/").slice(1).join("/");
+    if (!KEEP.test(rel)) continue;
+    files[rel] = data.toString("utf8");
+  }
+  return { comment, files };
+}
+
+async function download(url, fetchImpl) {
+  const response = await fetchImpl(url, { redirect: "follow" });
+  if (!response.ok) throw new Error("Download failed (" + response.status + "): " + url);
+  const length = Number(response.headers && response.headers.get && response.headers.get("content-length"));
+  if (length > MAX_ARCHIVE_BYTES) throw new Error("The download is too large: " + url);
+  const buf = Buffer.from(await response.arrayBuffer());
+  if (buf.length > MAX_ARCHIVE_BYTES) throw new Error("The download is too large: " + url);
+  return buf;
+}
+
+function writeTree(root, files) {
+  Object.keys(files).forEach((rel) => {
+    if (!KEEP.test(rel)) throw new Error("Refusing to write " + rel);
+    const dest = path.join(root, rel.split("/").join(path.sep));
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.writeFileSync(dest, files[rel]);
+  });
+}
+
+/**
+ * Returns { files, commit, digest, from } with files = { "mkdocs.yml": text, "docs/...md": text }.
+ *
+ * options.source     a local checkout to read instead of downloading
+ * options.cacheDir   shared cache root (default ~/.afterpresets/cache)
+ * options.fetch      fetch implementation (default global fetch)
+ * options.allowUnpinned  accept a local checkout whose digest differs (for bumping the pin)
+ */
+async function loadSource(pin, options) {
+  options = options || {};
+  if (options.source) {
+    const files = readTree(options.source);
+    if (!files["mkdocs.yml"]) throw new Error("No mkdocs.yml in " + options.source + " (expected a checkout of " + pin.repo + ").");
+    const digest = digestFiles(files);
+    if (digest !== pin.digest && !options.allowUnpinned) {
+      throw new Error("The guide in " + options.source + " is not the pinned version (digest " + digest + ", pinned " + pin.digest + "). Use --allow-unpinned to bump the pin.");
+    }
+    return { files, commit: digest === pin.digest ? pin.commit : "local", digest, from: options.source };
+  }
+
+  const cacheRoot = options.cacheDir || defaultCacheDir();
+  const dir = path.join(cacheRoot, "ae-plugin-guide-" + pin.commit);
+  if (fs.existsSync(path.join(dir, ".complete"))) {
+    const files = readTree(dir);
+    if (digestFiles(files) === pin.digest) return { files, commit: pin.commit, digest: pin.digest, from: dir };
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+
+  const url = "https://codeload.github.com/" + pin.repo + "/tar.gz/" + pin.commit;
+  const fetchImpl = options.fetch || ((u, init) => fetch(u, init));
+  const archive = await download(url, fetchImpl);
+  let tar;
+  try {
+    tar = zlib.gunzipSync(archive, { maxOutputLength: 4 * MAX_ARCHIVE_BYTES });
+  } catch (error) {
+    throw new Error("The downloaded guide is not a valid .tar.gz (" + error.message + ").");
+  }
+  const { comment, files } = readTar(tar);
+  if (comment !== pin.commit) throw new Error("The downloaded guide is commit " + comment + ", expected " + pin.commit + ".");
+  const digest = digestFiles(files);
+  if (digest !== pin.digest) throw new Error("The downloaded guide does not match the pinned digest (" + digest + ").");
+
+  fs.mkdirSync(cacheRoot, { recursive: true });
+  const tmp = fs.mkdtempSync(path.join(cacheRoot, ".ae-plugin-guide-"));
+  try {
+    writeTree(tmp, files);
+    fs.writeFileSync(path.join(tmp, ".complete"), pin.commit + "\n");
+    fs.rmSync(dir, { recursive: true, force: true });
+    fs.renameSync(tmp, dir);
+  } catch (error) {
+    fs.rmSync(tmp, { recursive: true, force: true });
+    // Another run may have filled the cache at the same time: that copy is as good.
+    if (!fs.existsSync(path.join(dir, ".complete"))) throw error;
+  }
+  return { files, commit: pin.commit, digest, from: url };
+}
+
+module.exports = { loadSource, digestFiles, readTar, readTree, defaultCacheDir, KEEP };

package/runtime/scripts/ae-sdk-docs/split.js

@@ -0,0 +1,170 @@
+"use strict";
+
+// Splits converted pages into topic files of bounded size, records which file
+// holds every heading anchor, and rewrites links to point at those files.
+//
+// A page under `singleMax` characters stays one file. A longer page becomes
+//   <name>.md            the page title, its introduction and a list of parts
+//   <name>--<part>.md    one per H2 section (H3 if a section is still too long,
+//                        then top-level blocks), small neighbours merged
+// so an agent opens only the part it needs.
+
+const { slugify, uniqueSlugger, stripInline, isFenceOpen, fenceEnd } = require("./convert");
+
+const HEADING = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
+
+function size(lines, start, end) {
+  let n = 0;
+  for (let i = start; i < end; i++) n += lines[i].length + 1;
+  return n;
+}
+
+/** Headings outside code: [{ index, level, text, slug }]. Slugs unique per page, in order. */
+function findHeadings(lines) {
+  const slug = uniqueSlugger();
+  const out = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (isFenceOpen(lines[i])) {
+      i = fenceEnd(lines, i) - 1;
+      continue;
+    }
+    const m = lines[i].match(HEADING);
+    if (m) out.push({ index: i, level: m[1].length, text: stripInline(m[2]), slug: slug(m[2]) });
+  }
+  return out;
+}
+
+/** Cuts [start, end) before each heading of `level`. Returns ranges; the first may have no heading. */
+function cutAtLevel(headings, start, end, level) {
+  const cuts = headings.filter((h) => h.level === level && h.index > start && h.index < end).map((h) => h.index);
+  const bounds = [start].concat(cuts, [end]);
+  const out = [];
+  for (let i = 0; i < bounds.length - 1; i++) if (bounds[i] < bounds[i + 1]) out.push({ start: bounds[i], end: bounds[i + 1] });
+  return out;
+}
+
+/** Cuts a range at top-level block starts (non-indented line after a blank, outside code) into pieces <= max. */
+function cutAtBlocks(lines, start, end, max) {
+  const starts = [];
+  for (let i = start + 1; i < end; i++) {
+    if (isFenceOpen(lines[i])) {
+      if (!lines[i - 1].trim() && !/^\s/.test(lines[i])) starts.push(i);
+      i = fenceEnd(lines, i) - 1;
+      continue;
+    }
+    if (lines[i].trim() && !lines[i - 1].trim() && !/^\s/.test(lines[i])) starts.push(i);
+  }
+  const out = [];
+  let from = start;
+  let lastGood = -1;
+  starts.concat([end]).forEach((cut) => {
+    if (size(lines, from, cut) > max && lastGood > from) {
+      out.push({ start: from, end: lastGood });
+      from = lastGood;
+    }
+    lastGood = cut;
+  });
+  if (from < end) out.push({ start: from, end });
+  return out;
+}
+
+function headingAt(headings, range) {
+  return headings.find((h) => h.index === range.start) || null;
+}
+
+/**
+ * Plans the files for one converted page. Returns [{ name, start, end, title, continued }];
+ * the first entry is the page itself (single file or the overview of a split page).
+ */
+function planPage(page, lines, headings, options) {
+  const { maxChars, singleMax, minChars, mergeMax } = options;
+  if (size(lines, 0, lines.length) <= singleMax) return [{ name: page.name, start: 0, end: lines.length, title: null, overview: false }];
+
+  const pieces = [];
+  const sections = cutAtLevel(headings, 0, lines.length, 2);
+  sections.forEach((section) => {
+    const subs = size(lines, section.start, section.end) > maxChars ? cutAtLevel(headings, section.start, section.end, 3) : [section];
+    subs.forEach((sub) => {
+      const blocks = size(lines, sub.start, sub.end) > maxChars ? cutAtBlocks(lines, sub.start, sub.end, maxChars) : [sub];
+      blocks.forEach((b) => pieces.push(b));
+    });
+  });
+
+  // The page's introduction (its first piece: everything before the first H2,
+  // or the first block run of a page without H2s) is the overview.
+  const overview = pieces.shift();
+
+  // Merge a small piece into its neighbour (each piece keeps its own file
+  // otherwise, so the file name says what is in it).
+  const merged = [];
+  pieces.forEach((p) => {
+    const last = merged[merged.length - 1];
+    const pSize = size(lines, p.start, p.end);
+    const small = pSize < minChars || (last && size(lines, last.start, last.end) < minChars);
+    if (last && small && size(lines, last.start, p.end) <= mergeMax) {
+      last.end = p.end;
+      last.members.push(p);
+    } else merged.push({ start: p.start, end: p.end, members: [p] });
+  });
+
+  // A piece's title: its heading, or for a piece cut between blocks, the
+  // nearest heading above it plus "(continued)".
+  const titleOf = (range) => {
+    const h = headingAt(headings, range);
+    if (h) return { text: h.text, continued: false };
+    const above = headings.filter((x) => x.index < range.start && x.level >= 2).pop();
+    return { text: above ? above.text : "", continued: true };
+  };
+
+  const used = new Set([page.name]);
+  const parts = merged.map((range) => {
+    // Named after its first piece of real size (else its largest), so a file
+    // of tiny sections is not named after a one-line heading; titled with
+    // every piece's heading.
+    const bySize = range.members.slice().sort((a, b) => size(lines, b.start, b.end) - size(lines, a.start, a.end));
+    const main = titleOf(range.members.find((m) => size(lines, m.start, m.end) >= minChars / 2) || bySize[0]);
+    let name = page.name + "--" + (slugify(main.text || "continued").replace(/_/g, "-") || "part");
+    if (name.length > 70) name = name.slice(0, 70).replace(/-+$/, "");
+    let unique = name;
+    let n = 2;
+    while (used.has(unique)) unique = name + "-" + n++;
+    used.add(unique);
+    const titles = range.members.map(titleOf).map((t) => (t.continued ? (t.text ? t.text + " (continued)" : "Continued") : t.text));
+    const title = titles.length > 4 ? titles.slice(0, 4).join(", ") + ", ..." : titles.join(", ");
+    return { name: unique, start: range.start, end: range.end, title, overview: false };
+  });
+  return [{ name: page.name, start: overview.start, end: overview.end, title: null, overview: true }].concat(parts);
+}
+
+/**
+ * Rewrites Markdown links in `lines` (outside code). `resolve(href)` returns
+ * the new href or null to keep it.
+ */
+function rewriteLinks(lines, resolve) {
+  const out = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (isFenceOpen(lines[i])) {
+      const end = fenceEnd(lines, i);
+      for (let k = i; k < end; k++) out.push(lines[k]);
+      i = end - 1;
+      continue;
+    }
+    // Leave inline code alone: split on backtick spans.
+    const parts = lines[i].split(/(`[^`]*`)/);
+    out.push(
+      parts
+        .map((part) =>
+          part[0] === "`"
+            ? part
+            : part.replace(/(\[(?:[^[\]]|\[[^[\]]*\])*\]\()([^)\s]+)((?:\s+"[^"]*")?\))/g, (m, open, href, close) => {
+                const next = resolve(href);
+                return next == null ? m : open + next + close;
+              })
+        )
+        .join("")
+    );
+  }
+  return out;
+}
+
+module.exports = { findHeadings, planPage, rewriteLinks, cutAtBlocks, cutAtLevel, size };