altalandi 0.4.0 → 0.6.0

package/segment.mjs

@@ -0,0 +1,1059 @@
+// Passages for folder and website runs: find the human text in a document,
+// and put refined text back without touching anything else.
+//
+// Every format returns units: a [start, end) slice of the source, the `text`
+// sent to Altalandi for it, and `place(refined)`, which turns refined text back
+// into source for that slice. Everything outside the units is copied as it is,
+// so a document whose passages all come back unchanged is written out byte for
+// byte as it came in (the tests hold every format to that).
+//
+//   segmentMarkdown(src)  paragraphs, list items, table cells, heading text and
+//                         a few front-matter values (title, description …)
+//   segmentHtml(src)      text in elements; inline tags travel as numbered
+//                         markers (<a1>…</a1>, <code2/>) and come back exactly
+//   segmentJson(src)      string values of an i18n catalog, never keys
+//   segmentText(src)      paragraphs or lines of plain text
+//
+// place() refuses ({ ok: false, reason }) when the refined text lost or gained
+// a protected token: code, a URL or link target, a placeholder such as {name}
+// or %s, an HTML tag, or the structure of an ICU plural/select. The caller then
+// keeps the original passage, so a passage is never written broken.
+//
+// Zero dependencies, like the rest of the CLI.
+
+/** The hosted API refuses a text longer than this (RITVIT_API_MAX_CHARS). */
+export const API_MAX_CHARS = 20_000;
+
+// ── protected tokens ────────────────────────────────────────────────────────
+//
+// Things inside a passage that must come back exactly as they went: the check
+// compares them as a multiset, so a translation may move a placeholder but not
+// drop, duplicate or "translate" it. Exactly one capturing group (the backtick
+// run of inline code), so the backreference stays \1.
+
+const BRACE = String.raw`\{[^{}\n]*\}`; // {name} {0} {count, number}
+// Inline code as CommonMark reads it: closed by the next backtick run of the
+// same length, so ``a `b` c`` is one span.
+const CODE_SPAN = String.raw`(?<!\`)(\`+)(?!\`)[\s\S]+?(?<!\`)\1(?!\`)`;
+const TOKEN_PARTS = [
+  String.raw`<!--[\s\S]*?-->`, // HTML comment
+  CODE_SPAN,
+  String.raw`\]\((?:[^()\s]|\([^()\s]*\))*(?:\s+(?:"[^"]*"|'[^']*'))?\)`, // Markdown link target
+  String.raw`\]\[[^\]\n]*\]`, // reference-link label
+  String.raw`\[\^[^\]\s]+\]`, // footnote reference
+  String.raw`<(?:https?|ftp|mailto):[^\s<>]+>`, // autolink
+  String.raw`<\/?[A-Za-z][\w.:-]*(?:\s[^<>]*?)?\/?>`, // HTML/XML tag, and our markers
+  String.raw`<\/?\d+\s*\/?>`, // react-i18next <0>…</0>
+  String.raw`\{\{[\s\S]*?\}\}`, // {{mustache}} and {{< shortcodes >}}
+  String.raw`\{%[\s\S]*?%\}`, // {% liquid %}
+  String.raw`<%[\s\S]*?%>`, // <% erb %>
+  String.raw`\$\{[^{}]*\}`, // ${template}
+  String.raw`\$t\([^)]*\)`, // i18next nesting
+  String.raw`@(?:\.[a-z]+)?:(?:\{[^}]*\}|[\w.-]*\w)`, // vue-i18n linked message
+  BRACE,
+  String.raw`%\([\w.-]+\)[-+0# ]*\d*(?:\.\d+)?[diouxXeEfFgGcrsa]`, // %(name)s
+  String.raw`%\{[\w.-]+\}`, // %{name}
+  String.raw`%%`,
+  String.raw`%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?(?:hh|h|ll|l|L|z|j|t)?[diouxXeEfFgGcsp@]`, // printf
+  String.raw`\b(?:https?|ftp):\/\/[^\s<>"'\`{}()\[\]]*[^\s<>"'\`{}()\[\].,;:!?]`, // URL
+  String.raw`\bwww\.[^\s<>"'\`{}()\[\]]*[^\s<>"'\`{}()\[\].,;:!?]`,
+  String.raw`[\w.+-]+@[\w-]+(?:\.[\w-]+)+`, // email
+  String.raw`&(?:[A-Za-z][A-Za-z0-9]{1,31}|#\d{1,7}|#[xX][0-9A-Fa-f]{1,6});`, // entity
+];
+const TOKEN_SOURCE = TOKEN_PARTS.join("|");
+// ICU messages nest braces ({count, plural, one {# item} …}); their structure is
+// compared by icuSkeleton() instead of as {…} tokens.
+const ICU_TOKEN_SOURCE = TOKEN_PARTS.filter((p) => p !== BRACE).join("|");
+const ICU_COMPLEX = /\{\s*[\w.-]+\s*,\s*(?:plural|selectordinal|select)\s*,/;
+
+const tokenRegex = (icu) => new RegExp(icu ? ICU_TOKEN_SOURCE : TOKEN_SOURCE, "g");
+
+export const isIcu = (text) => ICU_COMPLEX.test(text);
+
+/** The protected tokens in a passage, in order. */
+export function protectedTokens(text) {
+  return String(text).match(tokenRegex(isIcu(text))) || [];
+}
+
+/** Why `after` is not a safe replacement for `before`, or null when it is. */
+export function tokenProblem(before, after) {
+  const count = new Map();
+  for (const t of protectedTokens(before)) count.set(t, (count.get(t) || 0) + 1);
+  for (const t of protectedTokens(after)) {
+    const n = count.get(t) || 0;
+    if (!n) return `added or changed ${short(t)}`;
+    count.set(t, n - 1);
+  }
+  for (const [t, n] of count) if (n > 0) return `lost ${short(t)}`;
+  if (isIcu(before) && icuSkeleton(before) !== icuSkeleton(after)) {
+    return "changed the ICU plural/select structure";
+  }
+  return null;
+}
+
+const short = (t) => (t.length > 60 ? `${t.slice(0, 57)}…` : t);
+
+const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/** The engine's proofreading layer spaces out inline code (`x y` → ` x y `),
+ *  which changes nothing a reader needs: put each code span of the passage as
+ *  sent back exactly. Only spans that were in the passage are touched, and
+ *  only whitespace differences are repaired; tokenProblem() still checks the rest. */
+export function repairCodeSpans(before, after) {
+  let out = String(after);
+  for (const span of new Set(String(before).match(new RegExp(CODE_SPAN, "g")) || [])) {
+    const ticks = span.match(/^`+/)[0];
+    const inner = span.slice(ticks.length, span.length - ticks.length).trim();
+    if (!inner) continue;
+    const loose = new RegExp(`${ticks}\\s*${escapeRe(inner).replace(/\s+/g, "\\s+")}\\s*${ticks}`, "g");
+    out = out.replace(loose, span);
+  }
+  return out;
+}
+
+// A lone token that reads as code rather than as a word: a file name, a path, a
+// flag, camelCase, snake_case, a version. ALL CAPS words ("OK", "FAQ") are kept:
+// UI strings need them translated.
+function isCodeLike(word) {
+  return /^[-–]{1,2}[A-Za-z]/.test(word)
+    || /[A-Za-z0-9][._/\\:@][A-Za-z0-9]/.test(word)
+    || /^[a-z]+[A-Z]/.test(word)
+    || /^[A-Z][a-z]+[A-Z]/.test(word)
+    || /_/.test(word)
+    || (/\d/.test(word) && /[A-Za-z]/.test(word));
+}
+
+/** Does this passage hold anything to refine? Not when it is empty, only code,
+ *  numbers, URLs or placeholders, or a single code-like token. */
+export function needsWork(text) {
+  const t = String(text ?? "");
+  if (isIcu(t) && icuSkeleton(t) === null) return false; // can't verify it, so don't send it
+  const rest = t.replace(tokenRegex(isIcu(t)), " ").replace(/\\[!-/:-@[-`{-~]/g, " ");
+  if (!/\p{L}/u.test(rest)) return false;
+  const words = rest.trim().split(/\s+/);
+  if (words.length === 1) {
+    const word = words[0].replace(/^[("'“„«*_]+/, "").replace(/[.,;:!?…)"'”’»*_]+$/, "");
+    if (isCodeLike(word)) return false;
+  }
+  return true;
+}
+
+// ── ICU MessageFormat ───────────────────────────────────────────────────────
+
+/** The shape of an ICU message without its words: argument names and types,
+ *  plural/select selectors and `#`, sorted per level so a translation may
+ *  reorder them. null when the message does not parse. */
+export function icuSkeleton(text) {
+  const s = String(text);
+  let i = 0;
+  const fail = () => { throw new Error("icu"); };
+  const read = (re) => {
+    const sticky = new RegExp(re.source, "y");
+    sticky.lastIndex = i;
+    const m = sticky.exec(s);
+    if (!m) return null;
+    i += m[0].length;
+    return m[1];
+  };
+  const message = (inPlural, depth) => {
+    const items = [];
+    while (i < s.length) {
+      const ch = s[i];
+      if (ch === "'") {
+        if (s[i + 1] === "'") { i += 2; continue; }
+        if (/[{}#|]/.test(s[i + 1] || "")) {
+          const close = s.indexOf("'", i + 1);
+          i = close === -1 ? s.length : close + 1;
+          continue;
+        }
+        i++;
+        continue;
+      }
+      if (ch === "{") { i++; items.push(argument(inPlural, depth)); continue; }
+      if (ch === "}") { if (depth === 0) fail(); return items; }
+      if (ch === "#" && inPlural) items.push("#");
+      i++;
+    }
+    if (depth > 0) fail();
+    return items;
+  };
+  const argument = (inPlural, depth) => {
+    const name = read(/\s*([^\s,{}]+)\s*/) ?? fail();
+    if (s[i] === "}") { i++; return `{${name}}`; }
+    if (s[i] !== ",") fail();
+    i++;
+    const type = read(/\s*([A-Za-z]+)\s*/) ?? fail();
+    if (/^(?:plural|selectordinal|select)$/.test(type)) {
+      if (s[i] !== ",") fail();
+      i++;
+      const offset = read(/\s*(offset:\s*\d+)\s*/) || "";
+      const branches = [];
+      for (;;) {
+        const selector = read(/\s*(=?[\w-]+)\s*/);
+        if (selector === null) break;
+        if (s[i] !== "{") fail();
+        i++;
+        const sub = message(inPlural || type !== "select", depth + 1);
+        if (s[i] !== "}") fail();
+        i++;
+        branches.push(`${selector}[${sub.sort().join(",")}]`);
+      }
+      read(/(\s*)/);
+      if (s[i] !== "}") fail();
+      i++;
+      return `{${name},${type},${offset.replace(/\s+/g, "")}${branches.sort().join(";")}}`;
+    }
+    let style = "";
+    if (s[i] === ",") {
+      i++;
+      const start = i;
+      let d = 0;
+      while (i < s.length && !(s[i] === "}" && d === 0)) {
+        if (s[i] === "{") d++;
+        else if (s[i] === "}") d--;
+        i++;
+      }
+      style = s.slice(start, i).trim();
+    }
+    if (s[i] !== "}") fail();
+    i++;
+    return `{${name},${type}${style ? `,${style}` : ""}}`;
+  };
+  try {
+    return message(false, 0).sort().join(",");
+  } catch {
+    return null;
+  }
+}
+
+// ── shared helpers ──────────────────────────────────────────────────────────
+
+/** Lines with offsets: [start, end) excludes the line ending; `next` is where
+ *  the next line starts. */
+function toLines(src) {
+  const lines = [];
+  let start = 0;
+  while (start < src.length) {
+    const nl = src.indexOf("\n", start);
+    const next = nl === -1 ? src.length : nl + 1;
+    let end = nl === -1 ? src.length : nl;
+    if (end > start && src[end - 1] === "\r") end--;
+    lines.push({ start, end, next });
+    start = next;
+  }
+  return lines;
+}
+
+const eolOf = (src) => (src.includes("\r\n") ? "\r\n" : "\n");
+const oneLine = (s) => String(s).replace(/\s*[\r\n]+\s*/g, " ").trim();
+const widthOf = (ws) => [...ws].reduce((n, ch) => n + (ch === "\t" ? 4 - (n % 4) : 1), 0);
+
+// A refined paragraph must not turn into a different block (a heading, a list
+// item, a quote) just because its first word changed. Escape that first
+// character, unless the original began the same way.
+const BLOCK_START = /^(?:#{1,6}(?:\s|$)|>|[-+*](?:\s|$)|\d{1,9}[.)](?:\s|$)|={3,}|`{3,}|~{3,}|\|)/;
+function escapeBlockStart(text, original) {
+  if (!BLOCK_START.test(text) || BLOCK_START.test(original)) return text;
+  const num = /^(\d{1,9})([.)])/.exec(text);
+  return num ? `${num[1]}\\${text.slice(num[1].length)}` : `\\${text}`;
+}
+
+/** place() for a passage that is written back as one line of Markdown/text. */
+function inlinePlacer(sent, { escapeStart = false, escapePipes = false } = {}) {
+  return (refined) => {
+    let value = oneLine(refined);
+    const problem = tokenProblem(sent, value);
+    if (problem) return { ok: false, reason: problem };
+    if (escapeStart) value = escapeBlockStart(value, sent);
+    if (escapePipes) value = value.replace(/(?<!\\)\|/g, "\\|");
+    return { ok: true, value };
+  };
+}
+
+const shiftUnit = (unit, by) => ({ ...unit, start: unit.start + by, end: unit.end + by });
+
+// ── Markdown ────────────────────────────────────────────────────────────────
+
+// Front-matter keys whose values are text a reader sees. Everything else in
+// front matter (slugs, dates, layouts, tags) is left alone.
+const FRONT_MATTER_KEYS = new Set([
+  "title", "subtitle", "description", "summary", "excerpt", "abstract", "lead", "caption", "alt",
+  "linktitle", "sidebar_label", "seo_title", "seo_description", "meta_title", "meta_description",
+  "og_title", "og_description",
+]);
+
+// HTML block start tags (CommonMark type 6). A line that opens with one of them
+// starts an HTML block whatever follows; inline tags only do when alone.
+const HTML_BLOCK_TAGS = new Set([
+  "address", "article", "aside", "base", "blockquote", "body", "caption", "center", "col", "colgroup",
+  "dd", "details", "dialog", "dir", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer",
+  "form", "frame", "frameset", "h1", "h2", "h3", "h4", "h5", "h6", "head", "header", "hr", "html",
+  "iframe", "legend", "li", "link", "main", "menu", "nav", "ol", "optgroup", "option", "p", "param",
+  "picture", "section", "source", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "title",
+  "tr", "track", "ul",
+]);
+
+const FENCE = /^\s*(`{3,}|~{3,})/;
+const DELIMITER_ROW = /^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?\s*$/;
+const LIST_ITEM = /^([ \t]*)([-+*]|\d{1,9}[.)])([ \t]+|$)(\[[ xX]\][ \t]+)?/;
+const TEMPLATE_LINE = /^\s*(?:\{%.*%\}|\{\{.*\}\}|<%.*%>)\s*$/;
+
+function htmlBlockStart(c) {
+  if (/^ {0,3}<!--/.test(c)) return "comment";
+  const raw = /^ {0,3}<(script|pre|style|textarea)\b/i.exec(c);
+  if (raw) return raw[1].toLowerCase();
+  const tag = /^ {0,3}<\/?([A-Za-z][\w-]*)(?=[\s/>]|$)/.exec(c);
+  if (!tag) return null;
+  if (HTML_BLOCK_TAGS.has(tag[1].toLowerCase())) return "block";
+  return /^ {0,3}<\/?[A-Za-z][\w-]*(?:\s[^<>]*)?\/?>\s*$/.test(c) ? "block" : null;
+}
+
+/** Units of a Markdown document. */
+export function segmentMarkdown(src) {
+  const lines = toLines(src);
+  const units = [];
+  const n = lines.length;
+  const text = (k) => src.slice(lines[k].start, lines[k].end);
+  const info = (k) => {
+    const t = text(k);
+    const bq = /^(?: {0,3}>[ \t]?)*/.exec(t)[0];
+    return { bq, depth: (bq.match(/>/g) || []).length, c: t.slice(bq.length), cStart: lines[k].start + bq.length };
+  };
+  const blank = (c) => /^\s*$/.test(c);
+  const push = (start, end, sent, place) => {
+    if (end > start && needsWork(sent)) units.push({ start, end, text: sent, place });
+  };
+
+  let i = 0;
+  // Front matter: only the reader-facing values listed above, one line each.
+  const fm = n ? /^\uFEFF?(---|\+\+\+)[ \t]*$/.exec(text(0)) : null;
+  if (fm) {
+    const marker = fm[1];
+    let j = 1;
+    while (j < n && !(text(j).trim() === marker || (marker === "---" && text(j).trim() === "..."))) j++;
+    if (j < n) {
+      for (let k = 1; k < j; k++) frontMatterUnit(src, lines[k], marker === "+++", push);
+      i = j + 1;
+    }
+  }
+
+  let listIndent = -1; // content column of the open list item, or -1
+  // true when no paragraph text is open: after a blank line or a block that is
+  // not a paragraph. Only then can an indented line start code, and a line at
+  // the margin end the open list.
+  let fresh = true;
+
+  // A paragraph from line k, its text starting at `from`. Soft-wrapped lines
+  // become one passage; a hard line break (two trailing spaces or a backslash)
+  // ends one, so addresses and verse keep their lines.
+  const paragraph = (k, from, depth) => {
+    const segs = [];
+    let j = k;
+    for (;;) {
+      const t = text(j);
+      const start = j === k ? from : lines[j].start + info(j).bq.length + /^\s*/.exec(info(j).c)[0].length;
+      const trailing = /[ \t]*$/.exec(t)[0];
+      // An odd run of backslashes at the very end is a hard break; the
+      // backslash stays outside the passage, like the trailing spaces.
+      const backslash = !trailing && /\\*$/.exec(t)[0].length % 2 === 1;
+      const hard = backslash || / {2,}$/.test(t);
+      segs.push({ start, end: lines[j].end - trailing.length - (backslash ? 1 : 0), hard });
+      j++;
+      if (j >= n) break;
+      const next = info(j);
+      if (next.depth !== depth || blank(next.c) || interrupts(next.c)) break;
+    }
+    if (j < n && info(j).depth === depth && /^ {0,3}(?:=+|-+)[ \t]*$/.test(info(j).c)) j++; // setext underline, kept
+    let group = [];
+    const emit = () => {
+      if (!group.length) return;
+      const sent = group.map((g) => src.slice(g.start, g.end).trim()).join(" ");
+      push(group[0].start, group[group.length - 1].end, sent, inlinePlacer(sent, { escapeStart: true }));
+      group = [];
+    };
+    for (const seg of segs) {
+      group.push(seg);
+      if (seg.hard) emit();
+    }
+    emit();
+    return j;
+  };
+
+  const interrupts = (c) => FENCE.test(c)
+    || /^ {0,3}#{1,6}(?:[ \t]|$)/.test(c)
+    || /^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(c)
+    || /^\s*[-+*][ \t]+\S/.test(c)
+    || /^\s*1[.)][ \t]+\S/.test(c)
+    || htmlBlockStart(c) !== null
+    || TEMPLATE_LINE.test(c)
+    || /^\s*\$\$/.test(c);
+
+  while (i < n) {
+    const { depth, c, cStart } = info(i);
+    if (blank(c)) { fresh = true; i++; continue; }
+    const indent = widthOf(/^[ \t]*/.exec(c)[0]);
+    if (fresh && listIndent >= 0 && indent < Math.min(listIndent, 2) && !LIST_ITEM.test(c)) listIndent = -1;
+    let m;
+
+    // Fenced code, at any indentation: when in doubt, it is code and stays.
+    if ((m = FENCE.exec(c)) && !(m[1][0] === "`" && c.slice(m[0].length).includes("`"))) {
+      const ch = m[1][0];
+      const len = m[1].length;
+      i++;
+      while (i < n) {
+        const close = /^\s*(`{3,}|~{3,})\s*$/.exec(info(i).c);
+        i++;
+        if (close && close[1][0] === ch && close[1].length >= len) break;
+      }
+      fresh = true;
+      continue;
+    }
+    // $$ math $$
+    if (/^\s*\$\$/.test(c)) {
+      const oneLiner = /^\s*\$\$.*\$\$\s*$/.test(c) && c.trim().length > 4;
+      i++;
+      if (!oneLiner) {
+        while (i < n && !info(i).c.includes("$$")) i++;
+        i++;
+      }
+      fresh = true;
+      continue;
+    }
+    // Indented code: four columns past the open list item's content (or the
+    // margin), where no paragraph is open.
+    if (fresh && indent >= (listIndent >= 0 ? listIndent + 4 : 4)) {
+      const floor = listIndent >= 0 ? listIndent + 4 : 4;
+      i++;
+      while (i < n) {
+        const ci = info(i).c;
+        if (blank(ci)) {
+          let k = i;
+          while (k < n && blank(info(k).c)) k++;
+          if (k < n && widthOf(/^[ \t]*/.exec(info(k).c)[0]) >= floor) { i = k; continue; }
+          break;
+        }
+        if (widthOf(/^[ \t]*/.exec(ci)[0]) < floor) break;
+        i++;
+      }
+      fresh = true;
+      continue;
+    }
+    // HTML: comments and raw blocks stay; other blocks are segmented as HTML.
+    const html = htmlBlockStart(c);
+    if (html) {
+      const from = i;
+      if (html !== "block") {
+        // A comment or <script>/<pre>/<style>/<textarea>: kept whole, blank
+        // lines included, through the line that closes it.
+        const opener = html === "comment" ? "<!--" : `<${html}`;
+        const close = html === "comment" ? "-->" : `</${html}`;
+        while (i < n) {
+          const lower = text(i).toLowerCase();
+          const found = lower.includes(close, i === from ? lower.indexOf(opener) + opener.length : 0);
+          i++;
+          if (found) break;
+        }
+      } else {
+        while (i < n && !blank(info(i).c)) i++;
+        if (depth === 0) {
+          const blockStart = lines[from].start;
+          const blockEnd = lines[i - 1].end;
+          for (const u of segmentHtml(src.slice(blockStart, blockEnd)).units) units.push(shiftUnit(u, blockStart));
+        }
+      }
+      fresh = true;
+      continue;
+    }
+    if (TEMPLATE_LINE.test(c)) { i++; fresh = true; continue; }
+    // Thematic break
+    if (/^ {0,3}([-*_])(?:[ \t]*\1){2,}[ \t]*$/.test(c)) { i++; fresh = true; listIndent = -1; continue; }
+    // ATX heading: the text is refined, the #s and any {#id} stay.
+    if ((m = /^( {0,3}#{1,6})(?:[ \t]+|$)/.exec(c))) {
+      const body = c.slice(m[0].length);
+      const tail = /(?:[ \t]*\{[#.][^}]*\})?(?:[ \t]+#+)?[ \t]*$/.exec(body)[0];
+      const core = body.slice(0, body.length - tail.length);
+      const start = cStart + m[0].length;
+      push(start, start + core.length, core, inlinePlacer(core));
+      i++;
+      fresh = true;
+      listIndent = -1;
+      continue;
+    }
+    // Link reference definition
+    if (/^ {0,3}\[[^\]^][^\]]*\]:\s*\S/.test(c)) { i++; continue; }
+    // Table: each cell is a passage; pipes and the delimiter row stay.
+    if (c.includes("|") && i + 1 < n && info(i + 1).depth === depth && info(i + 1).c.includes("|") && DELIMITER_ROW.test(info(i + 1).c)) {
+      tableRow(src, info(i), push);
+      i += 2;
+      while (i < n && info(i).depth === depth && !blank(info(i).c) && info(i).c.includes("|")) {
+        tableRow(src, info(i), push);
+        i++;
+      }
+      fresh = false;
+      continue;
+    }
+    // List item: the marker (and a task box) stay, the item text is a passage.
+    if ((m = LIST_ITEM.exec(c))) {
+      listIndent = widthOf(m[1]) + m[2].length + (m[3] ? widthOf(m[3]) : 1);
+      if (blank(c.slice(m[0].length))) { i++; fresh = false; continue; }
+      i = paragraph(i, cStart + m[0].length, depth);
+      fresh = false;
+      continue;
+    }
+    // Footnote definition
+    if ((m = /^ {0,3}\[\^[^\]]+\]:[ \t]*/.exec(c))) {
+      i = paragraph(i, cStart + m[0].length, depth);
+      fresh = false;
+      continue;
+    }
+    i = paragraph(i, cStart + /^\s*/.exec(c)[0].length, depth);
+    fresh = false;
+  }
+  return { units };
+}
+
+function frontMatterUnit(src, line, toml, push) {
+  const t = src.slice(line.start, line.end);
+  const m = toml
+    ? /^([A-Za-z_][\w-]*)([ \t]*=[ \t]*)("(?:[^"\\]|\\.)*"|'[^']*')[ \t]*$/.exec(t)
+    : /^([A-Za-z_][\w-]*)(:[ \t]+)(.+?)[ \t]*$/.exec(t);
+  if (!m || !FRONT_MATTER_KEYS.has(m[1].toLowerCase())) return;
+  const at = line.start + m[1].length + m[2].length;
+  const raw = m[3];
+  if (raw.length >= 2 && raw[0] === '"' && raw.endsWith('"')) {
+    let value;
+    try { value = JSON.parse(raw); } catch { return; }
+    if (typeof value !== "string") return;
+    push(at + 1, at + raw.length - 1, value, (refined) => {
+      const r = oneLine(refined);
+      const problem = tokenProblem(value, r);
+      return problem ? { ok: false, reason: problem } : { ok: true, value: JSON.stringify(r).slice(1, -1) };
+    });
+  } else if (raw.length >= 2 && raw[0] === "'" && raw.endsWith("'")) {
+    const value = raw.slice(1, -1).replace(/''/g, "'");
+    push(at + 1, at + raw.length - 1, value, (refined) => {
+      const r = oneLine(refined);
+      const problem = tokenProblem(value, r);
+      return problem ? { ok: false, reason: problem } : { ok: true, value: r.replace(/'/g, "''") };
+    });
+  } else if (!toml && !/^[[{&*!|>%@`#"']/.test(raw) && !/\s#/.test(raw)) {
+    push(at, at + raw.length, raw, (refined) => {
+      const r = oneLine(refined);
+      const problem = tokenProblem(raw, r);
+      if (problem) return { ok: false, reason: problem };
+      // A plain YAML scalar must not grow a ": ", a " #", a leading marker or a
+      // boolean look; quote it when it would.
+      const plain = /^[\p{L}\p{N}(]/u.test(r) && !/:\s|:$|\s#|["'\\]/.test(r)
+        && !/^(?:true|false|yes|no|on|off|null|y|n|~)$/i.test(r);
+      return { ok: true, value: plain ? r : JSON.stringify(r) };
+    });
+  }
+}
+
+function tableRow(src, { c, cStart }, push) {
+  // GFM splits a row on every pipe that is not escaped, inside code spans too.
+  const cuts = [];
+  for (let k = 0; k < c.length; k++) {
+    if (c[k] === "\\") k++;
+    else if (c[k] === "|") cuts.push(k);
+  }
+  const bounds = [-1, ...cuts, c.length];
+  for (let b = 0; b < bounds.length - 1; b++) {
+    const from = bounds[b] + 1;
+    const to = bounds[b + 1];
+    const cell = c.slice(from, to);
+    const lead = /^\s*/.exec(cell)[0].length;
+    const core = cell.trim();
+    if (!core) continue;
+    const start = cStart + from + lead;
+    push(start, start + core.length, core, inlinePlacer(core, { escapePipes: true }));
+  }
+}
+
+// ── HTML ────────────────────────────────────────────────────────────────────
+
+const RAWTEXT = new Set(["script", "style", "textarea", "title", "xmp", "noembed", "noframes"]);
+export const VOID = new Set([
+  "area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr",
+]);
+// Phrasing elements: their tags travel inside a passage as markers.
+const INLINE = new Set([
+  "a", "abbr", "b", "bdi", "bdo", "big", "cite", "data", "del", "dfn", "em", "font", "i", "ins", "mark",
+  "q", "s", "small", "span", "strike", "strong", "sub", "sup", "time", "tt", "u", "wbr", "br", "img",
+  "nobr", "ruby", "rt", "rp",
+]);
+// Content that is never sent. Inline ones travel as a single marker; the rest
+// end a passage and are skipped whole.
+const ATOMIC_INLINE = new Set(["code", "kbd", "samp", "var", "svg", "math"]);
+const SKIP_BLOCK = new Set([
+  "pre", "script", "style", "textarea", "template", "noscript", "iframe", "object", "canvas", "video", "audio",
+  "listing", "plaintext", "xmp",
+]);
+
+const NAMED_ENTITIES = {
+  amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: "\u00a0", shy: "\u00ad", ensp: "\u2002",
+  emsp: "\u2003", thinsp: "\u2009", zwnj: "\u200c", zwj: "\u200d", lrm: "\u200e", rlm: "\u200f",
+  ndash: "–", mdash: "—", hellip: "…", lsquo: "‘", rsquo: "’", sbquo: "‚", ldquo: "“", rdquo: "”",
+  bdquo: "„", laquo: "«", raquo: "»", lsaquo: "‹", rsaquo: "›", bull: "•", middot: "·", prime: "′",
+  copy: "©", reg: "®", trade: "™", deg: "°", euro: "€", pound: "£", yen: "¥", cent: "¢", sect: "§",
+  para: "¶", times: "×", divide: "÷", plusmn: "±", minus: "−", frac12: "½", frac14: "¼", frac34: "¾",
+  sup1: "¹", sup2: "²", sup3: "³", iexcl: "¡", iquest: "¿", dagger: "†", Dagger: "‡", larr: "←",
+  rarr: "→", uarr: "↑", darr: "↓", harr: "↔", check: "✓", hearts: "♥", star: "☆",
+  aacute: "á", Aacute: "Á", eacute: "é", Eacute: "É", iacute: "í", Iacute: "Í", oacute: "ó", Oacute: "Ó",
+  uacute: "ú", Uacute: "Ú", yacute: "ý", Yacute: "Ý", eth: "ð", ETH: "Ð", thorn: "þ", THORN: "Þ",
+  aelig: "æ", AElig: "Æ", ouml: "ö", Ouml: "Ö", auml: "ä", Auml: "Ä", uuml: "ü", Uuml: "Ü", oslash: "ø",
+  Oslash: "Ø", aring: "å", Aring: "Å", ccedil: "ç", Ccedil: "Ç", ntilde: "ñ", Ntilde: "Ñ", szlig: "ß",
+  agrave: "à", Agrave: "À", egrave: "è", Egrave: "È", ecirc: "ê", Ecirc: "Ê", euml: "ë", ocirc: "ô",
+  acirc: "â", icirc: "î", iuml: "ï", ugrave: "ù", ucirc: "û",
+};
+
+/** Decode character references. Unknown names stay as written. */
+export function decodeEntities(s) {
+  return String(s).replace(/&(#\d{1,7}|#[xX][0-9a-fA-F]{1,6}|[A-Za-z][A-Za-z0-9]{1,31});/g, (m, e) => {
+    if (e[0] === "#") {
+      const cp = e[1] === "x" || e[1] === "X" ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10);
+      return cp > 0 && cp <= 0x10ffff && !(cp >= 0xd800 && cp <= 0xdfff) ? String.fromCodePoint(cp) : m;
+    }
+    return Object.prototype.hasOwnProperty.call(NAMED_ENTITIES, e) ? NAMED_ENTITIES[e] : m;
+  });
+}
+
+function encodeHtmlText(s, asciiOnly) {
+  let out = s
+    .replace(/&(?![A-Za-z][A-Za-z0-9]{1,31};|#\d{1,7};|#[xX][0-9a-fA-F]{1,6};)/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/\u00a0/g, "&nbsp;");
+  if (asciiOnly) out = out.replace(/[^\x00-\x7f]/gu, (ch) => `&#${ch.codePointAt(0)};`);
+  return out;
+}
+
+/** Tokens of an HTML document with offsets: text, tag, comment, rawtext (the
+ *  content of script/style/textarea/title), doctype. Forgiving: anything that
+ *  is not a well-formed tag is text. */
+export function tokenizeHtml(src) {
+  const tokens = [];
+  const lower = src.toLowerCase();
+  const tagStart = /<(\/?)([A-Za-z][A-Za-z0-9:-]*)/y;
+  let i = 0;
+  const text = (start, end) => { if (end > start) tokens.push({ type: "text", start, end }); };
+  while (i < src.length) {
+    const lt = src.indexOf("<", i);
+    if (lt === -1) { text(i, src.length); break; }
+    text(i, lt);
+    if (src.startsWith("<!--", lt)) {
+      const close = src.indexOf("-->", lt + 4);
+      const end = close === -1 ? src.length : close + 3;
+      tokens.push({ type: "comment", start: lt, end });
+      i = end;
+      continue;
+    }
+    if (src[lt + 1] === "!" || src[lt + 1] === "?") {
+      const close = src.indexOf(">", lt);
+      const end = close === -1 ? src.length : close + 1;
+      tokens.push({ type: "doctype", start: lt, end });
+      i = end;
+      continue;
+    }
+    tagStart.lastIndex = lt;
+    const m = tagStart.exec(src);
+    if (!m) { text(lt, lt + 1); i = lt + 1; continue; }
+    let j = lt + m[0].length;
+    let quote = null;
+    while (j < src.length) {
+      const ch = src[j];
+      if (quote) { if (ch === quote) quote = null; } else if (ch === '"' || ch === "'") quote = ch;
+      else if (ch === ">") break;
+      j++;
+    }
+    const end = Math.min(j + 1, src.length);
+    const name = m[2].toLowerCase();
+    const closing = m[1] === "/";
+    const raw = src.slice(lt, end);
+    tokens.push({ type: "tag", start: lt, end, name, closing, selfClosing: /\/\s*>$/.test(raw), raw });
+    i = end;
+    if (!closing && RAWTEXT.has(name)) {
+      const close = lower.indexOf(`</${name}`, i);
+      const stop = close === -1 ? src.length : close;
+      tokens.push({ type: "rawtext", start: i, end: stop, parent: name });
+      i = stop;
+    }
+  }
+  return tokens;
+}
+
+/** The attributes of a tag token, names lower-cased, values decoded. */
+export function tagAttributes(raw) {
+  const attrs = {};
+  const body = raw.replace(/^<\/?[A-Za-z][A-Za-z0-9:-]*/, "").replace(/\/?>$/, "");
+  const re = /([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+  let m;
+  while ((m = re.exec(body))) attrs[m[1].toLowerCase()] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? "");
+  return attrs;
+}
+
+const noTranslate = (tag) => {
+  const a = tagAttributes(tag.raw);
+  return a.translate === "no" || /(?:^|\s)notranslate(?:\s|$)/.test(a.class || "");
+};
+
+/** Index of the tag that closes tokens[i] (same name, nesting counted), or -1. */
+function closingIndex(tokens, i) {
+  const name = tokens[i].name;
+  let depth = 0;
+  for (let k = i + 1; k < tokens.length; k++) {
+    const t = tokens[k];
+    if (t.type !== "tag" || t.name !== name || VOID.has(name) || t.selfClosing) continue;
+    if (!t.closing) depth++;
+    else if (depth === 0) return k;
+    else depth--;
+  }
+  return -1;
+}
+
+/** Units of an HTML document: text in elements, never inside script, style,
+ *  pre, code or anything marked translate="no". */
+export function segmentHtml(src) {
+  const tokens = tokenizeHtml(src);
+  const asciiOnly = /<meta[^>]+charset\s*=\s*["']?\s*(?!utf-?8)[\w-]+/i.test(src);
+  const units = [];
+  let run = [];
+  const flush = () => {
+    const unit = htmlUnit(src, run, asciiOnly);
+    if (unit) units.push(unit);
+    run = [];
+  };
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+    if (t.type === "text") { run.push(t); continue; }
+    if (t.type === "comment") { run.push({ ...t, atomic: true }); continue; }
+    if (t.type === "rawtext") {
+      flush();
+      if (t.parent === "title") {
+        // The page title is text a reader sees (a browser tab, a search result).
+        const unit = htmlUnit(src, [{ type: "text", start: t.start, end: t.end }], asciiOnly);
+        if (unit) units.push(unit);
+      }
+      continue;
+    }
+    if (t.type !== "tag") { flush(); continue; }
+    if (t.closing) {
+      if (INLINE.has(t.name) || ATOMIC_INLINE.has(t.name)) run.push(t);
+      else flush();
+      continue;
+    }
+    const skip = SKIP_BLOCK.has(t.name) || ATOMIC_INLINE.has(t.name) || noTranslate(t);
+    if (skip && !VOID.has(t.name) && !t.selfClosing) {
+      const close = closingIndex(tokens, i);
+      const last = close === -1 ? tokens.length - 1 : close;
+      if (INLINE.has(t.name) || ATOMIC_INLINE.has(t.name)) {
+        run.push({ type: "atomic", name: t.name, start: t.start, end: tokens[last].end });
+      } else {
+        flush();
+      }
+      i = last;
+      continue;
+    }
+    if (INLINE.has(t.name)) run.push(t);
+    else flush();
+  }
+  flush();
+  return { units };
+}
+
+function htmlUnit(src, run, asciiOnly) {
+  // The passage runs from the first to the last visible thing in the run;
+  // whitespace around it stays where it is.
+  const significant = (t) => t.type !== "text" || /\S/.test(src.slice(t.start, t.end));
+  const first = run.findIndex(significant);
+  if (first === -1) return null;
+  let last = run.length - 1;
+  while (!significant(run[last])) last--;
+  const slice = run.slice(first, last + 1);
+  const firstT = slice[0];
+  const lastT = slice[slice.length - 1];
+  const start = firstT.type === "text" ? firstT.start + /^\s*/.exec(src.slice(firstT.start, firstT.end))[0].length : firstT.start;
+  const end = lastT.type === "text" ? lastT.end - /\s*$/.exec(src.slice(lastT.start, lastT.end))[0].length : lastT.end;
+
+  // Pair opening and closing inline tags inside the run; an unpaired one (the
+  // run starts or ends inside an element) travels as a single marker.
+  const pairOf = new Map();
+  const open = [];
+  slice.forEach((t, k) => {
+    if (t.type !== "tag" || VOID.has(t.name) || t.selfClosing) return;
+    if (!t.closing) open.push(k);
+    else {
+      const at = open.map((o) => slice[o].name).lastIndexOf(t.name);
+      if (at !== -1) {
+        const [o] = open.splice(at, 1);
+        open.length = Math.min(open.length, at);
+        pairOf.set(o, k);
+        pairOf.set(k, o);
+      }
+    }
+  });
+
+  const markers = new Map(); // marker → original source
+  const ids = new Map(); // token index → number
+  let next = 1;
+  let sent = "";
+  slice.forEach((t, k) => {
+    const s = Math.max(t.start, start);
+    const e = Math.min(t.end, end);
+    if (t.type === "text") {
+      sent += decodeEntities(src.slice(s, e)).replace(/\s+/g, " ");
+      return;
+    }
+    const name = (t.name || "x").replace(/[^a-z]/g, "") || "x";
+    let marker;
+    if (t.type === "tag" && pairOf.has(k)) {
+      const opening = !t.closing;
+      const id = opening ? next++ : ids.get(pairOf.get(k));
+      ids.set(k, id);
+      marker = opening ? `<${name}${id}>` : `</${name}${id}>`;
+    } else {
+      marker = `<${t.type === "comment" ? "x" : name}${next++}/>`;
+    }
+    markers.set(marker, src.slice(t.start, t.end));
+    sent += marker;
+  });
+  sent = sent.trim();
+  if (!needsWork(sent.replace(/<\/?[a-z]+\d+\/?>/g, " "))) return null;
+
+  const place = (refined) => {
+    const value = oneLine(refined);
+    const problem = tokenProblem(sent, value);
+    if (problem) return { ok: false, reason: problem };
+    for (const [marker] of markers) {
+      if (marker.startsWith("</")) {
+        const opening = `<${marker.slice(2)}`;
+        if (value.indexOf(opening) > value.indexOf(marker)) return { ok: false, reason: `moved ${marker} before ${opening}` };
+      }
+    }
+    let out = "";
+    let pos = 0;
+    const re = /<\/?[a-z]+\d+\/?>/g;
+    let m;
+    while ((m = re.exec(value))) {
+      if (!markers.has(m[0])) continue;
+      out += encodeHtmlText(value.slice(pos, m.index), asciiOnly) + markers.get(m[0]);
+      pos = m.index + m[0].length;
+    }
+    out += encodeHtmlText(value.slice(pos), asciiOnly);
+    return { ok: true, value: out };
+  };
+  return { start, end, text: sent, place };
+}
+
+// ── JSON (i18n catalogs) ────────────────────────────────────────────────────
+
+// Configuration that happens to be JSON. Its strings are commands, names and
+// paths, never prose, so it is copied unchanged.
+const CONFIG_JSON = /^(?:package(?:-lock)?|composer|tsconfig(?:\..+)?|jsconfig(?:\..+)?|manifest|deno|lerna|nx|turbo|vercel|netlify|firebase|renovate|angular|app|babel\.config|eslintrc|prettierrc|bower|project|launch|settings|extensions|devcontainer|.+\.schema)\.json$/i;
+
+export const isConfigJson = (fileName) => CONFIG_JSON.test(fileName);
+
+const metaKey = (k) => typeof k === "string" && /^[@$]/.test(k);
+
+function notCatalog(data) {
+  if (data === null || typeof data !== "object") return "not an object of strings";
+  let strings = 0;
+  let other = 0;
+  const walk = (v) => {
+    if (typeof v === "string") strings++;
+    else if (typeof v === "number" || typeof v === "boolean") other++;
+    else if (Array.isArray(v)) v.forEach(walk);
+    else if (v && typeof v === "object") for (const [k, x] of Object.entries(v)) if (!metaKey(k)) walk(x);
+  };
+  walk(data);
+  if (other) return "has numbers or true/false values, so it looks like configuration or data";
+  if (!strings) return "no strings";
+  return null;
+}
+
+/** Units of a JSON i18n catalog: string values, never keys, never metadata
+ *  (keys starting with @ or $). Formatting, key order and escapes elsewhere in
+ *  the file are untouched. `note` says why a file is not a catalog. */
+export function segmentJson(src) {
+  const bom = src.startsWith("\uFEFF") ? 1 : 0;
+  let data;
+  try { data = JSON.parse(src.slice(bom)); } catch { return { units: [], note: "not valid JSON" }; }
+  const why = notCatalog(data);
+  if (why) return { units: [], note: why };
+  // A file written with \u escapes and no raw non-ASCII keeps that style.
+  const asciiOnly = !/[^\x00-\x7f]/.test(src.slice(bom)) && /\\u[0-9a-fA-F]{4}/.test(src);
+  const units = [];
+  const stack = [];
+  let i = bom;
+  while (i < src.length) {
+    const ch = src[i];
+    if (ch === '"') {
+      let j = i + 1;
+      while (j < src.length && src[j] !== '"') j += src[j] === "\\" ? 2 : 1;
+      const end = j + 1;
+      const value = JSON.parse(src.slice(i, end));
+      const top = stack[stack.length - 1];
+      if (top && top.type === "object" && top.expectKey) {
+        top.key = value;
+        top.expectKey = false;
+      } else if (!stack.some((s) => s.type === "object" && metaKey(s.key))) {
+        const lead = /^\s*/.exec(value)[0];
+        const trail = /\s*$/.exec(value.slice(lead.length))[0];
+        const sent = value.trim();
+        if (needsWork(sent)) {
+          units.push({
+            start: i,
+            end,
+            text: sent,
+            place: (refined) => {
+              const r = String(refined).trim();
+              const problem = tokenProblem(sent, r);
+              if (problem) return { ok: false, reason: problem };
+              let out = JSON.stringify(lead + r + trail);
+              if (asciiOnly) out = out.replace(/[\u0080-\uffff]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);
+              return { ok: true, value: out };
+            },
+          });
+        }
+      }
+      i = end;
+      continue;
+    }
+    if (ch === "{") stack.push({ type: "object", key: null, expectKey: true });
+    else if (ch === "[") stack.push({ type: "array" });
+    else if (ch === "}" || ch === "]") stack.pop();
+    else if (ch === ",") {
+      const top = stack[stack.length - 1];
+      if (top && top.type === "object") top.expectKey = true;
+    }
+    i++;
+  }
+  return { units };
+}
+
+// ── plain text ──────────────────────────────────────────────────────────────
+
+const TEXT_LIST = /^([ \t]*(?:[-*•+]|\d{1,3}[.)])[ \t]+)/;
+
+/** Units of a plain-text file. A hard-wrapped paragraph is one passage and is
+ *  re-wrapped to its old width; short lines and list lines are one each. */
+export function segmentText(src) {
+  const lines = toLines(src);
+  const eol = eolOf(src);
+  const units = [];
+  const text = (k) => src.slice(lines[k].start, lines[k].end);
+  let i = 0;
+  while (i < lines.length) {
+    if (!text(i).trim()) { i++; continue; }
+    let j = i;
+    while (j < lines.length && text(j).trim()) j++;
+    const block = [];
+    for (let k = i; k < j; k++) block.push(k);
+    const listy = block.some((k) => TEXT_LIST.test(text(k)));
+    const wrapped = block.length >= 2 && !listy && block.slice(0, -1).every((k) => text(k).trim().length >= 40);
+    if (wrapped) {
+      const start = lines[i].start + /^\s*/.exec(text(i))[0].length;
+      const end = lines[j - 1].end - /\s*$/.exec(text(j - 1))[0].length;
+      const sent = block.map((k) => text(k).trim()).join(" ");
+      const width = Math.max(40, ...block.map((k) => text(k).replace(/\s+$/, "").length));
+      const indent = /^[ \t]*/.exec(text(i + 1))[0];
+      const firstIndent = /^[ \t]*/.exec(text(i))[0].length;
+      if (needsWork(sent)) {
+        units.push({
+          start, end, text: sent,
+          place: (refined) => {
+            const r = oneLine(refined);
+            const problem = tokenProblem(sent, r);
+            return problem ? { ok: false, reason: problem } : { ok: true, value: wrap(r, width, firstIndent, indent, eol) };
+          },
+        });
+      }
+    } else {
+      for (const k of block) {
+        const t = text(k);
+        const prefix = (TEXT_LIST.exec(t) || /^[ \t]*/.exec(t))[0];
+        const core = t.slice(prefix.length).replace(/\s+$/, "");
+        const start = lines[k].start + prefix.length;
+        if (needsWork(core)) units.push({ start, end: start + core.length, text: core, place: inlinePlacer(core) });
+      }
+    }
+    i = j;
+  }
+  return { units };
+}
+
+function wrap(text, width, firstIndent, indent, eol) {
+  const words = text.split(" ");
+  const out = [];
+  let line = "";
+  let room = width - firstIndent;
+  for (const w of words) {
+    if (line && line.length + 1 + w.length > room) {
+      out.push(line);
+      line = w;
+      room = width - indent.length;
+    } else {
+      line = line ? `${line} ${w}` : w;
+    }
+  }
+  if (line) out.push(line);
+  return out.join(eol + indent);
+}
+
+// ── long passages ───────────────────────────────────────────────────────────
+
+/** Split a passage longer than the API allows at sentence boundaries. Returns
+ *  the pieces and the whitespace between them, so that
+ *  pieces[0] + joins[0] + pieces[1] + … is the passage again. */
+export function splitForApi(text, max = API_MAX_CHARS) {
+  if (text.length <= max) return { pieces: [text], joins: [] };
+  const parts = text.split(/(?<=[.!?…]["'”’»)\]]*)(\s+)/);
+  const sentences = [];
+  for (let k = 0; k < parts.length; k += 2) sentences.push({ text: parts[k], sep: parts[k + 1] ?? "" });
+  const pieces = [];
+  const joins = [];
+  let current = "";
+  let pendingSep = "";
+  const emit = (sep) => { pieces.push(current); joins.push(sep); current = ""; };
+  for (const { text: sentence, sep } of sentences) {
+    let s = sentence;
+    while (s.length > max) {
+      // One sentence over the limit: cut at the last space that fits.
+      if (current) emit(pendingSep);
+      let cut = s.lastIndexOf(" ", max);
+      if (cut <= 0) cut = max - (/[\ud800-\udbff]/.test(s[max - 1]) ? 1 : 0);
+      current = s.slice(0, cut);
+      emit(s[cut] === " " ? " " : "");
+      s = s[cut] === " " ? s.slice(cut + 1) : s.slice(cut);
+    }
+    if (current && current.length + pendingSep.length + s.length > max) emit(pendingSep);
+    current = current ? current + pendingSep + s : s;
+    pendingSep = sep;
+  }
+  if (current || !pieces.length) { pieces.push(current); joins.push(pendingSep); }
+  joins.pop();
+  return { pieces, joins };
+}
+
+// ── documents ───────────────────────────────────────────────────────────────
+
+export const FORMATS = { ".md": "md", ".markdown": "md", ".txt": "txt", ".html": "html", ".htm": "html", ".json": "json" };
+
+/** Segment a document by format: { units, note? }. */
+export function segment(src, format) {
+  if (format === "md") return segmentMarkdown(src);
+  if (format === "html") return segmentHtml(src);
+  if (format === "json") return segmentJson(src);
+  return segmentText(src);
+}
+
+/** The document with each unit replaced by `values.get(unit)` when present. */
+export function assemble(src, units, values) {
+  let out = "";
+  let pos = 0;
+  for (const u of units) {
+    out += src.slice(pos, u.start) + (values.has(u) ? values.get(u) : src.slice(u.start, u.end));
+    pos = u.end;
+  }
+  return out + src.slice(pos);
+}