@firedrill-tools/unstructured 0.1.4

package/firedrill/tools/unstructured/lib/partition/html.mjs

@@ -0,0 +1,211 @@
+// HTML partitioning over the streamed token sequence: headings, paragraphs, lists, tables, code, images, header/footer,
+// links. Linear by construction: an explicit element stack with per-name open counts (an unmatched close tag costs
+// nothing, a matched one pops what was pushed), counters instead of stack scans for head/title/header/footer, text
+// accumulated in arrays and joined once, one open anchor at a time (as HTML parsers do) and one open mark per emphasis
+// tag. Every bound answers a declared error: nesting depth, table rows and cells per row.
+import { classifyBlock } from "./elements.mjs";
+import { tokenize } from "./html-tokens.mjs";
+import { collapseSpace, escapeHtml, hasClassToken } from "./html-util.mjs";
+
+export const MAX_DEPTH = 256;
+export const MAX_TABLE_ROWS = 10000;
+export const MAX_TABLE_COLUMNS = 200;
+
+const HEADING_DEPTH = new Map([["h1", 0], ["h2", 1], ["h3", 2], ["h4", 3], ["h5", 4], ["h6", 5]]);
+const TYPED = new Map([["li", "ListItem"], ["pre", "CodeSnippet"], ["address", "Address"], ["figcaption", "FigureCaption"], ["header", "Header"], ["footer", "Footer"]]);
+const BLOCKS = new Set(["p", "div", "section", "article", "main", "aside", "nav", "blockquote", "ul", "ol", "dl", "dt", "dd", "form", "figure", "body", "html", "tr", "td", "th", "thead", "tbody", "caption"]);
+const VOID = new Set(["br", "hr", "img", "input", "meta", "link", "source", "wbr", "col"]);
+const EMPHASIS = new Map([["b", "b"], ["strong", "b"], ["i", "i"], ["em", "i"]]);
+const CHROME = new Set(["head", "title", "header", "footer"]);
+const OVERFLOW = "File has too many text blocks";
+
+const hasPageBreak = (attrs, which) => typeof attrs.style === "string" && attrs.style.toLowerCase().includes(`page-break-${which}`);
+const stripLeadingNewlines = (text) => {
+  let k = 0;
+  while (k < text.length && text[k] === "\n") k += 1;
+  return text.slice(k);
+};
+
+export function partitionHtml(builder, html) {
+  const stack = [];
+  const openCount = new Map();
+  const inside = { head: 0, title: 0, header: 0, footer: 0 };
+  let buffer = null;
+  let table = null;
+
+  const push = (name) => {
+    stack.push(name);
+    openCount.set(name, (openCount.get(name) ?? 0) + 1);
+    if (CHROME.has(name)) inside[name] += 1;
+    return stack.length <= MAX_DEPTH;
+  };
+  const popTo = (at) => {
+    while (stack.length > at) {
+      const name = stack.pop();
+      const count = openCount.get(name) - 1;
+      if (count === 0) openCount.delete(name);
+      else openCount.set(name, count);
+      if (CHROME.has(name)) inside[name] -= 1;
+    }
+  };
+  const closeTag = (name) => {
+    if (openCount.has(name)) popTo(stack.lastIndexOf(name));
+  };
+
+  const openBuffer = (type, depth) => {
+    buffer = { type, depth, parts: [], linkTexts: [], linkUrls: [], emphasis: [], tags: [], anchor: null, marks: [] };
+  };
+  const ensureBuffer = () => {
+    if (buffer === null) openBuffer(null, null);
+  };
+  const since = (index) => collapseSpace(buffer.parts.slice(index).join(""));
+  const closeAnchor = () => {
+    const anchor = buffer.anchor;
+    buffer.anchor = null;
+    const label = since(anchor.start);
+    if (label.length > 0) {
+      buffer.linkTexts.push(label);
+      buffer.linkUrls.push(anchor.href);
+    }
+  };
+  const closeMark = (tag) => {
+    let at = buffer.marks.length - 1;
+    while (at >= 0 && buffer.marks[at].tag !== tag) at -= 1;
+    if (at < 0) return;
+    const [mark] = buffer.marks.splice(at, 1);
+    const content = since(mark.start);
+    if (content.length > 0) {
+      buffer.emphasis.push(content);
+      buffer.tags.push(mark.tag);
+    }
+  };
+  const flush = () => {
+    if (buffer === null) return;
+    const current = buffer;
+    if (current.anchor !== null) closeAnchor();
+    while (current.marks.length > 0) closeMark(current.marks[current.marks.length - 1].tag);
+    buffer = null;
+    const raw = current.parts.join("");
+    const text = current.type === "CodeSnippet" ? stripLeadingNewlines(raw).trimEnd() : collapseSpace(raw);
+    if (text.length === 0) return;
+    let type = current.type ?? classifyBlock(text);
+    const chrome = inside.header > 0 ? "Header" : inside.footer > 0 ? "Footer" : null;
+    if (chrome !== null && type !== "Title") type = chrome;
+    const meta = {};
+    if (type === "Title") meta.category_depth = current.depth ?? 0;
+    if (current.linkTexts.length > 0) {
+      meta.link_texts = current.linkTexts;
+      meta.link_urls = current.linkUrls;
+    }
+    if (current.emphasis.length > 0) {
+      meta.emphasized_text_contents = current.emphasis;
+      meta.emphasized_text_tags = current.tags;
+    }
+    builder.add(type, text, meta);
+  };
+
+  const closeCell = () => {
+    table.row.push(collapseSpace(table.cell.join("")));
+    table.cell = null;
+    return table.row.length <= MAX_TABLE_COLUMNS ? undefined : `File has a table row with more than ${MAX_TABLE_COLUMNS} cells`;
+  };
+  const closeRow = () => {
+    const problem = table.cell !== null ? closeCell() : undefined;
+    if (problem !== undefined) return problem;
+    table.rows.push(table.row);
+    table.row = null;
+    return table.rows.length <= MAX_TABLE_ROWS ? undefined : `File has a table with more than ${MAX_TABLE_ROWS} rows`;
+  };
+  const closeTable = () => {
+    const problem = table.row !== null ? closeRow() : undefined;
+    if (problem !== undefined) return problem;
+    if (table.rows.length > 0) {
+      const text = table.rows.map((row) => row.join(" ")).join("\n");
+      const markup = `<table>${table.rows.map((row) => `<tr>${row.map((cell) => `<td>${escapeHtml(cell)}</td>`).join("")}</tr>`).join("")}</table>`;
+      builder.add("Table", text, { text_as_html: markup });
+    }
+    table = null;
+    return undefined;
+  };
+
+  const visit = (token) => {
+    if (token.kind === "text") {
+      if (inside.head > 0 || inside.title > 0) return undefined;
+      if (table !== null) {
+        if (table.cell !== null) table.cell.push(token.text);
+      } else if (token.text.trim().length > 0 || buffer !== null) {
+        ensureBuffer();
+        buffer.parts.push(token.text);
+      }
+      return undefined;
+    }
+    const name = token.name;
+    if (token.kind === "open") {
+      if (hasPageBreak(token.attrs, "before") || (name === "hr" && typeof token.attrs.class === "string" && hasClassToken(token.attrs.class, "page-break"))) {
+        flush();
+        builder.pageBreak();
+      }
+      if (table !== null) {
+        let problem;
+        if (name === "tr") {
+          if (table.row !== null) problem = closeRow();
+          table.row = [];
+        } else if (name === "td" || name === "th") {
+          if (table.cell !== null) problem = closeCell();
+          if (table.row === null) table.row = [];
+          table.cell = [];
+        } else if (name === "br" && table.cell !== null) table.cell.push(" ");
+        if (problem !== undefined) return problem;
+      } else if (name === "table") {
+        flush();
+        table = { rows: [], row: null, cell: null };
+      } else if (name === "br") {
+        if (buffer !== null) buffer.parts.push("\n");
+      } else if (name === "img") {
+        flush();
+        builder.add("Image", collapseSpace(typeof token.attrs.alt === "string" ? token.attrs.alt : ""), { image_url: typeof token.attrs.src === "string" && token.attrs.src.length > 0 ? token.attrs.src : undefined });
+      } else if (HEADING_DEPTH.has(name)) {
+        flush();
+        openBuffer("Title", HEADING_DEPTH.get(name));
+      } else if (TYPED.has(name)) {
+        flush();
+        if (name !== "header" && name !== "footer") openBuffer(TYPED.get(name), null);
+      } else if (BLOCKS.has(name) || name === "hr") flush();
+      else if (name === "a") {
+        ensureBuffer();
+        if (buffer.anchor !== null) closeAnchor();
+        buffer.anchor = { start: buffer.parts.length, href: typeof token.attrs.href === "string" ? token.attrs.href : "" };
+      } else if (EMPHASIS.has(name)) {
+        ensureBuffer();
+        const tag = EMPHASIS.get(name);
+        if (!buffer.marks.some((mark) => mark.tag === tag)) buffer.marks.push({ start: buffer.parts.length, tag });
+      }
+      if (!token.selfClosing && !VOID.has(name) && !push(name)) return `File is not valid html: elements nested deeper than ${MAX_DEPTH} levels`;
+      return builder.overflow ? OVERFLOW : undefined;
+    }
+    if (table !== null) {
+      let problem;
+      if ((name === "td" || name === "th") && table.cell !== null) problem = closeCell();
+      else if (name === "tr" && table.row !== null) problem = closeRow();
+      else if (name === "table") problem = closeTable();
+      closeTag(name);
+      return problem ?? (builder.overflow ? OVERFLOW : undefined);
+    }
+    if (name === "a") {
+      if (buffer !== null && buffer.anchor !== null) closeAnchor();
+    } else if (EMPHASIS.has(name)) {
+      if (buffer !== null) closeMark(EMPHASIS.get(name));
+    } else if (HEADING_DEPTH.has(name) || TYPED.has(name) || BLOCKS.has(name)) flush();
+    closeTag(name);
+    return builder.overflow ? OVERFLOW : undefined;
+  };
+
+  const result = tokenize(html, visit);
+  if (result.error !== undefined) return { ok: false, message: result.error };
+  if (table !== null) {
+    const problem = closeTable();
+    if (problem !== undefined) return { ok: false, message: problem };
+  }
+  flush();
+  return builder.overflow ? { ok: false, message: OVERFLOW } : { ok: true };
+}

package/firedrill/tools/unstructured/lib/partition/index.mjs

@@ -0,0 +1,122 @@
+// File-type detection and dispatch: one document in, { ok, elements } or { ok: false, code, message } out.
+import { chunkElements } from "./chunk.mjs";
+import { Builder } from "./elements.mjs";
+import { partitionDelimited } from "./csv.mjs";
+import { partitionHtml } from "./html.mjs";
+import { partitionMarkdown } from "./markdown.mjs";
+import { partitionEmail } from "./email.mjs";
+import { partitionJson, partitionXml } from "./other.mjs";
+import { partitionText } from "./text.mjs";
+import { sha256Id } from "../sha256.mjs";
+import { uuid } from "../ids.mjs";
+import { utf8Length } from "../util.mjs";
+
+const BY_EXTENSION = new Map([
+  ["txt", "text/plain"], ["text", "text/plain"], ["log", "text/plain"], ["md", "text/markdown"], ["markdown", "text/markdown"], ["html", "text/html"],
+  ["htm", "text/html"], ["csv", "text/csv"], ["tsv", "text/tsv"], ["json", "application/json"], ["eml", "message/rfc822"], ["xml", "application/xml"],
+  ["rst", "text/x-rst"],
+]);
+const ALIASES = new Map([
+  ["text/x-markdown", "text/markdown"], ["application/xhtml+xml", "text/html"], ["text/xml", "application/xml"], ["text/tab-separated-values", "text/tsv"],
+  ["application/csv", "text/csv"], ["text/rst", "text/x-rst"], ["text/x-log", "text/plain"], ["text/json", "application/json"],
+]);
+export const SUPPORTED = new Set(["text/plain", "text/markdown", "text/html", "text/csv", "text/tsv", "application/json", "message/rfc822", "application/xml", "text/x-rst"]);
+const BINARY_EXTENSIONS = new Map([
+  ["pdf", "application/pdf"], ["docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"], ["doc", "application/msword"],
+  ["pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"], ["xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
+  ["png", "image/png"], ["jpg", "image/jpeg"], ["jpeg", "image/jpeg"], ["epub", "application/epub+zip"], ["zip", "application/zip"], ["odt", "application/vnd.oasis.opendocument.text"],
+]);
+
+/** Detected media type: explicit content_type first, then the part's type, then the filename extension. Null when unknown. */
+export function detectType(file, explicitType) {
+  const normalise = (raw) => {
+    if (typeof raw !== "string" || raw.length === 0) return null;
+    const base = raw.split(";")[0].trim().toLowerCase();
+    return ALIASES.get(base) ?? base;
+  };
+  const explicit = normalise(explicitType);
+  if (explicit !== null) return explicit;
+  const partType = normalise(file.content_type);
+  if (partType !== null && partType !== "application/octet-stream") return partType;
+  const name = typeof file.filename === "string" ? file.filename : "";
+  const dot = name.lastIndexOf(".");
+  const extension = dot >= 0 ? name.slice(dot + 1).toLowerCase() : "";
+  if (BY_EXTENSION.has(extension)) return BY_EXTENSION.get(extension);
+  if (BINARY_EXTENSIONS.has(extension)) return BINARY_EXTENSIONS.get(extension);
+  return null;
+}
+
+/** Response budget assumed when the caller passes none (`meta/limits.response_bytes` default). */
+const DEFAULT_BUDGET = 921600;
+const budgetOf = (options) => (typeof options.responseBudget === "number" ? options.responseBudget : DEFAULT_BUDGET);
+
+/**
+ * Byte bound for the running element-size estimate: the response budget when the elements themselves are the response
+ * (unchunked JSON or CSV with a known budget), three quarters of it when every element is serialised again into
+ * `orig_elements` (stored gzip never shrinks, base64 adds a third), and no bound when the chunks alone are returned.
+ */
+function elementBudget(options) {
+  if (options.chunk === null) return typeof options.responseBudget === "number" ? options.responseBudget : Infinity;
+  return options.chunk.includeOrigElements ? Math.floor((budgetOf(options) * 3) / 4) : Infinity;
+}
+
+/**
+ * Partitions one text document.
+ * @param file { filename, content, content_type?, last_modified? }
+ * @param options { elementSizing?: { bytes } shared across the files of one request, contentType, languages, includePageBreaks, startingPage, xmlKeepTags, uniqueIds, nowIso, chunk: null | chunk options }
+ * @param context Tool context (used only for UUID ids when uniqueIds)
+ */
+export function partitionFile(file, options, context) {
+  const filetype = detectType(file, options.contentType);
+  if (filetype === null || !SUPPORTED.has(filetype)) return { ok: false, code: "UNSUPPORTED_FILE_TYPE", message: `${filetype === null ? "None" : filetype} not currently supported` };
+  if (file.bad_transfer === true) return { ok: false, code: "INVALID_FILE", message: "File content is not valid base64 UTF-8 text" };
+  const builder = new Builder(
+    {
+      filename: file.filename,
+      filetype,
+      languages: options.languages,
+      startingPage: options.startingPage,
+      lastModified: typeof file.last_modified === "string" && file.last_modified.length > 0 ? file.last_modified : options.nowIso,
+      includePageBreaks: options.includePageBreaks,
+      sizing: options.elementSizing,
+      byteBudget: elementBudget(options),
+    },
+    options.uniqueIds ? context : null,
+  );
+  const text = file.content.startsWith("") ? file.content.slice(1) : file.content;
+  let result;
+  if (filetype === "text/plain" || filetype === "text/x-rst") result = partitionText(builder, text);
+  else if (filetype === "text/markdown") result = partitionMarkdown(builder, text);
+  else if (filetype === "text/html") result = partitionHtml(builder, text);
+  else if (filetype === "text/csv") result = partitionDelimited(builder, text, ",");
+  else if (filetype === "text/tsv") result = partitionDelimited(builder, text, "\t");
+  else if (filetype === "message/rfc822") result = partitionEmail(builder, text);
+  else if (filetype === "application/xml") result = partitionXml(builder, text, options.xmlKeepTags);
+  else result = partitionJson(builder, text);
+  if (builder.tooLarge) {
+    const orig = options.chunk !== null;
+    const floor = orig ? Math.ceil((builder.sizing.bytes * 4) / 3) : builder.sizing.bytes;
+    return { ok: false, code: "RESPONSE_TOO_LARGE", message: `Partition output${orig ? " with orig_elements" : ""} of at least ${floor} bytes exceeds the ${budgetOf(options)} byte response limit; upload fewer or smaller files${orig ? " or set include_orig_elements=false" : ""}` };
+  }
+  if (!result.ok) return { ok: false, code: result.code ?? "INVALID_FILE", message: result.message };
+  if (builder.overflow) return { ok: false, code: "INVALID_FILE", message: "File produces too many elements" };
+  let elements = builder.elements;
+  if (options.chunk !== null && options.chunk.includeOrigElements && typeof options.responseBudget === "number") {
+    // Every original element lands in some chunk's `orig_elements` (stored-block gzip never shrinks, base64 adds a third),
+    // so an element array whose JSON already passes three quarters of the budget cannot produce a response within it.
+    // Failing here skips compressing tens of megabytes that would be discarded by the response budget anyway.
+    const bytes = utf8Length(JSON.stringify(elements));
+    if (bytes * 4 > options.responseBudget * 3) return { ok: false, code: "RESPONSE_TOO_LARGE", message: `Partition output with orig_elements of at least ${Math.ceil((bytes * 4) / 3)} bytes exceeds the ${options.responseBudget} byte response limit; upload fewer or smaller files or set include_orig_elements=false` };
+  }
+  if (options.chunk !== null) {
+    let ordinal = 0;
+    const idOf = (chunkText) => {
+      ordinal += 1;
+      return options.uniqueIds ? uuid(context) : sha256Id(file.filename, "chunk", String(ordinal), chunkText);
+    };
+    const chunked = chunkElements(elements, { ...options.chunk, responseBudget: options.responseBudget, chunkSizing: options.chunkSizing }, idOf);
+    if (!chunked.ok) return chunked;
+    elements = chunked.elements;
+  }
+  return { ok: true, elements, filetype };
+}

package/firedrill/tools/unstructured/lib/partition/markdown.mjs

@@ -0,0 +1,220 @@
+// Markdown partitioning: headings, fenced code, pipe tables, lists, images, inline emphasis and links.
+import { classifyBlock, stripListMarker } from "./elements.mjs";
+import { tableHtml, tableText } from "./csv.mjs";
+import { MAX_TABLE_COLUMNS, MAX_TABLE_ROWS } from "./html.mjs";
+import { MAX_BLOCKS } from "./text.mjs";
+
+const LIST_RE = /^(\s*)([-*+]|\d{1,4}[.)])\s+(.*)$/;
+const CELL_RE = /^:?-+:?$/; // GFM: at least one dash per cell, optional alignment colons
+
+/** ATX heading: `#`{1,6} + whitespace + text with optional closing hashes. Linear scan, returns { depth, text } or null. */
+function parseHeading(line) {
+  let depth = 0;
+  while (depth < line.length && line[depth] === "#") depth += 1;
+  if (depth === 0 || depth > 6 || depth >= line.length || (line[depth] !== " " && line[depth] !== "\t")) return null;
+  let end = line.length;
+  while (end > depth && (line[end - 1] === " " || line[end - 1] === "\t")) end -= 1;
+  while (end > depth && line[end - 1] === "#") end -= 1;
+  while (end > depth && (line[end - 1] === " " || line[end - 1] === "\t")) end -= 1;
+  return { depth, text: line.slice(depth, end).trim() };
+}
+
+/** `![alt](url "title")` on its own line. Linear scan via indexOf, returns { alt, url } or null. */
+function parseImage(line) {
+  if (!line.startsWith("![")) return null;
+  const closeAlt = line.indexOf("](", 2);
+  if (closeAlt < 0 || line.indexOf("]", 2) !== closeAlt) return null;
+  const closeUrl = line.indexOf(")", closeAlt + 2);
+  if (closeUrl < 0 || line.slice(closeUrl + 1).trim().length > 0) return null;
+  const inner = line.slice(closeAlt + 2, closeUrl);
+  return { alt: line.slice(2, closeAlt), url: inner.split(/\s/)[0] };
+}
+
+const SEPARATOR_CHARS_RE = /^[\s|:-]*$/; // a separator row holds only pipes, dashes, colons and whitespace
+
+/**
+ * Pipe-table separator row (`| --- | :---: |`): the number of cells when every cell is `:?-+:?` after trimming, otherwise
+ * 0. Linear in the line at any width: ordinary text is rejected by one character-class scan before anything is split,
+ * and a real separator is never ignored for being long, so a table wider than the cell bound fails with the declared
+ * error instead of silently becoming text.
+ */
+function separatorCells(line) {
+  if (!line.includes("|") || !SEPARATOR_CHARS_RE.test(line)) return 0;
+  const cells = splitRow(line);
+  return cells.length > 0 && cells.every((cell) => CELL_RE.test(cell)) ? cells.length : 0;
+}
+
+/**
+ * A forward `indexOf` memo: `text` is scanned once per needle no matter how often it is asked, because every query
+ * position is at or after the previous one (the caller's index only moves forward), so a cached hit at or beyond
+ * the query is still the first occurrence. Returns Infinity when the needle does not occur again.
+ */
+function finder(text, needle) {
+  let at = -1;
+  return (from) => {
+    if (at < from) {
+      const next = text.indexOf(needle, from);
+      at = next < 0 ? Infinity : next;
+    }
+    return at;
+  };
+}
+
+/** Extracts links and emphasis from inline markup and returns the plain text plus metadata. Linear in the text. */
+export function inline(text) {
+  const meta = {};
+  const linkTexts = [];
+  const linkUrls = [];
+  const emphasis = [];
+  const tags = [];
+  const out = [];
+  const linkClose = finder(text, "](");
+  const linkEnd = finder(text, ")");
+  const tick = finder(text, "`");
+  const markers = new Map([["*", finder(text, "*")], ["**", finder(text, "**")], ["_", finder(text, "_")], ["__", finder(text, "__")]]);
+  let i = 0;
+  while (i < text.length) {
+    if (text[i] === "[") {
+      const close = linkClose(i);
+      const end = close !== Infinity ? linkEnd(close + 2) : Infinity;
+      if (end !== Infinity && end - i <= 2000) {
+        const label = text.slice(i + 1, close);
+        linkTexts.push(label);
+        linkUrls.push(text.slice(close + 2, end).split(/\s/)[0]);
+        out.push(label);
+        i = end + 1;
+        continue;
+      }
+    }
+    if (text[i] === "*" || text[i] === "_") {
+      const marker = text[i];
+      const double = text[i + 1] === marker;
+      const open = double ? marker + marker : marker;
+      const close = markers.get(open)(i + open.length);
+      if (close !== Infinity && close > i + open.length && close - i <= 2000) {
+        const content = text.slice(i + open.length, close);
+        if (!content.includes("\n") && content.trim().length > 0 && content[0] !== " ") {
+          emphasis.push(content);
+          tags.push(double ? "b" : "i");
+          out.push(content);
+          i = close + open.length;
+          continue;
+        }
+      }
+    }
+    if (text[i] === "`") {
+      const close = tick(i + 1);
+      if (close !== Infinity && close - i <= 2000) {
+        out.push(text.slice(i + 1, close));
+        i = close + 1;
+        continue;
+      }
+    }
+    out.push(text[i]);
+    i += 1;
+  }
+  if (linkTexts.length > 0) {
+    meta.link_texts = linkTexts;
+    meta.link_urls = linkUrls;
+  }
+  if (emphasis.length > 0) {
+    meta.emphasized_text_contents = emphasis;
+    meta.emphasized_text_tags = tags;
+  }
+  return { text: out.join(""), meta };
+}
+
+function flushParagraph(builder, lines) {
+  if (lines.length === 0) return;
+  const block = lines.join("\n");
+  const { text, meta } = inline(block);
+  const type = classifyBlock(text);
+  builder.add(type === "ListItem" ? "UncategorizedText" : type, text.split("\n").map((l) => l.trim()).join("\n"), type === "Title" ? { category_depth: 0, ...meta } : meta);
+  lines.length = 0;
+}
+
+const splitRow = (line) => {
+  let row = line.trim();
+  if (row.startsWith("|")) row = row.slice(1);
+  if (row.endsWith("|")) row = row.slice(0, -1);
+  return row.split("|").map((cell) => cell.trim());
+};
+
+export function partitionMarkdown(builder, text) {
+  const lines = text.split("\n").map((line) => (line.endsWith("\r") ? line.slice(0, -1) : line));
+  const paragraph = [];
+  let blocks = 0;
+  let i = 0;
+  while (i < lines.length) {
+    const line = lines[i];
+    // Blank lines and lines read while a paragraph is open never count against the bound (a line that closes the paragraph
+    // emits at most one more element, and elements have their own bound), so only real content can reach it.
+    if (paragraph.length === 0 && line.trim().length > 0 && (blocks += 1) > MAX_BLOCKS) return { ok: false, message: "File has too many text blocks" };
+    if (line.includes("\f")) {
+      flushParagraph(builder, paragraph);
+      builder.pageBreak();
+      i += 1;
+      continue;
+    }
+    if (line.trim().length === 0) {
+      flushParagraph(builder, paragraph);
+      i += 1;
+      continue;
+    }
+    if (line.startsWith("```") || line.startsWith("~~~")) {
+      flushParagraph(builder, paragraph);
+      const fence = line.slice(0, 3);
+      const code = [];
+      i += 1;
+      while (i < lines.length && !lines[i].startsWith(fence)) code.push(lines[i++]);
+      builder.add("CodeSnippet", code.join("\n"));
+      i += 1;
+      continue;
+    }
+    const heading = parseHeading(line);
+    if (heading !== null) {
+      flushParagraph(builder, paragraph);
+      const { text: title, meta } = inline(heading.text);
+      builder.add("Title", title, { category_depth: heading.depth - 1, ...meta });
+      i += 1;
+      continue;
+    }
+    const width = line.includes("|") && i + 1 < lines.length ? separatorCells(lines[i + 1]) : 0;
+    if (width > 0) {
+      flushParagraph(builder, paragraph);
+      // Same bounds as HTML tables: more rows or cells than the bound fail the file, nothing is dropped. The separator
+      // row counts too, so a header narrower than its separator cannot slip a wider table past the bound.
+      if (width > MAX_TABLE_COLUMNS) return { ok: false, message: `File has a table row with more than ${MAX_TABLE_COLUMNS} cells` };
+      const rows = [splitRow(line)];
+      if (rows[0].length > MAX_TABLE_COLUMNS) return { ok: false, message: `File has a table row with more than ${MAX_TABLE_COLUMNS} cells` };
+      i += 2;
+      while (i < lines.length && lines[i].includes("|") && lines[i].trim().length > 0) {
+        const row = splitRow(lines[i++]);
+        if (row.length > MAX_TABLE_COLUMNS) return { ok: false, message: `File has a table row with more than ${MAX_TABLE_COLUMNS} cells` };
+        rows.push(row);
+        if (rows.length > MAX_TABLE_ROWS) return { ok: false, message: `File has a table with more than ${MAX_TABLE_ROWS} rows` };
+      }
+      builder.add("Table", tableText(rows), { text_as_html: tableHtml(rows) });
+      continue;
+    }
+    const image = parseImage(line);
+    if (image !== null) {
+      flushParagraph(builder, paragraph);
+      builder.add("Image", image.alt, { image_url: image.url.length > 0 ? image.url : undefined });
+      i += 1;
+      continue;
+    }
+    const item = LIST_RE.exec(line);
+    if (item !== null && paragraph.length === 0) {
+      const depth = Math.min(Math.floor(item[1].replace(/\t/g, "  ").length / 2), 10);
+      const { text: itemText, meta } = inline(stripListMarker(line).trim());
+      builder.add("ListItem", itemText, { category_depth: depth, ...meta });
+      i += 1;
+      continue;
+    }
+    paragraph.push(line);
+    i += 1;
+  }
+  flushParagraph(builder, paragraph);
+  return { ok: true };
+}

package/firedrill/tools/unstructured/lib/partition/other.mjs

@@ -0,0 +1,118 @@
+// XML and pre-partitioned JSON documents (e-mail lives in email.mjs).
+import { ELEMENT_TYPES, MAX_TEXT, stringsBytes, valuesBytes } from "./elements.mjs";
+import { MAX_DEPTH } from "./html.mjs";
+import { tokenize } from "./html-tokens.mjs";
+import { emitPlainBlocks } from "./text.mjs";
+import { shapeProblem } from "../wire.mjs";
+
+export function partitionXml(builder, text, keepTags) {
+  const stack = [];
+  let leafText = [];
+  let leafOpen = null;
+  const emitLeaf = () => {
+    const leaf = leafText.join("").trim();
+    leafText = [];
+    if (leaf.length === 0) return undefined;
+    if (keepTags) builder.add("UncategorizedText", `<${leafOpen}>${leaf}</${leafOpen}>`);
+    else if (!emitPlainBlocks(builder, leaf)) return "File has too many text blocks";
+    return builder.overflow ? "File has too many text blocks" : undefined;
+  };
+  const visit = (token) => {
+    if (token.kind === "open") {
+      if (leafOpen !== null && !keepTags) {
+        const problem = emitLeaf();
+        if (problem !== undefined) return problem;
+      }
+      leafText = [];
+      if (!token.selfClosing) {
+        stack.push(token.name);
+        if (stack.length > MAX_DEPTH) return `File is not a valid xml: elements nested deeper than ${MAX_DEPTH} levels`;
+        leafOpen = token.name;
+      }
+      return undefined;
+    }
+    if (token.kind === "close") {
+      if (stack.length === 0 || stack[stack.length - 1] !== token.name) return "File is not a valid xml";
+      stack.pop();
+      const problem = leafOpen === token.name ? emitLeaf() : undefined;
+      leafText = [];
+      leafOpen = null;
+      return problem;
+    }
+    leafText.push(token.text);
+    return undefined;
+  };
+  const result = tokenize(text, visit);
+  if (result.error !== undefined) return { ok: false, message: result.error === "File is not valid html" ? "File is not a valid xml" : result.error };
+  if (stack.length > 0) return { ok: false, message: "File is not a valid xml" };
+  return { ok: true };
+}
+
+const SCHEMA_MESSAGE = "Json schema does not match the Unstructured schema";
+
+/** Pre-partitioned Unstructured JSON: elements are returned as given (ids regenerated when absent). */
+export function partitionJson(builder, text) {
+  let value;
+  try {
+    value = JSON.parse(text);
+  } catch {
+    return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+  }
+  if (!Array.isArray(value) || value.length > 100000 || shapeProblem(value) !== null) return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+  for (const item of value) {
+    if (item === null || typeof item !== "object" || Array.isArray(item)) return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+    if (typeof item.type !== "string" || !ELEMENT_TYPES.has(item.type) || typeof item.text !== "string" || item.text.length > MAX_TEXT) return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+    if (item.element_id !== undefined && (typeof item.element_id !== "string" || item.element_id.length === 0 || item.element_id.length > 128)) return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+    if (item.metadata !== undefined && (item.metadata === null || typeof item.metadata !== "object" || Array.isArray(item.metadata) || !metadataFits(item.metadata))) return { ok: false, code: "INVALID_REQUEST", message: SCHEMA_MESSAGE };
+  }
+  const fixed = 60 + String(builder.filename).length + String(builder.filetype).length + String(builder.lastModified).length + stringsBytes(builder.languages);
+  for (const item of value) {
+    if (!builder.account(fixed + item.type.length + item.text.length + 32 + (item.metadata === undefined ? 0 : valuesBytes(item.metadata)))) return { ok: true };
+    const metadata = { filename: builder.filename, filetype: builder.filetype, languages: builder.languages.slice(), page_number: builder.page, last_modified: builder.lastModified };
+    if (item.metadata !== undefined) copyMetadata(item.metadata, metadata);
+    builder.elements.push({ type: item.type, element_id: typeof item.element_id === "string" ? item.element_id : builder.id(item.text), text: item.text, metadata });
+    builder.ordinal += 1;
+  }
+  return { ok: true };
+}
+
+const TYPED = new Map([["filename", "string"], ["filetype", "string"], ["page_number", "integer"], ["languages", "strings"], ["category_depth", "integer"], ["parent_id", "string"], ["last_modified", "string"], ["is_continuation", "boolean"], ["text_as_html", "string"]]);
+const MAX_METADATA_KEYS = 100;
+const MAX_KEY = 100;
+const MAX_STRING = 10000;
+const MAX_ARRAY = 200;
+const scalar = (v) => v === null || typeof v === "boolean" || (typeof v === "number" && Number.isFinite(v)) || (typeof v === "string" && v.length <= MAX_STRING);
+
+/** True when one caller metadata value fits the output schema for its key (typed known keys, else a scalar or a scalar array). */
+function valueFits(key, value) {
+  const typed = TYPED.get(key);
+  if (typed === "string") return typeof value === "string" && value.length <= MAX_STRING;
+  if (typed === "integer") return Number.isInteger(value) && value >= 0;
+  if (typed === "boolean") return typeof value === "boolean";
+  if (typed === "strings") return Array.isArray(value) && value.length <= 50 && value.every((v) => typeof v === "string" && v.length <= 100);
+  return scalar(value) || (Array.isArray(value) && value.length <= MAX_ARRAY && value.every(scalar));
+}
+
+/**
+ * True when every caller metadata entry can be returned as given: at most 100 keys of at most 100 characters, none named
+ * `__proto__`/`constructor`/`prototype`, typed known keys with their type, other values scalars (strings ≤ 10,000 characters)
+ * or arrays of at most 200 scalars. A metadata object that does not fit fails the whole upload (400 schema mismatch);
+ * nothing is dropped from one that does.
+ */
+function metadataFits(source) {
+  const keys = Object.keys(source);
+  if (keys.length > MAX_METADATA_KEYS) return false;
+  for (const key of keys) {
+    if (key === "__proto__" || key === "constructor" || key === "prototype" || key.length > MAX_KEY) return false;
+    if (!valueFits(key, source[key])) return false;
+  }
+  return true;
+}
+
+/** Copies caller metadata (already checked by `metadataFits`) onto the element metadata; arrays are copied, nothing is cut. */
+function copyMetadata(source, target) {
+  for (const key of Object.keys(source)) {
+    const value = source[key];
+    target[key] = Array.isArray(value) ? value.slice() : value;
+  }
+}