@firedrill-tools/unstructured 0.1.4

package/firedrill/tools/unstructured/lib/partition/chunk.mjs

@@ -0,0 +1,243 @@
+// Chunking strategies basic / by_title / by_page over a partitioned element list.
+import { gzipBase64Json } from "../gzip.mjs";
+import { utf8Length } from "../util.mjs";
+
+/** Response budget assumed when the caller passes none (`meta/limits.response_bytes` default). */
+const DEFAULT_BUDGET = 921600;
+
+const MERGED_ARRAYS = ["link_texts", "link_urls", "emphasized_text_contents", "emphasized_text_tags"];
+
+/**
+ * The metadata one chunk group shares: built once per group (its gzip included), measured once, and copied by reference into
+ * every piece — the merged link/emphasis arrays, e-mail recipient lists and `orig_elements` are never rebuilt per piece.
+ * @param includeOrig true to compute the gzip+base64 `orig_elements` of the group, false to omit it
+ */
+function chunkMetadata(originals, includeOrig, budget) {
+  const first = originals[0].metadata;
+  const metadata = { filename: first.filename, filetype: first.filetype, languages: first.languages, page_number: first.page_number, last_modified: first.last_modified };
+  for (const key of ["parent_id", "sent_from", "sent_to", "cc_recipient", "bcc_recipient", "subject", "email_message_id"]) if (first[key] !== undefined) metadata[key] = first[key];
+  // Merged arrays are copied item by item (never spread as call arguments) and stop at the response budget: every item costs
+  // at least its characters plus a quote pair and a comma in each piece that carries it.
+  let bytes = 0;
+  for (const key of MERGED_ARRAYS) {
+    const merged = [];
+    for (const element of originals) {
+      const values = element.metadata[key];
+      if (!Array.isArray(values)) continue;
+      for (let i = 0; i < values.length; i += 1) {
+        const value = values[i];
+        bytes += (typeof value === "string" ? value.length : 1) + 3;
+        if (bytes > budget) return null;
+        merged.push(value);
+      }
+    }
+    if (merged.length > 0) metadata[key] = merged;
+  }
+  if (includeOrig) metadata.orig_elements = gzipBase64Json(originals);
+  return metadata;
+}
+
+/** UTF-8 bytes one copy of a chunk group's metadata costs in the response. */
+const metadataBytes = (metadata) => utf8Length(JSON.stringify(metadata));
+
+/** Splits `text` into pieces of at most `max` characters, each after the first prefixed with the previous tail. */
+function splitOversized(text, max, overlap) {
+  const pieces = [];
+  let start = 0;
+  let prefix = "";
+  while (start < text.length || pieces.length === 0) {
+    const room = Math.max(1, max - prefix.length);
+    const piece = prefix + text.slice(start, start + room);
+    pieces.push(piece);
+    start += room;
+    prefix = overlap > 0 ? piece.slice(-overlap) : "";
+  }
+  return pieces;
+}
+
+/** Sections: element groups that never share a chunk. */
+function sections(elements, options) {
+  const out = [];
+  let current = [];
+  let lastPage = null;
+  let lastTitleDepth = null;
+  for (const element of elements) {
+    if (element.type === "PageBreak") continue;
+    const page = element.metadata.page_number;
+    const pageChanged = lastPage !== null && page !== lastPage;
+    let boundary = false;
+    if (options.strategy === "by_title") boundary = element.type === "Title" || (pageChanged && !options.multipageSections);
+    else if (options.strategy === "by_page") boundary = pageChanged;
+    if (boundary && current.length > 0) {
+      out.push(current);
+      current = [];
+    }
+    current.push(element);
+    lastPage = page;
+    if (element.type === "Title") lastTitleDepth = element.metadata.category_depth ?? 0;
+  }
+  if (current.length > 0) out.push(current);
+  if (options.strategy === "by_title" && options.combineUnderNChars > 0) {
+    // Running length and table flag of the last combined section: O(total elements), never re-reduced.
+    const combined = [];
+    let prevLength = 0;
+    let prevHasTable = false;
+    for (const section of out) {
+      const previous = combined.length > 0 ? combined[combined.length - 1] : null;
+      let length = -2;
+      let hasTable = false;
+      for (const e of section) {
+        length += e.text.length + 2;
+        if (e.type === "Table") hasTable = true;
+      }
+      if (previous !== null && prevLength < options.combineUnderNChars && prevLength + 2 + length <= options.combineUnderNChars && !prevHasTable && !hasTable) {
+        for (const element of section) previous.push(element);
+        prevLength += 2 + length;
+      } else {
+        combined.push(section);
+        prevLength = length;
+        prevHasTable = hasTable;
+      }
+    }
+    return combined;
+  }
+  void lastTitleDepth;
+  return out;
+}
+
+/** @param metadata the table's shared chunk metadata (already accounted against the budget by the caller). */
+function tableChunks(element, options, idOf, metadata) {
+  const out = [];
+  const max = options.maxCharacters;
+  if (element.text.length <= max) {
+    out.push({ type: "Table", element_id: idOf(element.text), text: element.text, metadata: { ...metadata, text_as_html: element.metadata.text_as_html } });
+    return out;
+  }
+  const rows = element.text.split("\n");
+  const htmlRows = typeof element.metadata.text_as_html === "string" ? element.metadata.text_as_html.replace(/^<table>|<\/table>$/g, "").split("</tr>").filter((r) => r.length > 0).map((r) => `${r}</tr>`) : [];
+  let text = "";
+  let html = "";
+  let first = true;
+  const emit = () => {
+    const piece = { ...metadata };
+    if (html.length > 0) piece.text_as_html = `<table>${html}</table>`;
+    if (!first) piece.is_continuation = true;
+    out.push({ type: "TableChunk", element_id: idOf(text), text, metadata: piece });
+    first = false;
+    text = "";
+    html = "";
+  };
+  for (let i = 0; i < rows.length; i += 1) {
+    const row = rows[i];
+    if (row.length > max) {
+      // A row longer than max_characters becomes its own run of pieces; its markup rides with the first piece.
+      if (text.length > 0) emit();
+      for (let start = 0; start < row.length; start += max) {
+        text = row.slice(start, start + max);
+        if (start === 0 && htmlRows[i] !== undefined) html = htmlRows[i];
+        emit();
+      }
+      continue;
+    }
+    if (text.length > 0 && text.length + 1 + row.length > max) emit();
+    text = text.length > 0 ? `${text}\n${row}` : row;
+    if (htmlRows[i] !== undefined) html += htmlRows[i];
+  }
+  if (text.length > 0) emit();
+  return out;
+}
+
+/** Lower bound of the JSON bytes one chunk costs besides its text and metadata: `{"type":…,"element_id":"<32 hex>","text":"","metadata":},`. */
+const CHUNK_OVERHEAD = 90;
+
+/**
+ * Size of the pieces `splitOversized(text, max, overlap)` would build, computed arithmetically before any piece exists:
+ * the first piece holds `max` characters, every later one `max - overlap` new characters after an `overlap` prefix.
+ */
+export function splitSize(length, max, overlap) {
+  const count = length <= max ? 1 : Math.ceil((length - max) / (max - overlap)) + 1;
+  return { count, chars: length + (count - 1) * overlap };
+}
+
+/**
+ * @param elements partitioned elements (PageBreak elements are dropped)
+ * @param options { strategy, maxCharacters, newAfterNChars, overlap, overlapAll, combineUnderNChars, multipageSections, includeOrigElements, responseBudget, chunkSizing? }
+ * @param idOf (text) => element id for a chunk
+ * @returns { ok: true, elements } or { ok: false, code: "RESPONSE_TOO_LARGE", message } when the pieces of a chunk group (a
+ *   lower bound of the response, settled before any piece of that group is built) would pass the response budget. Two things
+ *   multiply: an overlap close to `max_characters` repeats the text up to `max_characters` times, and every piece carries a
+ *   copy of the group's metadata (merged link and emphasis arrays, e-mail recipients, `orig_elements`). The text size is
+ *   arithmetic; the metadata is built once per group — linear in that group's elements — measured once and multiplied by the
+ *   piece count. Per group the work before the check is bounded by the group's input, never by the piece count.
+ */
+export function chunkElements(elements, options, idOf) {
+  const out = [];
+  const includeOrig = options.includeOrigElements;
+  const budget = typeof options.responseBudget === "number" ? options.responseBudget : DEFAULT_BUDGET;
+  // The running chunk size is shared across the files of one request when the caller passes `chunkSizing`, so 32 files
+  // cannot each build a budget's worth of pieces before the response is measured.
+  const sizing = options.chunkSizing ?? { chars: 0, count: 0 };
+  let overflow = null;
+  const account = (count, chars, metaBytes) => {
+    sizing.count += count;
+    sizing.chars += chars + count * metaBytes;
+    const bytes = sizing.chars + sizing.count * CHUNK_OVERHEAD;
+    if (bytes > budget) overflow = { ok: false, code: "RESPONSE_TOO_LARGE", message: `Partition output of at least ${bytes} bytes exceeds the ${budget} byte response limit; upload fewer or smaller files, raise max_characters or lower overlap` };
+    return overflow === null;
+  };
+  const tooMany = () => {
+    overflow = { ok: false, code: "RESPONSE_TOO_LARGE", message: `Partition output with merged link and emphasis metadata exceeds the ${budget} byte response limit; upload fewer or smaller files or raise max_characters` };
+  };
+  for (const section of sections(elements, options)) {
+    if (overflow !== null) break;
+    let texts = [];
+    let originals = [];
+    let currentLength = 0; // texts.join("\n\n").length, maintained incrementally
+    let previousText = null;
+    const flush = () => {
+      if (originals.length === 0) return;
+      const joined = texts.join("\n\n");
+      const prefix = options.overlapAll && options.overlap > 0 && previousText !== null ? previousText.slice(-options.overlap) : "";
+      const size = splitSize(prefix.length + joined.length, options.maxCharacters, options.overlap);
+      const metadata = chunkMetadata(originals, includeOrig, budget);
+      if (metadata === null) return tooMany();
+      if (!account(size.count, size.chars, metadataBytes(metadata))) return;
+      const pieces = splitOversized(prefix + joined, options.maxCharacters, options.overlap);
+      for (let index = 0; index < pieces.length; index += 1) {
+        const piece = { ...metadata };
+        if (index > 0) piece.is_continuation = true;
+        out.push({ type: "CompositeElement", element_id: idOf(pieces[index]), text: pieces[index], metadata: piece });
+      }
+      previousText = pieces[pieces.length - 1];
+      texts = [];
+      originals = [];
+      currentLength = 0;
+    };
+    for (const element of section) {
+      if (element.type === "Table") {
+        flush();
+        if (overflow !== null) break;
+        // Table pieces never overlap (linear in the table text and its markup); at least one piece per max_characters window.
+        const tableSize = splitSize(element.text.length, options.maxCharacters, 0);
+        const metadata = chunkMetadata([element], includeOrig, budget);
+        if (metadata === null) {
+          tooMany();
+          break;
+        }
+        const html = typeof element.metadata.text_as_html === "string" ? element.metadata.text_as_html.length : 0;
+        if (!account(tableSize.count, tableSize.chars + html, metadataBytes(metadata))) break;
+        for (const piece of tableChunks(element, options, idOf, metadata)) out.push(piece);
+        previousText = null;
+        continue;
+      }
+      if (texts.length > 0 && (currentLength >= options.newAfterNChars || currentLength + 2 + element.text.length > options.maxCharacters)) flush();
+      if (overflow !== null) break;
+      currentLength += (texts.length === 0 ? 0 : 2) + element.text.length;
+      texts.push(element.text);
+      originals.push(element);
+    }
+    if (overflow === null) flush();
+  }
+  if (overflow !== null) return overflow;
+  return { ok: true, elements: out };
+}

package/firedrill/tools/unstructured/lib/partition/csv.mjs

@@ -0,0 +1,85 @@
+// RFC 4180 CSV/TSV parsing into a single Table element with `text_as_html`.
+import { escapeHtml } from "./html-util.mjs";
+
+const MAX_ROWS = 10000;
+const MAX_COLUMNS = 200;
+
+/** Returns { rows } or { error }. Quoted fields may contain delimiters, quotes ("") and newlines. */
+export function parseDelimited(text, delimiter) {
+  const rows = [];
+  let row = [];
+  let field = "";
+  let quoted = false;
+  let i = 0;
+  const n = text.length;
+  while (i < n) {
+    const ch = text[i];
+    if (quoted) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        quoted = false;
+        i += 1;
+        continue;
+      }
+      field += ch;
+      i += 1;
+      continue;
+    }
+    if (ch === '"' && field.length === 0) {
+      quoted = true;
+      i += 1;
+      continue;
+    }
+    if (ch === delimiter) {
+      row.push(field);
+      field = "";
+      if (row.length > MAX_COLUMNS) return { error: `File has more than ${MAX_COLUMNS} columns` };
+      i += 1;
+      continue;
+    }
+    if (ch === "\r" && text[i + 1] === "\n") i += 1;
+    if (ch === "\n" || ch === "\r") {
+      row.push(field);
+      field = "";
+      rows.push(row);
+      row = [];
+      if (rows.length > MAX_ROWS) return { error: `File has more than ${MAX_ROWS} rows` };
+      i += 1;
+      continue;
+    }
+    field += ch;
+    i += 1;
+  }
+  if (quoted) return { error: "File is not a valid csv" };
+  if (field.length > 0 || row.length > 0) {
+    row.push(field);
+    rows.push(row);
+  }
+  if (rows.length > MAX_ROWS) return { error: `File has more than ${MAX_ROWS} rows` };
+  return { rows: rows.filter((r) => !(r.length === 1 && r[0].length === 0)) };
+}
+
+export function tableHtml(rows) {
+  let html = "<table>";
+  for (const row of rows) {
+    html += "<tr>";
+    for (const cell of row) html += `<td>${escapeHtml(cell)}</td>`;
+    html += "</tr>";
+  }
+  return `${html}</table>`;
+}
+
+export function tableText(rows) {
+  return rows.map((row) => row.map((cell) => cell.replace(/[\r\n]+/g, " ")).join(" ")).join("\n");
+}
+
+export function partitionDelimited(builder, text, delimiter) {
+  const parsed = parseDelimited(text, delimiter);
+  if (parsed.error !== undefined) return { ok: false, message: parsed.error };
+  if (parsed.rows.length > 0) builder.add("Table", tableText(parsed.rows), { text_as_html: tableHtml(parsed.rows) });
+  return { ok: true };
+}

package/firedrill/tools/unstructured/lib/partition/csvout.mjs

@@ -0,0 +1,25 @@
+// `output_format=text/csv`: a fixed column set, arrays as JSON strings, RFC 4180 quoting.
+
+export const CSV_COLUMNS = [
+  "type", "element_id", "text", "filename", "filetype", "languages", "page_number", "parent_id", "category_depth", "text_as_html", "last_modified",
+  "link_urls", "link_texts", "emphasized_text_contents", "emphasized_text_tags", "is_continuation", "orig_elements",
+];
+
+function cell(value) {
+  if (value === undefined || value === null) return "";
+  const text = typeof value === "string" ? value : typeof value === "object" ? JSON.stringify(value) : String(value);
+  return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text;
+}
+
+export function elementsToCsv(elements) {
+  const lines = [CSV_COLUMNS.join(",")];
+  for (const element of elements) {
+    const row = [];
+    for (const column of CSV_COLUMNS) {
+      if (column === "type" || column === "element_id" || column === "text") row.push(cell(element[column]));
+      else row.push(cell(element.metadata[column]));
+    }
+    lines.push(row.join(","));
+  }
+  return `${lines.join("\r\n")}\r\n`;
+}

package/firedrill/tools/unstructured/lib/partition/elements.mjs

@@ -0,0 +1,171 @@
+// Element construction shared by every document type: ids, metadata, page tracking and the Title parent chain.
+import { sha256Id } from "../sha256.mjs";
+import { uuid } from "../ids.mjs";
+
+export const MAX_TEXT = 100000;
+export const MAX_ELEMENTS = 100000;
+
+export const ELEMENT_TYPES = new Set([
+  "Title", "NarrativeText", "ListItem", "UncategorizedText", "Table", "TableChunk", "CompositeElement", "Image", "Address", "EmailAddress",
+  "CodeSnippet", "Header", "Footer", "PageBreak", "FigureCaption", "Formula", "PageNumber", "Text",
+]);
+
+export class Builder {
+  /**
+   * @param {object} doc { filename, filetype, languages, startingPage, lastModified, includePageBreaks }
+   * @param {object|null} context when `unique_element_ids` is set, ids are UUIDs drawn from context.random
+   */
+  constructor(doc, context) {
+    this.filename = doc.filename;
+    this.filetype = doc.filetype;
+    this.languages = doc.languages;
+    this.page = doc.startingPage;
+    this.lastModified = doc.lastModified;
+    this.includePageBreaks = doc.includePageBreaks;
+    this.context = context;
+    this.elements = [];
+    this.ordinal = 0;
+    this.titles = [];
+    this.extraMetadata = null;
+    this.extraBytes = 0;
+    this.overflow = false;
+    // Running lower bound of the JSON (and CSV) bytes of the elements built so far, shared across the files of one request
+    // when `doc.sizing` is given; passing `doc.byteBudget` stops the build (`tooLarge`) before megabytes of elements exist.
+    this.sizing = doc.sizing ?? { bytes: 0 };
+    this.byteBudget = typeof doc.byteBudget === "number" ? doc.byteBudget : Infinity;
+    this.tooLarge = false;
+    this.fixedBytes = 60 + String(doc.filename).length + String(doc.filetype).length + String(doc.lastModified).length + stringsBytes(doc.languages);
+  }
+
+  /** E-mail metadata copied onto every element; its size is measured once. */
+  setExtraMetadata(extra) {
+    this.extraMetadata = extra;
+    this.extraBytes = valuesBytes(extra);
+  }
+
+  /** Adds `bytes` to the running size; false (and `tooLarge`, `overflow`) once the byte budget is passed. */
+  account(bytes) {
+    this.sizing.bytes += bytes;
+    if (this.sizing.bytes <= this.byteBudget) return true;
+    this.tooLarge = true;
+    this.overflow = true;
+    return false;
+  }
+
+  id(text) {
+    if (this.context !== null) return uuid(this.context);
+    return sha256Id(this.filename, String(this.page), String(this.ordinal), text);
+  }
+
+  /** Appends one element; text longer than MAX_TEXT is split into continuation elements. */
+  add(type, text, meta = {}) {
+    if (this.overflow) return;
+    const pieces = [];
+    const metaBytes = valuesBytes(meta) + this.extraBytes + this.fixedBytes + type.length + 32;
+    if (text.length <= MAX_TEXT) pieces.push(text);
+    else for (let i = 0; i < text.length; i += MAX_TEXT) pieces.push(text.slice(i, i + MAX_TEXT));
+    for (let index = 0; index < pieces.length; index += 1) {
+      if (this.elements.length >= MAX_ELEMENTS) {
+        this.overflow = true;
+        return;
+      }
+      const piece = pieces[index];
+      if (!this.account(metaBytes + piece.length)) return;
+      // `languages` is shared by every element of the document (never mutated; serialised identically).
+      const metadata = { filename: this.filename, filetype: this.filetype, languages: this.languages, page_number: this.page, last_modified: this.lastModified };
+      if (type === "Title") {
+        const depth = Number.isInteger(meta.category_depth) ? meta.category_depth : 0;
+        while (this.titles.length > 0 && this.titles[this.titles.length - 1].depth >= depth) this.titles.pop();
+        if (this.titles.length > 0) metadata.parent_id = this.titles[this.titles.length - 1].id;
+      } else if (this.titles.length > 0 && type !== "PageBreak") metadata.parent_id = this.titles[this.titles.length - 1].id;
+      for (const key in meta) if (Object.hasOwn(meta, key) && meta[key] !== undefined && meta[key] !== null) metadata[key] = meta[key];
+      if (index > 0) metadata.is_continuation = true;
+      if (this.extraMetadata !== null) Object.assign(metadata, this.extraMetadata);
+      const element = { type, element_id: this.id(piece), text: piece, metadata };
+      this.ordinal += 1;
+      this.elements.push(element);
+      if (type === "Title") this.titles.push({ id: element.element_id, depth: metadata.category_depth ?? 0 });
+    }
+  }
+
+  /** A page change; emits a PageBreak element when the caller asked for them. */
+  pageBreak() {
+    if (this.includePageBreaks) this.add("PageBreak", "");
+    this.page += 1;
+  }
+}
+
+/** Lower bound of the encoded bytes of an array of strings: every character is at least one byte, plus quotes or separators. */
+export function stringsBytes(values) {
+  let bytes = 2;
+  if (Array.isArray(values)) for (const value of values) bytes += (typeof value === "string" ? value.length : 1) + 1;
+  return bytes;
+}
+
+/** Lower bound of the encoded bytes of the values of a flat metadata object (strings, scalars and arrays of them). */
+export function valuesBytes(object) {
+  let bytes = 0;
+  if (object === null || typeof object !== "object") return bytes;
+  for (const key of Object.keys(object)) {
+    const value = object[key];
+    if (value === undefined || value === null) continue;
+    bytes += key.length;
+    if (typeof value === "string") bytes += value.length;
+    else if (Array.isArray(value)) bytes += stringsBytes(value);
+    else bytes += 1;
+  }
+  return bytes;
+}
+
+const ADDRESS_RE = /^\d{1,6}[A-Za-z]?\s+[A-Za-z0-9.' -]+\s(Street|St\.?|Avenue|Ave\.?|Road|Rd\.?|Lane|Ln\.?|Way|Blvd\.?|Boulevard|Drive|Dr\.?)\s*$/;
+const EMAIL_RE = /^[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9.-]{1,190}\.[A-Za-z]{2,24}$/;
+const LIST_LINE_RE = /^\s{0,8}(?:[-*•–]|\d{1,4}[.)])\s+\S/;
+
+export const isEmailBlock = (block) => block.length <= 260 && EMAIL_RE.test(block);
+export const isListLine = (line) => LIST_LINE_RE.test(line);
+export const stripListMarker = (line) => line.replace(/^\s{0,8}(?:[-*•–]|\d{1,4}[.)])\s+/, "");
+
+export function isAddressBlock(lines) {
+  if (lines.length < 2 || lines.length > 5) return false;
+  if (!lines.every((line) => line.length > 0 && line.length <= 120)) return false;
+  const street = lines.findIndex((line) => ADDRESS_RE.test(line));
+  return street >= 0 && street <= 1 && street < lines.length - 1;
+}
+
+const WORD_RE = /[A-Za-zÀ-ɏЀ-ӿ]+/g;
+const SMALL_WORDS = new Set(["a", "an", "the", "and", "or", "of", "to", "in", "on", "for", "with", "at", "by", "from", "de", "von", "und"]);
+
+/** A short single line without terminal punctuation in ALL CAPS or Title Case. */
+export function isTitleLine(line) {
+  if (line.length === 0 || line.length > 100 || /[.!?;:,]$/.test(line)) return false;
+  const words = line.match(WORD_RE);
+  if (words === null || words.length === 0) return false;
+  if (line === line.toUpperCase() && line !== line.toLowerCase()) return true;
+  let capitalised = 0;
+  for (const word of words) {
+    const lower = word.toLowerCase();
+    if (word[0] !== lower[0]) capitalised += 1;
+    else if (!SMALL_WORDS.has(lower)) return false;
+  }
+  return capitalised >= 1 && words[0][0] !== words[0][0].toLowerCase();
+}
+
+export const wordCount = (text) => {
+  let count = 0;
+  for (const part of text.split(/\s+/)) if (part.length > 0) count += 1;
+  return count;
+};
+
+/** Plain-text classification of one block (already split from its neighbours by blank lines). */
+export function classifyBlock(block) {
+  const lines = block.split("\n").map((line) => line.trimEnd());
+  if (lines.length === 1) {
+    const line = lines[0].trim();
+    if (isEmailBlock(line)) return "EmailAddress";
+    if (isTitleLine(line)) return "Title";
+  }
+  if (lines.length >= 2 && lines.length <= 5 && isAddressBlock(lines.map((l) => l.trim()))) return "Address";
+  if (lines.every(isListLine)) return "ListItem";
+  if (/[.!?:;]["')\]]?$/.test(block.trimEnd()) && wordCount(block) >= 5) return "NarrativeText";
+  return "UncategorizedText";
+}