codetrellis 0.0.0-stage → 0.2.0

package/src/backend/services/material-reader/read.js

@@ -0,0 +1,430 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var read_exports = {};
+__export(read_exports, {
+  MAX_OUTPUT_CHARS: () => MAX_OUTPUT_CHARS,
+  readMaterialBytes: () => readMaterialBytes
+});
+module.exports = __toCommonJS(read_exports);
+var import_mammoth = __toESM(require("mammoth"));
+var import_zip_reader = require("../../../shared/lib/zip-reader");
+var import_xlsx_xml = require("../../../shared/lib/xlsx-xml");
+var import_pptx_xml = require("../../../shared/lib/pptx-xml");
+var import_locator = require("../../../shared/lib/locator");
+var import_docx_markdown = require("./docx-markdown");
+var import_csv = require("../../../shared/lib/csv");
+const MAX_OUTPUT_CHARS = 1e5;
+const MAX_PART_BYTES = 30 * 1024 * 1024;
+const MAX_TOTAL_BYTES = 100 * 1024 * 1024;
+const SHEET_CAPS = { maxRows: 5e3, maxCols: 200, maxCells: 25e4 };
+const MAX_PDF_PAGES = 500;
+const MAX_SLIDES = 500;
+const TEXT_CONTEXT_LINES = 15;
+class ReadError extends Error {
+}
+const squash = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
+const dash = (a, b) => a === b ? String(a) : `${a}\u2013${b}`;
+function fit(sections, notes, rest) {
+  const out = [];
+  let left = MAX_OUTPUT_CHARS;
+  for (const [i, s] of sections.entries()) {
+    const cost = s.heading.length + s.body.length + 4;
+    if (cost <= left) {
+      out.push(s);
+      left -= cost;
+      continue;
+    }
+    let partly = false;
+    if (left > 2e3) {
+      const heading = `${s.heading} (cut short)`;
+      const cut = s.body.slice(0, left - heading.length - 4);
+      out.push({ heading, body: cut.slice(0, Math.max(0, cut.lastIndexOf("\n"))) });
+      partly = true;
+    }
+    const skipped = sections.slice(partly ? i + 1 : i).map((x) => x.heading);
+    notes.push(`Stopped at about ${MAX_OUTPUT_CHARS.toLocaleString("en-GB")} characters. ${rest(s.heading)}${skipped.length ? ` Not shown: ${skipped.slice(0, 20).join(", ")}${skipped.length > 20 ? ", \u2026" : ""}.` : ""}`);
+    break;
+  }
+  return out;
+}
+function numbered(lines, start, end) {
+  const width = String(end).length;
+  const out = [];
+  for (let n = start; n <= end; n++) out.push(`${String(n).padStart(width)}  ${lines[n - 1]}`);
+  return out.join("\n");
+}
+function readLines(lines, locator, opts) {
+  const total = lines.length;
+  const body = (a, b) => opts.number ? numbered(lines, a, b) : lines.slice(a - 1, b).join("\n");
+  if (locator.lines !== void 0) {
+    const span = (0, import_locator.toSpan)(locator.lines);
+    if (!span || span.start < 1 || span.end < span.start) {
+      throw new ReadError(`lines must be a line number or a span such as "10-20" \u2014 got ${JSON.stringify(locator.lines)}`);
+    }
+    if (span.start > total) throw new ReadError(`${opts.name} has ${total} lines; ${span.start} is past the end`);
+    const end = Math.min(span.end, total);
+    return { where: `lines ${dash(span.start, end)}`, section: { heading: `Lines ${dash(span.start, end)} of ${total}`, body: body(span.start, end) } };
+  }
+  if (locator.text !== void 0) {
+    const quote = squash(locator.text);
+    if (!quote) throw new ReadError("text must be the words to find");
+    const at = lines.findIndex((_, i) => squash(lines.slice(i, i + 8).join(" ")).includes(quote));
+    if (at < 0) throw new ReadError(`${opts.name} does not contain "${locator.text.slice(0, 80)}"`);
+    let first = at;
+    while (first + 1 < total && squash(lines.slice(first + 1, first + 9).join(" ")).includes(quote)) first++;
+    const a = Math.max(1, first + 1 - TEXT_CONTEXT_LINES);
+    const b = Math.min(total, first + 1 + TEXT_CONTEXT_LINES);
+    return { where: `the passage at line ${first + 1}`, section: { heading: `Lines ${dash(a, b)} of ${total}`, body: body(a, b) } };
+  }
+  return { where: null, section: { heading: `Lines 1\u2013${total}`, body: body(1, total) } };
+}
+function readTextFile(req, locator) {
+  const text = new TextDecoder("utf-8", { fatal: false }).decode(req.bytes);
+  const lines = text.split(/\r?\n/);
+  if (lines.length > 1 && lines[lines.length - 1] === "") lines.pop();
+  const notes = [];
+  refuseKeys(locator, req.ext === "csv" ? ["lines", "text", "range"] : ["lines", "text"], req);
+  const { where, section } = readLines(lines, locator, { number: true, name: req.name });
+  const sections = fit([section], notes, () => 'Ask for {"lines": "N-M"} to read further.');
+  return { ok: true, format: "text", where, outline: `${lines.length} lines`, sections, notes };
+}
+function readCsvRange(req, locator) {
+  refuseKeys(locator, ["range"], req);
+  const text = new TextDecoder("utf-8", { fatal: false }).decode(req.bytes);
+  const rows = (0, import_csv.parseCsv)(text);
+  const shape = (0, import_csv.csvShape)(rows);
+  const range = (0, import_locator.parseRange)(String(locator.range).trim());
+  if (!range) throw new ReadError(`"${locator.range}" is not a cell or range \u2014 write it as C14 or A1:F20`);
+  if (range.to.row > shape.rows || range.to.col > shape.cols) {
+    throw new ReadError(`${locator.range} is outside ${req.name}, which has ${shape.rows} rows and ${shape.cols} columns`);
+  }
+  const at = `${(0, import_xlsx_xml.colLetters)(range.from.col)}${range.from.row}:${(0, import_xlsx_xml.colLetters)(range.to.col)}${range.to.row}`;
+  const body = toCsv(rows.slice(range.from.row - 1, range.to.row).map((r) => r.slice(range.from.col - 1, range.to.col)));
+  const notes = [];
+  return {
+    ok: true,
+    format: "csv",
+    where: `cells ${at}`,
+    outline: `${shape.rows} rows, ${shape.cols} columns`,
+    sections: fit([{ heading: `Cells ${at}`, body }], notes, () => 'Ask for a smaller {"range"} to read the rest.'),
+    notes
+  };
+}
+function csvField(v) {
+  return /[",\r\n]/.test(v) ? `"${v.replace(/"/g, '""')}"` : v;
+}
+function toCsv(rows) {
+  let lastRow = rows.length;
+  while (lastRow > 0 && rows[lastRow - 1].every((c) => c === "")) lastRow--;
+  const width = Math.max(0, ...rows.slice(0, lastRow).map((r) => {
+    let w = r.length;
+    while (w > 0 && r[w - 1] === "") w--;
+    return w;
+  }));
+  return rows.slice(0, lastRow).map((r) => Array.from({ length: width }, (_, i) => csvField(r[i] ?? "")).join(",")).join("\n");
+}
+async function readWorkbook(req, locator) {
+  refuseKeys(locator, ["sheet", "range", "text"], req);
+  const view = new DataView(req.bytes.buffer, req.bytes.byteOffset, req.bytes.byteLength);
+  const entries = (0, import_zip_reader.listEntries)(view);
+  const budget = { maxPart: MAX_PART_BYTES, maxTotal: MAX_TOTAL_BYTES, used: 0 };
+  const part = (name) => (0, import_zip_reader.readText)(view, entries, name, budget);
+  const workbook = await part("xl/workbook.xml");
+  if (!workbook) throw new ReadError(`${req.name} is not an Excel workbook \u2014 it has no workbook part`);
+  const refs = (0, import_xlsx_xml.parseWorkbookSheets)(workbook, await part("xl/_rels/workbook.xml.rels") ?? "");
+  const shared = (0, import_xlsx_xml.parseSharedStrings)(await part("xl/sharedStrings.xml") ?? "");
+  const dates = (0, import_xlsx_xml.parseDateStyles)(await part("xl/styles.xml") ?? "");
+  const notes = ["Each cell is the value the spreadsheet app last saved; formulas are not re-calculated."];
+  if (entries.some((e) => /vbaProject\.bin$/i.test(e.name))) notes.push("This workbook carries macros. They were not run.");
+  let range = null;
+  if (locator.range !== void 0) {
+    range = (0, import_locator.parseRange)(String(locator.range).trim());
+    if (!range) throw new ReadError(`"${locator.range}" is not a cell or range \u2014 write it as C14 or A1:F20`);
+  }
+  let chosen = refs;
+  if (locator.sheet !== void 0) {
+    const want = String(locator.sheet);
+    const hit = refs.find((r) => r.name === want) ?? refs.find((r) => r.name.toLowerCase() === want.trim().toLowerCase());
+    if (!hit) throw new ReadError(`${req.name} has no sheet "${want}" \u2014 it has ${refs.map((r) => `"${r.name}"`).join(", ") || "none"}`);
+    chosen = [hit];
+  } else if (range) {
+    chosen = refs.slice(0, 1);
+  }
+  const dims = [];
+  const sections = [];
+  let cellsLeft = SHEET_CAPS.maxCells;
+  for (const ref of refs) {
+    const xml = ref.part ? await part(ref.part) : null;
+    const dimension = xml ? (0, import_xlsx_xml.sheetDimension)(xml) : null;
+    dims.push(`${ref.name}${dimension ? ` (${dimension})` : ""}`);
+    if (!chosen.includes(ref) || !xml) continue;
+    const caps = range ? { maxRows: range.to.row, maxCols: range.to.col, maxCells: cellsLeft } : { ...SHEET_CAPS, maxCells: cellsLeft };
+    const grid = (0, import_xlsx_xml.parseSheetGrid)(xml, shared, dates, caps);
+    cellsLeft = Math.max(0, cellsLeft - grid.cells);
+    let rows = grid.rows;
+    let heading = `Sheet "${ref.name}"`;
+    if (range) {
+      rows = rows.slice(range.from.row - 1, range.to.row).map((r) => r.slice(range.from.col - 1, range.to.col));
+      heading += `, ${(0, import_xlsx_xml.colLetters)(range.from.col)}${range.from.row}:${(0, import_xlsx_xml.colLetters)(range.to.col)}${range.to.row}`;
+    } else if (grid.truncated) {
+      notes.push(`Sheet "${ref.name}" goes past ${SHEET_CAPS.maxRows.toLocaleString("en-GB")} rows or ${SHEET_CAPS.maxCols} columns; ask for {"sheet": "${ref.name}", "range": "A${SHEET_CAPS.maxRows + 1}:\u2026"} to read further.`);
+    }
+    if (!range) heading += " from A1";
+    sections.push({ heading, body: toCsv(rows) });
+  }
+  let shown = sections;
+  let where = locator.sheet !== void 0 || range ? sections.map((s) => s.heading.replace(/^Sheet /, "sheet ").replace(/ from A1$/, "")).join("; ") : null;
+  if (locator.text !== void 0) {
+    const quote = squash(locator.text);
+    if (!quote) throw new ReadError("text must be the words to find");
+    shown = sections.filter((s) => squash(s.body.replace(/"/g, "")).includes(quote) || squash(s.body).includes(quote));
+    if (shown.length === 0) throw new ReadError(`No sheet in ${req.name} contains "${locator.text.slice(0, 80)}"`);
+    where = shown.map((s) => s.heading.replace(/^Sheet /, "sheet ").replace(/ from A1$/, "")).join("; ");
+  }
+  return {
+    ok: true,
+    format: "csv",
+    where,
+    outline: `${refs.length} sheet${refs.length === 1 ? "" : "s"}: ${dims.join(", ")}`,
+    sections: fit(shown, notes, () => 'Ask for one {"sheet"} and a {"range"} to read the rest.'),
+    notes
+  };
+}
+async function readWord(req, locator) {
+  if (req.rendition && locator.lines === void 0) {
+    refuseKeys(locator, ["page", "text", "lines"], req);
+    const read = await readPdfPages(req.rendition, req.name, locator, "page");
+    return pagesReply(read, locator, "page", [
+      "Read from the pages CodeTrellis shows: its rendition of this document, laid out as the person sees it.",
+      `To cite this document, quote it: {"text": "\u2026"}. A {"page"} says where the words are on these pages, but is not checked \u2014 a Word document's pages depend on layout.`,
+      'Ask for {"lines": "1-200"} to read it as markdown instead, with its headings and tables.'
+    ]);
+  }
+  if (locator.page !== void 0) {
+    throw new ReadError(`${req.name} is a Word document; its pages depend on layout. Ask for {"lines"} of what this returns, or {"text"}`);
+  }
+  refuseKeys(locator, ["lines", "text"], req);
+  const view = new DataView(req.bytes.buffer, req.bytes.byteOffset, req.bytes.byteLength);
+  const entries = (0, import_zip_reader.listEntries)(view);
+  if (!entries.some((e) => e.name === "word/document.xml")) {
+    throw new ReadError(`${req.name} is not a Word document \u2014 it has no document part`);
+  }
+  const budget = { maxPart: MAX_PART_BYTES, maxTotal: MAX_TOTAL_BYTES, used: 0 };
+  for (const e of entries) await (0, import_zip_reader.readEntry)(view, e, budget);
+  let images = 0;
+  const result = await import_mammoth.default.convertToHtml({ buffer: Buffer.from(req.bytes.buffer, req.bytes.byteOffset, req.bytes.byteLength) }, {
+    // A document can link an image by path instead of embedding it. Never follow one.
+    externalFileAccess: false,
+    convertImage: import_mammoth.default.images.imgElement(async () => {
+      images++;
+      return { src: "" };
+    })
+  });
+  const markdown = (0, import_docx_markdown.docxHtmlToMarkdown)(result.value);
+  const lines = markdown.split("\n");
+  while (lines.length > 1 && lines[lines.length - 1] === "") lines.pop();
+  const notes = [
+    "Converted to markdown: headings, lists, tables and emphasis survive; page layout does not.",
+    `To cite this document, quote it: {"text": "\u2026"}. Line numbers here are this markdown's, not the document's.`
+  ];
+  if (images) notes.push(`${images} image${images === 1 ? "" : "s"} shown as [image]; open the document in CodeTrellis to see them.`);
+  const { where, section } = readLines(lines, locator, { number: false, name: req.name });
+  return {
+    ok: true,
+    format: "markdown",
+    where,
+    outline: `${lines.length} lines of markdown`,
+    sections: fit([section], notes, () => 'Ask for {"lines": "N-M"} of this markdown to read further.'),
+    notes
+  };
+}
+function pageSpan(locator, count, noun, name) {
+  if (locator.page === void 0) return null;
+  const span = (0, import_locator.toSpan)(locator.page);
+  if (!span || span.start < 1 || span.end < span.start) {
+    throw new ReadError(`page must be a ${noun} number from 1, or a span such as "2-4" \u2014 got ${JSON.stringify(locator.page)}`);
+  }
+  if (span.start > count) throw new ReadError(`${name} has ${count} ${noun}s; ${span.start} is not one of them`);
+  return { start: span.start, end: Math.min(span.end, count) };
+}
+function narrowByText(sections, locator, name) {
+  if (locator.text === void 0) return sections;
+  const quote = squash(locator.text);
+  if (!quote) throw new ReadError("text must be the words to find");
+  const hits = sections.filter((s) => squash(`${s.heading} ${s.body}`).includes(quote));
+  if (hits.length === 0) throw new ReadError(`${name} does not contain "${locator.text.slice(0, 80)}"`);
+  return hits;
+}
+async function readDeck(req, locator) {
+  refuseKeys(locator, ["page", "text"], req);
+  const view = new DataView(req.bytes.buffer, req.bytes.byteOffset, req.bytes.byteLength);
+  const entries = (0, import_zip_reader.listEntries)(view);
+  const names = entries.map((e) => e.name);
+  if (!names.includes("ppt/presentation.xml")) throw new ReadError(`${req.name} is not a PowerPoint deck \u2014 it has no presentation part`);
+  const budget = { maxPart: MAX_PART_BYTES, maxTotal: MAX_TOTAL_BYTES, used: 0 };
+  const order = (0, import_pptx_xml.slideOrder)(
+    await (0, import_zip_reader.readText)(view, entries, "ppt/presentation.xml", budget),
+    await (0, import_zip_reader.readText)(view, entries, "ppt/_rels/presentation.xml.rels", budget),
+    names
+  );
+  const notes = ["Each slide's title and words in presentation order; pictures and charts are not described."];
+  if (req.rendition) {
+    const read = await readPdfPages(req.rendition, req.name, locator, "slide", order.length);
+    if (read) {
+      return pagesReply(read, locator, "slide", [
+        "Read from the slides as CodeTrellis shows them: the words on each, laid out as the person sees it. Pictures are not described."
+      ]);
+    }
+    notes.push("Read from the slides themselves: the view CodeTrellis shows leaves some out (hidden slides), so its pages are not numbered as the slides are.");
+  }
+  const span = pageSpan(locator, order.length, "slide", req.name);
+  const first = span?.start ?? 1;
+  const last = Math.min(span?.end ?? order.length, MAX_SLIDES);
+  if (order.length > MAX_SLIDES && !span) notes.push(`Only the first ${MAX_SLIDES} slides are read.`);
+  const sections = [];
+  for (let n = first; n <= last; n++) {
+    const xml = await (0, import_zip_reader.readText)(view, entries, order[n - 1], budget);
+    const s = xml ? (0, import_pptx_xml.slideText)(n, xml) : { n, title: null, paragraphs: [] };
+    sections.push({ heading: `Slide ${n}${s.title ? ` \u2014 ${s.title}` : ""}`, body: s.paragraphs.join("\n") || "(no words)" });
+  }
+  const shown = narrowByText(sections, locator, req.name);
+  const where = locator.text !== void 0 ? shown.map((s) => s.heading.split(" \u2014 ")[0].toLowerCase()).join(", ") : span ? `slide${span.start === span.end ? "" : "s"} ${dash(span.start, span.end)}` : null;
+  return {
+    ok: true,
+    format: "markdown",
+    where,
+    outline: `${order.length} slide${order.length === 1 ? "" : "s"}`,
+    sections: fit(shown, notes, () => 'Ask for {"page": "N-M"} to read further slides.'),
+    notes
+  };
+}
+let pdfjsLoaded = null;
+function loadPdfJs() {
+  pdfjsLoaded ??= (async () => {
+    globalThis.pdfjsWorker = await Promise.resolve().then(() => __toESM(require("pdfjs-dist/legacy/build/pdf.worker.mjs")));
+    return await Promise.resolve().then(() => __toESM(require("pdfjs-dist/legacy/build/pdf.mjs")));
+  })();
+  return pdfjsLoaded;
+}
+async function readPdfPages(bytes, name, locator, noun, expectPages) {
+  const Noun = noun === "page" ? "Page" : "Slide";
+  const pdfjs = await loadPdfJs();
+  const task = pdfjs.getDocument({
+    data: bytes,
+    isEvalSupported: false,
+    enableXfa: false,
+    disableFontFace: true,
+    useSystemFonts: false,
+    stopAtErrors: false,
+    verbosity: 0
+  });
+  try {
+    let doc;
+    try {
+      doc = await task.promise;
+    } catch (err) {
+      const errName = err.name;
+      throw new ReadError(errName === "PasswordException" ? `${name} is password-protected, so its text cannot be read` : `${name} could not be read as a PDF (${err.message})`);
+    }
+    if (expectPages !== void 0 && doc.numPages !== expectPages) return null;
+    const span = pageSpan(locator, doc.numPages, noun, name);
+    const notes = [];
+    const first = span?.start ?? 1;
+    const last = Math.min(span?.end ?? doc.numPages, first + MAX_PDF_PAGES - 1);
+    if (!span && doc.numPages > MAX_PDF_PAGES) notes.push(`Only the first ${MAX_PDF_PAGES} ${noun}s are read.`);
+    const sections = [];
+    let chars = 0;
+    for (let n = first; n <= last; n++) {
+      const page = await doc.getPage(n);
+      const content = await page.getTextContent();
+      const body = content.items.map((it) => (it.str ?? "") + (it.hasEOL ? "\n" : "")).join("").replace(/[ \t]+\n/g, "\n").trim();
+      page.cleanup();
+      sections.push({ heading: `${Noun} ${n}`, body: body || (noun === "page" ? "(no text on this page \u2014 it may be a scanned image)" : "(no text on this slide \u2014 it may be a picture)") });
+      chars += body.length;
+      if (chars > MAX_OUTPUT_CHARS * 2 && locator.text === void 0) break;
+    }
+    return { count: doc.numPages, span, sections: narrowByText(sections, locator, name), notes };
+  } finally {
+    await task.destroy().catch(() => {
+    });
+  }
+}
+function pagesReply(read, locator, noun, notes) {
+  const { count, span, sections } = read;
+  const all = [...notes, ...read.notes];
+  const where = locator.text !== void 0 ? sections.map((s) => s.heading.toLowerCase()).join(", ") : span ? `${noun}${span.start === span.end ? "" : "s"} ${dash(span.start, span.end)}` : null;
+  return {
+    ok: true,
+    format: "text",
+    where,
+    outline: `${count} ${noun}${count === 1 ? "" : "s"}`,
+    sections: fit(sections, all, () => `Ask for {"page": "N-M"} to read further ${noun}s.`),
+    notes: all
+  };
+}
+async function readPdf(req, locator) {
+  refuseKeys(locator, ["page", "text"], req);
+  const read = await readPdfPages(req.bytes, req.name, locator, "page");
+  return pagesReply(read, locator, "page", []);
+}
+function refuseKeys(locator, allowed, req) {
+  const extra = Object.keys(locator).filter((k) => locator[k] !== void 0 && !allowed.includes(k));
+  if (extra.length) {
+    throw new ReadError(`A .${req.ext} file is read by ${allowed.map((k) => `{${k}}`).join(", ")} \u2014 not by ${extra.map((k) => `{${k}}`).join(", ")}`);
+  }
+}
+async function readMaterialBytes(input) {
+  const view = (b) => new Uint8Array(b.buffer, b.byteOffset, b.byteLength);
+  const req = { ...input, bytes: view(input.bytes), rendition: input.rendition ? view(input.rendition) : null };
+  const locator = req.locator ?? {};
+  try {
+    switch (req.ext) {
+      case "xlsx":
+      case "xlsm":
+        return await readWorkbook(req, locator);
+      case "docx":
+        return await readWord(req, locator);
+      case "pptx":
+        return await readDeck(req, locator);
+      case "pdf":
+        return await readPdf(req, locator);
+      default:
+        if (req.ext === "csv" && locator.range !== void 0) return readCsvRange(req, locator);
+        if (import_locator.TEXT_EXTS.has(req.ext)) return readTextFile(req, locator);
+        return { ok: false, reason: `.${req.ext} files are not read as text` };
+    }
+  } catch (err) {
+    if (err instanceof ReadError || err instanceof import_zip_reader.CapError) return { ok: false, reason: err.message };
+    return { ok: false, reason: `${req.name} could not be read as a .${req.ext} file (${err.message})` };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  MAX_OUTPUT_CHARS,
+  readMaterialBytes
+});

package/src/backend/services/material-reader/reader-host.js

@@ -0,0 +1,202 @@
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var reader_host_exports = {};
+__export(reader_host_exports, {
+  READER_HEAP_MB: () => READER_HEAP_MB,
+  READ_TIMEOUT_MS: () => READ_TIMEOUT_MS,
+  readMaterial: () => readMaterial,
+  readerLaunch: () => readerLaunch,
+  readerScript: () => readerScript,
+  runReader: () => runReader
+});
+module.exports = __toCommonJS(reader_host_exports);
+var import_node_fs = __toESM(require("node:fs"));
+var import_node_path = __toESM(require("node:path"));
+var import_node_child_process = require("node:child_process");
+var import_artefact_content_service = require("../artefact-content-service");
+var import_rendition_service = require("../rendition/rendition-service");
+var import_command = require("../../mcp/connector/command");
+var import_confined_fs = require("../confined-fs");
+var import_locator = require("../../../shared/lib/locator");
+var import_cloud_files = require("../cloud-files");
+const READ_CAPS = {
+  pdf: 100 * 1024 * 1024,
+  pptx: 100 * 1024 * 1024,
+  xlsx: 25 * 1024 * 1024,
+  xlsm: 25 * 1024 * 1024,
+  docx: 25 * 1024 * 1024
+};
+const TEXT_CAP = 20 * 1024 * 1024;
+const IMAGE_CAP = 5 * 1024 * 1024;
+const IMAGE_TYPES = {
+  png: { mimeType: "image/png", magic: (b) => b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) },
+  jpg: { mimeType: "image/jpeg", magic: (b) => b[0] === 255 && b[1] === 216 && b[2] === 255 },
+  jpeg: { mimeType: "image/jpeg", magic: (b) => b[0] === 255 && b[1] === 216 && b[2] === 255 },
+  gif: { mimeType: "image/gif", magic: (b) => b.subarray(0, 4).toString("latin1") === "GIF8" },
+  webp: { mimeType: "image/webp", magic: (b) => b.subarray(0, 4).toString("latin1") === "RIFF" && b.subarray(8, 12).toString("latin1") === "WEBP" }
+};
+const VIDEO_EXTS = /* @__PURE__ */ new Set(["mp4", "webm", "mov"]);
+const READ_TIMEOUT_MS = 3e4;
+const READER_HEAP_MB = 768;
+function readerScript() {
+  const resourcesPath = process.resourcesPath;
+  if ((0, import_command.isPackagedElectron)({ execPath: process.execPath, electronVersion: process.versions.electron }) && resourcesPath) {
+    return import_node_path.default.join(resourcesPath, "reader", "material-reader.mjs");
+  }
+  const source = import_node_path.default.join(__dirname, "child.ts");
+  if (__filename.endsWith(".ts") && import_node_fs.default.existsSync(source)) return source;
+  const inPackage = import_node_path.default.resolve(__dirname, "..", "..", "..", "..", "reader", "material-reader.mjs");
+  if (import_node_fs.default.existsSync(inPackage)) return inPackage;
+  return import_node_path.default.join(process.cwd(), "out", "reader", "material-reader.mjs");
+}
+function readerLaunch(script, opts = {}) {
+  const heap = `--max-old-space-size=${opts.heapMb ?? READER_HEAP_MB}`;
+  const env = process.versions.electron ? { ELECTRON_RUN_AS_NODE: "1" } : {};
+  if (script.endsWith(".ts")) {
+    return { execArgv: [heap, "--import", "tsx"], cwd: import_node_path.default.resolve(__dirname, "..", "..", "..", ".."), env };
+  }
+  return { execArgv: [heap, "--permission", `--allow-fs-read=${script}`], cwd: import_node_path.default.dirname(script), env };
+}
+function runReader(req, opts = {}) {
+  return new Promise((resolve) => {
+    let settled = false;
+    let stderr = "";
+    let outOfMemory = false;
+    let child;
+    try {
+      const script = (0, import_confined_fs.canonicalRoot)(opts.script ?? readerScript());
+      const launch = readerLaunch(script, opts);
+      child = (0, import_node_child_process.fork)(script, [], {
+        execPath: process.execPath,
+        execArgv: launch.execArgv,
+        cwd: launch.cwd,
+        env: launch.env,
+        // pdf.js warns on every load that it cannot draw, which a reader never
+        // asks it to: stdout is dropped, and stderr kept only to tell a heap
+        // limit from any other death.
+        stdio: ["ignore", "ignore", "pipe", "ipc"],
+        serialization: "advanced"
+      });
+    } catch (err) {
+      resolve({ ok: false, reason: `The reader could not start (${err.message})` });
+      return;
+    }
+    child.stderr?.on("data", (d) => {
+      const joined = stderr + d.toString();
+      if (/heap limit|out of memory/i.test(joined)) outOfMemory = true;
+      stderr = joined.slice(-4e3);
+    });
+    const timeoutMs = opts.timeoutMs ?? READ_TIMEOUT_MS;
+    const done = (r) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+      resolve(r);
+    };
+    const timer = setTimeout(() => done({
+      ok: false,
+      timedOut: true,
+      reason: `Reading ${req.name} took longer than ${Math.round(timeoutMs / 1e3)}s and was stopped. Ask for a smaller part of it.`
+    }), timeoutMs);
+    child.once("message", (reply) => done(reply));
+    child.once("error", (err) => done({ ok: false, reason: `${req.name} could not be read (${err.message})` }));
+    child.once("exit", (code, signal) => {
+      const judge = () => setImmediate(() => {
+        if (settled) return;
+        if (!outOfMemory) {
+          console.warn(`[reader] stopped without answering (code ${code}, signal ${signal}): ${stderr.trim().split("\n").slice(-5).join(" | ") || "no output"}`);
+        }
+        done({
+          ok: false,
+          reason: outOfMemory ? `${req.name} needs more memory to read than CodeTrellis allows. Ask for a smaller part of it.` : `${req.name} could not be read (the reader stopped)`
+        });
+      });
+      if (!child.connected) {
+        judge();
+        return;
+      }
+      const cap = setTimeout(judge, 2e3);
+      child.once("disconnect", () => {
+        clearTimeout(cap);
+        judge();
+      });
+    });
+    child.send({ ...req, bytes: req.bytes });
+  });
+}
+async function readBytes(file, cap) {
+  try {
+    return await (0, import_artefact_content_service.readServableCapped)(file, cap);
+  } catch (err) {
+    return err instanceof import_cloud_files.NotOnDeviceError ? "cloud" : "missing";
+  }
+}
+async function readMaterial(attachmentUid, locator, opts = {}) {
+  const file = await (0, import_artefact_content_service.resolveServable)(attachmentUid);
+  if (!file) return { ok: false, status: 404, reason: "No material with that uid is a file CodeTrellis can read" };
+  const name = import_node_path.default.basename(file.rel);
+  const ext = import_node_path.default.extname(file.rel).slice(1).toLowerCase();
+  const base = { name, itemUid: file.itemUid };
+  const hasLocator = !!locator && Object.values(locator).some((v) => v !== void 0);
+  if (VIDEO_EXTS.has(ext)) {
+    return { ok: false, status: 415, reason: `${name} is a video; it can be watched in CodeTrellis but not read as text`, ...base };
+  }
+  if (ext === "xls") {
+    return { ok: false, status: 415, reason: `${name} is an old-format Excel workbook, which is not read as text; saved as .xlsx it can be`, ...base };
+  }
+  const image = IMAGE_TYPES[ext];
+  if (image) {
+    if (hasLocator) return { ok: false, status: 422, reason: `${name} is an image; it is read whole, without a locator`, ...base };
+    const bytes2 = await readBytes(file, IMAGE_CAP);
+    if (bytes2 === "cloud") return { ok: false, status: 409, reason: `${name} is not on this device yet: it is still only in the cloud`, ...base };
+    if (bytes2 === "missing") return { ok: false, status: 404, reason: `${name} is not there, or is not a regular file in the project`, ...base };
+    if (!bytes2) return { ok: false, status: 413, reason: `${name} is larger than the ${IMAGE_CAP / 1024 / 1024} MB an image read returns`, ...base };
+    if (!image.magic(bytes2)) return { ok: false, status: 422, reason: `${name} is named .${ext} but is not that kind of image`, ...base };
+    return { ok: true, kind: "image", ...base, mimeType: image.mimeType, base64: bytes2.toString("base64"), bytes: bytes2.length };
+  }
+  const cap = READ_CAPS[ext] ?? (import_locator.TEXT_EXTS.has(ext) ? TEXT_CAP : 0);
+  if (!cap) return { ok: false, status: 415, reason: `.${ext} files are not read as text`, ...base };
+  const bytes = await readBytes(file, cap);
+  if (bytes === "cloud") return { ok: false, status: 409, reason: `${name} is not on this device yet: it is still only in the cloud`, ...base };
+  if (bytes === "missing") return { ok: false, status: 404, reason: `${name} is not there, or is not a regular file in the project`, ...base };
+  if (!bytes) return { ok: false, status: 413, reason: `${name} is larger than the ${Math.round(cap / 1024 / 1024)} MB CodeTrellis reads of a .${ext} file`, ...base };
+  const rendition = ext === "docx" || ext === "pptx" ? await (0, import_rendition_service.existingRendition)(bytes, ext, opts.engine) : null;
+  const reply = await runReader({ name, ext, bytes, locator, rendition }, opts);
+  if (!reply.ok) return { ok: false, status: "timedOut" in reply ? 504 : 422, reason: reply.reason, ...base };
+  return { ok: true, kind: "text", ...base, reply };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  READER_HEAP_MB,
+  READ_TIMEOUT_MS,
+  readMaterial,
+  readerLaunch,
+  readerScript,
+  runReader
+});