@bevel-software/platform-core-backend 0.11.2 → 0.12.0

package/src/modules/workspace/file-readers/file-reader.ts

@@ -0,0 +1,104 @@
+/**
+ * The FileReader contract: ONE interface every read-path consumer goes
+ * through, with per-extension implementations picked by a single registry
+ * lookup — the same shape as the frontend's renderer registry.
+ *
+ * Adding a file format is ONE new reader entry in `createFileReaderRegistry`
+ * (file-reader.registry.ts). The three consumers — `read_file`, `grep` and the
+ * write-refusal in workspace.tools.ts — dispatch through `readerFor` and never
+ * branch on extensions themselves.
+ */
+import { fileExtension } from './doc-extract.types.js';
+
+/**
+ * What reading a file produces, before the tool layer shapes it for MCP:
+ * text (the honest `[extracted text of …]` marker is ALREADY prepended for
+ * extractions, so line numbers match between read_file and grep), an image
+ * payload (the tool layer wraps it in the `McpImageResult` sentinel), or a
+ * refusal — legacy formats, oversized images, unreadable binary, corrupt
+ * documents — whose message is returned as the file's text content.
+ */
+export type ReadResult =
+  | { kind: 'text'; text: string }
+  | { kind: 'image'; data: string; mimeType: string; note: string }
+  | { kind: 'refusal'; message: string };
+
+/**
+ * `path` as interpolated into a ONE-LINE notice or refusal: CR/LF are shown
+ * as escapes rather than obeyed (the same rule as `extractionMarker`), so a
+ * filename cannot forge extra output lines.
+ */
+export function displayPath(path: string): string {
+  return path.replace(/[\r\n]/g, (c) => (c === '\r' ? '\\r' : '\\n'));
+}
+
+/**
+ * Any text interpolated into a ONE-LINE notice — a path, or an extractor's
+ * own failure message, which quotes bytes from the file and can therefore
+ * carry CR/LF of the document's choosing.
+ */
+export const oneLine = displayPath;
+
+/** A per-format file reader. Register implementations in `createFileReaderRegistry`. */
+export interface FileReader {
+  /** The extensions this reader owns — lowercase, with the dot. Empty for the default (fallback) reader. */
+  readonly extensions: readonly string[];
+  /** Read `bytes` (read at `path`) into what the read tools return. Must not throw for bad file content. */
+  read(bytes: Buffer, path: string): Promise<ReadResult>;
+  /**
+   * Text `grep` may search, or null when there is nothing (cheaply) searchable.
+   * Default readers: the decoded text content (TextReader; null on NUL bytes) /
+   * the CACHED extraction (DocumentReader — cold extraction is grep's call,
+   * under its per-walk budget) / absent = never greppable (ImageReader).
+   */
+  greppableText?(bytes: Buffer, path: string): Promise<string | null>;
+  /** May the agent TEXT-editing tools (write_file/write_files/edit_file) touch this file? */
+  readonly textEditable: boolean;
+  /**
+   * Format-specific copy for the write-refusal thrown when `textEditable` is
+   * false (see `assertNotDocumentEdit` in workspace.tools.ts). Absent = the
+   * generic extracted-text/round-trip explanation.
+   */
+  editRefusal?(path: string): string;
+  /**
+   * Refusal for OVERWRITING what this file ALREADY holds, or null to allow.
+   * Consulted only when `textEditable` is true: that answer covers the
+   * FORMAT, this one covers the CONTENT. The fallback reader needs it because
+   * an extensionless file may hold anything — `read_file` refuses binary
+   * content, and a write gate that did not ask would let an agent overwrite
+   * bytes it was never allowed to see.
+   */
+  editRefusalForExisting?(bytes: Buffer, path: string): string | null;
+}
+
+/**
+ * Extension → reader, with a default fallback (the text reader). Built once
+ * per deployment by `createFileReaderRegistry`; `readerFor` is the ONE lookup
+ * every consumer routes through.
+ */
+export class FileReaderRegistry {
+  private readonly byExtension = new Map<string, FileReader>();
+
+  constructor(
+    readers: readonly FileReader[],
+    private readonly fallback: FileReader,
+  ) {
+    for (const reader of readers) {
+      for (const raw of reader.extensions) {
+        // Lookups arrive lowercased (`fileExtension`), so a key stored with any
+        // other casing could never match and its reader would silently fall back
+        // to the default — routing a .PDF-registered reader nowhere.
+        const ext = raw.toLowerCase();
+        // Duplicate claims fail LOUDLY: were the later registration to win
+        // silently, adding a format could disable an existing reader.
+        if (this.byExtension.has(ext)) throw new Error(`two file readers claim the extension "${ext}"`);
+        this.byExtension.set(ext, reader);
+      }
+    }
+  }
+
+  /** The reader owning `path`'s extension (lowercased by `fileExtension`), or the fallback. */
+  readerFor(path: string): FileReader {
+    return this.byExtension.get(fileExtension(path)) ?? this.fallback;
+  }
+}

package/src/modules/workspace/file-readers/image-read.ts

@@ -0,0 +1,122 @@
+/**
+ * `read_file` image support: which files come back as a native MCP image
+ * content block instead of text, the size cap that keeps the encoded payload
+ * under what Claude-family clients accept, and the self-describing note that
+ * rides beside the image so the transcript still says what was read.
+ *
+ * Deliberately dependency-free: dimensions are parsed straight from the
+ * container headers (PNG/GIF/JPEG) rather than through a native image library.
+ * No downscaling in this increment — `sharp` is a heavy native dependency, so
+ * an oversized image gets an honest refusal telling the caller to downscale
+ * locally or upload a smaller export.
+ */
+import { fileExtension } from './doc-extract.types.js';
+
+/** The image types `read_file` returns as a native MCP image content block. `.svg` is TEXT and stays on the text path. */
+const IMAGE_MIME_BY_EXT: Record<string, string> = {
+  '.png': 'image/png',
+  '.jpg': 'image/jpeg',
+  '.jpeg': 'image/jpeg',
+  // GIF passes through whole under the same cap; clients render the first frame.
+  '.gif': 'image/gif',
+  '.webp': 'image/webp',
+};
+
+/** The extensions the `ImageReader` registers for — exactly the map above. */
+export const IMAGE_EXTENSIONS: readonly string[] = Object.keys(IMAGE_MIME_BY_EXT);
+
+/** Is this a file `read_file` should return as an image? */
+export function isImageFile(path: string): boolean {
+  return fileExtension(path) in IMAGE_MIME_BY_EXT;
+}
+
+/** The MCP mime type for an image path — call only after `isImageFile`. */
+export function imageMimeType(path: string): string {
+  return IMAGE_MIME_BY_EXT[fileExtension(path)]!;
+}
+
+/**
+ * Cap on the RAW bytes of an image `read_file` will return.
+ *
+ * The arithmetic: Claude-family clients reject a base64 image payload around
+ * 5 MB encoded (5 * 1024 * 1024 = 5,242,880 chars). Base64 inflates by 4/3,
+ * so the raw ceiling for that limit is 5,242,880 * 3/4 = 3,932,160 bytes. We
+ * cap at 3.5 MiB raw = 3,670,016 bytes, which encodes to
+ * ceil(3,670,016 / 3) * 4 = 4,893,356 base64 chars (~4.67 MiB) — under the
+ * reject line with headroom for the JSON envelope the sentinel travels in.
+ */
+export const IMAGE_MAX_RAW_BYTES = 3.5 * 1024 * 1024; // = 3,670,016
+
+/** Width×height parsed from the container header, when the format makes that cheap. */
+export interface ImageDimensions {
+  width: number;
+  height: number;
+}
+
+/**
+ * Best-effort dimensions straight from the header bytes: PNG (IHDR), GIF
+ * (logical screen descriptor), JPEG (SOFn scan). WebP is left dimensionless —
+ * its three sub-formats (VP8/VP8L/VP8X) each encode size differently, and the
+ * note is honest without it. Returns undefined on anything unexpected; never
+ * throws (a corrupt image must still be delivered or refused, not 500).
+ */
+export function imageDimensions(bytes: Buffer): ImageDimensions | undefined {
+  try {
+    // PNG: 8-byte signature, then the IHDR chunk — width/height at 16/20 (BE).
+    if (bytes.length >= 24 && bytes.readUInt32BE(0) === 0x89504e47 && bytes.toString('latin1', 12, 16) === 'IHDR') {
+      return { width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
+    }
+    // GIF: the FULL "GIF87a"/"GIF89a" signature, then the logical screen
+    // size at 6/8 (LE) — a bare "GIF" prefix is not a GIF header.
+    if (bytes.length >= 10) {
+      const sig = bytes.toString('latin1', 0, 6);
+      if (sig === 'GIF87a' || sig === 'GIF89a') {
+        return { width: bytes.readUInt16LE(6), height: bytes.readUInt16LE(8) };
+      }
+    }
+    // JPEG: walk the marker segments to the first SOFn frame header.
+    if (bytes.length >= 4 && bytes[0] === 0xff && bytes[1] === 0xd8) {
+      let i = 2;
+      while (i + 9 < bytes.length) {
+        if (bytes[i] !== 0xff) {
+          i += 1; // stray fill byte — resync
+          continue;
+        }
+        const marker = bytes[i + 1]!;
+        // Standalone markers (no length field): padding, restarts, SOI/EOI.
+        if (marker === 0xff) {
+          i += 1;
+          continue;
+        }
+        if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd9)) {
+          i += 2;
+          continue;
+        }
+        // SOFn (C0–CF except the non-frame C4/C8/CC): height at +5, width at +7.
+        if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+          return { height: bytes.readUInt16BE(i + 5), width: bytes.readUInt16BE(i + 7) };
+        }
+        i += 2 + bytes.readUInt16BE(i + 2);
+      }
+    }
+  } catch {
+    // Truncated/corrupt header — the note simply omits dimensions.
+  }
+  return undefined;
+}
+
+/** The one-line note emitted as a text block beside the image, so the transcript names what it shows. */
+export function imageNote(path: string, mime: string, sizeBytes: number, dims: ImageDimensions | undefined): string {
+  const dimsPart = dims ? `, ${dims.width}×${dims.height} px` : '';
+  return `[image: ${path} — ${mime}, ${sizeBytes} bytes${dimsPart}]`;
+}
+
+/** The honest refusal for an image over {@link IMAGE_MAX_RAW_BYTES} — returned as the file's text content. */
+export function oversizedImageNotice(path: string, mime: string, sizeBytes: number): string {
+  return (
+    `[${path} is a ${mime} image of ${sizeBytes} bytes — too large to return over MCP. The cap is ` +
+    `${IMAGE_MAX_RAW_BYTES} bytes (3.5 MiB) of raw image data, because base64 encoding inflates it by 4/3 and ` +
+    'Claude-family clients reject images near 5 MB encoded. Downscale the image locally or upload a smaller ' +
+    'export, then read that file instead.]'
+  );
+}

package/src/modules/workspace/file-readers/image-reader.ts

@@ -0,0 +1,39 @@
+import type { FileReader, ReadResult } from './file-reader.js';
+import {
+  IMAGE_EXTENSIONS,
+  IMAGE_MAX_RAW_BYTES,
+  imageDimensions,
+  imageMimeType,
+  imageNote,
+  oversizedImageNotice,
+} from './image-read.js';
+
+/**
+ * FileReader over the image types `read_file` returns as a native MCP image
+ * content block (`.svg` is text and stays on the text path). Within the raw
+ * cap the read yields the picture itself (base64 + mime + the self-describing
+ * note); over it, the honest downscale refusal — see image-read.ts for the
+ * cap arithmetic and header-parsing details.
+ *
+ * No `greppableText`: a picture is never text-searchable. And images stay
+ * `textEditable` — the write tools only refuse formats whose reads are lossy
+ * EXTRACTIONS (documents); an image read is the real bytes, and image writes
+ * were never gated.
+ */
+export class ImageReader implements FileReader {
+  readonly extensions: readonly string[] = IMAGE_EXTENSIONS;
+  readonly textEditable = true;
+
+  async read(bytes: Buffer, path: string): Promise<ReadResult> {
+    const mime = imageMimeType(path);
+    if (bytes.length > IMAGE_MAX_RAW_BYTES) {
+      return { kind: 'refusal', message: oversizedImageNotice(path, mime, bytes.length) };
+    }
+    return {
+      kind: 'image',
+      data: bytes.toString('base64'),
+      mimeType: mime,
+      note: imageNote(path, mime, bytes.length, imageDimensions(bytes)),
+    };
+  }
+}

package/src/modules/workspace/file-readers/odf-text.ts

@@ -0,0 +1,123 @@
+import AdmZip from 'adm-zip';
+import { Parser } from 'htmlparser2';
+import { attrByLocalName, localElementBlocks, localName, zipEntryOversize } from './ooxml-text.js';
+
+/**
+ * ODF (OpenDocument) text helpers shared by the odt/odp/ods extractors.
+ *
+ * Elements are matched by LOCAL name — `p`, not `text:p`. A prefix is the
+ * document's own choice, and naming `text:` literally meant an ODF file that
+ * defaulted the namespace, or bound it to any other prefix, extracted as
+ * empty. This module used to answer that by REWRITING every non-conventional
+ * prefix across the whole of `content.xml` before scanning it, a pass that had
+ * to be bounded against crafted alias lists and could still corrupt ordinary
+ * paragraph text that happened to look like an alias. Matching local names
+ * needs none of it.
+ */
+
+/**
+ * The text of one ODF paragraph (`<text:p>` / `<text:h>` content). Character
+ * data is concatenated with NO separator — formatting runs (`<text:span>`)
+ * split words exactly like OOXML runs do, so ignoring the span tags joins them
+ * back. Three ODF whitespace elements are REAL characters and render as such:
+ *
+ *  - `<text:tab/>`          → a tab
+ *  - `<text:line-break/>`   → a newline
+ *  - `<text:s text:c="N"/>` → N spaces (no `text:c` attribute = 1)
+ *
+ * Read through the parser, so entities decode, CDATA sections are the text
+ * they hold rather than markup, and a comment is not paragraph content.
+ */
+/**
+ * Cap on the characters the whitespace ELEMENTS may add to ONE paragraph.
+ * Each `<text:s text:c="N"/>` is clamped on its own below, but nothing else
+ * bounds how many such elements a paragraph may hold — a content.xml well
+ * under the 50 MB part limit could still expand to gigabytes of spaces. The
+ * budget caps the SUM per paragraph; real layout whitespace is nowhere near it.
+ */
+const MAX_PARAGRAPH_WHITESPACE_CHARS = 10_000;
+
+export function odfParagraphText(paragraphXml: string): string {
+  let out = '';
+  let whitespaceBudget = MAX_PARAGRAPH_WHITESPACE_CHARS;
+  const parser = new Parser(
+    {
+      onopentag(name, attributes) {
+        const local = localName(name);
+        if (local === 'tab') {
+          if (whitespaceBudget > 0) {
+            out += '\t';
+            whitespaceBudget--;
+          }
+        } else if (local === 'line-break') {
+          if (whitespaceBudget > 0) {
+            out += '\n';
+            whitespaceBudget--;
+          }
+        } else if (local === 's') {
+          const raw = attrByLocalName(attributes, 'c');
+          const count = raw !== undefined ? parseInt(raw, 10) : 1;
+          // Bounded defensively — a corrupt attribute must not balloon the
+          // extraction — and again by the per-paragraph budget above.
+          const spaces = Math.min(Number.isFinite(count) ? Math.min(Math.max(count, 0), 1000) : 1, whitespaceBudget);
+          out += ' '.repeat(spaces);
+          whitespaceBudget -= spaces;
+        }
+      },
+      ontext(text) {
+        out += text;
+      },
+    },
+    { xmlMode: true, decodeEntities: true },
+  );
+  parser.write(paragraphXml);
+  parser.end();
+  return out;
+}
+
+/**
+ * The `<text:p>` / `<text:h>` paragraph bodies of an XML fragment, in DOCUMENT
+ * order (headings interleaved with paragraphs, as written). Self-closing
+ * elements (an empty paragraph) yield ''. `<text:page-number>` and friends do
+ * not count as paragraphs.
+ */
+export function odfParagraphBlocks(xml: string): string[] {
+  return localElementBlocks(xml, ['p', 'h']).map((e) => e.body ?? '');
+}
+
+
+/** Non-empty paragraph texts of an ODF fragment — what odp slides/notes render. */
+export function odfParagraphLines(xml: string): string[] {
+  const out: string[] = [];
+  for (const p of odfParagraphBlocks(xml)) {
+    const text = odfParagraphText(p);
+    if (text.trim() !== '') out.push(text);
+  }
+  return out;
+}
+
+/**
+ * `content.xml` of an ODF package or a typed could-not-parse failure message
+ * (`kind` names the extension for the message, e.g. '.odt').
+ *
+ * Bounded: the entry's DECLARED uncompressed size is checked against
+ * `MAX_DOC_PART_BYTES` (50 MB) before anything inflates, so a zip bomb is a
+ * typed refusal, never an allocation.
+ */
+export function readOdfContentXml(
+  bytes: Buffer,
+  kind: '.odt' | '.odp' | '.ods',
+): { ok: true; xml: string } | { ok: false; message: string } {
+  try {
+    const zip = new AdmZip(bytes);
+    const entry = zip.getEntry('content.xml');
+    if (!entry) {
+      return { ok: false, message: `could not be parsed as a ${kind} (no content.xml inside the archive)` };
+    }
+    const oversize = zipEntryOversize(entry);
+    if (oversize) return { ok: false, message: `could not be extracted as a ${kind} (${oversize})` };
+    return { ok: true, xml: entry.getData().toString('utf8') };
+  } catch (err) {
+    return { ok: false, message: `could not be parsed as a ${kind} (${(err as Error).message})` };
+  }
+}