@abot-ai/runtime 1.0.0

package/plugins/document-reader/source/index.ts

@@ -0,0 +1,11 @@
+import { defineRuntimePlugin } from "../../../src/plugin-sdk/index.js";
+
+import { createDocumentReaderHandler } from "./handler.js";
+
+export default defineRuntimePlugin((context) =>
+  Object.freeze({
+    handlers: Object.freeze({
+      document_reader: createDocumentReaderHandler(context),
+    }),
+  }),
+);

package/plugins/document-reader/source/ooxml.ts

@@ -0,0 +1,182 @@
+import {
+  archiveMetadata,
+  loadOfficeArchive,
+  readBoundedXml,
+  type LoadedOfficeArchive,
+} from "./archive.js";
+import {
+  BoundedTextAccumulator,
+  normalizeExtractedText,
+} from "./text-accumulator.js";
+import type { ExtractedDocument } from "./types.js";
+
+function decodeCodePoint(value: string, radix: number): string {
+  const parsed = Number.parseInt(value, radix);
+  if (!Number.isSafeInteger(parsed) || parsed < 0 || parsed > 0x10ffff) {
+    return "\ufffd";
+  }
+  return String.fromCodePoint(parsed);
+}
+
+function decodeXml(value: string): string {
+  return value
+    .replace(/&#x([0-9a-f]+);/giu, (_, hex: string) => decodeCodePoint(hex, 16))
+    .replace(/&#(\d+);/gu, (_, decimal: string) => decodeCodePoint(decimal, 10))
+    .replaceAll("&lt;", "<")
+    .replaceAll("&gt;", ">")
+    .replaceAll("&quot;", '"')
+    .replaceAll("&apos;", "'")
+    .replaceAll("&amp;", "&");
+}
+
+function extractTextNodes(xml: string, tag: string): string[] {
+  const values: string[] = [];
+  const pattern = new RegExp(
+    `<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`,
+    "giu",
+  );
+  for (const match of xml.matchAll(pattern)) {
+    values.push(decodeXml((match[1] ?? "").replace(/<[^>]+>/gu, "")));
+  }
+  return values;
+}
+
+function numericSuffix(name: string): number {
+  return Number(name.match(/(\d+)\.xml$/u)?.[1] ?? Number.MAX_SAFE_INTEGER);
+}
+
+function archiveResult(
+  accumulator: BoundedTextAccumulator,
+  archive: LoadedOfficeArchive,
+): ExtractedDocument {
+  return accumulator.finish(
+    Object.freeze({
+      kind: "archive",
+      archive: archiveMetadata(archive),
+    }),
+  );
+}
+
+export async function extractDocxText(
+  bytes: Uint8Array,
+  signal?: AbortSignal,
+): Promise<ExtractedDocument> {
+  const archive = await loadOfficeArchive(bytes);
+  const names = Object.keys(archive.zip.files)
+    .filter((name) =>
+      /^word\/(?:document|header\d+|footer\d+|footnotes|endnotes)\.xml$/u.test(
+        name,
+      ),
+    )
+    .sort((left, right) =>
+      left === "word/document.xml"
+        ? -1
+        : right === "word/document.xml"
+          ? 1
+          : left.localeCompare(right),
+    );
+  const accumulator = new BoundedTextAccumulator();
+  for (const name of names) {
+    const entry = archive.zip.file(name);
+    if (!entry) continue;
+    const xml = await readBoundedXml(entry, archive.budget, signal);
+    const normalized = xml
+      .replace(/<w:tab\b[^>]*\/>/giu, "\t")
+      .replace(/<w:br\b[^>]*\/>/giu, "\n")
+      .replace(/<\/w:tc>/giu, "\t")
+      .replace(/<\/w:p>/giu, "\n");
+    const text = normalizeExtractedText(
+      decodeXml(
+        normalized
+          .replace(/<(?!\/?w:t\b)[^>]+>/giu, "")
+          .replace(/<\/?w:t\b[^>]*>/giu, ""),
+      )
+        .replace(/[ \t]+\n/gu, "\n")
+        .replace(/\n{3,}/gu, "\n\n"),
+    );
+    accumulator.append(text);
+  }
+  return archiveResult(accumulator, archive);
+}
+
+export async function extractPptxText(
+  bytes: Uint8Array,
+  signal?: AbortSignal,
+): Promise<ExtractedDocument> {
+  const archive = await loadOfficeArchive(bytes);
+  const names = Object.keys(archive.zip.files)
+    .filter((name) => /^ppt\/slides\/slide\d+\.xml$/u.test(name))
+    .sort((left, right) => numericSuffix(left) - numericSuffix(right));
+  const accumulator = new BoundedTextAccumulator();
+  for (const [index, name] of names.entries()) {
+    const entry = archive.zip.file(name);
+    if (!entry) continue;
+    const xml = await readBoundedXml(entry, archive.budget, signal);
+    const content = normalizeExtractedText(
+      extractTextNodes(xml, "a:t").filter(Boolean).join("\n"),
+    );
+    accumulator.append(`[Slide ${index + 1}]${content ? `\n${content}` : ""}`);
+  }
+  return archiveResult(accumulator, archive);
+}
+
+function cellValue(
+  attributes: string,
+  body: string,
+  sharedStrings: readonly string[],
+): string {
+  const type = attributes.match(/\bt="([^"]+)"/u)?.[1];
+  const rawValue = body.match(/<v(?:\s[^>]*)?>([\s\S]*?)<\/v>/iu)?.[1];
+  if (type === "s" && rawValue !== undefined) {
+    const index = Number(rawValue);
+    return Number.isSafeInteger(index) && index >= 0
+      ? (sharedStrings[index] ?? rawValue)
+      : rawValue;
+  }
+  if (type === "inlineStr") return extractTextNodes(body, "t").join("");
+  return decodeXml(rawValue ?? "");
+}
+
+export async function extractXlsxText(
+  bytes: Uint8Array,
+  signal?: AbortSignal,
+): Promise<ExtractedDocument> {
+  const archive = await loadOfficeArchive(bytes);
+  const sharedStrings: string[] = [];
+  const sharedEntry = archive.zip.file("xl/sharedStrings.xml");
+  if (sharedEntry) {
+    const sharedXml = await readBoundedXml(sharedEntry, archive.budget, signal);
+    for (const match of sharedXml.matchAll(
+      /<si(?:\s[^>]*)?>([\s\S]*?)<\/si>/giu,
+    )) {
+      sharedStrings.push(extractTextNodes(match[1] ?? "", "t").join(""));
+    }
+  }
+
+  const sheetNames = Object.keys(archive.zip.files)
+    .filter((name) => /^xl\/worksheets\/sheet\d+\.xml$/u.test(name))
+    .sort((left, right) => numericSuffix(left) - numericSuffix(right));
+  const accumulator = new BoundedTextAccumulator();
+  for (const [index, name] of sheetNames.entries()) {
+    const entry = archive.zip.file(name);
+    if (!entry) continue;
+    const xml = await readBoundedXml(entry, archive.budget, signal);
+    const rows: string[] = [];
+    for (const rowMatch of xml.matchAll(
+      /<row(?:\s[^>]*)?>([\s\S]*?)<\/row>/giu,
+    )) {
+      const cells: string[] = [];
+      for (const cellMatch of (rowMatch[1] ?? "").matchAll(
+        /<c([^>]*)>([\s\S]*?)<\/c>/giu,
+      )) {
+        cells.push(
+          cellValue(cellMatch[1] ?? "", cellMatch[2] ?? "", sharedStrings),
+        );
+      }
+      rows.push(cells.join("\t"));
+    }
+    const content = normalizeExtractedText(rows.join("\n"));
+    accumulator.append(`[Sheet ${index + 1}]${content ? `\n${content}` : ""}`);
+  }
+  return archiveResult(accumulator, archive);
+}

package/plugins/document-reader/source/pdf-worker.ts

@@ -0,0 +1,425 @@
+import { createRequire } from "node:module";
+import { Worker } from "node:worker_threads";
+
+import {
+  MAX_PDF_ERROR_MESSAGE_CHARS,
+  MAX_PDF_PAGES,
+  MAX_RETAINED_TEXT_CHARS,
+  PDF_EXTRACTION_TIMEOUT_MS,
+  PDF_MAX_OLD_GENERATION_MB,
+  PDF_MAX_STACK_MB,
+  PDF_MAX_YOUNG_GENERATION_MB,
+  PDF_PAGE_BATCH_SIZE,
+} from "./constants.js";
+import { DocumentReaderError } from "./errors.js";
+import type { ExtractedDocument } from "./types.js";
+
+export type PdfExtractionLimits = Readonly<{
+  maxPages: number;
+  pageBatchSize: number;
+  maxRetainedCharacters: number;
+  timeoutMs: number;
+  maxOldGenerationSizeMb: number;
+  maxYoungGenerationSizeMb: number;
+  stackSizeMb: number;
+}>;
+
+export const PDF_EXTRACTION_LIMITS: PdfExtractionLimits = Object.freeze({
+  maxPages: MAX_PDF_PAGES,
+  pageBatchSize: PDF_PAGE_BATCH_SIZE,
+  maxRetainedCharacters: MAX_RETAINED_TEXT_CHARS,
+  timeoutMs: PDF_EXTRACTION_TIMEOUT_MS,
+  maxOldGenerationSizeMb: PDF_MAX_OLD_GENERATION_MB,
+  maxYoungGenerationSizeMb: PDF_MAX_YOUNG_GENERATION_MB,
+  stackSizeMb: PDF_MAX_STACK_MB,
+});
+
+type PdfWorkerSuccess = Readonly<{
+  kind: "success";
+  text: string;
+  totalCharacters: number;
+  returnedCharacters: number;
+  totalPages: number;
+  processedPages: number;
+}>;
+
+type PdfWorkerFailure = Readonly<{
+  kind: "failure";
+  code?: string;
+  message: string;
+}>;
+
+type PdfWorkerMessage = PdfWorkerSuccess | PdfWorkerFailure;
+
+const CONTROLLED_PDF_ERROR_CODES = new Set([
+  "document_content_invalid",
+  "document_pdf_page_limit",
+  "document_pdf_resource_limit",
+]);
+
+function pdfExtractionWorkerMain(): void {
+  const { parentPort, workerData } =
+    require("node:worker_threads") as typeof import("node:worker_threads");
+  const data = workerData as {
+    bytes: ArrayBuffer;
+    pdfParseEntrypoint: string;
+    limits: {
+      maxPages: number;
+      pageBatchSize: number;
+      maxRetainedCharacters: number;
+      maxErrorMessageCharacters: number;
+    };
+  };
+
+  const fail = (code: string, message: string): never => {
+    const error = new Error(message) as Error & { code: string };
+    error.code = code;
+    throw error;
+  };
+
+  const boundedMessage = (value: unknown): string => {
+    const message = value instanceof Error ? value.message : String(value);
+    return message.slice(0, data.limits.maxErrorMessageCharacters);
+  };
+
+  const run = async (): Promise<void> => {
+    if (!parentPort) {
+      throw new Error("PDF extraction worker has no parent port.");
+    }
+    let parser:
+      | InstanceType<(typeof import("pdf-parse"))["PDFParse"]>
+      | undefined;
+    try {
+      const { PDFParse } = require(
+        data.pdfParseEntrypoint,
+      ) as typeof import("pdf-parse");
+      parser = new PDFParse({
+        data: new Uint8Array(data.bytes),
+        isEvalSupported: false,
+        stopAtErrors: true,
+      });
+      const info = await parser.getInfo();
+      const totalPages = info.total;
+      if (!Number.isSafeInteger(totalPages) || totalPages < 0) {
+        fail("document_content_invalid", "The PDF page index is invalid.");
+      }
+      if (totalPages > data.limits.maxPages) {
+        fail(
+          "document_pdf_page_limit",
+          `The PDF exceeds the ${data.limits.maxPages}-page parsing limit.`,
+        );
+      }
+
+      const retainedParts: string[] = [];
+      let retainedCharacters = 0;
+      let totalCharacters = 0;
+      let processedPages = 0;
+      let hasContent = false;
+
+      const append = (rawValue: string): void => {
+        const value = rawValue.replaceAll("\u0000", "").trim();
+        if (value.length === 0) return;
+        const separator = hasContent ? "\n\n" : "";
+        hasContent = true;
+        const nextTotal = totalCharacters + separator.length + value.length;
+        if (!Number.isSafeInteger(nextTotal)) {
+          fail(
+            "document_pdf_resource_limit",
+            "The PDF text exceeds the supported character range.",
+          );
+        }
+        totalCharacters = nextTotal;
+        let remaining = data.limits.maxRetainedCharacters - retainedCharacters;
+        if (remaining <= 0) return;
+        if (separator.length > 0) {
+          const retainedSeparator = separator.slice(0, remaining);
+          retainedParts.push(retainedSeparator);
+          retainedCharacters += retainedSeparator.length;
+          remaining -= retainedSeparator.length;
+        }
+        if (remaining > 0) {
+          const retainedValue = value.slice(0, remaining);
+          retainedParts.push(retainedValue);
+          retainedCharacters += retainedValue.length;
+        }
+      };
+
+      for (
+        let firstPage = 1;
+        firstPage <= totalPages;
+        firstPage += data.limits.pageBatchSize
+      ) {
+        const pageNumbers = Array.from(
+          {
+            length: Math.min(
+              data.limits.pageBatchSize,
+              totalPages - firstPage + 1,
+            ),
+          },
+          (_, index) => firstPage + index,
+        );
+        const batch = await parser.getText({
+          partial: pageNumbers,
+          pageJoiner: "",
+        });
+        if (!Array.isArray(batch.pages)) {
+          fail(
+            "document_content_invalid",
+            "The PDF page set could not be extracted.",
+          );
+        }
+        for (const page of batch.pages) {
+          if (typeof page?.text !== "string") {
+            fail("document_content_invalid", "The PDF page text is invalid.");
+          }
+          processedPages += 1;
+          append(page.text);
+        }
+      }
+      if (processedPages !== totalPages) {
+        fail(
+          "document_content_invalid",
+          "The PDF page set could not be extracted completely.",
+        );
+      }
+      parentPort.postMessage({
+        kind: "success",
+        text: retainedParts.join(""),
+        totalCharacters,
+        returnedCharacters: retainedCharacters,
+        totalPages,
+        processedPages,
+      } satisfies PdfWorkerSuccess);
+    } catch (error) {
+      const candidateCode =
+        error instanceof Error &&
+        typeof (error as Error & { code?: unknown }).code === "string"
+          ? (error as Error & { code: string }).code
+          : undefined;
+      parentPort.postMessage({
+        kind: "failure",
+        ...(candidateCode ? { code: candidateCode } : {}),
+        message: boundedMessage(error),
+      } satisfies PdfWorkerFailure);
+    } finally {
+      if (parser) {
+        await parser.destroy().catch(() => undefined);
+      }
+      parentPort.close();
+    }
+  };
+
+  void run();
+}
+
+export function resolvePdfParseEntrypoint(pluginEntrypoint: string): string {
+  return createRequire(pluginEntrypoint).resolve("pdf-parse");
+}
+
+export function createPdfExtractionWorker(
+  bytes: Uint8Array,
+  pdfParseEntrypoint: string,
+  limits: PdfExtractionLimits = PDF_EXTRACTION_LIMITS,
+): Worker {
+  const transferredBytes = new ArrayBuffer(bytes.byteLength);
+  new Uint8Array(transferredBytes).set(bytes);
+  return new Worker(`(${pdfExtractionWorkerMain.toString()})()`, {
+    eval: true,
+    transferList: [transferredBytes],
+    workerData: {
+      bytes: transferredBytes,
+      pdfParseEntrypoint,
+      limits: {
+        maxPages: limits.maxPages,
+        pageBatchSize: limits.pageBatchSize,
+        maxRetainedCharacters: limits.maxRetainedCharacters,
+        maxErrorMessageCharacters: MAX_PDF_ERROR_MESSAGE_CHARS,
+      },
+    },
+    resourceLimits: {
+      maxOldGenerationSizeMb: limits.maxOldGenerationSizeMb,
+      maxYoungGenerationSizeMb: limits.maxYoungGenerationSizeMb,
+      stackSizeMb: limits.stackSizeMb,
+    },
+  });
+}
+
+function isSafeNonNegativeInteger(value: unknown): value is number {
+  return Number.isSafeInteger(value) && typeof value === "number" && value >= 0;
+}
+
+function isPdfWorkerMessage(value: unknown): value is PdfWorkerMessage {
+  if (!value || typeof value !== "object") return false;
+  const candidate = value as Partial<PdfWorkerMessage>;
+  if (candidate.kind === "failure") {
+    return (
+      typeof candidate.message === "string" &&
+      candidate.message.length <= MAX_PDF_ERROR_MESSAGE_CHARS &&
+      (candidate.code === undefined || typeof candidate.code === "string")
+    );
+  }
+  return (
+    candidate.kind === "success" &&
+    typeof candidate.text === "string" &&
+    isSafeNonNegativeInteger(candidate.totalCharacters) &&
+    isSafeNonNegativeInteger(candidate.returnedCharacters) &&
+    isSafeNonNegativeInteger(candidate.totalPages) &&
+    isSafeNonNegativeInteger(candidate.processedPages)
+  );
+}
+
+function resourceLimitError(message: string): DocumentReaderError {
+  return new DocumentReaderError("document_pdf_resource_limit", message);
+}
+
+export async function extractPdfInWorker(
+  bytes: Uint8Array,
+  params: Readonly<{
+    pdfParseEntrypoint: string;
+    signal?: AbortSignal;
+    limits?: PdfExtractionLimits;
+  }>,
+): Promise<ExtractedDocument> {
+  const limits = params.limits ?? PDF_EXTRACTION_LIMITS;
+  if (params.signal?.aborted) {
+    throw new DocumentReaderError(
+      "document_read_cancelled",
+      "Document reading was cancelled.",
+    );
+  }
+  const worker = createPdfExtractionWorker(
+    bytes,
+    params.pdfParseEntrypoint,
+    limits,
+  );
+  return new Promise<ExtractedDocument>((resolve, reject) => {
+    let settled = false;
+    const timeout = setTimeout(() => {
+      settle(
+        () =>
+          reject(
+            resourceLimitError(
+              `PDF extraction exceeded the ${limits.timeoutMs}-millisecond time limit.`,
+            ),
+          ),
+        true,
+      );
+    }, limits.timeoutMs);
+    timeout.unref();
+
+    const cleanup = (): void => {
+      clearTimeout(timeout);
+      params.signal?.removeEventListener("abort", onAbort);
+      worker.removeAllListeners();
+    };
+    const settle = (complete: () => void, terminate: boolean): void => {
+      if (settled) return;
+      settled = true;
+      cleanup();
+      if (terminate) void worker.terminate();
+      complete();
+    };
+    const onAbort = (): void => {
+      settle(
+        () =>
+          reject(
+            new DocumentReaderError(
+              "document_read_cancelled",
+              "Document reading was cancelled.",
+            ),
+          ),
+        true,
+      );
+    };
+
+    worker.once("message", (message: unknown) => {
+      if (!isPdfWorkerMessage(message)) {
+        settle(
+          () =>
+            reject(new Error("PDF extraction worker returned invalid data.")),
+          true,
+        );
+        return;
+      }
+      if (message.kind === "failure") {
+        settle(() => {
+          if (message.code && CONTROLLED_PDF_ERROR_CODES.has(message.code)) {
+            reject(new DocumentReaderError(message.code, message.message));
+            return;
+          }
+          reject(new Error(message.message));
+        }, true);
+        return;
+      }
+      const validSuccess =
+        message.text.length === message.returnedCharacters &&
+        message.returnedCharacters <= limits.maxRetainedCharacters &&
+        message.returnedCharacters <= message.totalCharacters &&
+        message.totalPages <= limits.maxPages &&
+        message.processedPages === message.totalPages;
+      if (!validSuccess) {
+        settle(
+          () =>
+            reject(new Error("PDF extraction worker exceeded its contract.")),
+          true,
+        );
+        return;
+      }
+      settle(
+        () =>
+          resolve(
+            Object.freeze({
+              text: message.text,
+              textBounds: Object.freeze({
+                truncated: message.totalCharacters > message.returnedCharacters,
+                totalCharacters: message.totalCharacters,
+                returnedCharacters: message.returnedCharacters,
+                omittedCharacters:
+                  message.totalCharacters - message.returnedCharacters,
+                maxCharacters: limits.maxRetainedCharacters,
+              }),
+              format: Object.freeze({
+                kind: "pdf",
+                totalPages: message.totalPages,
+                processedPages: message.processedPages,
+                maxPages: limits.maxPages,
+              }),
+            }),
+          ),
+        true,
+      );
+    });
+    worker.once("error", (error: Error & { code?: string }) => {
+      settle(
+        () =>
+          reject(
+            error.code === "ERR_WORKER_OUT_OF_MEMORY"
+              ? resourceLimitError("PDF extraction exceeded the memory limit.")
+              : error,
+          ),
+        false,
+      );
+    });
+    worker.once("exit", (code) => {
+      if (code === 0) {
+        settle(
+          () =>
+            reject(new Error("PDF extraction worker exited without a result.")),
+          false,
+        );
+        return;
+      }
+      settle(
+        () =>
+          reject(
+            resourceLimitError(
+              "PDF extraction stopped after exceeding a resource limit.",
+            ),
+          ),
+        false,
+      );
+    });
+    params.signal?.addEventListener("abort", onAbort, { once: true });
+    if (params.signal?.aborted) onAbort();
+  });
+}

package/plugins/document-reader/source/pdf.ts

@@ -0,0 +1,13 @@
+import { extractPdfInWorker } from "./pdf-worker.js";
+import type { ExtractedDocument } from "./types.js";
+
+export function extractPdfText(
+  bytes: Uint8Array,
+  pdfParseEntrypoint: string,
+  signal?: AbortSignal,
+): Promise<ExtractedDocument> {
+  return extractPdfInWorker(bytes, {
+    pdfParseEntrypoint,
+    ...(signal ? { signal } : {}),
+  });
+}

package/plugins/document-reader/source/reader.ts

@@ -0,0 +1,55 @@
+import { readBoundedRegularFile } from "../../../src/plugin-sdk/index.js";
+
+import { MAX_DOCUMENT_BYTES } from "./constants.js";
+import { failDocument, rethrowDocumentIoError } from "./errors.js";
+import { extractDocumentText } from "./extract.js";
+import type { DocumentReference, ExtractedDocument } from "./types.js";
+
+export type ReadDocumentResult = Readonly<{
+  bytes: number;
+  extracted: ExtractedDocument;
+}>;
+
+export async function readDocument(
+  document: DocumentReference,
+  pdfParseEntrypoint: string,
+  signal?: AbortSignal,
+): Promise<ReadDocumentResult> {
+  try {
+    const file = await readBoundedRegularFile(document.absolutePath, {
+      maxBytes: MAX_DOCUMENT_BYTES,
+      rootPath: document.rootPath,
+      ...(signal ? { signal } : {}),
+    });
+    if (!file.ok && file.reason === "not_regular_file") {
+      failDocument(
+        "document_target_not_file",
+        "The requested document is not a regular file.",
+      );
+    }
+    if (!file.ok && file.reason === "too_large") {
+      failDocument(
+        "document_file_too_large",
+        `The requested document exceeds the ${MAX_DOCUMENT_BYTES}-byte parsing limit.`,
+      );
+    }
+    if (!file.ok) {
+      failDocument(
+        "document_changed_during_read",
+        "The requested document changed while it was being read.",
+      );
+    }
+    const bytes = new Uint8Array(file.bytes);
+    return Object.freeze({
+      bytes: bytes.byteLength,
+      extracted: await extractDocumentText(
+        document,
+        bytes,
+        pdfParseEntrypoint,
+        signal,
+      ),
+    });
+  } catch (error) {
+    rethrowDocumentIoError(error);
+  }
+}