@crewhaus/tool-document-ingest 0.1.0

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@crewhaus/tool-document-ingest",
+  "version": "0.1.0",
+  "type": "module",
+  "description": "M4.3 — IngestDocument tool. Reads .txt/.md/.csv/.json/.yaml inline; PDF/docx/xlsx via pluggable parsers (deferred to operator-provided handlers).",
+  "main": "src/index.ts",
+  "types": "src/index.ts",
+  "exports": {
+    ".": "./src/index.ts"
+  },
+  "scripts": {
+    "test": "bun test src"
+  },
+  "dependencies": {
+    "@crewhaus/errors": "0.0.0",
+    "@crewhaus/tool-builder": "0.0.0",
+    "@crewhaus/tool-catalog": "0.0.0",
+    "zod": "^3.23.8"
+  },
+  "license": "Apache-2.0",
+  "author": {
+    "name": "Max Meier",
+    "email": "max@studiomax.io",
+    "url": "https://studiomax.io"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/crewhaus/factory.git",
+    "directory": "packages/tool-document-ingest"
+  },
+  "homepage": "https://github.com/crewhaus/factory/tree/main/packages/tool-document-ingest#readme",
+  "bugs": {
+    "url": "https://github.com/crewhaus/factory/issues"
+  },
+  "publishConfig": {
+    "access": "restricted"
+  },
+  "files": [
+    "src",
+    "README.md",
+    "LICENSE",
+    "NOTICE"
+  ]
+}

package/src/index.test.ts

@@ -0,0 +1,147 @@
+/**
+ * Tests for tool-document-ingest. Built-in handling for plain text /
+ * structured / tabular formats, plus operator-registered parsers.
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+  DocumentIngestError,
+  clearDocumentParsers,
+  ingestDocument,
+  registerDocumentParser,
+} from "./index";
+
+let tmp: string;
+
+beforeEach(() => {
+  tmp = mkdtempSync(join(tmpdir(), "doc-ingest-"));
+  clearDocumentParsers();
+});
+
+afterEach(() => {
+  rmSync(tmp, { recursive: true, force: true });
+  clearDocumentParsers();
+});
+
+function writeFile(name: string, content: string): string {
+  const path = join(tmp, name);
+  writeFileSync(path, content);
+  return path;
+}
+
+describe("ingestDocument — basics", () => {
+  test("tool flags: read-only, non-destructive, named 'IngestDocument'", () => {
+    expect(ingestDocument.name).toBe("IngestDocument");
+    expect(ingestDocument.readOnly).toBe(true);
+    expect(ingestDocument.destructive).toBe(false);
+  });
+
+  test("throws when file does not exist", async () => {
+    await expect(ingestDocument.execute({ path: "/does/not/exist.txt" })).rejects.toThrow(
+      DocumentIngestError,
+    );
+  });
+
+  test("ingests a plain .txt file with the document envelope", async () => {
+    const path = writeFile("note.txt", "hello\nworld\n");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toContain("<document path=");
+    expect(result).toContain("hello");
+    expect(result).toContain("</document>");
+    expect(result).toContain("metadata:");
+  });
+
+  test("emits line count + ext in metadata for .md", async () => {
+    const path = writeFile("doc.md", "# title\n\nbody.\n");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toMatch(/"ext":"\.md"/);
+    expect(result).toMatch(/"lines":3/);
+  });
+});
+
+describe("ingestDocument — tabular formats", () => {
+  test("counts rows + columns in CSV", async () => {
+    const path = writeFile("data.csv", "a,b,c\n1,2,3\n4,5,6\n");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toMatch(/"rows":3/);
+    expect(result).toMatch(/"columns":3/);
+  });
+
+  test("uses tab delimiter for .tsv", async () => {
+    const path = writeFile("data.tsv", "a\tb\n1\t2\n");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toMatch(/"columns":2/);
+  });
+});
+
+describe("ingestDocument — structured formats", () => {
+  test("validates JSON parse for .json files", async () => {
+    const path = writeFile("config.json", '{"k":1}');
+    const result = await ingestDocument.execute({ path });
+    expect(result).toMatch(/"valid_json":true/);
+  });
+
+  test("flags malformed JSON without throwing", async () => {
+    const path = writeFile("config.json", "{not json}");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toMatch(/"valid_json":false/);
+    expect(result).toContain("parse_error");
+  });
+});
+
+describe("ingestDocument — stubbed extensions", () => {
+  test(".pdf raises with a pointer to registerDocumentParser", async () => {
+    const path = writeFile("doc.pdf", "%PDF-1.4 fake content");
+    await expect(ingestDocument.execute({ path })).rejects.toThrow(
+      /needs a parser registered via registerDocumentParser/,
+    );
+  });
+
+  test(".docx and .xlsx similarly throw with extension-specific message", async () => {
+    const docx = writeFile("a.docx", "fake docx");
+    await expect(ingestDocument.execute({ path: docx })).rejects.toThrow(/"\.docx"/);
+    const xlsx = writeFile("b.xlsx", "fake xlsx");
+    await expect(ingestDocument.execute({ path: xlsx })).rejects.toThrow(/"\.xlsx"/);
+  });
+});
+
+describe("ingestDocument — operator-registered parsers", () => {
+  test("registerDocumentParser overrides built-in handling", async () => {
+    registerDocumentParser(".pdf", async (path) => ({
+      content: `parsed PDF text from ${path}`,
+      metadata: { pages: 42 },
+    }));
+    const path = writeFile("doc.pdf", "fake content");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toContain("parsed PDF text from");
+    expect(result).toMatch(/"pages":42/);
+  });
+
+  test("registerDocumentParser rejects extensions without leading dot", () => {
+    expect(() => registerDocumentParser("pdf", async () => ({ content: "" }))).toThrow(
+      DocumentIngestError,
+    );
+  });
+
+  test("ext matching is case-insensitive", async () => {
+    registerDocumentParser(".PDF", async () => ({ content: "uppercase ext" }));
+    const path = writeFile("doc.pdf", "x");
+    const result = await ingestDocument.execute({ path });
+    expect(result).toContain("uppercase ext");
+  });
+});
+
+describe("ingestDocument — size cap", () => {
+  test("truncates content above maxBytes with a TRUNCATED notice", async () => {
+    const path = writeFile("big.txt", "x".repeat(200));
+    const result = await ingestDocument.execute({ path, maxBytes: 50 });
+    expect(result).toContain("TRUNCATED to 50 bytes");
+  });
+
+  test("default maxBytes is 1MB", () => {
+    const parsed = ingestDocument.inputSchema.parse({ path: "x" });
+    expect(parsed.maxBytes).toBeUndefined();
+  });
+});

package/src/index.ts

@@ -0,0 +1,205 @@
+/**
+ * Catalog R3 — tool-document-ingest. M4.3 of the heavy-hitter plan.
+ *
+ * `IngestDocument(path)` reads a file from the user's host and returns
+ * its content plus structured metadata (line count, byte size, MIME
+ * guess, optional chunks).
+ *
+ * v0 supported formats — handled inline, zero extra deps:
+ *   - .txt, .md, .mdx — plain UTF-8 text
+ *   - .csv, .tsv — text plus row count
+ *   - .json, .yaml, .yml — text plus parse-validation
+ *   - .log, .out — plain text
+ *
+ * Stubbed formats — return a clear "needs operator-registered parser"
+ * error pointing at `registerDocumentParser(ext, parser)`:
+ *   - .pdf, .docx, .doc, .xlsx, .xls, .pptx, .epub
+ *
+ * Why no pdf-parse / mammoth / xlsx deps in v0: those packages weigh
+ * several MB each and have native sub-deps. Operators who need them
+ * can register their own parser via `registerDocumentParser`; the
+ * tool's contract stays the same.
+ *
+ * Security note: the path is a Bun.file() read of a user-controlled
+ * string. The runtime should classify the OUTPUT via boundary-classifier
+ * with origin "user" — files in the user's host are developer-trusted,
+ * but the contents may contain anything (e.g. a prompt-injecting PDF).
+ * Pillar 3 compliance is satisfied by the existing `tool` origin
+ * classifier in runtime-core.
+ */
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { basename, extname, resolve } from "node:path";
+import { CrewhausError } from "@crewhaus/errors";
+import { buildTool } from "@crewhaus/tool-builder";
+import type { RegisteredTool } from "@crewhaus/tool-catalog";
+import { z } from "zod";
+
+export type DocumentParserResult = {
+  readonly content: string;
+  /** Optional structured metadata to surface to the model. */
+  readonly metadata?: Record<string, unknown>;
+};
+
+export type DocumentParser = (path: string) => Promise<DocumentParserResult> | DocumentParserResult;
+
+export class DocumentIngestError extends CrewhausError {
+  override readonly name = "DocumentIngestError";
+  constructor(message: string, cause?: unknown) {
+    super("tool", message, cause);
+  }
+}
+
+const TEXT_EXTENSIONS = new Set([".txt", ".md", ".mdx", ".log", ".out", ".rst"]);
+const TABULAR_EXTENSIONS = new Set([".csv", ".tsv"]);
+const STRUCTURED_EXTENSIONS = new Set([".json", ".yaml", ".yml"]);
+const STUB_EXTENSIONS = new Set([".pdf", ".docx", ".doc", ".xlsx", ".xls", ".pptx", ".epub"]);
+
+const customParsers = new Map<string, DocumentParser>();
+
+/**
+ * Register a parser for a file extension. Operators who need PDF/docx
+ * support wire their preferred library here:
+ *
+ *   import pdfParse from "pdf-parse";
+ *   registerDocumentParser(".pdf", async (path) => {
+ *     const buf = await fs.readFile(path);
+ *     const { text, numpages } = await pdfParse(buf);
+ *     return { content: text, metadata: { pages: numpages } };
+ *   });
+ *
+ * Extension is matched case-insensitively. Must start with ".".
+ */
+export function registerDocumentParser(ext: string, parser: DocumentParser): void {
+  if (!ext.startsWith(".")) {
+    throw new DocumentIngestError(`extension must start with "." (got "${ext}")`);
+  }
+  customParsers.set(ext.toLowerCase(), parser);
+}
+
+/**
+ * Clear all registered parsers. For tests.
+ */
+export function clearDocumentParsers(): void {
+  customParsers.clear();
+}
+
+const inputSchema = z.object({
+  path: z.string().min(1).describe("Absolute or cwd-relative path to the file."),
+  maxBytes: z
+    .number()
+    .int()
+    .positive()
+    .max(10_000_000)
+    .optional()
+    .describe("Hard cap. Default 1MB. Files larger than this are truncated with a notice."),
+});
+
+const DEFAULT_MAX_BYTES = 1_000_000;
+
+export const ingestDocument: RegisteredTool = buildTool({
+  name: "IngestDocument",
+  description:
+    "Read a file from the host filesystem and return its content with structured metadata. Supports plain text, CSV/TSV, JSON, YAML out of the box; PDF/docx/xlsx need an operator-registered parser.",
+  inputSchema,
+  readOnly: true,
+  destructive: false,
+  execute: async (input) => {
+    const abs = resolve(input.path);
+    if (!existsSync(abs)) {
+      throw new DocumentIngestError(`file not found: ${abs}`);
+    }
+    const stat = statSync(abs);
+    if (!stat.isFile()) {
+      throw new DocumentIngestError(`not a regular file: ${abs}`);
+    }
+    const ext = extname(abs).toLowerCase();
+    const maxBytes = input.maxBytes ?? DEFAULT_MAX_BYTES;
+
+    // Operator-registered parser takes priority over built-in handling.
+    const customParser = customParsers.get(ext);
+    if (customParser !== undefined) {
+      const result = await customParser(abs);
+      return renderResult({
+        path: abs,
+        content: result.content,
+        metadata: { ext, size: stat.size, ...result.metadata },
+        maxBytes,
+      });
+    }
+
+    if (STUB_EXTENSIONS.has(ext)) {
+      throw new DocumentIngestError(
+        `extension "${ext}" needs a parser registered via registerDocumentParser(). See @crewhaus/tool-document-ingest README for the pdf-parse / mammoth / xlsx setup.`,
+      );
+    }
+
+    if (
+      TEXT_EXTENSIONS.has(ext) ||
+      TABULAR_EXTENSIONS.has(ext) ||
+      STRUCTURED_EXTENSIONS.has(ext) ||
+      ext === ""
+    ) {
+      const raw = readFileSync(abs, "utf-8");
+      const metadata: Record<string, unknown> = {
+        ext: ext || "(none)",
+        size: stat.size,
+        lines: countLines(raw),
+      };
+      if (TABULAR_EXTENSIONS.has(ext)) {
+        const delim = ext === ".tsv" ? "\t" : ",";
+        const lines = raw.split("\n").filter((l) => l.length > 0);
+        metadata["rows"] = lines.length;
+        metadata["columns"] = (lines[0] ?? "").split(delim).length;
+      }
+      if (STRUCTURED_EXTENSIONS.has(ext)) {
+        if (ext === ".json") {
+          try {
+            JSON.parse(raw);
+            metadata["valid_json"] = true;
+          } catch (err) {
+            metadata["valid_json"] = false;
+            metadata["parse_error"] = (err as Error).message.slice(0, 200);
+          }
+        }
+      }
+      return renderResult({ path: abs, content: raw, metadata, maxBytes });
+    }
+
+    throw new DocumentIngestError(
+      `extension "${ext}" is not handled by built-in ingest. Register a parser via registerDocumentParser("${ext}", …), or rename to .txt/.md if it's plain text.`,
+    );
+  },
+});
+
+function renderResult(args: {
+  readonly path: string;
+  readonly content: string;
+  readonly metadata: Record<string, unknown>;
+  readonly maxBytes: number;
+}): string {
+  let content = args.content;
+  let truncated = false;
+  if (Buffer.byteLength(content, "utf8") > args.maxBytes) {
+    content = Buffer.from(content, "utf-8").subarray(0, args.maxBytes).toString("utf-8");
+    truncated = true;
+  }
+  const lines: string[] = [
+    `<document path="${args.path}" name="${basename(args.path)}">`,
+    `metadata: ${JSON.stringify(args.metadata)}${truncated ? ` (TRUNCATED to ${args.maxBytes} bytes)` : ""}`,
+    "---",
+    content,
+    "</document>",
+  ];
+  return lines.join("\n");
+}
+
+function countLines(s: string): number {
+  if (s.length === 0) return 0;
+  let n = 1;
+  for (const ch of s) {
+    if (ch === "\n") n++;
+  }
+  // A trailing newline is a line terminator, not a 4th empty line.
+  if (s.endsWith("\n")) n--;
+  return n;
+}