@gresmcp/mcp 1.2.0 → 1.3.0

package/dist/cli.js

@@ -325,8 +325,8 @@ async function probeModel(baseUrl, model) {
 init_config();
 
 // src/ingest.ts
-import { promises as fs } from "fs";
-import path from "path";
+import { promises as fs2 } from "fs";
+import path2 from "path";
 
 // src/chunk.ts
 var HEADING_RE = /^(#{1,6})\s+(.+)$/;
@@ -390,8 +390,8 @@ function chunkMarkdown(text, opts = {}) {
     const m = HEADING_RE.exec(line);
     if (m) {
       const level = m[1].length;
-      const path3 = sections[sections.length - 1].path;
-      const parent = path3.slice(0, level - 1);
+      const path4 = sections[sections.length - 1].path;
+      const parent = path4.slice(0, level - 1);
       sections.push({ path: [...parent, m[2].trim()], body: [line] });
     } else {
       sections[sections.length - 1].body.push(line);
@@ -576,9 +576,337 @@ function extractPage(html, opts = {}) {
   };
 }
 
+// src/doc.ts
+import mammoth from "mammoth";
+import {
+  Doc97File,
+  InvalidKeyError,
+  OfficeFile,
+  OleFileIO,
+  isEncrypted
+} from "office-crypto";
+import WordExtractor from "word-extractor";
+var DocError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "DocError";
+  }
+  kind;
+};
+var OLE_MAGIC = [208, 207, 17, 224, 161, 177, 26, 225];
+function hasMagic(data, magic) {
+  if (data.length < magic.length) return false;
+  return magic.every((byte, i) => data[i] === byte);
+}
+function isOle(data) {
+  return hasMagic(data, OLE_MAGIC);
+}
+function isZip(data) {
+  return data.length >= 4 && data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
+}
+function toDocError(err, prefix) {
+  if (err instanceof DocError) return err;
+  if (err instanceof InvalidKeyError) {
+    return new DocError("wrong password for encrypted Word document", "password");
+  }
+  const message = err instanceof Error ? err.message : String(err);
+  return new DocError(`${prefix}: ${message.slice(0, 200)}`, "corrupt");
+}
+function requirePassword() {
+  return new DocError("password-protected Word document (use --file-password)", "password");
+}
+function oleHasStream(data, name) {
+  try {
+    return new OleFileIO(data).exists(name);
+  } catch {
+    return false;
+  }
+}
+function decryptOoxml(data, password) {
+  try {
+    if (!password) throw requirePassword();
+    const file = OfficeFile(data);
+    file.loadKey({ password, verifyPassword: true });
+    return file.decrypt();
+  } catch (err) {
+    throw toDocError(err, "cannot decrypt Word document");
+  }
+}
+function decryptDoc97(data, password) {
+  try {
+    if (!password) throw requirePassword();
+    const file = new Doc97File(new OleFileIO(data));
+    file.loadKey({ password });
+    return file.decrypt();
+  } catch (err) {
+    throw toDocError(err, "cannot decrypt Word document");
+  }
+}
+async function docxToMarkdown(data, encrypted) {
+  let html;
+  try {
+    const result = await mammoth.convertToHtml({ buffer: Buffer.from(data) });
+    html = result.value;
+  } catch (err) {
+    throw toDocError(err, "cannot parse docx");
+  }
+  html = html.replace(/<img\b[^>]*>/gi, "").replace(
+    /<t([dh])\b([^>]*)>([\s\S]*?)<\/t\1>/gi,
+    (_m, letter, attrs, inner) => `<t${letter}${attrs}>${inner.replace(/<\/?p(?:\s[^>]*)?>/gi, "").replace(/<br(?:\s[^>]*)?>/gi, " ")}</t${letter}>`
+  ).replace(
+    /<table\b[^>]*>([\s\S]*?)<\/table>/gi,
+    (m, inner) => m.replace(
+      /(<tr\b[^>]*>)([\s\S]*?)(<\/tr>)/i,
+      (row, open, cells) => open + cells.replace(/<td\b/gi, "<th").replace(/<\/td>/gi, "</th>") + "</tr>"
+    )
+  );
+  const { markdown, title } = htmlToMarkdown(html);
+  return {
+    markdown: markdown.replace(/\n{3,}/g, "\n\n").trim(),
+    title: title || void 0,
+    format: "docx",
+    ...encrypted ? { encrypted: true } : {}
+  };
+}
+async function docToText(data, encrypted) {
+  try {
+    const doc = await new WordExtractor().extract(Buffer.from(data));
+    const normalize = (s) => s.replace(/\r\n?/g, "\n").replace(/\f/g, "\n\n").replace(/\n{3,}/g, "\n\n").trim();
+    const parts = [doc.getBody(), doc.getTextboxes(), doc.getFootnotes(), doc.getEndnotes()].map(normalize).filter(Boolean);
+    return { markdown: parts.join("\n\n"), format: "doc", ...encrypted ? { encrypted: true } : {} };
+  } catch (err) {
+    throw toDocError(err, "cannot parse doc");
+  }
+}
+async function docToDoc(data, opts = {}) {
+  if (isZip(data)) return docxToMarkdown(data, false);
+  if (isOle(data)) {
+    if (oleHasStream(data, "EncryptionInfo")) {
+      if (oleHasStream(data, "Workbook") || oleHasStream(data, "PowerPoint Document")) {
+        throw new DocError("not a Word document (encrypted Excel/PowerPoint package)", "corrupt");
+      }
+      const plain = decryptOoxml(data, opts.password);
+      return isZip(plain) ? docxToMarkdown(plain, true) : docToText(plain, true);
+    }
+    if (oleHasStream(data, "WordDocument") || oleHasStream(data, "wordDocument")) {
+      if (isEncrypted(data)) {
+        const plain = decryptDoc97(data, opts.password);
+        return docToText(plain, true);
+      }
+      return docToText(data, false);
+    }
+    throw new DocError("not a Word document (unsupported OLE compound file)", "corrupt");
+  }
+  throw new DocError("not a Word document (missing docx/doc signature)", "corrupt");
+}
+
+// src/epub.ts
+import { promises as fs } from "fs";
+import { mkdtempSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { EPub } from "epub2";
+var EpubError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "EpubError";
+  }
+  kind;
+};
+var REWRITE_ROOT_RE = /^\/(?:links|images)\//i;
+var ENCRYPTION_ENTRY = "meta-inf/encryption.xml";
+function toEpubError(err) {
+  if (err instanceof EpubError) return err;
+  const message = err instanceof Error ? err.message : String(err);
+  return new EpubError(`cannot parse EPUB: ${message.slice(0, 200)}`, "corrupt");
+}
+function hasEncryptionEntry(names) {
+  return (names ?? []).some((n) => n.toLowerCase() === ENCRYPTION_ENTRY);
+}
+function normalizeHref(href) {
+  const raw = href.split("#")[0].trim();
+  if (!raw) return "";
+  let decoded = raw;
+  try {
+    decoded = decodeURIComponent(raw);
+  } catch {
+  }
+  return path.posix.normalize(decoded).toLowerCase();
+}
+function cleanChapterHtml(html) {
+  return html.replace(/<img\b[^>]*>/gi, "").replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (m, attrs, inner) => {
+    const href = /\shref\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(attrs);
+    const url = (href?.[1] ?? href?.[2] ?? "").trim();
+    return REWRITE_ROOT_RE.test(url) ? inner : m;
+  });
+}
+function tocTitleMap(epub) {
+  const titles = /* @__PURE__ */ new Map();
+  for (const entry of epub.toc ?? []) {
+    const title = entry?.title?.trim();
+    if (!title) continue;
+    if (entry.id && epub.manifest[entry.id]) {
+      const idKey = `id:${entry.id}`;
+      if (!titles.has(idKey)) titles.set(idKey, title);
+      continue;
+    }
+    if (entry.href) {
+      const hrefKey = normalizeHref(entry.href);
+      if (hrefKey && !titles.has(hrefKey)) titles.set(hrefKey, title);
+    }
+  }
+  return titles;
+}
+async function epubToDoc(filePath) {
+  let epub;
+  try {
+    epub = await EPub.createAsync(filePath);
+  } catch (err) {
+    throw toEpubError(err);
+  }
+  const encrypted = hasEncryptionEntry(epub.zip?.names);
+  const titles = tocTitleMap(epub);
+  const parts = [];
+  for (const item of epub.flow ?? []) {
+    const id = item?.id;
+    if (!id) continue;
+    let html;
+    try {
+      html = await epub.getChapterAsync(id);
+    } catch {
+      continue;
+    }
+    const { markdown: markdown2 } = htmlToMarkdown(cleanChapterHtml(html));
+    if (!markdown2.trim()) continue;
+    const heading = titles.get(`id:${id}`) ?? (item.href ? titles.get(normalizeHref(item.href)) : void 0);
+    parts.push(heading ? `## ${heading}
+
+${markdown2}` : markdown2);
+  }
+  const markdown = parts.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  if (!markdown) {
+    throw new EpubError(
+      encrypted ? "DRM-protected EPUB (encryption.xml; no readable text)" : "no extractable text from EPUB",
+      encrypted ? "drm" : "corrupt"
+    );
+  }
+  return {
+    markdown,
+    title: epub.metadata?.title?.trim() || void 0,
+    author: epub.metadata?.creator?.trim() || void 0,
+    language: epub.metadata?.language?.trim() || void 0,
+    chapters: parts.length,
+    ...encrypted ? { encrypted: true } : {}
+  };
+}
+async function epubFromBuffer(data) {
+  const dir = mkdtempSync(path.join(tmpdir(), "gresmcp-epub-"));
+  try {
+    const file = path.join(dir, "book.epub");
+    await fs.writeFile(file, data);
+    return await epubToDoc(file);
+  } finally {
+    await fs.rm(dir, { recursive: true, force: true }).catch(() => {
+    });
+  }
+}
+
 // src/ingest.ts
 init_db();
 
+// src/pdf.ts
+import pdf2md from "@opendocsg/pdf2md";
+import { extractText, getMeta } from "unpdf";
+var PAGE_BREAK_MARK = "<!-- PAGE_BREAK -->";
+var PdfError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "PdfError";
+  }
+  kind;
+};
+function isPasswordException(err) {
+  return err instanceof Error && err.name === "PasswordException";
+}
+function toPdfError(err) {
+  if (err instanceof PdfError) return err;
+  if (isPasswordException(err)) {
+    return new PdfError("wrong password for encrypted PDF", "password");
+  }
+  const message = err instanceof Error ? err.message : String(err);
+  return new PdfError(`cannot parse PDF: ${message.slice(0, 200)}`, "corrupt");
+}
+function titleFromInfo(info) {
+  const raw = info?.Title;
+  const value = typeof raw === "string" ? raw.trim() : "";
+  return value || void 0;
+}
+function imageOnlyFromPages(pages) {
+  const out = [];
+  for (const page of pages) {
+    const hasText = page.items.some((it) => String(it.text ?? "").trim().length > 0);
+    if (!hasText) out.push(page.index + 1);
+  }
+  return out;
+}
+async function extractEncrypted(data, password) {
+  try {
+    const extractOpts = { mergePages: false, password };
+    const metaOpts = { password };
+    const [result, meta] = await Promise.all([
+      extractText(data, extractOpts),
+      getMeta(data, metaOpts).catch(() => void 0)
+    ]);
+    const pages = result.text;
+    const imageOnlyPages = [];
+    pages.forEach((text, i) => {
+      if (!text.trim()) imageOnlyPages.push(i + 1);
+    });
+    const markdown = pages.map((p) => p.trim()).filter(Boolean).join("\n\n").trim();
+    return {
+      markdown,
+      title: titleFromInfo(meta?.info),
+      pageCount: result.totalPages,
+      imageOnlyPages
+    };
+  } catch (err) {
+    throw toPdfError(err);
+  }
+}
+async function pdfToDoc(data, opts = {}) {
+  let title;
+  let pages = [];
+  let markdown;
+  try {
+    markdown = await pdf2md(data, {
+      metadataParsed: (meta) => {
+        title = titleFromInfo(meta.info);
+      },
+      pageParsed: (parsed) => {
+        pages = parsed;
+      }
+    });
+  } catch (err) {
+    if (isPasswordException(err)) {
+      if (opts.password) {
+        return extractEncrypted(data, opts.password);
+      }
+      throw new PdfError("password-protected PDF (use --file-password)", "password");
+    }
+    throw toPdfError(err);
+  }
+  markdown = markdown.split(PAGE_BREAK_MARK).map((part) => part.replace(/\s+$/, "")).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  return {
+    markdown,
+    title,
+    pageCount: pages.length,
+    imageOnlyPages: imageOnlyFromPages(pages)
+  };
+}
+
 // src/progress.ts
 var BAR_WIDTH = 20;
 var PLAIN_STEP = 32;
@@ -718,6 +1046,9 @@ var FeedProgress = class {
 // src/ingest.ts
 var MD_EXT = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
 var HTML_EXT = /* @__PURE__ */ new Set([".html", ".htm", ".xhtml"]);
+var PDF_EXT = /* @__PURE__ */ new Set([".pdf"]);
+var DOC_EXT = /* @__PURE__ */ new Set([".doc", ".docx"]);
+var EPUB_EXT = /* @__PURE__ */ new Set([".epub"]);
 var TEXT_EXT = /* @__PURE__ */ new Set([
   ".txt",
   ".text",
@@ -805,13 +1136,13 @@ var SKIP_DIRS = /* @__PURE__ */ new Set([
   ".cache"
 ]);
 function isAllowed(ext) {
-  return MD_EXT.has(ext) || HTML_EXT.has(ext) || TEXT_EXT.has(ext) || CODE_EXT.has(ext);
+  return MD_EXT.has(ext) || HTML_EXT.has(ext) || TEXT_EXT.has(ext) || CODE_EXT.has(ext) || PDF_EXT.has(ext) || DOC_EXT.has(ext) || EPUB_EXT.has(ext);
 }
 async function walk(root) {
   const out = [];
-  const entries = await fs.readdir(root, { withFileTypes: true });
+  const entries = await fs2.readdir(root, { withFileTypes: true });
   for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
-    const full = path.join(root, entry.name);
+    const full = path2.join(root, entry.name);
     if (entry.isDirectory()) {
       if (SKIP_DIRS.has(entry.name)) continue;
       out.push(...await walk(full));
@@ -822,50 +1153,110 @@ async function walk(root) {
   return out;
 }
 async function loadTarget(target, opts) {
-  const stat = await fs.stat(target);
+  const stat = await fs2.stat(target);
   const files = stat.isDirectory() ? await walk(target) : [target];
   const docs = [];
   const skipped = [];
   for (const file of files) {
-    const ext = path.extname(file).toLowerCase();
+    const ext = path2.extname(file).toLowerCase();
     if (!isAllowed(ext)) {
       skipped.push({ path: file, reason: `unsupported extension '${ext || "(none)"}'` });
       continue;
     }
-    let raw;
+    let buf;
     try {
-      raw = await fs.readFile(file, "utf8");
+      buf = await fs2.readFile(file);
     } catch {
       skipped.push({ path: file, reason: "unreadable" });
       continue;
     }
-    if (raw.includes("\0")) {
-      skipped.push({ path: file, reason: "binary content" });
-      continue;
-    }
-    let text = raw;
+    let text = "";
     let title = "";
-    if (HTML_EXT.has(ext)) {
-      const conv = htmlToMarkdown(raw, { filter: opts.htmlFilter });
-      text = conv.markdown;
-      title = conv.title ?? "";
-    } else if (MD_EXT.has(ext)) {
-      title = mdTitle(raw) ?? "";
-    }
-    if (!text.trim()) {
-      skipped.push({
-        path: file,
-        reason: HTML_EXT.has(ext) && opts.htmlFilter ? `html filter '${opts.htmlFilter}' matched nothing` : "empty"
-      });
-      continue;
+    let docMeta;
+    let chunks = [];
+    if (PDF_EXT.has(ext)) {
+      let pdf;
+      try {
+        pdf = await pdfToDoc(new Uint8Array(buf), { password: opts.filePassword });
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "PDF parse failed" });
+        continue;
+      }
+      text = pdf.markdown;
+      title = pdf.title ?? "";
+      if (pdf.pageCount > 0) docMeta = { pdf_pages: pdf.pageCount };
+      if (pdf.imageOnlyPages.length > 0) {
+        docMeta = { ...docMeta, pdf_image_only_pages: pdf.imageOnlyPages.length };
+      }
+      if (!text.trim()) {
+        skipped.push({
+          path: file,
+          reason: pdf.pageCount > 0 && pdf.imageOnlyPages.length === pdf.pageCount ? "image-only PDF (no text layer; likely scanned)" : "no extractable text from PDF"
+        });
+        continue;
+      }
+      chunks = chunkMarkdown(text, opts);
+    } else if (DOC_EXT.has(ext)) {
+      let word;
+      try {
+        word = await docToDoc(new Uint8Array(buf), { password: opts.filePassword });
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "Word document parse failed" });
+        continue;
+      }
+      text = word.markdown;
+      title = word.title ?? "";
+      docMeta = { doc_format: word.format };
+      if (word.encrypted) docMeta.doc_encrypted = true;
+      if (!text.trim()) {
+        skipped.push({ path: file, reason: "no extractable text from Word document" });
+        continue;
+      }
+      chunks = word.format === "docx" ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
+    } else if (EPUB_EXT.has(ext)) {
+      let epub;
+      try {
+        epub = await epubToDoc(file);
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "EPUB parse failed" });
+        continue;
+      }
+      text = epub.markdown;
+      title = epub.title ?? "";
+      docMeta = { epub_chapters: epub.chapters };
+      if (epub.author) docMeta.epub_author = epub.author;
+      if (epub.language) docMeta.epub_language = epub.language;
+      if (epub.encrypted) docMeta.epub_encrypted = true;
+      chunks = chunkMarkdown(text, opts);
+    } else {
+      const raw = buf.toString("utf8");
+      if (raw.includes("\0")) {
+        skipped.push({ path: file, reason: "binary content" });
+        continue;
+      }
+      text = raw;
+      if (HTML_EXT.has(ext)) {
+        const conv = htmlToMarkdown(raw, { filter: opts.htmlFilter });
+        text = conv.markdown;
+        title = conv.title ?? "";
+      } else if (MD_EXT.has(ext)) {
+        title = mdTitle(raw) ?? "";
+      }
+      if (!text.trim()) {
+        skipped.push({
+          path: file,
+          reason: HTML_EXT.has(ext) && opts.htmlFilter ? `html filter '${opts.htmlFilter}' matched nothing` : "empty"
+        });
+        continue;
+      }
+      chunks = HTML_EXT.has(ext) || MD_EXT.has(ext) ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
     }
-    if (!title) title = path.basename(file);
-    const chunks = HTML_EXT.has(ext) || MD_EXT.has(ext) ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
     if (chunks.length === 0) {
       skipped.push({ path: file, reason: "no chunks produced" });
       continue;
     }
-    docs.push({ source: path.resolve(file), title, chunks });
+    if (!title) title = path2.basename(file);
+    docs.push({ source: path2.resolve(file), title, chunks, metadata: docMeta });
   }
   return { docs, fileCount: files.length, skipped };
 }
@@ -966,7 +1357,7 @@ async function feedDocs(ks, docs, opts = {}) {
   };
 }
 async function feedTarget(ks, target, opts = {}) {
-  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180, htmlFilter: opts.htmlFilter };
+  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180, htmlFilter: opts.htmlFilter, filePassword: opts.filePassword };
   const { docs, skipped } = await loadTarget(target, loadOpts);
   for (const s of skipped) {
     (opts.log ?? ((m) => console.log(m)))(`skipped ${s.path}: ${s.reason}`);
@@ -986,9 +1377,9 @@ async function feedText(ks, text, opts = {}) {
 }
 
 // src/scrape.ts
-import { mkdtempSync } from "fs";
-import { tmpdir } from "os";
-import path2 from "path";
+import { mkdtempSync as mkdtempSync2 } from "fs";
+import { tmpdir as tmpdir2 } from "os";
+import path3 from "path";
 import { randomUUID } from "crypto";
 import {
   CheerioCrawler,
@@ -1014,8 +1405,11 @@ function parseSitemapOption(value) {
   if (key === "html-only") return { kind: "html-only" };
   return { kind: "selector", selector: trimmed };
 }
-var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|docx?|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
-process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync(path2.join(tmpdir(), "gresmcp-crawl-"));
+var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
+var PDF_URL_RE = /\.pdf(\?|#|$)/i;
+var DOC_URL_RE = /\.docx?(\?|#|$)/i;
+var EPUB_URL_RE = /\.epub(\?|#|$)/i;
+process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync2(path3.join(tmpdir2(), "gresmcp-crawl-"));
 async function freshQueue() {
   return RequestQueue.open(`gresmcp-${randomUUID()}`);
 }
@@ -1090,18 +1484,84 @@ function dedupeDocs(docs) {
   }
   return [...bySource.values()];
 }
+async function buildPdfDoc(url, data, deps) {
+  try {
+    const pdf = await pdfToDoc(data, { password: deps.filePassword });
+    if (!pdf.markdown.trim()) {
+      const reason = pdf.pageCount > 0 && pdf.imageOnlyPages.length === pdf.pageCount ? "image-only PDF (no text layer; likely scanned)" : "no extractable text from PDF";
+      return { reason };
+    }
+    const chunks = chunkMarkdown(pdf.markdown, { maxLen: deps.maxLen, overlap: deps.overlap });
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = { source_url: url, crawled_at: (/* @__PURE__ */ new Date()).toISOString() };
+    if (pdf.pageCount > 0) metadata.pdf_pages = pdf.pageCount;
+    if (pdf.imageOnlyPages.length > 0) metadata.pdf_image_only_pages = pdf.imageOnlyPages.length;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: pdf.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "PDF parse failed" };
+  }
+}
+async function buildEpubDoc(url, data, deps) {
+  try {
+    const epub = await epubFromBuffer(data);
+    if (!epub.markdown.trim()) {
+      return {
+        reason: epub.encrypted ? "DRM-protected EPUB (encryption.xml; no readable text)" : "no extractable text from EPUB"
+      };
+    }
+    const chunks = chunkMarkdown(epub.markdown, { maxLen: deps.maxLen, overlap: deps.overlap });
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = {
+      source_url: url,
+      crawled_at: (/* @__PURE__ */ new Date()).toISOString(),
+      epub_chapters: epub.chapters
+    };
+    if (epub.author) metadata.epub_author = epub.author;
+    if (epub.language) metadata.epub_language = epub.language;
+    if (epub.encrypted) metadata.epub_encrypted = true;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: epub.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "EPUB parse failed" };
+  }
+}
 function shortReason(err) {
   const message = (err?.message ?? String(err)).replace(/\s+/g, " ").trim();
   const contentType = /served Content-Type ([^,\s]+),/.exec(message);
   if (contentType) return `unsupported content-type '${contentType[1]}'`;
   return message.slice(0, 200);
 }
+async function buildDocDoc(url, data, deps) {
+  try {
+    const word = await docToDoc(data, { password: deps.filePassword });
+    if (!word.markdown.trim()) {
+      return { reason: "no extractable text from Word document" };
+    }
+    const chunkOpts = { maxLen: deps.maxLen, overlap: deps.overlap };
+    const chunks = word.format === "docx" ? chunkMarkdown(word.markdown, chunkOpts) : chunkPlain(word.markdown, chunkOpts);
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = {
+      source_url: url,
+      crawled_at: (/* @__PURE__ */ new Date()).toISOString(),
+      doc_format: word.format
+    };
+    if (word.encrypted) metadata.doc_encrypted = true;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: word.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "Word document parse failed" };
+  }
+}
 async function crawlWithCheerio(queue, seedUrls, opts, sink) {
   const crawler = new CheerioCrawler({
     requestQueue: queue,
     maxRequestsPerCrawl: Math.max(1, opts.maxPages),
     maxCrawlDepth: Math.max(0, opts.depth),
-    additionalMimeTypes: ["text/plain", "text/markdown"],
+    additionalMimeTypes: ["text/plain", "text/markdown", "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/epub+zip", "application/zip", "application/octet-stream"],
     async requestHandler(ctx) {
       const { request, response, contentType, body, $ } = ctx;
       const url = request.loadedUrl ?? request.url;
@@ -1169,6 +1629,47 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
           return;
         }
         sink.docs.push(doc);
+      } else if (type === "application/pdf") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildPdfDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "PDF produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/msword" || type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildDocDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/epub+zip" || type === "application/zip") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildEpubDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "EPUB produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/octet-stream") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        if (EPUB_URL_RE.test(url)) {
+          const { doc: doc2, reason: reason2 } = await buildEpubDoc(url, data, sinkDeps(opts));
+          if (reason2 || !doc2) {
+            sink.skipped.push({ path: url, reason: reason2 ?? "EPUB produced no document" });
+            return;
+          }
+          sink.docs.push(doc2);
+          return;
+        }
+        const { doc, reason } = await buildDocDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
       } else {
         sink.skipped.push({ path: url, reason: `unsupported content-type '${type}'` });
       }
@@ -1192,6 +1693,7 @@ function sinkDeps(opts) {
     cliTags: opts.tags,
     urlFilter: opts.urlFilter,
     htmlFilter: opts.htmlFilter,
+    filePassword: opts.filePassword,
     followLinks: opts.followLinks,
     linkSelector: opts.linkSelector,
     selectorStats: opts.selectorStats
@@ -1212,9 +1714,75 @@ async function requireBrowserEngine(kind) {
     throw new Error(`--crawler ${kind} requires ${moduleName}, which is not installed (install it with: ${install})`);
   }
 }
+async function handleBrowserPdf(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildPdfDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "PDF produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "PDF fetch failed" });
+  }
+}
+async function handleBrowserDoc(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildDocDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "Word document fetch failed" });
+  }
+}
+async function handleBrowserEpub(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildEpubDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "EPUB produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "EPUB fetch failed" });
+  }
+}
 async function handleBrowserPage(ctx, deps) {
   const request = ctx.request;
   const url = request.loadedUrl ?? request.url;
+  if (PDF_URL_RE.test(url)) {
+    await handleBrowserPdf(url, deps);
+    return;
+  }
+  if (DOC_URL_RE.test(url)) {
+    await handleBrowserDoc(url, deps);
+    return;
+  }
+  if (EPUB_URL_RE.test(url)) {
+    await handleBrowserEpub(url, deps);
+    return;
+  }
   const status = responseStatus(ctx.response);
   if (status === void 0 || status < 200 || status >= 300) {
     deps.sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
@@ -1426,6 +1994,7 @@ async function feedUrl(ks, url, opts = {}) {
     tags: opts.tags,
     urlFilter: opts.urlFilter,
     htmlFilter: opts.htmlFilter,
+    filePassword: opts.filePassword,
     sitemap: opts.sitemap,
     log: log2
   });
@@ -1670,7 +2239,7 @@ async function runChecks(opts) {
 }
 
 // src/version.ts
-var VERSION = "1.2.0";
+var VERSION = "1.3.0";
 
 // src/cli.ts
 async function run(fn) {
@@ -1850,7 +2419,7 @@ ksCmd.command("delete <name>").description("Delete a knowledge source and all of
     console.log(`Deleted knowledge source '${name}'`);
   })
 );
-program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option(
+program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html, pdf, doc, docx, epub)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option(
   "--url-filter <glob>",
   "Only follow --url links whose URL matches this glob (repeatable; a link is followed if it matches any glob, e.g. '**/docs/**')",
   ((v, prev) => [...prev, v]),
@@ -1861,6 +2430,9 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
 ).option(
   "--sitemap <mode>",
   "Link discovery for --url: 'auto' (default; sitemap.xml, then robots.txt sitemaps, else follow links), 'sitemap-only' (only sitemap.xml / robots.txt sitemaps), 'html-only' (follow links), or a CSS selector to only follow links inside it (e.g. 'nav.docs a')"
+).option(
+  "--file-password <password>",
+  "Password for encrypted files (PDF, DOC/DOCX); applies to --path and --url ingestion. Owner-restricted PDFs without a user password are read without it"
 ).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
   (name, opts) => run(async () => {
     await initDb();
@@ -1889,7 +2461,8 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       overlap: opts.overlap,
       tags: parseTags(opts.tags),
       metadata: parseKeyValue(opts.metadata ?? []),
-      htmlFilter: opts.htmlFilter
+      htmlFilter: opts.htmlFilter,
+      filePassword: opts.filePassword
     };
     let summary;
     const since = await dbClockTimestamp();