@gresmcp/mcp 1.1.0 → 1.3.0

package/dist/cli.js

@@ -325,8 +325,8 @@ async function probeModel(baseUrl, model) {
 init_config();
 
 // src/ingest.ts
-import { promises as fs } from "fs";
-import path from "path";
+import { promises as fs2 } from "fs";
+import path2 from "path";
 
 // src/chunk.ts
 var HEADING_RE = /^(#{1,6})\s+(.+)$/;
@@ -390,8 +390,8 @@ function chunkMarkdown(text, opts = {}) {
     const m = HEADING_RE.exec(line);
     if (m) {
       const level = m[1].length;
-      const path3 = sections[sections.length - 1].path;
-      const parent = path3.slice(0, level - 1);
+      const path4 = sections[sections.length - 1].path;
+      const parent = path4.slice(0, level - 1);
       sections.push({ path: [...parent, m[2].trim()], body: [line] });
     } else {
       sections[sections.length - 1].body.push(line);
@@ -446,17 +446,57 @@ function turndown() {
   }
   return service;
 }
-function htmlToMarkdown(html) {
+function parseHtmlFilter(spec) {
+  const steps = spec.split(/\s*->\s*/).map((s) => s.trim()).filter(Boolean);
+  if (steps.length === 0) {
+    throw new Error(`invalid --html-filter '${spec.trim()}': expected a CSS selector chain, e.g. '#article -> .content'`);
+  }
+  return steps;
+}
+function validateCssSelector(selector) {
+  const dom = new JSDOM();
+  try {
+    dom.window.document.querySelector(selector);
+  } catch (err) {
+    throw new Error(`invalid CSS selector '${selector}': ${err instanceof Error ? err.message : err}`);
+  }
+}
+function selectChain(scope, steps) {
+  for (const step of steps.slice(0, -1)) {
+    const next = scope.querySelector(step);
+    if (!next) return void 0;
+    scope = next;
+  }
+  const matches = [...scope.querySelectorAll(steps[steps.length - 1])];
+  return matches.length > 0 ? matches : void 0;
+}
+function elementsToMarkdown(elements) {
+  const parts = [];
+  for (const el of elements) {
+    try {
+      parts.push(turndown().turndown(el.innerHTML));
+    } catch {
+      parts.push(el.textContent ?? "");
+    }
+  }
+  return parts.join("\n\n");
+}
+function htmlToMarkdown(html, opts = {}) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
   doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
   const title = doc.title?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
-  const root = doc.body ?? doc.documentElement;
   let markdown = "";
-  try {
-    markdown = turndown().turndown(root.innerHTML);
-  } catch {
-    markdown = root.textContent ?? "";
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
+    const root = doc.body ?? doc.documentElement;
+    try {
+      markdown = turndown().turndown(root.innerHTML);
+    } catch {
+      markdown = root.textContent ?? "";
+    }
   }
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
   return { markdown, title: title || void 0 };
@@ -490,7 +530,7 @@ function collectTags(doc) {
     return true;
   });
 }
-function extractPage(html) {
+function extractPage(html, opts = {}) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
   const title = doc.title?.trim() || doc.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
@@ -501,20 +541,27 @@ function extractPage(html) {
   const tags = collectTags(doc);
   doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
   let markdown = "";
-  try {
-    if (isProbablyReaderable(doc)) {
-      const article = new Readability(doc).parse();
-      if (article?.content) markdown = turndown().turndown(article.content);
-    }
-  } catch {
-    markdown = "";
-  }
-  if (!markdown.trim()) {
-    const root = doc.body ?? doc.documentElement;
+  let filterMatched;
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    filterMatched = elements !== void 0;
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
     try {
-      markdown = turndown().turndown(root.innerHTML);
+      if (isProbablyReaderable(doc)) {
+        const article = new Readability(doc).parse();
+        if (article?.content) markdown = turndown().turndown(article.content);
+      }
     } catch {
-      markdown = root.textContent ?? "";
+      markdown = "";
+    }
+    if (!markdown.trim()) {
+      const root = doc.body ?? doc.documentElement;
+      try {
+        markdown = turndown().turndown(root.innerHTML);
+      } catch {
+        markdown = root.textContent ?? "";
+      }
     }
   }
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
@@ -524,13 +571,342 @@ function extractPage(html) {
     title,
     description,
     tags: tags.length > 0 ? tags : void 0,
-    likelyShell
+    likelyShell,
+    filterMatched
+  };
+}
+
+// src/doc.ts
+import mammoth from "mammoth";
+import {
+  Doc97File,
+  InvalidKeyError,
+  OfficeFile,
+  OleFileIO,
+  isEncrypted
+} from "office-crypto";
+import WordExtractor from "word-extractor";
+var DocError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "DocError";
+  }
+  kind;
+};
+var OLE_MAGIC = [208, 207, 17, 224, 161, 177, 26, 225];
+function hasMagic(data, magic) {
+  if (data.length < magic.length) return false;
+  return magic.every((byte, i) => data[i] === byte);
+}
+function isOle(data) {
+  return hasMagic(data, OLE_MAGIC);
+}
+function isZip(data) {
+  return data.length >= 4 && data[0] === 80 && data[1] === 75 && data[2] === 3 && data[3] === 4;
+}
+function toDocError(err, prefix) {
+  if (err instanceof DocError) return err;
+  if (err instanceof InvalidKeyError) {
+    return new DocError("wrong password for encrypted Word document", "password");
+  }
+  const message = err instanceof Error ? err.message : String(err);
+  return new DocError(`${prefix}: ${message.slice(0, 200)}`, "corrupt");
+}
+function requirePassword() {
+  return new DocError("password-protected Word document (use --file-password)", "password");
+}
+function oleHasStream(data, name) {
+  try {
+    return new OleFileIO(data).exists(name);
+  } catch {
+    return false;
+  }
+}
+function decryptOoxml(data, password) {
+  try {
+    if (!password) throw requirePassword();
+    const file = OfficeFile(data);
+    file.loadKey({ password, verifyPassword: true });
+    return file.decrypt();
+  } catch (err) {
+    throw toDocError(err, "cannot decrypt Word document");
+  }
+}
+function decryptDoc97(data, password) {
+  try {
+    if (!password) throw requirePassword();
+    const file = new Doc97File(new OleFileIO(data));
+    file.loadKey({ password });
+    return file.decrypt();
+  } catch (err) {
+    throw toDocError(err, "cannot decrypt Word document");
+  }
+}
+async function docxToMarkdown(data, encrypted) {
+  let html;
+  try {
+    const result = await mammoth.convertToHtml({ buffer: Buffer.from(data) });
+    html = result.value;
+  } catch (err) {
+    throw toDocError(err, "cannot parse docx");
+  }
+  html = html.replace(/<img\b[^>]*>/gi, "").replace(
+    /<t([dh])\b([^>]*)>([\s\S]*?)<\/t\1>/gi,
+    (_m, letter, attrs, inner) => `<t${letter}${attrs}>${inner.replace(/<\/?p(?:\s[^>]*)?>/gi, "").replace(/<br(?:\s[^>]*)?>/gi, " ")}</t${letter}>`
+  ).replace(
+    /<table\b[^>]*>([\s\S]*?)<\/table>/gi,
+    (m, inner) => m.replace(
+      /(<tr\b[^>]*>)([\s\S]*?)(<\/tr>)/i,
+      (row, open, cells) => open + cells.replace(/<td\b/gi, "<th").replace(/<\/td>/gi, "</th>") + "</tr>"
+    )
+  );
+  const { markdown, title } = htmlToMarkdown(html);
+  return {
+    markdown: markdown.replace(/\n{3,}/g, "\n\n").trim(),
+    title: title || void 0,
+    format: "docx",
+    ...encrypted ? { encrypted: true } : {}
   };
 }
+async function docToText(data, encrypted) {
+  try {
+    const doc = await new WordExtractor().extract(Buffer.from(data));
+    const normalize = (s) => s.replace(/\r\n?/g, "\n").replace(/\f/g, "\n\n").replace(/\n{3,}/g, "\n\n").trim();
+    const parts = [doc.getBody(), doc.getTextboxes(), doc.getFootnotes(), doc.getEndnotes()].map(normalize).filter(Boolean);
+    return { markdown: parts.join("\n\n"), format: "doc", ...encrypted ? { encrypted: true } : {} };
+  } catch (err) {
+    throw toDocError(err, "cannot parse doc");
+  }
+}
+async function docToDoc(data, opts = {}) {
+  if (isZip(data)) return docxToMarkdown(data, false);
+  if (isOle(data)) {
+    if (oleHasStream(data, "EncryptionInfo")) {
+      if (oleHasStream(data, "Workbook") || oleHasStream(data, "PowerPoint Document")) {
+        throw new DocError("not a Word document (encrypted Excel/PowerPoint package)", "corrupt");
+      }
+      const plain = decryptOoxml(data, opts.password);
+      return isZip(plain) ? docxToMarkdown(plain, true) : docToText(plain, true);
+    }
+    if (oleHasStream(data, "WordDocument") || oleHasStream(data, "wordDocument")) {
+      if (isEncrypted(data)) {
+        const plain = decryptDoc97(data, opts.password);
+        return docToText(plain, true);
+      }
+      return docToText(data, false);
+    }
+    throw new DocError("not a Word document (unsupported OLE compound file)", "corrupt");
+  }
+  throw new DocError("not a Word document (missing docx/doc signature)", "corrupt");
+}
+
+// src/epub.ts
+import { promises as fs } from "fs";
+import { mkdtempSync } from "fs";
+import { tmpdir } from "os";
+import path from "path";
+import { EPub } from "epub2";
+var EpubError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "EpubError";
+  }
+  kind;
+};
+var REWRITE_ROOT_RE = /^\/(?:links|images)\//i;
+var ENCRYPTION_ENTRY = "meta-inf/encryption.xml";
+function toEpubError(err) {
+  if (err instanceof EpubError) return err;
+  const message = err instanceof Error ? err.message : String(err);
+  return new EpubError(`cannot parse EPUB: ${message.slice(0, 200)}`, "corrupt");
+}
+function hasEncryptionEntry(names) {
+  return (names ?? []).some((n) => n.toLowerCase() === ENCRYPTION_ENTRY);
+}
+function normalizeHref(href) {
+  const raw = href.split("#")[0].trim();
+  if (!raw) return "";
+  let decoded = raw;
+  try {
+    decoded = decodeURIComponent(raw);
+  } catch {
+  }
+  return path.posix.normalize(decoded).toLowerCase();
+}
+function cleanChapterHtml(html) {
+  return html.replace(/<img\b[^>]*>/gi, "").replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (m, attrs, inner) => {
+    const href = /\shref\s*=\s*(?:"([^"]*)"|'([^']*)')/i.exec(attrs);
+    const url = (href?.[1] ?? href?.[2] ?? "").trim();
+    return REWRITE_ROOT_RE.test(url) ? inner : m;
+  });
+}
+function tocTitleMap(epub) {
+  const titles = /* @__PURE__ */ new Map();
+  for (const entry of epub.toc ?? []) {
+    const title = entry?.title?.trim();
+    if (!title) continue;
+    if (entry.id && epub.manifest[entry.id]) {
+      const idKey = `id:${entry.id}`;
+      if (!titles.has(idKey)) titles.set(idKey, title);
+      continue;
+    }
+    if (entry.href) {
+      const hrefKey = normalizeHref(entry.href);
+      if (hrefKey && !titles.has(hrefKey)) titles.set(hrefKey, title);
+    }
+  }
+  return titles;
+}
+async function epubToDoc(filePath) {
+  let epub;
+  try {
+    epub = await EPub.createAsync(filePath);
+  } catch (err) {
+    throw toEpubError(err);
+  }
+  const encrypted = hasEncryptionEntry(epub.zip?.names);
+  const titles = tocTitleMap(epub);
+  const parts = [];
+  for (const item of epub.flow ?? []) {
+    const id = item?.id;
+    if (!id) continue;
+    let html;
+    try {
+      html = await epub.getChapterAsync(id);
+    } catch {
+      continue;
+    }
+    const { markdown: markdown2 } = htmlToMarkdown(cleanChapterHtml(html));
+    if (!markdown2.trim()) continue;
+    const heading = titles.get(`id:${id}`) ?? (item.href ? titles.get(normalizeHref(item.href)) : void 0);
+    parts.push(heading ? `## ${heading}
+
+${markdown2}` : markdown2);
+  }
+  const markdown = parts.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  if (!markdown) {
+    throw new EpubError(
+      encrypted ? "DRM-protected EPUB (encryption.xml; no readable text)" : "no extractable text from EPUB",
+      encrypted ? "drm" : "corrupt"
+    );
+  }
+  return {
+    markdown,
+    title: epub.metadata?.title?.trim() || void 0,
+    author: epub.metadata?.creator?.trim() || void 0,
+    language: epub.metadata?.language?.trim() || void 0,
+    chapters: parts.length,
+    ...encrypted ? { encrypted: true } : {}
+  };
+}
+async function epubFromBuffer(data) {
+  const dir = mkdtempSync(path.join(tmpdir(), "gresmcp-epub-"));
+  try {
+    const file = path.join(dir, "book.epub");
+    await fs.writeFile(file, data);
+    return await epubToDoc(file);
+  } finally {
+    await fs.rm(dir, { recursive: true, force: true }).catch(() => {
+    });
+  }
+}
 
 // src/ingest.ts
 init_db();
 
+// src/pdf.ts
+import pdf2md from "@opendocsg/pdf2md";
+import { extractText, getMeta } from "unpdf";
+var PAGE_BREAK_MARK = "<!-- PAGE_BREAK -->";
+var PdfError = class extends Error {
+  constructor(message, kind) {
+    super(message);
+    this.kind = kind;
+    this.name = "PdfError";
+  }
+  kind;
+};
+function isPasswordException(err) {
+  return err instanceof Error && err.name === "PasswordException";
+}
+function toPdfError(err) {
+  if (err instanceof PdfError) return err;
+  if (isPasswordException(err)) {
+    return new PdfError("wrong password for encrypted PDF", "password");
+  }
+  const message = err instanceof Error ? err.message : String(err);
+  return new PdfError(`cannot parse PDF: ${message.slice(0, 200)}`, "corrupt");
+}
+function titleFromInfo(info) {
+  const raw = info?.Title;
+  const value = typeof raw === "string" ? raw.trim() : "";
+  return value || void 0;
+}
+function imageOnlyFromPages(pages) {
+  const out = [];
+  for (const page of pages) {
+    const hasText = page.items.some((it) => String(it.text ?? "").trim().length > 0);
+    if (!hasText) out.push(page.index + 1);
+  }
+  return out;
+}
+async function extractEncrypted(data, password) {
+  try {
+    const extractOpts = { mergePages: false, password };
+    const metaOpts = { password };
+    const [result, meta] = await Promise.all([
+      extractText(data, extractOpts),
+      getMeta(data, metaOpts).catch(() => void 0)
+    ]);
+    const pages = result.text;
+    const imageOnlyPages = [];
+    pages.forEach((text, i) => {
+      if (!text.trim()) imageOnlyPages.push(i + 1);
+    });
+    const markdown = pages.map((p) => p.trim()).filter(Boolean).join("\n\n").trim();
+    return {
+      markdown,
+      title: titleFromInfo(meta?.info),
+      pageCount: result.totalPages,
+      imageOnlyPages
+    };
+  } catch (err) {
+    throw toPdfError(err);
+  }
+}
+async function pdfToDoc(data, opts = {}) {
+  let title;
+  let pages = [];
+  let markdown;
+  try {
+    markdown = await pdf2md(data, {
+      metadataParsed: (meta) => {
+        title = titleFromInfo(meta.info);
+      },
+      pageParsed: (parsed) => {
+        pages = parsed;
+      }
+    });
+  } catch (err) {
+    if (isPasswordException(err)) {
+      if (opts.password) {
+        return extractEncrypted(data, opts.password);
+      }
+      throw new PdfError("password-protected PDF (use --file-password)", "password");
+    }
+    throw toPdfError(err);
+  }
+  markdown = markdown.split(PAGE_BREAK_MARK).map((part) => part.replace(/\s+$/, "")).join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  return {
+    markdown,
+    title,
+    pageCount: pages.length,
+    imageOnlyPages: imageOnlyFromPages(pages)
+  };
+}
+
 // src/progress.ts
 var BAR_WIDTH = 20;
 var PLAIN_STEP = 32;
@@ -544,14 +920,16 @@ function bar(pct2) {
 function pct(p) {
   return `${Math.round(Math.max(0, Math.min(1, p)) * 100)}%`;
 }
+var NOOP_HANDLE = { handled: () => {
+} };
 var FeedProgress = class {
   write;
   interactive;
   total;
   rejected;
+  slots = [];
   filesDone = 0;
   totalChars = 0;
-  current;
   blockLines = 0;
   finished = false;
   constructor(opts) {
@@ -564,26 +942,13 @@ var FeedProgress = class {
     return this.interactive;
   }
   startFile(source, totalChunks, totalChars) {
-    if (this.finished) return;
-    this.current = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    if (this.finished) return NOOP_HANDLE;
+    const slot = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    this.slots.push(slot);
     this.render();
-  }
-  handled(chunks, chars) {
-    if (this.finished || !this.current || chunks <= 0) return;
-    const c = this.current;
-    c.doneChunks += chunks;
-    c.doneChars += chars;
-    this.totalChars += chars;
-    if (c.doneChunks >= c.totalChunks) {
-      this.filesDone++;
-      this.printFileLine();
-      return;
-    }
-    if (!this.interactive && c.doneChunks - c.printedChunks >= PLAIN_STEP) {
-      this.printFileLine();
-    } else if (this.interactive) {
-      this.render();
-    }
+    return {
+      handled: (chunks, chars) => this.handleSlot(slot, chunks, chars)
+    };
   }
   log(msg) {
     if (this.finished) return;
@@ -594,9 +959,9 @@ var FeedProgress = class {
   finish() {
     if (this.finished) return;
     this.finished = true;
-    if (!this.current && this.total === 0 && this.rejected === 0) return;
+    if (this.slots.length === 0 && this.total === 0 && this.rejected === 0) return;
     if (this.interactive) {
-      this.redraw([this.fileLine(), this.totalLine()]);
+      this.redraw(this.lines());
       this.write("\n");
       this.blockLines = 0;
     } else {
@@ -609,25 +974,42 @@ var FeedProgress = class {
     this.finished = true;
     this.clearBlock();
   }
+  lines() {
+    return [...this.slots.map((slot) => this.fileLine(slot)), this.totalLine()];
+  }
+  handleSlot(slot, chunks, chars) {
+    if (this.finished || chunks <= 0 || slot.doneChunks >= slot.totalChunks) return;
+    slot.doneChunks += chunks;
+    slot.doneChars += chars;
+    this.totalChars += chars;
+    if (slot.doneChunks >= slot.totalChunks) {
+      this.filesDone++;
+      if (this.interactive) {
+        this.render();
+      } else {
+        this.printFileLine(slot);
+      }
+      this.slots.splice(this.slots.indexOf(slot), 1);
+      return;
+    }
+    if (!this.interactive && slot.doneChunks - slot.printedChunks >= PLAIN_STEP) {
+      this.printFileLine(slot);
+    } else if (this.interactive) {
+      this.render();
+    }
+  }
   render() {
     if (!this.interactive) return;
-    this.redraw([this.fileLine(), this.totalLine()]);
+    this.redraw(this.lines());
   }
-  printFileLine() {
-    if (!this.current) return;
-    this.current.printedChunks = this.current.doneChunks;
-    if (this.interactive) {
-      this.render();
-    } else {
-      this.write(`${this.fileLine()}
+  printFileLine(slot) {
+    slot.printedChunks = slot.doneChunks;
+    this.write(`${this.fileLine(slot)}
 `);
-    }
   }
-  fileLine() {
-    const c = this.current;
-    if (!c) return "";
-    const p = c.totalChunks > 0 ? c.doneChunks / c.totalChunks : 1;
-    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(c.doneChunks)}/${fmt(c.totalChunks)} chunks  ${fmt(c.doneChars)}/${fmt(c.totalChars)} chars  ${c.source}`;
+  fileLine(slot) {
+    const p = slot.totalChunks > 0 ? slot.doneChunks / slot.totalChunks : 1;
+    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(slot.doneChunks)}/${fmt(slot.totalChunks)} chunks  ${fmt(slot.doneChars)}/${fmt(slot.totalChars)} chars  ${slot.source}`;
   }
   totalLine() {
     const p = this.total > 0 ? this.filesDone / this.total : 0;
@@ -664,6 +1046,9 @@ var FeedProgress = class {
 // src/ingest.ts
 var MD_EXT = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
 var HTML_EXT = /* @__PURE__ */ new Set([".html", ".htm", ".xhtml"]);
+var PDF_EXT = /* @__PURE__ */ new Set([".pdf"]);
+var DOC_EXT = /* @__PURE__ */ new Set([".doc", ".docx"]);
+var EPUB_EXT = /* @__PURE__ */ new Set([".epub"]);
 var TEXT_EXT = /* @__PURE__ */ new Set([
   ".txt",
   ".text",
@@ -751,13 +1136,13 @@ var SKIP_DIRS = /* @__PURE__ */ new Set([
   ".cache"
 ]);
 function isAllowed(ext) {
-  return MD_EXT.has(ext) || HTML_EXT.has(ext) || TEXT_EXT.has(ext) || CODE_EXT.has(ext);
+  return MD_EXT.has(ext) || HTML_EXT.has(ext) || TEXT_EXT.has(ext) || CODE_EXT.has(ext) || PDF_EXT.has(ext) || DOC_EXT.has(ext) || EPUB_EXT.has(ext);
 }
 async function walk(root) {
   const out = [];
-  const entries = await fs.readdir(root, { withFileTypes: true });
+  const entries = await fs2.readdir(root, { withFileTypes: true });
   for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
-    const full = path.join(root, entry.name);
+    const full = path2.join(root, entry.name);
     if (entry.isDirectory()) {
       if (SKIP_DIRS.has(entry.name)) continue;
       out.push(...await walk(full));
@@ -768,50 +1153,114 @@ async function walk(root) {
   return out;
 }
 async function loadTarget(target, opts) {
-  const stat = await fs.stat(target);
+  const stat = await fs2.stat(target);
   const files = stat.isDirectory() ? await walk(target) : [target];
   const docs = [];
   const skipped = [];
   for (const file of files) {
-    const ext = path.extname(file).toLowerCase();
+    const ext = path2.extname(file).toLowerCase();
     if (!isAllowed(ext)) {
       skipped.push({ path: file, reason: `unsupported extension '${ext || "(none)"}'` });
       continue;
     }
-    let raw;
+    let buf;
     try {
-      raw = await fs.readFile(file, "utf8");
+      buf = await fs2.readFile(file);
     } catch {
       skipped.push({ path: file, reason: "unreadable" });
       continue;
     }
-    if (raw.includes("\0")) {
-      skipped.push({ path: file, reason: "binary content" });
-      continue;
-    }
-    let text = raw;
+    let text = "";
     let title = "";
-    if (HTML_EXT.has(ext)) {
-      const conv = htmlToMarkdown(raw);
-      text = conv.markdown;
-      title = conv.title ?? "";
-    } else if (MD_EXT.has(ext)) {
-      title = mdTitle(raw) ?? "";
-    }
-    if (!text.trim()) {
-      skipped.push({ path: file, reason: "empty" });
-      continue;
+    let docMeta;
+    let chunks = [];
+    if (PDF_EXT.has(ext)) {
+      let pdf;
+      try {
+        pdf = await pdfToDoc(new Uint8Array(buf), { password: opts.filePassword });
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "PDF parse failed" });
+        continue;
+      }
+      text = pdf.markdown;
+      title = pdf.title ?? "";
+      if (pdf.pageCount > 0) docMeta = { pdf_pages: pdf.pageCount };
+      if (pdf.imageOnlyPages.length > 0) {
+        docMeta = { ...docMeta, pdf_image_only_pages: pdf.imageOnlyPages.length };
+      }
+      if (!text.trim()) {
+        skipped.push({
+          path: file,
+          reason: pdf.pageCount > 0 && pdf.imageOnlyPages.length === pdf.pageCount ? "image-only PDF (no text layer; likely scanned)" : "no extractable text from PDF"
+        });
+        continue;
+      }
+      chunks = chunkMarkdown(text, opts);
+    } else if (DOC_EXT.has(ext)) {
+      let word;
+      try {
+        word = await docToDoc(new Uint8Array(buf), { password: opts.filePassword });
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "Word document parse failed" });
+        continue;
+      }
+      text = word.markdown;
+      title = word.title ?? "";
+      docMeta = { doc_format: word.format };
+      if (word.encrypted) docMeta.doc_encrypted = true;
+      if (!text.trim()) {
+        skipped.push({ path: file, reason: "no extractable text from Word document" });
+        continue;
+      }
+      chunks = word.format === "docx" ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
+    } else if (EPUB_EXT.has(ext)) {
+      let epub;
+      try {
+        epub = await epubToDoc(file);
+      } catch (err) {
+        skipped.push({ path: file, reason: err instanceof Error ? err.message : "EPUB parse failed" });
+        continue;
+      }
+      text = epub.markdown;
+      title = epub.title ?? "";
+      docMeta = { epub_chapters: epub.chapters };
+      if (epub.author) docMeta.epub_author = epub.author;
+      if (epub.language) docMeta.epub_language = epub.language;
+      if (epub.encrypted) docMeta.epub_encrypted = true;
+      chunks = chunkMarkdown(text, opts);
+    } else {
+      const raw = buf.toString("utf8");
+      if (raw.includes("\0")) {
+        skipped.push({ path: file, reason: "binary content" });
+        continue;
+      }
+      text = raw;
+      if (HTML_EXT.has(ext)) {
+        const conv = htmlToMarkdown(raw, { filter: opts.htmlFilter });
+        text = conv.markdown;
+        title = conv.title ?? "";
+      } else if (MD_EXT.has(ext)) {
+        title = mdTitle(raw) ?? "";
+      }
+      if (!text.trim()) {
+        skipped.push({
+          path: file,
+          reason: HTML_EXT.has(ext) && opts.htmlFilter ? `html filter '${opts.htmlFilter}' matched nothing` : "empty"
+        });
+        continue;
+      }
+      chunks = HTML_EXT.has(ext) || MD_EXT.has(ext) ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
     }
-    if (!title) title = path.basename(file);
-    const chunks = HTML_EXT.has(ext) || MD_EXT.has(ext) ? chunkMarkdown(text, opts) : chunkPlain(text, opts);
     if (chunks.length === 0) {
       skipped.push({ path: file, reason: "no chunks produced" });
       continue;
     }
-    docs.push({ source: path.resolve(file), title, chunks });
+    if (!title) title = path2.basename(file);
+    docs.push({ source: path2.resolve(file), title, chunks, metadata: docMeta });
   }
   return { docs, fileCount: files.length, skipped };
 }
+var DOC_CONCURRENCY = 3;
 async function feedDocs(ks, docs, opts = {}) {
   const log2 = opts.log ?? ((m) => console.log(m));
   const progress = opts.progress;
@@ -823,61 +1272,78 @@ async function feedDocs(ks, docs, opts = {}) {
   let skippedExisting = 0;
   let inserted = 0;
   let announcedEmbed = false;
-  try {
-    for (const doc of docs) {
-      progress?.startFile(
-        doc.source,
-        doc.chunks.length,
-        doc.chunks.reduce((n, c) => n + c.content.length, 0)
-      );
-      if (opts.replace) {
-        deletedForReplace += await deleteBySource(ks, doc.source);
-      }
-      const existing = await sourceChunkHashes(ks, doc.source);
-      const meta = { ...baseMeta, ...doc.metadata ?? {} };
-      if (!("tags" in meta) && cliTags) meta.tags = cliTags;
-      const todo = [];
-      for (const chunk of doc.chunks) {
-        if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
-          skippedExisting++;
-          progress?.handled(1, chunk.content.length);
-          continue;
-        }
-        todo.push(chunk);
-      }
-      if (todo.length === 0) continue;
-      if (opts.dryRun) {
-        for (const chunk of todo) progress?.handled(1, chunk.content.length);
+  const processDoc = async (doc) => {
+    const file = progress?.startFile(
+      doc.source,
+      doc.chunks.length,
+      doc.chunks.reduce((n, c) => n + c.content.length, 0)
+    );
+    if (opts.replace) {
+      const deleted = await deleteBySource(ks, doc.source);
+      deletedForReplace += deleted;
+    }
+    const existing = await sourceChunkHashes(ks, doc.source);
+    const meta = { ...baseMeta, ...doc.metadata ?? {} };
+    if (!("tags" in meta) && cliTags) meta.tags = cliTags;
+    const todo = [];
+    for (const chunk of doc.chunks) {
+      if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
+        skippedExisting++;
+        file?.handled(1, chunk.content.length);
         continue;
       }
-      if (!announcedEmbed) {
-        say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
-        announcedEmbed = true;
+      todo.push(chunk);
+    }
+    if (todo.length === 0) return;
+    if (opts.dryRun) {
+      for (const chunk of todo) file?.handled(1, chunk.content.length);
+      return;
+    }
+    if (!announcedEmbed) {
+      announcedEmbed = true;
+      say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
+    }
+    const contents = todo.map((c) => c.content);
+    const charLens = contents.map((c) => c.length);
+    let reported = 0;
+    const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
+      const delta = done - reported;
+      if (delta <= 0) return;
+      let chars = 0;
+      for (let i = reported; i < done; i++) chars += charLens[i];
+      reported = done;
+      file?.handled(delta, chars);
+    });
+    const rows = todo.map((chunk, i) => ({
+      source: doc.source,
+      title: doc.title,
+      chunk_index: chunk.index,
+      content: chunk.content,
+      metadata: meta,
+      embedding: embeddings[i]
+    }));
+    const rowCount = await insertChunks(ks, rows);
+    inserted += rowCount;
+  };
+  let cursor = 0;
+  let firstError;
+  const runWorker = async () => {
+    while (firstError === void 0) {
+      const index = cursor++;
+      if (index >= docs.length) return;
+      try {
+        await processDoc(docs[index]);
+      } catch (err) {
+        firstError ??= err;
+        return;
       }
-      const contents = todo.map((c) => c.content);
-      const charLens = contents.map((c) => c.length);
-      let reported = 0;
-      const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
-        const delta = done - reported;
-        if (delta <= 0) return;
-        let chars = 0;
-        for (let i = reported; i < done; i++) chars += charLens[i];
-        reported = done;
-        progress?.handled(delta, chars);
-      });
-      const rows = todo.map((chunk, i) => ({
-        source: doc.source,
-        title: doc.title,
-        chunk_index: chunk.index,
-        content: chunk.content,
-        metadata: meta,
-        embedding: embeddings[i]
-      }));
-      inserted += await insertChunks(ks, rows);
     }
-  } catch (err) {
+  };
+  const workerCount = Math.min(DOC_CONCURRENCY, docs.length);
+  await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
+  if (firstError !== void 0) {
     progress?.close();
-    throw err;
+    throw firstError;
   }
   progress?.finish();
   return {
@@ -891,7 +1357,7 @@ async function feedDocs(ks, docs, opts = {}) {
   };
 }
 async function feedTarget(ks, target, opts = {}) {
-  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
+  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180, htmlFilter: opts.htmlFilter, filePassword: opts.filePassword };
   const { docs, skipped } = await loadTarget(target, loadOpts);
   for (const s of skipped) {
     (opts.log ?? ((m) => console.log(m)))(`skipped ${s.path}: ${s.reason}`);
@@ -911,14 +1377,39 @@ async function feedText(ks, text, opts = {}) {
 }
 
 // src/scrape.ts
-import { mkdtempSync } from "fs";
-import { tmpdir } from "os";
-import path2 from "path";
+import { mkdtempSync as mkdtempSync2 } from "fs";
+import { tmpdir as tmpdir2 } from "os";
+import path3 from "path";
 import { randomUUID } from "crypto";
-import { CheerioCrawler, PlaywrightCrawler, PuppeteerCrawler, RequestQueue, log } from "crawlee";
+import {
+  CheerioCrawler,
+  PlaywrightCrawler,
+  PuppeteerCrawler,
+  Request as CrawleeRequest,
+  RequestQueue,
+  RobotsTxtFile,
+  Sitemap,
+  constructGlobObjectsFromGlobs,
+  filterRequestsByPatterns,
+  log
+} from "crawlee";
 var CRAWLER_KINDS = ["auto", "crawlee", "playwright", "puppeteer"];
-var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|docx?|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
-process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync(path2.join(tmpdir(), "gresmcp-crawl-"));
+function parseSitemapOption(value) {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    throw new Error("invalid --sitemap '': use auto, sitemap-only, html-only or a CSS selector");
+  }
+  const key = trimmed.toLowerCase();
+  if (key === "auto") return { kind: "auto" };
+  if (key === "sitemap-only") return { kind: "sitemap-only" };
+  if (key === "html-only") return { kind: "html-only" };
+  return { kind: "selector", selector: trimmed };
+}
+var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
+var PDF_URL_RE = /\.pdf(\?|#|$)/i;
+var DOC_URL_RE = /\.docx?(\?|#|$)/i;
+var EPUB_URL_RE = /\.epub(\?|#|$)/i;
+process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync2(path3.join(tmpdir2(), "gresmcp-crawl-"));
 async function freshQueue() {
   return RequestQueue.open(`gresmcp-${randomUUID()}`);
 }
@@ -993,20 +1484,86 @@ function dedupeDocs(docs) {
   }
   return [...bySource.values()];
 }
+async function buildPdfDoc(url, data, deps) {
+  try {
+    const pdf = await pdfToDoc(data, { password: deps.filePassword });
+    if (!pdf.markdown.trim()) {
+      const reason = pdf.pageCount > 0 && pdf.imageOnlyPages.length === pdf.pageCount ? "image-only PDF (no text layer; likely scanned)" : "no extractable text from PDF";
+      return { reason };
+    }
+    const chunks = chunkMarkdown(pdf.markdown, { maxLen: deps.maxLen, overlap: deps.overlap });
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = { source_url: url, crawled_at: (/* @__PURE__ */ new Date()).toISOString() };
+    if (pdf.pageCount > 0) metadata.pdf_pages = pdf.pageCount;
+    if (pdf.imageOnlyPages.length > 0) metadata.pdf_image_only_pages = pdf.imageOnlyPages.length;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: pdf.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "PDF parse failed" };
+  }
+}
+async function buildEpubDoc(url, data, deps) {
+  try {
+    const epub = await epubFromBuffer(data);
+    if (!epub.markdown.trim()) {
+      return {
+        reason: epub.encrypted ? "DRM-protected EPUB (encryption.xml; no readable text)" : "no extractable text from EPUB"
+      };
+    }
+    const chunks = chunkMarkdown(epub.markdown, { maxLen: deps.maxLen, overlap: deps.overlap });
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = {
+      source_url: url,
+      crawled_at: (/* @__PURE__ */ new Date()).toISOString(),
+      epub_chapters: epub.chapters
+    };
+    if (epub.author) metadata.epub_author = epub.author;
+    if (epub.language) metadata.epub_language = epub.language;
+    if (epub.encrypted) metadata.epub_encrypted = true;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: epub.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "EPUB parse failed" };
+  }
+}
 function shortReason(err) {
   const message = (err?.message ?? String(err)).replace(/\s+/g, " ").trim();
   const contentType = /served Content-Type ([^,\s]+),/.exec(message);
   if (contentType) return `unsupported content-type '${contentType[1]}'`;
   return message.slice(0, 200);
 }
+async function buildDocDoc(url, data, deps) {
+  try {
+    const word = await docToDoc(data, { password: deps.filePassword });
+    if (!word.markdown.trim()) {
+      return { reason: "no extractable text from Word document" };
+    }
+    const chunkOpts = { maxLen: deps.maxLen, overlap: deps.overlap };
+    const chunks = word.format === "docx" ? chunkMarkdown(word.markdown, chunkOpts) : chunkPlain(word.markdown, chunkOpts);
+    if (chunks.length === 0) return { reason: "no chunks produced" };
+    const metadata = {
+      source_url: url,
+      crawled_at: (/* @__PURE__ */ new Date()).toISOString(),
+      doc_format: word.format
+    };
+    if (word.encrypted) metadata.doc_encrypted = true;
+    const tags = unionTags(void 0, deps.cliTags);
+    if (tags) metadata.tags = tags;
+    return { doc: { source: url, title: word.title?.trim() || titleFor(url), chunks, metadata } };
+  } catch (err) {
+    return { reason: err instanceof Error ? err.message : "Word document parse failed" };
+  }
+}
 async function crawlWithCheerio(queue, seedUrls, opts, sink) {
   const crawler = new CheerioCrawler({
     requestQueue: queue,
     maxRequestsPerCrawl: Math.max(1, opts.maxPages),
     maxCrawlDepth: Math.max(0, opts.depth),
-    additionalMimeTypes: ["text/plain", "text/markdown"],
+    additionalMimeTypes: ["text/plain", "text/markdown", "application/pdf", "application/msword", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/epub+zip", "application/zip", "application/octet-stream"],
     async requestHandler(ctx) {
-      const { request, response, contentType, body } = ctx;
+      const { request, response, contentType, body, $ } = ctx;
       const url = request.loadedUrl ?? request.url;
       const status = responseStatus(response);
       if (status === void 0 || status < 200 || status >= 300) {
@@ -1016,12 +1573,16 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
       const type = contentType?.type ?? "";
       if (type === "text/html" || type === "application/xhtml+xml") {
         const html = typeof body === "string" ? body : toText(body);
-        const page = extractPage(html);
+        const page = extractPage(html, { filter: opts.htmlFilter });
         if (page.likelyShell) {
           sink.shellUrls?.push(url);
           sink.skipped.push({ path: url, reason: "likely JS-rendered shell" });
           return;
         }
+        if (opts.htmlFilter && !page.filterMatched) {
+          sink.skipped.push({ path: url, reason: `html filter '${opts.htmlFilter}' matched nothing` });
+          return;
+        }
         if (!page.markdown.trim()) {
           sink.skipped.push({ path: url, reason: "empty" });
           return;
@@ -1032,7 +1593,28 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
           return;
         }
         sink.docs.push(doc);
-        await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+        if (opts.followLinks !== false) {
+          const selector = opts.linkSelector;
+          if (selector) {
+            if ($(selector).length > 0) {
+              opts.selectorStats.hits += 1;
+              await ctx.enqueueLinks({
+                selector,
+                strategy: "same-hostname",
+                exclude: [BINARY_LINK_RE],
+                globs: opts.urlFilter
+              });
+            } else {
+              opts.selectorStats.misses += 1;
+            }
+          } else {
+            await ctx.enqueueLinks({
+              strategy: "same-hostname",
+              exclude: [BINARY_LINK_RE],
+              globs: opts.urlFilter
+            });
+          }
+        }
       } else if (type === "text/markdown") {
         const doc = buildDoc(url, "md", toText(body), void 0, sinkDeps(opts));
         if (!doc) {
@@ -1047,6 +1629,47 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
           return;
         }
         sink.docs.push(doc);
+      } else if (type === "application/pdf") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildPdfDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "PDF produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/msword" || type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildDocDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/epub+zip" || type === "application/zip") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        const { doc, reason } = await buildEpubDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "EPUB produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "application/octet-stream") {
+        const data = typeof body === "string" ? new TextEncoder().encode(body) : new Uint8Array(body);
+        if (EPUB_URL_RE.test(url)) {
+          const { doc: doc2, reason: reason2 } = await buildEpubDoc(url, data, sinkDeps(opts));
+          if (reason2 || !doc2) {
+            sink.skipped.push({ path: url, reason: reason2 ?? "EPUB produced no document" });
+            return;
+          }
+          sink.docs.push(doc2);
+          return;
+        }
+        const { doc, reason } = await buildDocDoc(url, data, sinkDeps(opts));
+        if (reason || !doc) {
+          sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+          return;
+        }
+        sink.docs.push(doc);
       } else {
         sink.skipped.push({ path: url, reason: `unsupported content-type '${type}'` });
       }
@@ -1064,7 +1687,17 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
   }
 }
 function sinkDeps(opts) {
-  return { maxLen: opts.maxLen, overlap: opts.overlap, cliTags: opts.tags };
+  return {
+    maxLen: opts.maxLen,
+    overlap: opts.overlap,
+    cliTags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    filePassword: opts.filePassword,
+    followLinks: opts.followLinks,
+    linkSelector: opts.linkSelector,
+    selectorStats: opts.selectorStats
+  };
 }
 async function importOptional(name) {
   try {
@@ -1081,16 +1714,86 @@ async function requireBrowserEngine(kind) {
     throw new Error(`--crawler ${kind} requires ${moduleName}, which is not installed (install it with: ${install})`);
   }
 }
+async function handleBrowserPdf(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildPdfDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "PDF produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "PDF fetch failed" });
+  }
+}
+async function handleBrowserDoc(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildDocDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "Word document produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "Word document fetch failed" });
+  }
+}
+async function handleBrowserEpub(url, deps) {
+  try {
+    const res = await fetch(url, { redirect: "follow" });
+    if (!res.ok) {
+      deps.sink.skipped.push({ path: url, reason: `HTTP ${res.status}` });
+      return;
+    }
+    const data = new Uint8Array(await res.arrayBuffer());
+    const { doc, reason } = await buildEpubDoc(url, data, deps);
+    if (reason || !doc) {
+      deps.sink.skipped.push({ path: url, reason: reason ?? "EPUB produced no document" });
+      return;
+    }
+    deps.sink.docs.push(doc);
+  } catch (err) {
+    deps.sink.skipped.push({ path: url, reason: err instanceof Error ? err.message : "EPUB fetch failed" });
+  }
+}
 async function handleBrowserPage(ctx, deps) {
   const request = ctx.request;
   const url = request.loadedUrl ?? request.url;
+  if (PDF_URL_RE.test(url)) {
+    await handleBrowserPdf(url, deps);
+    return;
+  }
+  if (DOC_URL_RE.test(url)) {
+    await handleBrowserDoc(url, deps);
+    return;
+  }
+  if (EPUB_URL_RE.test(url)) {
+    await handleBrowserEpub(url, deps);
+    return;
+  }
   const status = responseStatus(ctx.response);
   if (status === void 0 || status < 200 || status >= 300) {
     deps.sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
     return;
   }
   const html = await ctx.page.content();
-  const page = extractPage(html);
+  const page = extractPage(html, { filter: deps.htmlFilter });
+  if (deps.htmlFilter && !page.filterMatched) {
+    deps.sink.skipped.push({ path: url, reason: `html filter '${deps.htmlFilter}' matched nothing` });
+    return;
+  }
   if (!page.markdown.trim()) {
     deps.sink.skipped.push({ path: url, reason: "empty after render" });
     return;
@@ -1101,7 +1804,29 @@ async function handleBrowserPage(ctx, deps) {
     return;
   }
   deps.sink.docs.push(doc);
-  await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+  if (deps.followLinks !== false) {
+    const selector = deps.linkSelector;
+    if (selector) {
+      const elements = await ctx.page.$$(selector);
+      if (elements.length > 0) {
+        deps.selectorStats.hits += 1;
+        await ctx.enqueueLinks({
+          selector,
+          strategy: "same-hostname",
+          exclude: [BINARY_LINK_RE],
+          globs: deps.urlFilter
+        });
+      } else {
+        deps.selectorStats.misses += 1;
+      }
+    } else {
+      await ctx.enqueueLinks({
+        strategy: "same-hostname",
+        exclude: [BINARY_LINK_RE],
+        globs: deps.urlFilter
+      });
+    }
+  }
 }
 async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
   const crawler = kind === "playwright" ? new PlaywrightCrawler({
@@ -1142,20 +1867,66 @@ async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
     });
   }
 }
-async function crawlWebsite(seed, opts) {
-  const seedUrl = normalizeSeedUrl(seed);
-  const kind = opts.crawler;
+function normalizeSitemapUrls(urls) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const raw of urls) {
+    let url;
+    try {
+      url = new URL(raw);
+    } catch {
+      continue;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") continue;
+    url.hash = "";
+    const key = url.toString();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+function applyUrlFilter(urls, globs) {
+  if (!globs || globs.length === 0) return urls;
+  const requests = urls.map((url) => new CrawleeRequest({ url }));
+  const patterns = constructGlobObjectsFromGlobs(globs);
+  return filterRequestsByPatterns(requests, patterns).map((r) => r.url);
+}
+async function loadSitemap(urls) {
+  try {
+    const sitemap = await Sitemap.load(urls, void 0, { reportNetworkErrors: false });
+    return normalizeSitemapUrls(sitemap.urls);
+  } catch {
+    return [];
+  }
+}
+async function discoverSitemapUrls(seedUrl) {
+  const origin = new URL(seedUrl).origin;
+  const fromSitemapXml = await loadSitemap([`${origin}/sitemap.xml`]);
+  if (fromSitemapXml.length > 0) return fromSitemapXml;
+  try {
+    const robots = await RobotsTxtFile.find(seedUrl);
+    const referenced = robots.getSitemaps();
+    if (referenced.length > 0) {
+      const fromRobots = await loadSitemap(referenced);
+      if (fromRobots.length > 0) return fromRobots;
+    }
+  } catch {
+  }
+  return void 0;
+}
+async function runCrawl(kind, seedUrls, opts) {
   if (kind === "playwright" || kind === "puppeteer") {
     await requireBrowserEngine(kind);
     const sink2 = { docs: [], skipped: [] };
     const queue = await freshQueue();
-    await crawlWithBrowser(kind, queue, [seedUrl], opts, sink2);
+    await crawlWithBrowser(kind, queue, seedUrls, opts, sink2);
     return { docs: dedupeDocs(sink2.docs), skipped: sink2.skipped };
   }
   const shellUrls = [];
   const sink = { docs: [], skipped: [], shellUrls };
   const staticQueue = await freshQueue();
-  await crawlWithCheerio(staticQueue, [seedUrl], opts, sink);
+  await crawlWithCheerio(staticQueue, seedUrls, opts, sink);
   if (kind !== "auto" || shellUrls.length === 0) {
     return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
   }
@@ -1174,6 +1945,36 @@ async function crawlWebsite(seed, opts) {
     skipped: [...sink.skipped, ...browserSink.skipped]
   };
 }
+async function crawlWebsite(seed, opts) {
+  const seedUrl = normalizeSeedUrl(seed);
+  const kind = opts.crawler;
+  const mode = opts.sitemap ?? { kind: "auto" };
+  const selectorStats = { hits: 0, misses: 0 };
+  if (mode.kind === "auto" || mode.kind === "sitemap-only") {
+    const sitemapUrls = await discoverSitemapUrls(seedUrl);
+    if (sitemapUrls) {
+      const filtered = applyUrlFilter(sitemapUrls, opts.urlFilter);
+      const detail = filtered.length < sitemapUrls.length ? `${filtered.length} matching --url-filter` : `${filtered.length} URL(s)`;
+      opts.log(`sitemap: ${sitemapUrls.length} URL(s) discovered, ${detail} to crawl`);
+      return runCrawl(kind, filtered, { ...opts, followLinks: false, selectorStats });
+    }
+    if (mode.kind === "sitemap-only") {
+      opts.log(`warning: no sitemap.xml and no sitemaps in robots.txt found for ${seedUrl}; nothing to ingest`);
+      return { docs: [], skipped: [] };
+    }
+    opts.log("no sitemap found; falling back to HTML link crawling");
+  }
+  const result = await runCrawl(kind, [seedUrl], {
+    ...opts,
+    followLinks: true,
+    linkSelector: mode.kind === "selector" ? mode.selector : void 0,
+    selectorStats
+  });
+  if (mode.kind === "selector" && selectorStats.hits === 0 && selectorStats.misses > 0) {
+    opts.log(`warning: --sitemap selector '${mode.selector}' matched no links`);
+  }
+  return result;
+}
 async function feedUrl(ks, url, opts = {}) {
   const log2 = opts.log ?? ((m) => console.log(m));
   const kind = opts.crawler ?? "auto";
@@ -1191,6 +1992,10 @@ async function feedUrl(ks, url, opts = {}) {
     maxLen,
     overlap,
     tags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    filePassword: opts.filePassword,
+    sitemap: opts.sitemap,
     log: log2
   });
   for (const s of skipped) {
@@ -1434,7 +2239,7 @@ async function runChecks(opts) {
 }
 
 // src/version.ts
-var VERSION = "1.1.0";
+var VERSION = "1.3.0";
 
 // src/cli.ts
 async function run(fn) {
@@ -1475,6 +2280,11 @@ function parseTags(tags) {
   const list = tags.split(",").map((t) => t.trim()).filter(Boolean);
   return list.length > 0 ? list : void 0;
 }
+function parseGlobs(globs) {
+  if (!globs || globs.length === 0) return void 0;
+  const list = [...new Set(globs.map((g) => g.trim()).filter(Boolean))];
+  return list.length > 0 ? list : void 0;
+}
 var program = new Command();
 program.name("gresmcp").description("Manage Postgres-backed knowledge sources for the gresmcp MCP server").version(VERSION);
 program.command("init").description("Initialize the database schema (extension, ks table; repairs entry tables)").action(
@@ -1609,12 +2419,40 @@ ksCmd.command("delete <name>").description("Delete a knowledge source and all of
     console.log(`Deleted knowledge source '${name}'`);
   })
 );
-program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
+program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html, pdf, doc, docx, epub)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option(
+  "--url-filter <glob>",
+  "Only follow --url links whose URL matches this glob (repeatable; a link is followed if it matches any glob, e.g. '**/docs/**')",
+  ((v, prev) => [...prev, v]),
+  []
+).option(
+  "--html-filter <selector>",
+  "CSS selector for HTML content \u2014 any DOM selector works (e.g. 'article.post', '#article .content:first-child'); optionally chain steps with ' -> ' (e.g. '#article -> .content'); applies to --path HTML files and --url pages"
+).option(
+  "--sitemap <mode>",
+  "Link discovery for --url: 'auto' (default; sitemap.xml, then robots.txt sitemaps, else follow links), 'sitemap-only' (only sitemap.xml / robots.txt sitemaps), 'html-only' (follow links), or a CSS selector to only follow links inside it (e.g. 'nav.docs a')"
+).option(
+  "--file-password <password>",
+  "Password for encrypted files (PDF, DOC/DOCX); applies to --path and --url ingestion. Owner-restricted PDFs without a user password are read without it"
+).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
   (name, opts) => run(async () => {
     await initDb();
     const ks = requireKs(await getKs(name), name);
     const modes = [opts.path, opts.url, opts.text, opts.stdin].filter(Boolean).length;
     if (modes !== 1) throw new Error("provide exactly one of --path, --url, --text or --stdin");
+    const urlFilter = parseGlobs(opts.urlFilter);
+    if (urlFilter && !opts.url) {
+      console.error("warning: --url-filter has no effect without --url");
+    }
+    if (opts.htmlFilter) parseHtmlFilter(opts.htmlFilter);
+    if (opts.htmlFilter && !opts.path && !opts.url) {
+      console.error("warning: --html-filter has no effect without --path or --url");
+    }
+    let sitemap;
+    if (opts.sitemap !== void 0) {
+      sitemap = parseSitemapOption(opts.sitemap);
+      if (sitemap.kind === "selector") validateCssSelector(sitemap.selector);
+      if (!opts.url) console.error("warning: --sitemap has no effect without --url");
+    }
     const common = {
       replace: opts.replace,
       dryRun: opts.dryRun,
@@ -1622,7 +2460,9 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       maxLen: opts.chunkSize && opts.chunkSize > 0 ? opts.chunkSize : void 0,
       overlap: opts.overlap,
       tags: parseTags(opts.tags),
-      metadata: parseKeyValue(opts.metadata ?? [])
+      metadata: parseKeyValue(opts.metadata ?? []),
+      htmlFilter: opts.htmlFilter,
+      filePassword: opts.filePassword
     };
     let summary;
     const since = await dbClockTimestamp();
@@ -1633,7 +2473,7 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       if (crawler && !CRAWLER_KINDS.includes(crawler)) {
         throw new Error(`unknown crawler '${crawler}' (use auto, crawlee, playwright or puppeteer)`);
       }
-      summary = await feedUrl(ks, opts.url, { ...common, crawler, maxPages: opts.maxPages, depth: opts.depth });
+      summary = await feedUrl(ks, opts.url, { ...common, crawler, sitemap, maxPages: opts.maxPages, depth: opts.depth, urlFilter });
     } else {
       const text = opts.stdin ? await readStdin() : opts.text;
       if (!text.trim()) throw new Error("empty input text");