@gresmcp/mcp 1.1.0 → 1.2.0

package/dist/cli.js

@@ -446,17 +446,57 @@ function turndown() {
   }
   return service;
 }
-function htmlToMarkdown(html) {
+function parseHtmlFilter(spec) {
+  const steps = spec.split(/\s*->\s*/).map((s) => s.trim()).filter(Boolean);
+  if (steps.length === 0) {
+    throw new Error(`invalid --html-filter '${spec.trim()}': expected a CSS selector chain, e.g. '#article -> .content'`);
+  }
+  return steps;
+}
+function validateCssSelector(selector) {
+  const dom = new JSDOM();
+  try {
+    dom.window.document.querySelector(selector);
+  } catch (err) {
+    throw new Error(`invalid CSS selector '${selector}': ${err instanceof Error ? err.message : err}`);
+  }
+}
+function selectChain(scope, steps) {
+  for (const step of steps.slice(0, -1)) {
+    const next = scope.querySelector(step);
+    if (!next) return void 0;
+    scope = next;
+  }
+  const matches = [...scope.querySelectorAll(steps[steps.length - 1])];
+  return matches.length > 0 ? matches : void 0;
+}
+function elementsToMarkdown(elements) {
+  const parts = [];
+  for (const el of elements) {
+    try {
+      parts.push(turndown().turndown(el.innerHTML));
+    } catch {
+      parts.push(el.textContent ?? "");
+    }
+  }
+  return parts.join("\n\n");
+}
+function htmlToMarkdown(html, opts = {}) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
   doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
   const title = doc.title?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
-  const root = doc.body ?? doc.documentElement;
   let markdown = "";
-  try {
-    markdown = turndown().turndown(root.innerHTML);
-  } catch {
-    markdown = root.textContent ?? "";
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
+    const root = doc.body ?? doc.documentElement;
+    try {
+      markdown = turndown().turndown(root.innerHTML);
+    } catch {
+      markdown = root.textContent ?? "";
+    }
   }
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
   return { markdown, title: title || void 0 };
@@ -490,7 +530,7 @@ function collectTags(doc) {
     return true;
   });
 }
-function extractPage(html) {
+function extractPage(html, opts = {}) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
   const title = doc.title?.trim() || doc.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
@@ -501,20 +541,27 @@ function extractPage(html) {
   const tags = collectTags(doc);
   doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
   let markdown = "";
-  try {
-    if (isProbablyReaderable(doc)) {
-      const article = new Readability(doc).parse();
-      if (article?.content) markdown = turndown().turndown(article.content);
-    }
-  } catch {
-    markdown = "";
-  }
-  if (!markdown.trim()) {
-    const root = doc.body ?? doc.documentElement;
+  let filterMatched;
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    filterMatched = elements !== void 0;
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
     try {
-      markdown = turndown().turndown(root.innerHTML);
+      if (isProbablyReaderable(doc)) {
+        const article = new Readability(doc).parse();
+        if (article?.content) markdown = turndown().turndown(article.content);
+      }
     } catch {
-      markdown = root.textContent ?? "";
+      markdown = "";
+    }
+    if (!markdown.trim()) {
+      const root = doc.body ?? doc.documentElement;
+      try {
+        markdown = turndown().turndown(root.innerHTML);
+      } catch {
+        markdown = root.textContent ?? "";
+      }
     }
   }
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
@@ -524,7 +571,8 @@ function extractPage(html) {
     title,
     description,
     tags: tags.length > 0 ? tags : void 0,
-    likelyShell
+    likelyShell,
+    filterMatched
   };
 }
 
@@ -544,14 +592,16 @@ function bar(pct2) {
 function pct(p) {
   return `${Math.round(Math.max(0, Math.min(1, p)) * 100)}%`;
 }
+var NOOP_HANDLE = { handled: () => {
+} };
 var FeedProgress = class {
   write;
   interactive;
   total;
   rejected;
+  slots = [];
   filesDone = 0;
   totalChars = 0;
-  current;
   blockLines = 0;
   finished = false;
   constructor(opts) {
@@ -564,26 +614,13 @@ var FeedProgress = class {
     return this.interactive;
   }
   startFile(source, totalChunks, totalChars) {
-    if (this.finished) return;
-    this.current = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    if (this.finished) return NOOP_HANDLE;
+    const slot = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    this.slots.push(slot);
     this.render();
-  }
-  handled(chunks, chars) {
-    if (this.finished || !this.current || chunks <= 0) return;
-    const c = this.current;
-    c.doneChunks += chunks;
-    c.doneChars += chars;
-    this.totalChars += chars;
-    if (c.doneChunks >= c.totalChunks) {
-      this.filesDone++;
-      this.printFileLine();
-      return;
-    }
-    if (!this.interactive && c.doneChunks - c.printedChunks >= PLAIN_STEP) {
-      this.printFileLine();
-    } else if (this.interactive) {
-      this.render();
-    }
+    return {
+      handled: (chunks, chars) => this.handleSlot(slot, chunks, chars)
+    };
   }
   log(msg) {
     if (this.finished) return;
@@ -594,9 +631,9 @@ var FeedProgress = class {
   finish() {
     if (this.finished) return;
     this.finished = true;
-    if (!this.current && this.total === 0 && this.rejected === 0) return;
+    if (this.slots.length === 0 && this.total === 0 && this.rejected === 0) return;
     if (this.interactive) {
-      this.redraw([this.fileLine(), this.totalLine()]);
+      this.redraw(this.lines());
       this.write("\n");
       this.blockLines = 0;
     } else {
@@ -609,25 +646,42 @@ var FeedProgress = class {
     this.finished = true;
     this.clearBlock();
   }
+  lines() {
+    return [...this.slots.map((slot) => this.fileLine(slot)), this.totalLine()];
+  }
+  handleSlot(slot, chunks, chars) {
+    if (this.finished || chunks <= 0 || slot.doneChunks >= slot.totalChunks) return;
+    slot.doneChunks += chunks;
+    slot.doneChars += chars;
+    this.totalChars += chars;
+    if (slot.doneChunks >= slot.totalChunks) {
+      this.filesDone++;
+      if (this.interactive) {
+        this.render();
+      } else {
+        this.printFileLine(slot);
+      }
+      this.slots.splice(this.slots.indexOf(slot), 1);
+      return;
+    }
+    if (!this.interactive && slot.doneChunks - slot.printedChunks >= PLAIN_STEP) {
+      this.printFileLine(slot);
+    } else if (this.interactive) {
+      this.render();
+    }
+  }
   render() {
     if (!this.interactive) return;
-    this.redraw([this.fileLine(), this.totalLine()]);
+    this.redraw(this.lines());
   }
-  printFileLine() {
-    if (!this.current) return;
-    this.current.printedChunks = this.current.doneChunks;
-    if (this.interactive) {
-      this.render();
-    } else {
-      this.write(`${this.fileLine()}
+  printFileLine(slot) {
+    slot.printedChunks = slot.doneChunks;
+    this.write(`${this.fileLine(slot)}
 `);
-    }
   }
-  fileLine() {
-    const c = this.current;
-    if (!c) return "";
-    const p = c.totalChunks > 0 ? c.doneChunks / c.totalChunks : 1;
-    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(c.doneChunks)}/${fmt(c.totalChunks)} chunks  ${fmt(c.doneChars)}/${fmt(c.totalChars)} chars  ${c.source}`;
+  fileLine(slot) {
+    const p = slot.totalChunks > 0 ? slot.doneChunks / slot.totalChunks : 1;
+    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(slot.doneChunks)}/${fmt(slot.totalChunks)} chunks  ${fmt(slot.doneChars)}/${fmt(slot.totalChars)} chars  ${slot.source}`;
   }
   totalLine() {
     const p = this.total > 0 ? this.filesDone / this.total : 0;
@@ -792,14 +846,17 @@ async function loadTarget(target, opts) {
     let text = raw;
     let title = "";
     if (HTML_EXT.has(ext)) {
-      const conv = htmlToMarkdown(raw);
+      const conv = htmlToMarkdown(raw, { filter: opts.htmlFilter });
       text = conv.markdown;
       title = conv.title ?? "";
     } else if (MD_EXT.has(ext)) {
       title = mdTitle(raw) ?? "";
     }
     if (!text.trim()) {
-      skipped.push({ path: file, reason: "empty" });
+      skipped.push({
+        path: file,
+        reason: HTML_EXT.has(ext) && opts.htmlFilter ? `html filter '${opts.htmlFilter}' matched nothing` : "empty"
+      });
       continue;
     }
     if (!title) title = path.basename(file);
@@ -812,6 +869,7 @@ async function loadTarget(target, opts) {
   }
   return { docs, fileCount: files.length, skipped };
 }
+var DOC_CONCURRENCY = 3;
 async function feedDocs(ks, docs, opts = {}) {
   const log2 = opts.log ?? ((m) => console.log(m));
   const progress = opts.progress;
@@ -823,61 +881,78 @@ async function feedDocs(ks, docs, opts = {}) {
   let skippedExisting = 0;
   let inserted = 0;
   let announcedEmbed = false;
-  try {
-    for (const doc of docs) {
-      progress?.startFile(
-        doc.source,
-        doc.chunks.length,
-        doc.chunks.reduce((n, c) => n + c.content.length, 0)
-      );
-      if (opts.replace) {
-        deletedForReplace += await deleteBySource(ks, doc.source);
-      }
-      const existing = await sourceChunkHashes(ks, doc.source);
-      const meta = { ...baseMeta, ...doc.metadata ?? {} };
-      if (!("tags" in meta) && cliTags) meta.tags = cliTags;
-      const todo = [];
-      for (const chunk of doc.chunks) {
-        if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
-          skippedExisting++;
-          progress?.handled(1, chunk.content.length);
-          continue;
-        }
-        todo.push(chunk);
-      }
-      if (todo.length === 0) continue;
-      if (opts.dryRun) {
-        for (const chunk of todo) progress?.handled(1, chunk.content.length);
+  const processDoc = async (doc) => {
+    const file = progress?.startFile(
+      doc.source,
+      doc.chunks.length,
+      doc.chunks.reduce((n, c) => n + c.content.length, 0)
+    );
+    if (opts.replace) {
+      const deleted = await deleteBySource(ks, doc.source);
+      deletedForReplace += deleted;
+    }
+    const existing = await sourceChunkHashes(ks, doc.source);
+    const meta = { ...baseMeta, ...doc.metadata ?? {} };
+    if (!("tags" in meta) && cliTags) meta.tags = cliTags;
+    const todo = [];
+    for (const chunk of doc.chunks) {
+      if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
+        skippedExisting++;
+        file?.handled(1, chunk.content.length);
         continue;
       }
-      if (!announcedEmbed) {
-        say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
-        announcedEmbed = true;
+      todo.push(chunk);
+    }
+    if (todo.length === 0) return;
+    if (opts.dryRun) {
+      for (const chunk of todo) file?.handled(1, chunk.content.length);
+      return;
+    }
+    if (!announcedEmbed) {
+      announcedEmbed = true;
+      say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
+    }
+    const contents = todo.map((c) => c.content);
+    const charLens = contents.map((c) => c.length);
+    let reported = 0;
+    const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
+      const delta = done - reported;
+      if (delta <= 0) return;
+      let chars = 0;
+      for (let i = reported; i < done; i++) chars += charLens[i];
+      reported = done;
+      file?.handled(delta, chars);
+    });
+    const rows = todo.map((chunk, i) => ({
+      source: doc.source,
+      title: doc.title,
+      chunk_index: chunk.index,
+      content: chunk.content,
+      metadata: meta,
+      embedding: embeddings[i]
+    }));
+    const rowCount = await insertChunks(ks, rows);
+    inserted += rowCount;
+  };
+  let cursor = 0;
+  let firstError;
+  const runWorker = async () => {
+    while (firstError === void 0) {
+      const index = cursor++;
+      if (index >= docs.length) return;
+      try {
+        await processDoc(docs[index]);
+      } catch (err) {
+        firstError ??= err;
+        return;
       }
-      const contents = todo.map((c) => c.content);
-      const charLens = contents.map((c) => c.length);
-      let reported = 0;
-      const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
-        const delta = done - reported;
-        if (delta <= 0) return;
-        let chars = 0;
-        for (let i = reported; i < done; i++) chars += charLens[i];
-        reported = done;
-        progress?.handled(delta, chars);
-      });
-      const rows = todo.map((chunk, i) => ({
-        source: doc.source,
-        title: doc.title,
-        chunk_index: chunk.index,
-        content: chunk.content,
-        metadata: meta,
-        embedding: embeddings[i]
-      }));
-      inserted += await insertChunks(ks, rows);
     }
-  } catch (err) {
+  };
+  const workerCount = Math.min(DOC_CONCURRENCY, docs.length);
+  await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
+  if (firstError !== void 0) {
     progress?.close();
-    throw err;
+    throw firstError;
   }
   progress?.finish();
   return {
@@ -891,7 +966,7 @@ async function feedDocs(ks, docs, opts = {}) {
   };
 }
 async function feedTarget(ks, target, opts = {}) {
-  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
+  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180, htmlFilter: opts.htmlFilter };
   const { docs, skipped } = await loadTarget(target, loadOpts);
   for (const s of skipped) {
     (opts.log ?? ((m) => console.log(m)))(`skipped ${s.path}: ${s.reason}`);
@@ -915,8 +990,30 @@ import { mkdtempSync } from "fs";
 import { tmpdir } from "os";
 import path2 from "path";
 import { randomUUID } from "crypto";
-import { CheerioCrawler, PlaywrightCrawler, PuppeteerCrawler, RequestQueue, log } from "crawlee";
+import {
+  CheerioCrawler,
+  PlaywrightCrawler,
+  PuppeteerCrawler,
+  Request as CrawleeRequest,
+  RequestQueue,
+  RobotsTxtFile,
+  Sitemap,
+  constructGlobObjectsFromGlobs,
+  filterRequestsByPatterns,
+  log
+} from "crawlee";
 var CRAWLER_KINDS = ["auto", "crawlee", "playwright", "puppeteer"];
+function parseSitemapOption(value) {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    throw new Error("invalid --sitemap '': use auto, sitemap-only, html-only or a CSS selector");
+  }
+  const key = trimmed.toLowerCase();
+  if (key === "auto") return { kind: "auto" };
+  if (key === "sitemap-only") return { kind: "sitemap-only" };
+  if (key === "html-only") return { kind: "html-only" };
+  return { kind: "selector", selector: trimmed };
+}
 var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|docx?|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
 process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync(path2.join(tmpdir(), "gresmcp-crawl-"));
 async function freshQueue() {
@@ -1006,7 +1103,7 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
     maxCrawlDepth: Math.max(0, opts.depth),
     additionalMimeTypes: ["text/plain", "text/markdown"],
     async requestHandler(ctx) {
-      const { request, response, contentType, body } = ctx;
+      const { request, response, contentType, body, $ } = ctx;
       const url = request.loadedUrl ?? request.url;
       const status = responseStatus(response);
       if (status === void 0 || status < 200 || status >= 300) {
@@ -1016,12 +1113,16 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
       const type = contentType?.type ?? "";
       if (type === "text/html" || type === "application/xhtml+xml") {
         const html = typeof body === "string" ? body : toText(body);
-        const page = extractPage(html);
+        const page = extractPage(html, { filter: opts.htmlFilter });
         if (page.likelyShell) {
           sink.shellUrls?.push(url);
           sink.skipped.push({ path: url, reason: "likely JS-rendered shell" });
           return;
         }
+        if (opts.htmlFilter && !page.filterMatched) {
+          sink.skipped.push({ path: url, reason: `html filter '${opts.htmlFilter}' matched nothing` });
+          return;
+        }
         if (!page.markdown.trim()) {
           sink.skipped.push({ path: url, reason: "empty" });
           return;
@@ -1032,7 +1133,28 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
           return;
         }
         sink.docs.push(doc);
-        await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+        if (opts.followLinks !== false) {
+          const selector = opts.linkSelector;
+          if (selector) {
+            if ($(selector).length > 0) {
+              opts.selectorStats.hits += 1;
+              await ctx.enqueueLinks({
+                selector,
+                strategy: "same-hostname",
+                exclude: [BINARY_LINK_RE],
+                globs: opts.urlFilter
+              });
+            } else {
+              opts.selectorStats.misses += 1;
+            }
+          } else {
+            await ctx.enqueueLinks({
+              strategy: "same-hostname",
+              exclude: [BINARY_LINK_RE],
+              globs: opts.urlFilter
+            });
+          }
+        }
       } else if (type === "text/markdown") {
         const doc = buildDoc(url, "md", toText(body), void 0, sinkDeps(opts));
         if (!doc) {
@@ -1064,7 +1186,16 @@ async function crawlWithCheerio(queue, seedUrls, opts, sink) {
   }
 }
 function sinkDeps(opts) {
-  return { maxLen: opts.maxLen, overlap: opts.overlap, cliTags: opts.tags };
+  return {
+    maxLen: opts.maxLen,
+    overlap: opts.overlap,
+    cliTags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    followLinks: opts.followLinks,
+    linkSelector: opts.linkSelector,
+    selectorStats: opts.selectorStats
+  };
 }
 async function importOptional(name) {
   try {
@@ -1090,7 +1221,11 @@ async function handleBrowserPage(ctx, deps) {
     return;
   }
   const html = await ctx.page.content();
-  const page = extractPage(html);
+  const page = extractPage(html, { filter: deps.htmlFilter });
+  if (deps.htmlFilter && !page.filterMatched) {
+    deps.sink.skipped.push({ path: url, reason: `html filter '${deps.htmlFilter}' matched nothing` });
+    return;
+  }
   if (!page.markdown.trim()) {
     deps.sink.skipped.push({ path: url, reason: "empty after render" });
     return;
@@ -1101,7 +1236,29 @@ async function handleBrowserPage(ctx, deps) {
     return;
   }
   deps.sink.docs.push(doc);
-  await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+  if (deps.followLinks !== false) {
+    const selector = deps.linkSelector;
+    if (selector) {
+      const elements = await ctx.page.$$(selector);
+      if (elements.length > 0) {
+        deps.selectorStats.hits += 1;
+        await ctx.enqueueLinks({
+          selector,
+          strategy: "same-hostname",
+          exclude: [BINARY_LINK_RE],
+          globs: deps.urlFilter
+        });
+      } else {
+        deps.selectorStats.misses += 1;
+      }
+    } else {
+      await ctx.enqueueLinks({
+        strategy: "same-hostname",
+        exclude: [BINARY_LINK_RE],
+        globs: deps.urlFilter
+      });
+    }
+  }
 }
 async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
   const crawler = kind === "playwright" ? new PlaywrightCrawler({
@@ -1142,20 +1299,66 @@ async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
     });
   }
 }
-async function crawlWebsite(seed, opts) {
-  const seedUrl = normalizeSeedUrl(seed);
-  const kind = opts.crawler;
+function normalizeSitemapUrls(urls) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const raw of urls) {
+    let url;
+    try {
+      url = new URL(raw);
+    } catch {
+      continue;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") continue;
+    url.hash = "";
+    const key = url.toString();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+function applyUrlFilter(urls, globs) {
+  if (!globs || globs.length === 0) return urls;
+  const requests = urls.map((url) => new CrawleeRequest({ url }));
+  const patterns = constructGlobObjectsFromGlobs(globs);
+  return filterRequestsByPatterns(requests, patterns).map((r) => r.url);
+}
+async function loadSitemap(urls) {
+  try {
+    const sitemap = await Sitemap.load(urls, void 0, { reportNetworkErrors: false });
+    return normalizeSitemapUrls(sitemap.urls);
+  } catch {
+    return [];
+  }
+}
+async function discoverSitemapUrls(seedUrl) {
+  const origin = new URL(seedUrl).origin;
+  const fromSitemapXml = await loadSitemap([`${origin}/sitemap.xml`]);
+  if (fromSitemapXml.length > 0) return fromSitemapXml;
+  try {
+    const robots = await RobotsTxtFile.find(seedUrl);
+    const referenced = robots.getSitemaps();
+    if (referenced.length > 0) {
+      const fromRobots = await loadSitemap(referenced);
+      if (fromRobots.length > 0) return fromRobots;
+    }
+  } catch {
+  }
+  return void 0;
+}
+async function runCrawl(kind, seedUrls, opts) {
   if (kind === "playwright" || kind === "puppeteer") {
     await requireBrowserEngine(kind);
     const sink2 = { docs: [], skipped: [] };
     const queue = await freshQueue();
-    await crawlWithBrowser(kind, queue, [seedUrl], opts, sink2);
+    await crawlWithBrowser(kind, queue, seedUrls, opts, sink2);
     return { docs: dedupeDocs(sink2.docs), skipped: sink2.skipped };
   }
   const shellUrls = [];
   const sink = { docs: [], skipped: [], shellUrls };
   const staticQueue = await freshQueue();
-  await crawlWithCheerio(staticQueue, [seedUrl], opts, sink);
+  await crawlWithCheerio(staticQueue, seedUrls, opts, sink);
   if (kind !== "auto" || shellUrls.length === 0) {
     return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
   }
@@ -1174,6 +1377,36 @@ async function crawlWebsite(seed, opts) {
     skipped: [...sink.skipped, ...browserSink.skipped]
   };
 }
+async function crawlWebsite(seed, opts) {
+  const seedUrl = normalizeSeedUrl(seed);
+  const kind = opts.crawler;
+  const mode = opts.sitemap ?? { kind: "auto" };
+  const selectorStats = { hits: 0, misses: 0 };
+  if (mode.kind === "auto" || mode.kind === "sitemap-only") {
+    const sitemapUrls = await discoverSitemapUrls(seedUrl);
+    if (sitemapUrls) {
+      const filtered = applyUrlFilter(sitemapUrls, opts.urlFilter);
+      const detail = filtered.length < sitemapUrls.length ? `${filtered.length} matching --url-filter` : `${filtered.length} URL(s)`;
+      opts.log(`sitemap: ${sitemapUrls.length} URL(s) discovered, ${detail} to crawl`);
+      return runCrawl(kind, filtered, { ...opts, followLinks: false, selectorStats });
+    }
+    if (mode.kind === "sitemap-only") {
+      opts.log(`warning: no sitemap.xml and no sitemaps in robots.txt found for ${seedUrl}; nothing to ingest`);
+      return { docs: [], skipped: [] };
+    }
+    opts.log("no sitemap found; falling back to HTML link crawling");
+  }
+  const result = await runCrawl(kind, [seedUrl], {
+    ...opts,
+    followLinks: true,
+    linkSelector: mode.kind === "selector" ? mode.selector : void 0,
+    selectorStats
+  });
+  if (mode.kind === "selector" && selectorStats.hits === 0 && selectorStats.misses > 0) {
+    opts.log(`warning: --sitemap selector '${mode.selector}' matched no links`);
+  }
+  return result;
+}
 async function feedUrl(ks, url, opts = {}) {
   const log2 = opts.log ?? ((m) => console.log(m));
   const kind = opts.crawler ?? "auto";
@@ -1191,6 +1424,9 @@ async function feedUrl(ks, url, opts = {}) {
     maxLen,
     overlap,
     tags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    sitemap: opts.sitemap,
     log: log2
   });
   for (const s of skipped) {
@@ -1434,7 +1670,7 @@ async function runChecks(opts) {
 }
 
 // src/version.ts
-var VERSION = "1.1.0";
+var VERSION = "1.2.0";
 
 // src/cli.ts
 async function run(fn) {
@@ -1475,6 +1711,11 @@ function parseTags(tags) {
   const list = tags.split(",").map((t) => t.trim()).filter(Boolean);
   return list.length > 0 ? list : void 0;
 }
+function parseGlobs(globs) {
+  if (!globs || globs.length === 0) return void 0;
+  const list = [...new Set(globs.map((g) => g.trim()).filter(Boolean))];
+  return list.length > 0 ? list : void 0;
+}
 var program = new Command();
 program.name("gresmcp").description("Manage Postgres-backed knowledge sources for the gresmcp MCP server").version(VERSION);
 program.command("init").description("Initialize the database schema (extension, ks table; repairs entry tables)").action(
@@ -1609,12 +1850,37 @@ ksCmd.command("delete <name>").description("Delete a knowledge source and all of
     console.log(`Deleted knowledge source '${name}'`);
   })
 );
-program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
+program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option(
+  "--url-filter <glob>",
+  "Only follow --url links whose URL matches this glob (repeatable; a link is followed if it matches any glob, e.g. '**/docs/**')",
+  ((v, prev) => [...prev, v]),
+  []
+).option(
+  "--html-filter <selector>",
+  "CSS selector for HTML content \u2014 any DOM selector works (e.g. 'article.post', '#article .content:first-child'); optionally chain steps with ' -> ' (e.g. '#article -> .content'); applies to --path HTML files and --url pages"
+).option(
+  "--sitemap <mode>",
+  "Link discovery for --url: 'auto' (default; sitemap.xml, then robots.txt sitemaps, else follow links), 'sitemap-only' (only sitemap.xml / robots.txt sitemaps), 'html-only' (follow links), or a CSS selector to only follow links inside it (e.g. 'nav.docs a')"
+).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
   (name, opts) => run(async () => {
     await initDb();
     const ks = requireKs(await getKs(name), name);
     const modes = [opts.path, opts.url, opts.text, opts.stdin].filter(Boolean).length;
     if (modes !== 1) throw new Error("provide exactly one of --path, --url, --text or --stdin");
+    const urlFilter = parseGlobs(opts.urlFilter);
+    if (urlFilter && !opts.url) {
+      console.error("warning: --url-filter has no effect without --url");
+    }
+    if (opts.htmlFilter) parseHtmlFilter(opts.htmlFilter);
+    if (opts.htmlFilter && !opts.path && !opts.url) {
+      console.error("warning: --html-filter has no effect without --path or --url");
+    }
+    let sitemap;
+    if (opts.sitemap !== void 0) {
+      sitemap = parseSitemapOption(opts.sitemap);
+      if (sitemap.kind === "selector") validateCssSelector(sitemap.selector);
+      if (!opts.url) console.error("warning: --sitemap has no effect without --url");
+    }
     const common = {
       replace: opts.replace,
       dryRun: opts.dryRun,
@@ -1622,7 +1888,8 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       maxLen: opts.chunkSize && opts.chunkSize > 0 ? opts.chunkSize : void 0,
       overlap: opts.overlap,
       tags: parseTags(opts.tags),
-      metadata: parseKeyValue(opts.metadata ?? [])
+      metadata: parseKeyValue(opts.metadata ?? []),
+      htmlFilter: opts.htmlFilter
     };
     let summary;
     const since = await dbClockTimestamp();
@@ -1633,7 +1900,7 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       if (crawler && !CRAWLER_KINDS.includes(crawler)) {
         throw new Error(`unknown crawler '${crawler}' (use auto, crawlee, playwright or puppeteer)`);
       }
-      summary = await feedUrl(ks, opts.url, { ...common, crawler, maxPages: opts.maxPages, depth: opts.depth });
+      summary = await feedUrl(ks, opts.url, { ...common, crawler, sitemap, maxPages: opts.maxPages, depth: opts.depth, urlFilter });
     } else {
       const text = opts.stdin ? await readStdin() : opts.text;
       if (!text.trim()) throw new Error("empty input text");