@gresmcp/mcp 1.0.0 → 1.2.0

package/dist/cli.js

@@ -37,6 +37,7 @@ __export(db_exports, {
   contentHash: () => contentHash,
   countEntries: () => countEntries,
   createKs: () => createKs,
+  dbClockTimestamp: () => dbClockTimestamp,
   deleteBySource: () => deleteBySource,
   deleteKs: () => deleteKs,
   ensureEntryTable: () => ensureEntryTable,
@@ -45,6 +46,7 @@ __export(db_exports, {
   getPool: () => getPool,
   initDb: () => initDb,
   insertChunks: () => insertChunks,
+  ksStats: () => ksStats,
   listKs: () => listKs,
   sourceChunkHashes: () => sourceChunkHashes,
   updateKs: () => updateKs
@@ -211,6 +213,41 @@ async function countEntries(ks) {
   const res = await getPool().query(`SELECT count(*)::int AS n FROM ${tbl}`);
   return Number(res.rows[0]?.n ?? 0);
 }
+async function dbClockTimestamp() {
+  const res = await getPool().query(`SELECT clock_timestamp() AS ts`);
+  return res.rows[0].ts;
+}
+async function ksStats(ks, since) {
+  const tbl = entryTableName(ks);
+  const params = [];
+  let where = "";
+  if (since) {
+    params.push(since);
+    where = ` WHERE created_at >= $1`;
+  }
+  const res = await getPool().query(
+    `SELECT count(*)::int AS entries,
+            count(DISTINCT source)::int AS sources,
+            COALESCE(sum(length(content)), 0)::bigint AS chars,
+            round(avg(length(content)))::int AS avg_chunk_chars,
+            min(length(content))::int AS min_chunk_chars,
+            max(length(content))::int AS max_chunk_chars,
+            max(created_at) AS last_fed_at
+     FROM ${tbl}${where}`,
+    params
+  );
+  const r = res.rows[0];
+  return {
+    ks,
+    entries: Number(r.entries ?? 0),
+    sources: Number(r.sources ?? 0),
+    chars: Number(r.chars ?? 0),
+    avg_chunk_chars: r.avg_chunk_chars === null ? null : Number(r.avg_chunk_chars),
+    min_chunk_chars: r.min_chunk_chars === null ? null : Number(r.min_chunk_chars),
+    max_chunk_chars: r.max_chunk_chars === null ? null : Number(r.max_chunk_chars),
+    last_fed_at: r.last_fed_at === null ? null : r.last_fed_at
+  };
+}
 var pool;
 var init_db = __esm({
   "src/db.ts"() {
@@ -353,8 +390,8 @@ function chunkMarkdown(text, opts = {}) {
     const m = HEADING_RE.exec(line);
     if (m) {
       const level = m[1].length;
-      const path2 = sections[sections.length - 1].path;
-      const parent = path2.slice(0, level - 1);
+      const path3 = sections[sections.length - 1].path;
+      const parent = path3.slice(0, level - 1);
       sections.push({ path: [...parent, m[2].trim()], body: [line] });
     } else {
       sections[sections.length - 1].body.push(line);
@@ -388,9 +425,12 @@ function mdTitle(text) {
 // src/html.ts
 import { JSDOM } from "jsdom";
 import TurndownService from "turndown";
+import { Readability, isProbablyReaderable } from "@mozilla/readability";
 import gfmModule from "turndown-plugin-gfm";
 var gfm = typeof gfmModule === "function" ? gfmModule : gfmModule.gfm;
 var REMOVE_SELECTORS = "script, style, noscript, template, iframe, svg, nav, footer, aside, form, button";
+var SHELL_MARKER_RE = /id=["'](root|app|__next|__nuxt|q-app)["']|data-reactroot|ng-version=/i;
+var SHELL_TEXT_THRESHOLD = 250;
 var service;
 function turndown() {
   if (!service) {
@@ -406,25 +446,276 @@ function turndown() {
   }
   return service;
 }
-function htmlToMarkdown(html) {
+function parseHtmlFilter(spec) {
+  const steps = spec.split(/\s*->\s*/).map((s) => s.trim()).filter(Boolean);
+  if (steps.length === 0) {
+    throw new Error(`invalid --html-filter '${spec.trim()}': expected a CSS selector chain, e.g. '#article -> .content'`);
+  }
+  return steps;
+}
+function validateCssSelector(selector) {
+  const dom = new JSDOM();
+  try {
+    dom.window.document.querySelector(selector);
+  } catch (err) {
+    throw new Error(`invalid CSS selector '${selector}': ${err instanceof Error ? err.message : err}`);
+  }
+}
+function selectChain(scope, steps) {
+  for (const step of steps.slice(0, -1)) {
+    const next = scope.querySelector(step);
+    if (!next) return void 0;
+    scope = next;
+  }
+  const matches = [...scope.querySelectorAll(steps[steps.length - 1])];
+  return matches.length > 0 ? matches : void 0;
+}
+function elementsToMarkdown(elements) {
+  const parts = [];
+  for (const el of elements) {
+    try {
+      parts.push(turndown().turndown(el.innerHTML));
+    } catch {
+      parts.push(el.textContent ?? "");
+    }
+  }
+  return parts.join("\n\n");
+}
+function htmlToMarkdown(html, opts = {}) {
   const dom = new JSDOM(html);
   const doc = dom.window.document;
   doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
   const title = doc.title?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
-  const root = doc.body ?? doc.documentElement;
   let markdown = "";
-  try {
-    markdown = turndown().turndown(root.innerHTML);
-  } catch {
-    markdown = root.textContent ?? "";
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
+    const root = doc.body ?? doc.documentElement;
+    try {
+      markdown = turndown().turndown(root.innerHTML);
+    } catch {
+      markdown = root.textContent ?? "";
+    }
   }
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
   return { markdown, title: title || void 0 };
 }
+function metaContent(doc, selectors) {
+  for (const selector of selectors) {
+    const el = doc.querySelector(selector);
+    const value = (el?.getAttribute("content") ?? el?.textContent ?? "").trim();
+    if (value) return value;
+  }
+  return void 0;
+}
+function collectTags(doc) {
+  const tags = [];
+  const keywords = metaContent(doc, ['meta[name="keywords"]']);
+  if (keywords) {
+    for (const part of keywords.split(",")) {
+      const tag = part.trim();
+      if (tag) tags.push(tag);
+    }
+  }
+  doc.querySelectorAll('meta[property="og:article:tag"]').forEach((el) => {
+    const value = (el.getAttribute("content") ?? "").trim();
+    if (value) tags.push(value);
+  });
+  const seen = /* @__PURE__ */ new Set();
+  return tags.filter((t) => {
+    const key = t.toLowerCase();
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+function extractPage(html, opts = {}) {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+  const title = doc.title?.trim() || doc.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
+  const description = metaContent(doc, [
+    'meta[name="description"]',
+    'meta[property="og:description"]'
+  ]);
+  const tags = collectTags(doc);
+  doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
+  let markdown = "";
+  let filterMatched;
+  if (opts.filter) {
+    const elements = selectChain(doc, parseHtmlFilter(opts.filter));
+    filterMatched = elements !== void 0;
+    if (elements) markdown = elementsToMarkdown(elements);
+  } else {
+    try {
+      if (isProbablyReaderable(doc)) {
+        const article = new Readability(doc).parse();
+        if (article?.content) markdown = turndown().turndown(article.content);
+      }
+    } catch {
+      markdown = "";
+    }
+    if (!markdown.trim()) {
+      const root = doc.body ?? doc.documentElement;
+      try {
+        markdown = turndown().turndown(root.innerHTML);
+      } catch {
+        markdown = root.textContent ?? "";
+      }
+    }
+  }
+  markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
+  const likelyShell = markdown.length < SHELL_TEXT_THRESHOLD && SHELL_MARKER_RE.test(html);
+  return {
+    markdown,
+    title,
+    description,
+    tags: tags.length > 0 ? tags : void 0,
+    likelyShell,
+    filterMatched
+  };
+}
 
 // src/ingest.ts
 init_db();
-init_config();
+
+// src/progress.ts
+var BAR_WIDTH = 20;
+var PLAIN_STEP = 32;
+function fmt(n) {
+  return n.toLocaleString("en-US");
+}
+function bar(pct2) {
+  const filled = Math.max(0, Math.min(BAR_WIDTH, Math.round(pct2 * BAR_WIDTH)));
+  return "#".repeat(filled) + ".".repeat(BAR_WIDTH - filled);
+}
+function pct(p) {
+  return `${Math.round(Math.max(0, Math.min(1, p)) * 100)}%`;
+}
+var NOOP_HANDLE = { handled: () => {
+} };
+var FeedProgress = class {
+  write;
+  interactive;
+  total;
+  rejected;
+  slots = [];
+  filesDone = 0;
+  totalChars = 0;
+  blockLines = 0;
+  finished = false;
+  constructor(opts) {
+    this.total = Math.max(0, Math.floor(opts.total));
+    this.rejected = Math.max(0, Math.floor(opts.rejected));
+    this.write = opts.out ?? ((s) => process.stderr.write(s));
+    this.interactive = opts.interactive ?? (opts.out === void 0 && !!process.stderr.isTTY);
+  }
+  get isInteractive() {
+    return this.interactive;
+  }
+  startFile(source, totalChunks, totalChars) {
+    if (this.finished) return NOOP_HANDLE;
+    const slot = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    this.slots.push(slot);
+    this.render();
+    return {
+      handled: (chunks, chars) => this.handleSlot(slot, chunks, chars)
+    };
+  }
+  log(msg) {
+    if (this.finished) return;
+    this.clearBlock();
+    this.write(`${msg}
+`);
+  }
+  finish() {
+    if (this.finished) return;
+    this.finished = true;
+    if (this.slots.length === 0 && this.total === 0 && this.rejected === 0) return;
+    if (this.interactive) {
+      this.redraw(this.lines());
+      this.write("\n");
+      this.blockLines = 0;
+    } else {
+      this.write(`${this.totalLine()}
+`);
+    }
+  }
+  close() {
+    if (this.finished) return;
+    this.finished = true;
+    this.clearBlock();
+  }
+  lines() {
+    return [...this.slots.map((slot) => this.fileLine(slot)), this.totalLine()];
+  }
+  handleSlot(slot, chunks, chars) {
+    if (this.finished || chunks <= 0 || slot.doneChunks >= slot.totalChunks) return;
+    slot.doneChunks += chunks;
+    slot.doneChars += chars;
+    this.totalChars += chars;
+    if (slot.doneChunks >= slot.totalChunks) {
+      this.filesDone++;
+      if (this.interactive) {
+        this.render();
+      } else {
+        this.printFileLine(slot);
+      }
+      this.slots.splice(this.slots.indexOf(slot), 1);
+      return;
+    }
+    if (!this.interactive && slot.doneChunks - slot.printedChunks >= PLAIN_STEP) {
+      this.printFileLine(slot);
+    } else if (this.interactive) {
+      this.render();
+    }
+  }
+  render() {
+    if (!this.interactive) return;
+    this.redraw(this.lines());
+  }
+  printFileLine(slot) {
+    slot.printedChunks = slot.doneChunks;
+    this.write(`${this.fileLine(slot)}
+`);
+  }
+  fileLine(slot) {
+    const p = slot.totalChunks > 0 ? slot.doneChunks / slot.totalChunks : 1;
+    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(slot.doneChunks)}/${fmt(slot.totalChunks)} chunks  ${fmt(slot.doneChars)}/${fmt(slot.totalChars)} chars  ${slot.source}`;
+  }
+  totalLine() {
+    const p = this.total > 0 ? this.filesDone / this.total : 0;
+    const rejected = this.rejected > 0 ? `  |  rejected: ${this.rejected} (excluded)` : "";
+    return `total: ${this.filesDone}/${this.total} files (${pct(p)})  ${fmt(this.totalChars)} chars${rejected}`;
+  }
+  redraw(lines) {
+    let out = "";
+    if (this.blockLines > 1) out += `\x1B[${this.blockLines - 1}A`;
+    out += "\r";
+    for (let i = 0; i < lines.length; i++) {
+      out += `\x1B[2K${lines[i]}${i < lines.length - 1 ? "\n" : ""}`;
+    }
+    this.write(out);
+    this.blockLines = lines.length;
+  }
+  clearBlock() {
+    if (!this.interactive || this.blockLines === 0) {
+      this.blockLines = 0;
+      return;
+    }
+    let out = "";
+    if (this.blockLines > 1) out += `\x1B[${this.blockLines - 1}A`;
+    out += "\r";
+    for (let i = 0; i < this.blockLines; i++) {
+      out += `\x1B[2K${i < this.blockLines - 1 ? "\n" : ""}`;
+    }
+    out += "\n";
+    this.write(out);
+    this.blockLines = 0;
+  }
+};
+
+// src/ingest.ts
 var MD_EXT = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
 var HTML_EXT = /* @__PURE__ */ new Set([".html", ".htm", ".xhtml"]);
 var TEXT_EXT = /* @__PURE__ */ new Set([
@@ -555,14 +846,17 @@ async function loadTarget(target, opts) {
     let text = raw;
     let title = "";
     if (HTML_EXT.has(ext)) {
-      const conv = htmlToMarkdown(raw);
+      const conv = htmlToMarkdown(raw, { filter: opts.htmlFilter });
       text = conv.markdown;
       title = conv.title ?? "";
     } else if (MD_EXT.has(ext)) {
       title = mdTitle(raw) ?? "";
     }
     if (!text.trim()) {
-      skipped.push({ path: file, reason: "empty" });
+      skipped.push({
+        path: file,
+        reason: HTML_EXT.has(ext) && opts.htmlFilter ? `html filter '${opts.htmlFilter}' matched nothing` : "empty"
+      });
       continue;
     }
     if (!title) title = path.basename(file);
@@ -575,115 +869,587 @@ async function loadTarget(target, opts) {
   }
   return { docs, fileCount: files.length, skipped };
 }
-async function embedAndInsert(ks, pending, opts) {
-  if (pending.length === 0 || opts.dryRun) return { inserted: 0 };
-  const contents = pending.map((p) => p.chunk.content);
-  opts.log?.(`Embedding ${contents.length} chunk(s) with ${ks.embedding_model} at ${opts.url} ...`);
-  const embeddings = await embedBatch(opts.url, ks.embedding_model, contents, opts.batchSize, (done, total) => {
-    opts.log?.(`  embedded ${done}/${total}`);
-  });
-  const byDoc = /* @__PURE__ */ new Map();
-  pending.forEach((p, i) => {
-    const list = byDoc.get(p.doc) ?? [];
-    list.push({
-      source: p.doc.source,
-      title: p.doc.title,
-      chunk_index: p.chunk.index,
-      content: p.chunk.content,
-      metadata: p.metadata,
-      embedding: embeddings[i]
-    });
-    byDoc.set(p.doc, list);
-  });
-  let inserted = 0;
-  for (const [, rows] of byDoc) {
-    inserted += await insertChunks(ks, rows);
-  }
-  return { inserted };
-}
-async function feedTarget(ks, target, opts = {}) {
-  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
-  const { docs, fileCount, skipped } = await loadTarget(target, loadOpts);
-  const log = opts.log ?? ((m) => console.log(m));
-  for (const doc of docs) {
-    log(`${doc.source}: ${doc.chunks.length} chunk(s)`);
-  }
-  for (const s of skipped) {
-    log(`skipped ${s.path}: ${s.reason}`);
-  }
-  const meta = { ...opts.metadata };
-  if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
+var DOC_CONCURRENCY = 3;
+async function feedDocs(ks, docs, opts = {}) {
+  const log2 = opts.log ?? ((m) => console.log(m));
+  const progress = opts.progress;
+  const batchSize = opts.batchSize ?? 32;
+  const baseMeta = { ...opts.metadata ?? {} };
+  const cliTags = opts.tags && opts.tags.length > 0 ? opts.tags : void 0;
+  const say = (msg) => progress ? progress.log(msg) : log2(msg);
   let deletedForReplace = 0;
-  const pending = [];
   let skippedExisting = 0;
-  for (const doc of docs) {
+  let inserted = 0;
+  let announcedEmbed = false;
+  const processDoc = async (doc) => {
+    const file = progress?.startFile(
+      doc.source,
+      doc.chunks.length,
+      doc.chunks.reduce((n, c) => n + c.content.length, 0)
+    );
     if (opts.replace) {
-      deletedForReplace += await deleteBySource(ks, doc.source);
+      const deleted = await deleteBySource(ks, doc.source);
+      deletedForReplace += deleted;
     }
     const existing = await sourceChunkHashes(ks, doc.source);
+    const meta = { ...baseMeta, ...doc.metadata ?? {} };
+    if (!("tags" in meta) && cliTags) meta.tags = cliTags;
+    const todo = [];
     for (const chunk of doc.chunks) {
       if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
         skippedExisting++;
+        file?.handled(1, chunk.content.length);
         continue;
       }
-      pending.push({ doc, chunk, metadata: meta });
+      todo.push(chunk);
+    }
+    if (todo.length === 0) return;
+    if (opts.dryRun) {
+      for (const chunk of todo) file?.handled(1, chunk.content.length);
+      return;
     }
+    if (!announcedEmbed) {
+      announcedEmbed = true;
+      say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
+    }
+    const contents = todo.map((c) => c.content);
+    const charLens = contents.map((c) => c.length);
+    let reported = 0;
+    const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
+      const delta = done - reported;
+      if (delta <= 0) return;
+      let chars = 0;
+      for (let i = reported; i < done; i++) chars += charLens[i];
+      reported = done;
+      file?.handled(delta, chars);
+    });
+    const rows = todo.map((chunk, i) => ({
+      source: doc.source,
+      title: doc.title,
+      chunk_index: chunk.index,
+      content: chunk.content,
+      metadata: meta,
+      embedding: embeddings[i]
+    }));
+    const rowCount = await insertChunks(ks, rows);
+    inserted += rowCount;
+  };
+  let cursor = 0;
+  let firstError;
+  const runWorker = async () => {
+    while (firstError === void 0) {
+      const index = cursor++;
+      if (index >= docs.length) return;
+      try {
+        await processDoc(docs[index]);
+      } catch (err) {
+        firstError ??= err;
+        return;
+      }
+    }
+  };
+  const workerCount = Math.min(DOC_CONCURRENCY, docs.length);
+  await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
+  if (firstError !== void 0) {
+    progress?.close();
+    throw firstError;
   }
-  const { inserted } = await embedAndInsert(ks, pending, {
-    url: ollamaUrl(opts.url),
-    dryRun: opts.dryRun ?? false,
-    batchSize: opts.batchSize ?? 32,
-    log
-  });
+  progress?.finish();
   return {
     documents: docs.length,
     chunks: docs.reduce((n, d) => n + d.chunks.length, 0),
     inserted,
     skippedExisting,
     deletedForReplace,
-    skippedFiles: skipped,
+    skippedFiles: [],
     dryRun: opts.dryRun ?? false
   };
 }
+async function feedTarget(ks, target, opts = {}) {
+  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180, htmlFilter: opts.htmlFilter };
+  const { docs, skipped } = await loadTarget(target, loadOpts);
+  for (const s of skipped) {
+    (opts.log ?? ((m) => console.log(m)))(`skipped ${s.path}: ${s.reason}`);
+  }
+  const progress = opts.progress ?? new FeedProgress({ total: docs.length, rejected: skipped.length });
+  const summary = await feedDocs(ks, docs, { ...opts, progress });
+  return { ...summary, skippedFiles: skipped };
+}
 async function feedText(ks, text, opts = {}) {
-  const log = opts.log ?? ((m) => console.log(m));
+  const log2 = opts.log ?? ((m) => console.log(m));
   const source = opts.sourceName?.trim() || "manual";
   const title = opts.title?.trim() || source;
   const chunks = chunkMarkdown(text, { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 });
   const doc = { source, title, chunks };
-  log(`${source}: ${chunks.length} chunk(s)`);
-  const meta = { ...opts.metadata };
-  if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
-  let deletedForReplace = 0;
-  const pending = [];
-  let skippedExisting = 0;
-  if (opts.replace) {
-    deletedForReplace += await deleteBySource(ks, source);
+  const progress = opts.progress ?? new FeedProgress({ total: 1, rejected: 0 });
+  return feedDocs(ks, [doc], { ...opts, progress });
+}
+
+// src/scrape.ts
+import { mkdtempSync } from "fs";
+import { tmpdir } from "os";
+import path2 from "path";
+import { randomUUID } from "crypto";
+import {
+  CheerioCrawler,
+  PlaywrightCrawler,
+  PuppeteerCrawler,
+  Request as CrawleeRequest,
+  RequestQueue,
+  RobotsTxtFile,
+  Sitemap,
+  constructGlobObjectsFromGlobs,
+  filterRequestsByPatterns,
+  log
+} from "crawlee";
+var CRAWLER_KINDS = ["auto", "crawlee", "playwright", "puppeteer"];
+function parseSitemapOption(value) {
+  const trimmed = value.trim();
+  if (!trimmed) {
+    throw new Error("invalid --sitemap '': use auto, sitemap-only, html-only or a CSS selector");
   }
-  const existing = await sourceChunkHashes(ks, source);
-  for (const chunk of chunks) {
-    if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
-      skippedExisting++;
-      continue;
+  const key = trimmed.toLowerCase();
+  if (key === "auto") return { kind: "auto" };
+  if (key === "sitemap-only") return { kind: "sitemap-only" };
+  if (key === "html-only") return { kind: "html-only" };
+  return { kind: "selector", selector: trimmed };
+}
+var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|docx?|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
+process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync(path2.join(tmpdir(), "gresmcp-crawl-"));
+async function freshQueue() {
+  return RequestQueue.open(`gresmcp-${randomUUID()}`);
+}
+function normalizeSeedUrl(seed) {
+  let url;
+  try {
+    url = new URL(seed.trim());
+  } catch {
+    throw new Error(`invalid URL '${seed}'`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`unsupported URL protocol '${url.protocol.replace(":", "")}' (use http or https)`);
+  }
+  url.hash = "";
+  return url.toString();
+}
+function responseStatus(response) {
+  if (!response || typeof response !== "object") return void 0;
+  const res = response;
+  if (typeof res.status === "number") return res.status;
+  if (typeof res.status === "function") {
+    const value = res.status();
+    return typeof value === "number" ? value : void 0;
+  }
+  if (typeof res.statusCode === "number") return res.statusCode;
+  return void 0;
+}
+function toText(body) {
+  return typeof body === "string" ? body : body.toString("utf-8");
+}
+function titleFor(url, fallback) {
+  const trimmed = fallback?.trim();
+  if (trimmed) return trimmed;
+  try {
+    const segment = new URL(url).pathname.split("/").filter(Boolean).pop();
+    const name = segment ? decodeURIComponent(segment) : "";
+    return name || url;
+  } catch {
+    return url;
+  }
+}
+function unionTags(...lists) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const list of lists) {
+    for (const tag of list ?? []) {
+      const value = tag.trim();
+      if (!value) continue;
+      const key = value.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(value);
     }
-    pending.push({ doc, chunk, metadata: meta });
   }
-  const { inserted } = await embedAndInsert(ks, pending, {
-    url: ollamaUrl(opts.url),
-    dryRun: opts.dryRun ?? false,
-    batchSize: opts.batchSize ?? 32,
-    log
+  return out.length > 0 ? out : void 0;
+}
+function buildDoc(url, kind, text, page, deps) {
+  const chunkOpts = { maxLen: deps.maxLen, overlap: deps.overlap };
+  const chunks = kind === "plain" ? chunkPlain(text, chunkOpts) : chunkMarkdown(text, chunkOpts);
+  if (chunks.length === 0) return void 0;
+  const title = kind === "html" ? titleFor(url, page?.title) : kind === "md" ? mdTitle(text) ?? titleFor(url) : titleFor(url);
+  const metadata = { source_url: url, crawled_at: (/* @__PURE__ */ new Date()).toISOString() };
+  if (page?.description) metadata.description = page.description;
+  const tags = unionTags(page?.tags, deps.cliTags);
+  if (tags) metadata.tags = tags;
+  return { source: url, title, chunks, metadata };
+}
+function dedupeDocs(docs) {
+  const bySource = /* @__PURE__ */ new Map();
+  for (const doc of docs) {
+    if (!bySource.has(doc.source)) bySource.set(doc.source, doc);
+  }
+  return [...bySource.values()];
+}
+function shortReason(err) {
+  const message = (err?.message ?? String(err)).replace(/\s+/g, " ").trim();
+  const contentType = /served Content-Type ([^,\s]+),/.exec(message);
+  if (contentType) return `unsupported content-type '${contentType[1]}'`;
+  return message.slice(0, 200);
+}
+async function crawlWithCheerio(queue, seedUrls, opts, sink) {
+  const crawler = new CheerioCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    additionalMimeTypes: ["text/plain", "text/markdown"],
+    async requestHandler(ctx) {
+      const { request, response, contentType, body, $ } = ctx;
+      const url = request.loadedUrl ?? request.url;
+      const status = responseStatus(response);
+      if (status === void 0 || status < 200 || status >= 300) {
+        sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
+        return;
+      }
+      const type = contentType?.type ?? "";
+      if (type === "text/html" || type === "application/xhtml+xml") {
+        const html = typeof body === "string" ? body : toText(body);
+        const page = extractPage(html, { filter: opts.htmlFilter });
+        if (page.likelyShell) {
+          sink.shellUrls?.push(url);
+          sink.skipped.push({ path: url, reason: "likely JS-rendered shell" });
+          return;
+        }
+        if (opts.htmlFilter && !page.filterMatched) {
+          sink.skipped.push({ path: url, reason: `html filter '${opts.htmlFilter}' matched nothing` });
+          return;
+        }
+        if (!page.markdown.trim()) {
+          sink.skipped.push({ path: url, reason: "empty" });
+          return;
+        }
+        const doc = buildDoc(url, "html", page.markdown, page, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+        if (opts.followLinks !== false) {
+          const selector = opts.linkSelector;
+          if (selector) {
+            if ($(selector).length > 0) {
+              opts.selectorStats.hits += 1;
+              await ctx.enqueueLinks({
+                selector,
+                strategy: "same-hostname",
+                exclude: [BINARY_LINK_RE],
+                globs: opts.urlFilter
+              });
+            } else {
+              opts.selectorStats.misses += 1;
+            }
+          } else {
+            await ctx.enqueueLinks({
+              strategy: "same-hostname",
+              exclude: [BINARY_LINK_RE],
+              globs: opts.urlFilter
+            });
+          }
+        }
+      } else if (type === "text/markdown") {
+        const doc = buildDoc(url, "md", toText(body), void 0, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "text/plain") {
+        const doc = buildDoc(url, "plain", toText(body), void 0, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else {
+        sink.skipped.push({ path: url, reason: `unsupported content-type '${type}'` });
+      }
+    },
+    async failedRequestHandler(ctx, error) {
+      const url = ctx.request.loadedUrl ?? ctx.request.url;
+      sink.skipped.push({ path: url, reason: shortReason(error) });
+    }
   });
+  try {
+    await crawler.run(seedUrls);
+  } finally {
+    await crawler.teardown().catch(() => {
+    });
+  }
+}
+function sinkDeps(opts) {
   return {
-    documents: 1,
-    chunks: chunks.length,
-    inserted,
-    skippedExisting,
-    deletedForReplace,
-    skippedFiles: [],
-    dryRun: opts.dryRun ?? false
+    maxLen: opts.maxLen,
+    overlap: opts.overlap,
+    cliTags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    followLinks: opts.followLinks,
+    linkSelector: opts.linkSelector,
+    selectorStats: opts.selectorStats
+  };
+}
+async function importOptional(name) {
+  try {
+    return await import(name);
+  } catch {
+    return void 0;
+  }
+}
+async function requireBrowserEngine(kind) {
+  const moduleName = kind === "playwright" ? "playwright" : "puppeteer";
+  const mod = await importOptional(moduleName);
+  if (!mod) {
+    const install = kind === "playwright" ? "npm i -g playwright && npx playwright install chromium" : "npm i -g puppeteer";
+    throw new Error(`--crawler ${kind} requires ${moduleName}, which is not installed (install it with: ${install})`);
+  }
+}
+async function handleBrowserPage(ctx, deps) {
+  const request = ctx.request;
+  const url = request.loadedUrl ?? request.url;
+  const status = responseStatus(ctx.response);
+  if (status === void 0 || status < 200 || status >= 300) {
+    deps.sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
+    return;
+  }
+  const html = await ctx.page.content();
+  const page = extractPage(html, { filter: deps.htmlFilter });
+  if (deps.htmlFilter && !page.filterMatched) {
+    deps.sink.skipped.push({ path: url, reason: `html filter '${deps.htmlFilter}' matched nothing` });
+    return;
+  }
+  if (!page.markdown.trim()) {
+    deps.sink.skipped.push({ path: url, reason: "empty after render" });
+    return;
+  }
+  const doc = buildDoc(url, "html", page.markdown, page, deps);
+  if (!doc) {
+    deps.sink.skipped.push({ path: url, reason: "no chunks produced" });
+    return;
+  }
+  deps.sink.docs.push(doc);
+  if (deps.followLinks !== false) {
+    const selector = deps.linkSelector;
+    if (selector) {
+      const elements = await ctx.page.$$(selector);
+      if (elements.length > 0) {
+        deps.selectorStats.hits += 1;
+        await ctx.enqueueLinks({
+          selector,
+          strategy: "same-hostname",
+          exclude: [BINARY_LINK_RE],
+          globs: deps.urlFilter
+        });
+      } else {
+        deps.selectorStats.misses += 1;
+      }
+    } else {
+      await ctx.enqueueLinks({
+        strategy: "same-hostname",
+        exclude: [BINARY_LINK_RE],
+        globs: deps.urlFilter
+      });
+    }
+  }
+}
+async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
+  const crawler = kind === "playwright" ? new PlaywrightCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    preNavigationHooks: [
+      (_ctx, gotoOptions) => {
+        gotoOptions.waitUntil = "networkidle";
+        gotoOptions.timeout = 3e4;
+      }
+    ],
+    requestHandler: (ctx) => handleBrowserPage(ctx, { ...sinkDeps(opts), sink }),
+    failedRequestHandler: (ctx, error) => {
+      const request = ctx.request;
+      sink.skipped.push({ path: request.loadedUrl ?? request.url, reason: shortReason(error) });
+    }
+  }) : new PuppeteerCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    preNavigationHooks: [
+      (_ctx, gotoOptions) => {
+        gotoOptions.waitUntil = "networkidle2";
+        gotoOptions.timeout = 3e4;
+      }
+    ],
+    requestHandler: (ctx) => handleBrowserPage(ctx, { ...sinkDeps(opts), sink }),
+    failedRequestHandler: (ctx, error) => {
+      const request = ctx.request;
+      sink.skipped.push({ path: request.loadedUrl ?? request.url, reason: shortReason(error) });
+    }
+  });
+  try {
+    await crawler.run(seedUrls);
+  } finally {
+    await crawler.teardown().catch(() => {
+    });
+  }
+}
+function normalizeSitemapUrls(urls) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const raw of urls) {
+    let url;
+    try {
+      url = new URL(raw);
+    } catch {
+      continue;
+    }
+    if (url.protocol !== "http:" && url.protocol !== "https:") continue;
+    url.hash = "";
+    const key = url.toString();
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(key);
+  }
+  return out;
+}
+function applyUrlFilter(urls, globs) {
+  if (!globs || globs.length === 0) return urls;
+  const requests = urls.map((url) => new CrawleeRequest({ url }));
+  const patterns = constructGlobObjectsFromGlobs(globs);
+  return filterRequestsByPatterns(requests, patterns).map((r) => r.url);
+}
+async function loadSitemap(urls) {
+  try {
+    const sitemap = await Sitemap.load(urls, void 0, { reportNetworkErrors: false });
+    return normalizeSitemapUrls(sitemap.urls);
+  } catch {
+    return [];
+  }
+}
+async function discoverSitemapUrls(seedUrl) {
+  const origin = new URL(seedUrl).origin;
+  const fromSitemapXml = await loadSitemap([`${origin}/sitemap.xml`]);
+  if (fromSitemapXml.length > 0) return fromSitemapXml;
+  try {
+    const robots = await RobotsTxtFile.find(seedUrl);
+    const referenced = robots.getSitemaps();
+    if (referenced.length > 0) {
+      const fromRobots = await loadSitemap(referenced);
+      if (fromRobots.length > 0) return fromRobots;
+    }
+  } catch {
+  }
+  return void 0;
+}
+async function runCrawl(kind, seedUrls, opts) {
+  if (kind === "playwright" || kind === "puppeteer") {
+    await requireBrowserEngine(kind);
+    const sink2 = { docs: [], skipped: [] };
+    const queue = await freshQueue();
+    await crawlWithBrowser(kind, queue, seedUrls, opts, sink2);
+    return { docs: dedupeDocs(sink2.docs), skipped: sink2.skipped };
+  }
+  const shellUrls = [];
+  const sink = { docs: [], skipped: [], shellUrls };
+  const staticQueue = await freshQueue();
+  await crawlWithCheerio(staticQueue, seedUrls, opts, sink);
+  if (kind !== "auto" || shellUrls.length === 0) {
+    return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
+  }
+  opts.log(
+    `${shellUrls.length} page(s) look JS-rendered; retrying with playwright (install: npm i -g playwright && npx playwright install chromium) ...`
+  );
+  const mod = await importOptional("playwright");
+  if (!mod) {
+    return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
+  }
+  const browserSink = { docs: [], skipped: [] };
+  const browserQueue = await freshQueue();
+  await crawlWithBrowser("playwright", browserQueue, shellUrls, opts, browserSink);
+  return {
+    docs: dedupeDocs([...sink.docs, ...browserSink.docs]),
+    skipped: [...sink.skipped, ...browserSink.skipped]
   };
 }
+async function crawlWebsite(seed, opts) {
+  const seedUrl = normalizeSeedUrl(seed);
+  const kind = opts.crawler;
+  const mode = opts.sitemap ?? { kind: "auto" };
+  const selectorStats = { hits: 0, misses: 0 };
+  if (mode.kind === "auto" || mode.kind === "sitemap-only") {
+    const sitemapUrls = await discoverSitemapUrls(seedUrl);
+    if (sitemapUrls) {
+      const filtered = applyUrlFilter(sitemapUrls, opts.urlFilter);
+      const detail = filtered.length < sitemapUrls.length ? `${filtered.length} matching --url-filter` : `${filtered.length} URL(s)`;
+      opts.log(`sitemap: ${sitemapUrls.length} URL(s) discovered, ${detail} to crawl`);
+      return runCrawl(kind, filtered, { ...opts, followLinks: false, selectorStats });
+    }
+    if (mode.kind === "sitemap-only") {
+      opts.log(`warning: no sitemap.xml and no sitemaps in robots.txt found for ${seedUrl}; nothing to ingest`);
+      return { docs: [], skipped: [] };
+    }
+    opts.log("no sitemap found; falling back to HTML link crawling");
+  }
+  const result = await runCrawl(kind, [seedUrl], {
+    ...opts,
+    followLinks: true,
+    linkSelector: mode.kind === "selector" ? mode.selector : void 0,
+    selectorStats
+  });
+  if (mode.kind === "selector" && selectorStats.hits === 0 && selectorStats.misses > 0) {
+    opts.log(`warning: --sitemap selector '${mode.selector}' matched no links`);
+  }
+  return result;
+}
+async function feedUrl(ks, url, opts = {}) {
+  const log2 = opts.log ?? ((m) => console.log(m));
+  const kind = opts.crawler ?? "auto";
+  if (!CRAWLER_KINDS.includes(kind)) {
+    throw new Error(`unknown crawler '${kind}' (use auto, crawlee, playwright or puppeteer)`);
+  }
+  const maxPages = opts.maxPages && opts.maxPages > 0 ? Math.floor(opts.maxPages) : 999;
+  const depth = typeof opts.depth === "number" && Number.isFinite(opts.depth) && opts.depth >= 0 ? Math.floor(opts.depth) : 5;
+  const maxLen = opts.maxLen ?? 1200;
+  const overlap = opts.overlap ?? 180;
+  const { docs, skipped } = await crawlWebsite(url, {
+    crawler: kind,
+    maxPages,
+    depth,
+    maxLen,
+    overlap,
+    tags: opts.tags,
+    urlFilter: opts.urlFilter,
+    htmlFilter: opts.htmlFilter,
+    sitemap: opts.sitemap,
+    log: log2
+  });
+  for (const s of skipped) {
+    log2(`skipped ${s.path}: ${s.reason}`);
+  }
+  const progress = opts.progress ?? new FeedProgress({ total: docs.length, rejected: skipped.length });
+  const summary = await feedDocs(ks, docs, {
+    replace: opts.replace,
+    dryRun: opts.dryRun,
+    batchSize: opts.batchSize,
+    tags: opts.tags,
+    metadata: opts.metadata,
+    log: log2,
+    progress
+  });
+  return { ...summary, skippedFiles: [...summary.skippedFiles, ...skipped] };
+}
+try {
+  if (typeof log?.setLevel === "function" && log.LEVELS) {
+    log.setLevel(log.LEVELS.ERROR);
+  }
+} catch {
+}
 
 // src/check.ts
 init_config();
@@ -754,7 +1520,7 @@ async function checkOneModel(id, model, baseUrl, names, ks, probe) {
   }
   return { id, status: "ok", detail: `model '${model}' available at ${baseUrl}${scope}` };
 }
-async function runChecks(opts = {}) {
+async function runChecks(opts) {
   const results = [];
   const major = Number(process.versions.node.split(".")[0]);
   results.push({
@@ -876,44 +1642,35 @@ async function runChecks(opts = {}) {
   results.push(
     "error" in defaultTags ? { id: "ollama", status: "fail", detail: `Ollama not reachable at ${defaultUrl} \u2014 ${defaultTags.error}` } : { id: "ollama", status: "ok", detail: `Ollama reachable at ${defaultUrl} (${defaultTags.names.length} model(s))` }
   );
-  if (opts.model) {
-    const tags = await getTags(defaultUrl);
-    if ("error" in tags) {
-      results.push({
-        id: "model",
-        status: "skip",
-        detail: `Ollama unreachable at ${defaultUrl} \u2014 cannot verify model '${opts.model}'`
-      });
-    } else {
-      results.push(await checkOneModel("model", opts.model, defaultUrl, tags.names, void 0, opts.probe ?? false));
-    }
+  const tags = await getTags(defaultUrl);
+  if ("error" in tags) {
+    results.push({
+      id: "model",
+      status: "skip",
+      detail: `Ollama unreachable at ${defaultUrl} \u2014 cannot verify model '${opts.model}'`
+    });
+  } else {
+    results.push(await checkOneModel("model", opts.model, defaultUrl, tags.names, void 0, opts.probe ?? false));
   }
   for (const ks of kss) {
-    const tags = await getTags(ks.ollama_url);
-    if ("error" in tags) {
+    const tags2 = await getTags(ks.ollama_url);
+    if ("error" in tags2) {
       results.push({
         id: `model:${ks.name}`,
         status: "fail",
-        detail: `Ollama not reachable at ${ks.ollama_url} (required by ks '${ks.name}') \u2014 ${tags.error}`
+        detail: `Ollama not reachable at ${ks.ollama_url} (required by ks '${ks.name}') \u2014 ${tags2.error}`
       });
       continue;
     }
     results.push(
-      await checkOneModel(`model:${ks.name}`, ks.embedding_model, ks.ollama_url, tags.names, ks, opts.probe ?? false)
+      await checkOneModel(`model:${ks.name}`, ks.embedding_model, ks.ollama_url, tags2.names, ks, opts.probe ?? false)
     );
   }
-  if (!opts.model && kss.length === 0) {
-    results.push({
-      id: "model",
-      status: "warn",
-      detail: "no embedding model to verify \u2014 pass --model <model> or create a knowledge source"
-    });
-  }
   return results;
 }
 
 // src/version.ts
-var VERSION = "1.0.0";
+var VERSION = "1.2.0";
 
 // src/cli.ts
 async function run(fn) {
@@ -954,6 +1711,11 @@ function parseTags(tags) {
   const list = tags.split(",").map((t) => t.trim()).filter(Boolean);
   return list.length > 0 ? list : void 0;
 }
+function parseGlobs(globs) {
+  if (!globs || globs.length === 0) return void 0;
+  const list = [...new Set(globs.map((g) => g.trim()).filter(Boolean))];
+  return list.length > 0 ? list : void 0;
+}
 var program = new Command();
 program.name("gresmcp").description("Manage Postgres-backed knowledge sources for the gresmcp MCP server").version(VERSION);
 program.command("init").description("Initialize the database schema (extension, ks table; repairs entry tables)").action(
@@ -966,7 +1728,7 @@ program.command("init").description("Initialize the database schema (extension,
     console.log("database ready");
   })
 );
-program.command("check").description("Verify environment requirements (Node, Postgres, pgvector, schema, Ollama models)").option("--model <model>", "Verify this Ollama model is available (e.g. nomic-embed-text)").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--probe", "Embed a test string with each model to verify dimensions (slower)").option("--json", "Output as JSON").action(
+program.command("check").description("Verify environment requirements (Node, Postgres, pgvector, schema, Ollama models)").requiredOption("--model <model>", "Verify this Ollama model is available (e.g. nomic-embed-text)").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--probe", "Embed a test string with each model to verify dimensions (slower)").option("--json", "Output as JSON").action(
   (opts) => run(async () => {
     const results = await runChecks({ model: opts.model, url: opts.url, probe: opts.probe });
     if (opts.json) {
@@ -1044,6 +1806,34 @@ ksCmd.command("list").description("List knowledge sources").option("--json", "Ou
     }
   })
 );
+ksCmd.command("stats <name>").description("Show statistics for a knowledge source (entries, sources, content volume)").option("--json", "Output as JSON").action(
+  (name, opts) => run(async () => {
+    await initDb();
+    const ks = requireKs(await getKs(name), name);
+    const stats = await ksStats(ks);
+    if (opts.json) {
+      console.log(JSON.stringify(stats, null, 2));
+      return;
+    }
+    console.log(
+      `Knowledge source '${stats.ks.name}' (model=${stats.ks.embedding_model}@${stats.ks.ollama_url}, dim=${stats.ks.embedding_dim})`
+    );
+    if (stats.ks.description) console.log(`  description: ${stats.ks.description}`);
+    console.log(`  created:      ${stats.ks.created_at.toISOString()}`);
+    console.log(`  last fed:     ${stats.last_fed_at ? stats.last_fed_at.toISOString() : "never"}`);
+    console.log("");
+    if (stats.entries === 0) {
+      console.log(`  no entries yet; feed it with: gresmcp feed ${stats.ks.name}`);
+      return;
+    }
+    const size = (n) => n === null ? "-" : n.toLocaleString("en-US");
+    console.log(`  entries (chunks): ${stats.entries.toLocaleString("en-US")}`);
+    console.log(`  distinct sources: ${stats.sources.toLocaleString("en-US")}`);
+    console.log(
+      `  content:          ${stats.chars.toLocaleString("en-US")} chars (avg ${size(stats.avg_chunk_chars)}, min ${size(stats.min_chunk_chars)}, max ${size(stats.max_chunk_chars)} per chunk)`
+    );
+  })
+);
 ksCmd.command("delete <name>").description("Delete a knowledge source and all of its entries").option("--yes", "Do not ask for confirmation").action(
   (name, opts) => run(async () => {
     await initDb();
@@ -1060,30 +1850,61 @@ ksCmd.command("delete <name>").description("Delete a knowledge source and all of
     console.log(`Deleted knowledge source '${name}'`);
   })
 );
-program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--url <url>", "Ollama base URL override for this run").option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
+program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option(
+  "--url-filter <glob>",
+  "Only follow --url links whose URL matches this glob (repeatable; a link is followed if it matches any glob, e.g. '**/docs/**')",
+  ((v, prev) => [...prev, v]),
+  []
+).option(
+  "--html-filter <selector>",
+  "CSS selector for HTML content \u2014 any DOM selector works (e.g. 'article.post', '#article .content:first-child'); optionally chain steps with ' -> ' (e.g. '#article -> .content'); applies to --path HTML files and --url pages"
+).option(
+  "--sitemap <mode>",
+  "Link discovery for --url: 'auto' (default; sitemap.xml, then robots.txt sitemaps, else follow links), 'sitemap-only' (only sitemap.xml / robots.txt sitemaps), 'html-only' (follow links), or a CSS selector to only follow links inside it (e.g. 'nav.docs a')"
+).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
   (name, opts) => run(async () => {
     await initDb();
     const ks = requireKs(await getKs(name), name);
-    const modes = [opts.path, opts.text, opts.stdin].filter(Boolean).length;
-    if (modes !== 1) throw new Error("provide exactly one of --path, --text or --stdin");
+    const modes = [opts.path, opts.url, opts.text, opts.stdin].filter(Boolean).length;
+    if (modes !== 1) throw new Error("provide exactly one of --path, --url, --text or --stdin");
+    const urlFilter = parseGlobs(opts.urlFilter);
+    if (urlFilter && !opts.url) {
+      console.error("warning: --url-filter has no effect without --url");
+    }
+    if (opts.htmlFilter) parseHtmlFilter(opts.htmlFilter);
+    if (opts.htmlFilter && !opts.path && !opts.url) {
+      console.error("warning: --html-filter has no effect without --path or --url");
+    }
+    let sitemap;
+    if (opts.sitemap !== void 0) {
+      sitemap = parseSitemapOption(opts.sitemap);
+      if (sitemap.kind === "selector") validateCssSelector(sitemap.selector);
+      if (!opts.url) console.error("warning: --sitemap has no effect without --url");
+    }
     const common = {
-      title: opts.title,
-      url: opts.url,
       replace: opts.replace,
       dryRun: opts.dryRun,
       batchSize: 32,
       maxLen: opts.chunkSize && opts.chunkSize > 0 ? opts.chunkSize : void 0,
       overlap: opts.overlap,
       tags: parseTags(opts.tags),
-      metadata: parseKeyValue(opts.metadata ?? [])
+      metadata: parseKeyValue(opts.metadata ?? []),
+      htmlFilter: opts.htmlFilter
     };
     let summary;
+    const since = await dbClockTimestamp();
     if (opts.path) {
       summary = await feedTarget(ks, opts.path, common);
+    } else if (opts.url) {
+      const crawler = opts.crawler;
+      if (crawler && !CRAWLER_KINDS.includes(crawler)) {
+        throw new Error(`unknown crawler '${crawler}' (use auto, crawlee, playwright or puppeteer)`);
+      }
+      summary = await feedUrl(ks, opts.url, { ...common, crawler, sitemap, maxPages: opts.maxPages, depth: opts.depth, urlFilter });
     } else {
       const text = opts.stdin ? await readStdin() : opts.text;
       if (!text.trim()) throw new Error("empty input text");
-      summary = await feedText(ks, text, { ...common, sourceName: opts.sourceName });
+      summary = await feedText(ks, text, { ...common, title: opts.title, sourceName: opts.sourceName });
     }
     if (summary.dryRun) {
       console.log(`dry run: ${summary.documents} document(s), ${summary.chunks} chunk(s), ${summary.skippedFiles.length} file(s) skipped`);
@@ -1091,9 +1912,22 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       console.log(
         `fed '${ks.name}': ${summary.inserted} chunk(s) inserted` + (summary.skippedExisting ? `, ${summary.skippedExisting} unchanged chunk(s) skipped` : "") + (summary.deletedForReplace ? `, ${summary.deletedForReplace} old chunk(s) replaced` : "") + `, ${summary.skippedFiles.length} file(s) skipped`
       );
+      await printRunStats(ks, since);
     }
   })
 );
+async function printRunStats(ks, since) {
+  const stats = await ksStats(ks, since);
+  if (stats.entries === 0) return;
+  const size = (n) => n === null ? "-" : n.toLocaleString("en-US");
+  console.log("");
+  console.log(
+    `this run: ${stats.entries.toLocaleString("en-US")} chunk(s) across ${stats.sources.toLocaleString("en-US")} source(s)`
+  );
+  console.log(
+    `  content:          ${stats.chars.toLocaleString("en-US")} chars (avg ${size(stats.avg_chunk_chars)}, min ${size(stats.min_chunk_chars)}, max ${size(stats.max_chunk_chars)} per chunk)`
+  );
+}
 async function readStdin() {
   const chunks = [];
   for await (const chunk of stdin) chunks.push(chunk);