@gresmcp/mcp 1.0.0 → 1.1.0

package/dist/cli.js

@@ -37,6 +37,7 @@ __export(db_exports, {
   contentHash: () => contentHash,
   countEntries: () => countEntries,
   createKs: () => createKs,
+  dbClockTimestamp: () => dbClockTimestamp,
   deleteBySource: () => deleteBySource,
   deleteKs: () => deleteKs,
   ensureEntryTable: () => ensureEntryTable,
@@ -45,6 +46,7 @@ __export(db_exports, {
   getPool: () => getPool,
   initDb: () => initDb,
   insertChunks: () => insertChunks,
+  ksStats: () => ksStats,
   listKs: () => listKs,
   sourceChunkHashes: () => sourceChunkHashes,
   updateKs: () => updateKs
@@ -211,6 +213,41 @@ async function countEntries(ks) {
   const res = await getPool().query(`SELECT count(*)::int AS n FROM ${tbl}`);
   return Number(res.rows[0]?.n ?? 0);
 }
+async function dbClockTimestamp() {
+  const res = await getPool().query(`SELECT clock_timestamp() AS ts`);
+  return res.rows[0].ts;
+}
+async function ksStats(ks, since) {
+  const tbl = entryTableName(ks);
+  const params = [];
+  let where = "";
+  if (since) {
+    params.push(since);
+    where = ` WHERE created_at >= $1`;
+  }
+  const res = await getPool().query(
+    `SELECT count(*)::int AS entries,
+            count(DISTINCT source)::int AS sources,
+            COALESCE(sum(length(content)), 0)::bigint AS chars,
+            round(avg(length(content)))::int AS avg_chunk_chars,
+            min(length(content))::int AS min_chunk_chars,
+            max(length(content))::int AS max_chunk_chars,
+            max(created_at) AS last_fed_at
+     FROM ${tbl}${where}`,
+    params
+  );
+  const r = res.rows[0];
+  return {
+    ks,
+    entries: Number(r.entries ?? 0),
+    sources: Number(r.sources ?? 0),
+    chars: Number(r.chars ?? 0),
+    avg_chunk_chars: r.avg_chunk_chars === null ? null : Number(r.avg_chunk_chars),
+    min_chunk_chars: r.min_chunk_chars === null ? null : Number(r.min_chunk_chars),
+    max_chunk_chars: r.max_chunk_chars === null ? null : Number(r.max_chunk_chars),
+    last_fed_at: r.last_fed_at === null ? null : r.last_fed_at
+  };
+}
 var pool;
 var init_db = __esm({
   "src/db.ts"() {
@@ -353,8 +390,8 @@ function chunkMarkdown(text, opts = {}) {
     const m = HEADING_RE.exec(line);
     if (m) {
       const level = m[1].length;
-      const path2 = sections[sections.length - 1].path;
-      const parent = path2.slice(0, level - 1);
+      const path3 = sections[sections.length - 1].path;
+      const parent = path3.slice(0, level - 1);
       sections.push({ path: [...parent, m[2].trim()], body: [line] });
     } else {
       sections[sections.length - 1].body.push(line);
@@ -388,9 +425,12 @@ function mdTitle(text) {
 // src/html.ts
 import { JSDOM } from "jsdom";
 import TurndownService from "turndown";
+import { Readability, isProbablyReaderable } from "@mozilla/readability";
 import gfmModule from "turndown-plugin-gfm";
 var gfm = typeof gfmModule === "function" ? gfmModule : gfmModule.gfm;
 var REMOVE_SELECTORS = "script, style, noscript, template, iframe, svg, nav, footer, aside, form, button";
+var SHELL_MARKER_RE = /id=["'](root|app|__next|__nuxt|q-app)["']|data-reactroot|ng-version=/i;
+var SHELL_TEXT_THRESHOLD = 250;
 var service;
 function turndown() {
   if (!service) {
@@ -421,10 +461,207 @@ function htmlToMarkdown(html) {
   markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
   return { markdown, title: title || void 0 };
 }
+function metaContent(doc, selectors) {
+  for (const selector of selectors) {
+    const el = doc.querySelector(selector);
+    const value = (el?.getAttribute("content") ?? el?.textContent ?? "").trim();
+    if (value) return value;
+  }
+  return void 0;
+}
+function collectTags(doc) {
+  const tags = [];
+  const keywords = metaContent(doc, ['meta[name="keywords"]']);
+  if (keywords) {
+    for (const part of keywords.split(",")) {
+      const tag = part.trim();
+      if (tag) tags.push(tag);
+    }
+  }
+  doc.querySelectorAll('meta[property="og:article:tag"]').forEach((el) => {
+    const value = (el.getAttribute("content") ?? "").trim();
+    if (value) tags.push(value);
+  });
+  const seen = /* @__PURE__ */ new Set();
+  return tags.filter((t) => {
+    const key = t.toLowerCase();
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+}
+function extractPage(html) {
+  const dom = new JSDOM(html);
+  const doc = dom.window.document;
+  const title = doc.title?.trim() || doc.querySelector('meta[property="og:title"]')?.getAttribute("content")?.trim() || doc.querySelector("h1")?.textContent?.trim() || void 0;
+  const description = metaContent(doc, [
+    'meta[name="description"]',
+    'meta[property="og:description"]'
+  ]);
+  const tags = collectTags(doc);
+  doc.querySelectorAll(REMOVE_SELECTORS).forEach((el) => el.remove());
+  let markdown = "";
+  try {
+    if (isProbablyReaderable(doc)) {
+      const article = new Readability(doc).parse();
+      if (article?.content) markdown = turndown().turndown(article.content);
+    }
+  } catch {
+    markdown = "";
+  }
+  if (!markdown.trim()) {
+    const root = doc.body ?? doc.documentElement;
+    try {
+      markdown = turndown().turndown(root.innerHTML);
+    } catch {
+      markdown = root.textContent ?? "";
+    }
+  }
+  markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
+  const likelyShell = markdown.length < SHELL_TEXT_THRESHOLD && SHELL_MARKER_RE.test(html);
+  return {
+    markdown,
+    title,
+    description,
+    tags: tags.length > 0 ? tags : void 0,
+    likelyShell
+  };
+}
 
 // src/ingest.ts
 init_db();
-init_config();
+
+// src/progress.ts
+var BAR_WIDTH = 20;
+var PLAIN_STEP = 32;
+function fmt(n) {
+  return n.toLocaleString("en-US");
+}
+function bar(pct2) {
+  const filled = Math.max(0, Math.min(BAR_WIDTH, Math.round(pct2 * BAR_WIDTH)));
+  return "#".repeat(filled) + ".".repeat(BAR_WIDTH - filled);
+}
+function pct(p) {
+  return `${Math.round(Math.max(0, Math.min(1, p)) * 100)}%`;
+}
+var FeedProgress = class {
+  write;
+  interactive;
+  total;
+  rejected;
+  filesDone = 0;
+  totalChars = 0;
+  current;
+  blockLines = 0;
+  finished = false;
+  constructor(opts) {
+    this.total = Math.max(0, Math.floor(opts.total));
+    this.rejected = Math.max(0, Math.floor(opts.rejected));
+    this.write = opts.out ?? ((s) => process.stderr.write(s));
+    this.interactive = opts.interactive ?? (opts.out === void 0 && !!process.stderr.isTTY);
+  }
+  get isInteractive() {
+    return this.interactive;
+  }
+  startFile(source, totalChunks, totalChars) {
+    if (this.finished) return;
+    this.current = { source, totalChunks, totalChars, doneChunks: 0, doneChars: 0, printedChunks: 0 };
+    this.render();
+  }
+  handled(chunks, chars) {
+    if (this.finished || !this.current || chunks <= 0) return;
+    const c = this.current;
+    c.doneChunks += chunks;
+    c.doneChars += chars;
+    this.totalChars += chars;
+    if (c.doneChunks >= c.totalChunks) {
+      this.filesDone++;
+      this.printFileLine();
+      return;
+    }
+    if (!this.interactive && c.doneChunks - c.printedChunks >= PLAIN_STEP) {
+      this.printFileLine();
+    } else if (this.interactive) {
+      this.render();
+    }
+  }
+  log(msg) {
+    if (this.finished) return;
+    this.clearBlock();
+    this.write(`${msg}
+`);
+  }
+  finish() {
+    if (this.finished) return;
+    this.finished = true;
+    if (!this.current && this.total === 0 && this.rejected === 0) return;
+    if (this.interactive) {
+      this.redraw([this.fileLine(), this.totalLine()]);
+      this.write("\n");
+      this.blockLines = 0;
+    } else {
+      this.write(`${this.totalLine()}
+`);
+    }
+  }
+  close() {
+    if (this.finished) return;
+    this.finished = true;
+    this.clearBlock();
+  }
+  render() {
+    if (!this.interactive) return;
+    this.redraw([this.fileLine(), this.totalLine()]);
+  }
+  printFileLine() {
+    if (!this.current) return;
+    this.current.printedChunks = this.current.doneChunks;
+    if (this.interactive) {
+      this.render();
+    } else {
+      this.write(`${this.fileLine()}
+`);
+    }
+  }
+  fileLine() {
+    const c = this.current;
+    if (!c) return "";
+    const p = c.totalChunks > 0 ? c.doneChunks / c.totalChunks : 1;
+    return `[${bar(p)}] ${pct(p).padStart(4)}  ${fmt(c.doneChunks)}/${fmt(c.totalChunks)} chunks  ${fmt(c.doneChars)}/${fmt(c.totalChars)} chars  ${c.source}`;
+  }
+  totalLine() {
+    const p = this.total > 0 ? this.filesDone / this.total : 0;
+    const rejected = this.rejected > 0 ? `  |  rejected: ${this.rejected} (excluded)` : "";
+    return `total: ${this.filesDone}/${this.total} files (${pct(p)})  ${fmt(this.totalChars)} chars${rejected}`;
+  }
+  redraw(lines) {
+    let out = "";
+    if (this.blockLines > 1) out += `\x1B[${this.blockLines - 1}A`;
+    out += "\r";
+    for (let i = 0; i < lines.length; i++) {
+      out += `\x1B[2K${lines[i]}${i < lines.length - 1 ? "\n" : ""}`;
+    }
+    this.write(out);
+    this.blockLines = lines.length;
+  }
+  clearBlock() {
+    if (!this.interactive || this.blockLines === 0) {
+      this.blockLines = 0;
+      return;
+    }
+    let out = "";
+    if (this.blockLines > 1) out += `\x1B[${this.blockLines - 1}A`;
+    out += "\r";
+    for (let i = 0; i < this.blockLines; i++) {
+      out += `\x1B[2K${i < this.blockLines - 1 ? "\n" : ""}`;
+    }
+    out += "\n";
+    this.write(out);
+    this.blockLines = 0;
+  }
+};
+
+// src/ingest.ts
 var MD_EXT = /* @__PURE__ */ new Set([".md", ".markdown", ".mdx"]);
 var HTML_EXT = /* @__PURE__ */ new Set([".html", ".htm", ".xhtml"]);
 var TEXT_EXT = /* @__PURE__ */ new Set([
@@ -575,115 +812,408 @@ async function loadTarget(target, opts) {
   }
   return { docs, fileCount: files.length, skipped };
 }
-async function embedAndInsert(ks, pending, opts) {
-  if (pending.length === 0 || opts.dryRun) return { inserted: 0 };
-  const contents = pending.map((p) => p.chunk.content);
-  opts.log?.(`Embedding ${contents.length} chunk(s) with ${ks.embedding_model} at ${opts.url} ...`);
-  const embeddings = await embedBatch(opts.url, ks.embedding_model, contents, opts.batchSize, (done, total) => {
-    opts.log?.(`  embedded ${done}/${total}`);
-  });
-  const byDoc = /* @__PURE__ */ new Map();
-  pending.forEach((p, i) => {
-    const list = byDoc.get(p.doc) ?? [];
-    list.push({
-      source: p.doc.source,
-      title: p.doc.title,
-      chunk_index: p.chunk.index,
-      content: p.chunk.content,
-      metadata: p.metadata,
-      embedding: embeddings[i]
-    });
-    byDoc.set(p.doc, list);
-  });
-  let inserted = 0;
-  for (const [, rows] of byDoc) {
-    inserted += await insertChunks(ks, rows);
-  }
-  return { inserted };
-}
-async function feedTarget(ks, target, opts = {}) {
-  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
-  const { docs, fileCount, skipped } = await loadTarget(target, loadOpts);
-  const log = opts.log ?? ((m) => console.log(m));
-  for (const doc of docs) {
-    log(`${doc.source}: ${doc.chunks.length} chunk(s)`);
-  }
-  for (const s of skipped) {
-    log(`skipped ${s.path}: ${s.reason}`);
-  }
-  const meta = { ...opts.metadata };
-  if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
+async function feedDocs(ks, docs, opts = {}) {
+  const log2 = opts.log ?? ((m) => console.log(m));
+  const progress = opts.progress;
+  const batchSize = opts.batchSize ?? 32;
+  const baseMeta = { ...opts.metadata ?? {} };
+  const cliTags = opts.tags && opts.tags.length > 0 ? opts.tags : void 0;
+  const say = (msg) => progress ? progress.log(msg) : log2(msg);
   let deletedForReplace = 0;
-  const pending = [];
   let skippedExisting = 0;
-  for (const doc of docs) {
-    if (opts.replace) {
-      deletedForReplace += await deleteBySource(ks, doc.source);
-    }
-    const existing = await sourceChunkHashes(ks, doc.source);
-    for (const chunk of doc.chunks) {
-      if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
-        skippedExisting++;
+  let inserted = 0;
+  let announcedEmbed = false;
+  try {
+    for (const doc of docs) {
+      progress?.startFile(
+        doc.source,
+        doc.chunks.length,
+        doc.chunks.reduce((n, c) => n + c.content.length, 0)
+      );
+      if (opts.replace) {
+        deletedForReplace += await deleteBySource(ks, doc.source);
+      }
+      const existing = await sourceChunkHashes(ks, doc.source);
+      const meta = { ...baseMeta, ...doc.metadata ?? {} };
+      if (!("tags" in meta) && cliTags) meta.tags = cliTags;
+      const todo = [];
+      for (const chunk of doc.chunks) {
+        if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
+          skippedExisting++;
+          progress?.handled(1, chunk.content.length);
+          continue;
+        }
+        todo.push(chunk);
+      }
+      if (todo.length === 0) continue;
+      if (opts.dryRun) {
+        for (const chunk of todo) progress?.handled(1, chunk.content.length);
         continue;
       }
-      pending.push({ doc, chunk, metadata: meta });
+      if (!announcedEmbed) {
+        say(`Embedding with ${ks.embedding_model} at ${ks.ollama_url} ...`);
+        announcedEmbed = true;
+      }
+      const contents = todo.map((c) => c.content);
+      const charLens = contents.map((c) => c.length);
+      let reported = 0;
+      const embeddings = await embedBatch(ks.ollama_url, ks.embedding_model, contents, batchSize, (done) => {
+        const delta = done - reported;
+        if (delta <= 0) return;
+        let chars = 0;
+        for (let i = reported; i < done; i++) chars += charLens[i];
+        reported = done;
+        progress?.handled(delta, chars);
+      });
+      const rows = todo.map((chunk, i) => ({
+        source: doc.source,
+        title: doc.title,
+        chunk_index: chunk.index,
+        content: chunk.content,
+        metadata: meta,
+        embedding: embeddings[i]
+      }));
+      inserted += await insertChunks(ks, rows);
     }
+  } catch (err) {
+    progress?.close();
+    throw err;
   }
-  const { inserted } = await embedAndInsert(ks, pending, {
-    url: ollamaUrl(opts.url),
-    dryRun: opts.dryRun ?? false,
-    batchSize: opts.batchSize ?? 32,
-    log
-  });
+  progress?.finish();
   return {
     documents: docs.length,
     chunks: docs.reduce((n, d) => n + d.chunks.length, 0),
     inserted,
     skippedExisting,
     deletedForReplace,
-    skippedFiles: skipped,
+    skippedFiles: [],
     dryRun: opts.dryRun ?? false
   };
 }
+async function feedTarget(ks, target, opts = {}) {
+  const loadOpts = { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 };
+  const { docs, skipped } = await loadTarget(target, loadOpts);
+  for (const s of skipped) {
+    (opts.log ?? ((m) => console.log(m)))(`skipped ${s.path}: ${s.reason}`);
+  }
+  const progress = opts.progress ?? new FeedProgress({ total: docs.length, rejected: skipped.length });
+  const summary = await feedDocs(ks, docs, { ...opts, progress });
+  return { ...summary, skippedFiles: skipped };
+}
 async function feedText(ks, text, opts = {}) {
-  const log = opts.log ?? ((m) => console.log(m));
+  const log2 = opts.log ?? ((m) => console.log(m));
   const source = opts.sourceName?.trim() || "manual";
   const title = opts.title?.trim() || source;
   const chunks = chunkMarkdown(text, { maxLen: opts.maxLen ?? 1200, overlap: opts.overlap ?? 180 });
   const doc = { source, title, chunks };
-  log(`${source}: ${chunks.length} chunk(s)`);
-  const meta = { ...opts.metadata };
-  if (opts.tags && opts.tags.length > 0) meta.tags = opts.tags;
-  let deletedForReplace = 0;
-  const pending = [];
-  let skippedExisting = 0;
-  if (opts.replace) {
-    deletedForReplace += await deleteBySource(ks, source);
+  const progress = opts.progress ?? new FeedProgress({ total: 1, rejected: 0 });
+  return feedDocs(ks, [doc], { ...opts, progress });
+}
+
+// src/scrape.ts
+import { mkdtempSync } from "fs";
+import { tmpdir } from "os";
+import path2 from "path";
+import { randomUUID } from "crypto";
+import { CheerioCrawler, PlaywrightCrawler, PuppeteerCrawler, RequestQueue, log } from "crawlee";
+var CRAWLER_KINDS = ["auto", "crawlee", "playwright", "puppeteer"];
+var BINARY_LINK_RE = /\.(css|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|tiff?|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|tar|rar|7z|bz2|xz|zst|mp3|mp4|m4a|webm|mov|avi|wmv|flv|docx?|xlsx?|pptx?|odt|ods|odp|rtf|dmg|exe|msi|apk|iso|bin|wasm|deb|rpm)(\?.*)?$/i;
+process.env.CRAWLEE_STORAGE_DIR ??= mkdtempSync(path2.join(tmpdir(), "gresmcp-crawl-"));
+async function freshQueue() {
+  return RequestQueue.open(`gresmcp-${randomUUID()}`);
+}
+function normalizeSeedUrl(seed) {
+  let url;
+  try {
+    url = new URL(seed.trim());
+  } catch {
+    throw new Error(`invalid URL '${seed}'`);
   }
-  const existing = await sourceChunkHashes(ks, source);
-  for (const chunk of chunks) {
-    if (!opts.replace && existing.get(chunk.index) === contentHash(chunk.content)) {
-      skippedExisting++;
-      continue;
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`unsupported URL protocol '${url.protocol.replace(":", "")}' (use http or https)`);
+  }
+  url.hash = "";
+  return url.toString();
+}
+function responseStatus(response) {
+  if (!response || typeof response !== "object") return void 0;
+  const res = response;
+  if (typeof res.status === "number") return res.status;
+  if (typeof res.status === "function") {
+    const value = res.status();
+    return typeof value === "number" ? value : void 0;
+  }
+  if (typeof res.statusCode === "number") return res.statusCode;
+  return void 0;
+}
+function toText(body) {
+  return typeof body === "string" ? body : body.toString("utf-8");
+}
+function titleFor(url, fallback) {
+  const trimmed = fallback?.trim();
+  if (trimmed) return trimmed;
+  try {
+    const segment = new URL(url).pathname.split("/").filter(Boolean).pop();
+    const name = segment ? decodeURIComponent(segment) : "";
+    return name || url;
+  } catch {
+    return url;
+  }
+}
+function unionTags(...lists) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const list of lists) {
+    for (const tag of list ?? []) {
+      const value = tag.trim();
+      if (!value) continue;
+      const key = value.toLowerCase();
+      if (seen.has(key)) continue;
+      seen.add(key);
+      out.push(value);
+    }
+  }
+  return out.length > 0 ? out : void 0;
+}
+function buildDoc(url, kind, text, page, deps) {
+  const chunkOpts = { maxLen: deps.maxLen, overlap: deps.overlap };
+  const chunks = kind === "plain" ? chunkPlain(text, chunkOpts) : chunkMarkdown(text, chunkOpts);
+  if (chunks.length === 0) return void 0;
+  const title = kind === "html" ? titleFor(url, page?.title) : kind === "md" ? mdTitle(text) ?? titleFor(url) : titleFor(url);
+  const metadata = { source_url: url, crawled_at: (/* @__PURE__ */ new Date()).toISOString() };
+  if (page?.description) metadata.description = page.description;
+  const tags = unionTags(page?.tags, deps.cliTags);
+  if (tags) metadata.tags = tags;
+  return { source: url, title, chunks, metadata };
+}
+function dedupeDocs(docs) {
+  const bySource = /* @__PURE__ */ new Map();
+  for (const doc of docs) {
+    if (!bySource.has(doc.source)) bySource.set(doc.source, doc);
+  }
+  return [...bySource.values()];
+}
+function shortReason(err) {
+  const message = (err?.message ?? String(err)).replace(/\s+/g, " ").trim();
+  const contentType = /served Content-Type ([^,\s]+),/.exec(message);
+  if (contentType) return `unsupported content-type '${contentType[1]}'`;
+  return message.slice(0, 200);
+}
+async function crawlWithCheerio(queue, seedUrls, opts, sink) {
+  const crawler = new CheerioCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    additionalMimeTypes: ["text/plain", "text/markdown"],
+    async requestHandler(ctx) {
+      const { request, response, contentType, body } = ctx;
+      const url = request.loadedUrl ?? request.url;
+      const status = responseStatus(response);
+      if (status === void 0 || status < 200 || status >= 300) {
+        sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
+        return;
+      }
+      const type = contentType?.type ?? "";
+      if (type === "text/html" || type === "application/xhtml+xml") {
+        const html = typeof body === "string" ? body : toText(body);
+        const page = extractPage(html);
+        if (page.likelyShell) {
+          sink.shellUrls?.push(url);
+          sink.skipped.push({ path: url, reason: "likely JS-rendered shell" });
+          return;
+        }
+        if (!page.markdown.trim()) {
+          sink.skipped.push({ path: url, reason: "empty" });
+          return;
+        }
+        const doc = buildDoc(url, "html", page.markdown, page, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+        await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+      } else if (type === "text/markdown") {
+        const doc = buildDoc(url, "md", toText(body), void 0, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else if (type === "text/plain") {
+        const doc = buildDoc(url, "plain", toText(body), void 0, sinkDeps(opts));
+        if (!doc) {
+          sink.skipped.push({ path: url, reason: "no chunks produced" });
+          return;
+        }
+        sink.docs.push(doc);
+      } else {
+        sink.skipped.push({ path: url, reason: `unsupported content-type '${type}'` });
+      }
+    },
+    async failedRequestHandler(ctx, error) {
+      const url = ctx.request.loadedUrl ?? ctx.request.url;
+      sink.skipped.push({ path: url, reason: shortReason(error) });
     }
-    pending.push({ doc, chunk, metadata: meta });
+  });
+  try {
+    await crawler.run(seedUrls);
+  } finally {
+    await crawler.teardown().catch(() => {
+    });
   }
-  const { inserted } = await embedAndInsert(ks, pending, {
-    url: ollamaUrl(opts.url),
-    dryRun: opts.dryRun ?? false,
-    batchSize: opts.batchSize ?? 32,
-    log
+}
+function sinkDeps(opts) {
+  return { maxLen: opts.maxLen, overlap: opts.overlap, cliTags: opts.tags };
+}
+async function importOptional(name) {
+  try {
+    return await import(name);
+  } catch {
+    return void 0;
+  }
+}
+async function requireBrowserEngine(kind) {
+  const moduleName = kind === "playwright" ? "playwright" : "puppeteer";
+  const mod = await importOptional(moduleName);
+  if (!mod) {
+    const install = kind === "playwright" ? "npm i -g playwright && npx playwright install chromium" : "npm i -g puppeteer";
+    throw new Error(`--crawler ${kind} requires ${moduleName}, which is not installed (install it with: ${install})`);
+  }
+}
+async function handleBrowserPage(ctx, deps) {
+  const request = ctx.request;
+  const url = request.loadedUrl ?? request.url;
+  const status = responseStatus(ctx.response);
+  if (status === void 0 || status < 200 || status >= 300) {
+    deps.sink.skipped.push({ path: url, reason: status === void 0 ? "no HTTP response" : `HTTP ${status}` });
+    return;
+  }
+  const html = await ctx.page.content();
+  const page = extractPage(html);
+  if (!page.markdown.trim()) {
+    deps.sink.skipped.push({ path: url, reason: "empty after render" });
+    return;
+  }
+  const doc = buildDoc(url, "html", page.markdown, page, deps);
+  if (!doc) {
+    deps.sink.skipped.push({ path: url, reason: "no chunks produced" });
+    return;
+  }
+  deps.sink.docs.push(doc);
+  await ctx.enqueueLinks({ strategy: "same-hostname", exclude: [BINARY_LINK_RE] });
+}
+async function crawlWithBrowser(kind, queue, seedUrls, opts, sink) {
+  const crawler = kind === "playwright" ? new PlaywrightCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    preNavigationHooks: [
+      (_ctx, gotoOptions) => {
+        gotoOptions.waitUntil = "networkidle";
+        gotoOptions.timeout = 3e4;
+      }
+    ],
+    requestHandler: (ctx) => handleBrowserPage(ctx, { ...sinkDeps(opts), sink }),
+    failedRequestHandler: (ctx, error) => {
+      const request = ctx.request;
+      sink.skipped.push({ path: request.loadedUrl ?? request.url, reason: shortReason(error) });
+    }
+  }) : new PuppeteerCrawler({
+    requestQueue: queue,
+    maxRequestsPerCrawl: Math.max(1, opts.maxPages),
+    maxCrawlDepth: Math.max(0, opts.depth),
+    preNavigationHooks: [
+      (_ctx, gotoOptions) => {
+        gotoOptions.waitUntil = "networkidle2";
+        gotoOptions.timeout = 3e4;
+      }
+    ],
+    requestHandler: (ctx) => handleBrowserPage(ctx, { ...sinkDeps(opts), sink }),
+    failedRequestHandler: (ctx, error) => {
+      const request = ctx.request;
+      sink.skipped.push({ path: request.loadedUrl ?? request.url, reason: shortReason(error) });
+    }
   });
+  try {
+    await crawler.run(seedUrls);
+  } finally {
+    await crawler.teardown().catch(() => {
+    });
+  }
+}
+async function crawlWebsite(seed, opts) {
+  const seedUrl = normalizeSeedUrl(seed);
+  const kind = opts.crawler;
+  if (kind === "playwright" || kind === "puppeteer") {
+    await requireBrowserEngine(kind);
+    const sink2 = { docs: [], skipped: [] };
+    const queue = await freshQueue();
+    await crawlWithBrowser(kind, queue, [seedUrl], opts, sink2);
+    return { docs: dedupeDocs(sink2.docs), skipped: sink2.skipped };
+  }
+  const shellUrls = [];
+  const sink = { docs: [], skipped: [], shellUrls };
+  const staticQueue = await freshQueue();
+  await crawlWithCheerio(staticQueue, [seedUrl], opts, sink);
+  if (kind !== "auto" || shellUrls.length === 0) {
+    return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
+  }
+  opts.log(
+    `${shellUrls.length} page(s) look JS-rendered; retrying with playwright (install: npm i -g playwright && npx playwright install chromium) ...`
+  );
+  const mod = await importOptional("playwright");
+  if (!mod) {
+    return { docs: dedupeDocs(sink.docs), skipped: sink.skipped };
+  }
+  const browserSink = { docs: [], skipped: [] };
+  const browserQueue = await freshQueue();
+  await crawlWithBrowser("playwright", browserQueue, shellUrls, opts, browserSink);
   return {
-    documents: 1,
-    chunks: chunks.length,
-    inserted,
-    skippedExisting,
-    deletedForReplace,
-    skippedFiles: [],
-    dryRun: opts.dryRun ?? false
+    docs: dedupeDocs([...sink.docs, ...browserSink.docs]),
+    skipped: [...sink.skipped, ...browserSink.skipped]
   };
 }
+async function feedUrl(ks, url, opts = {}) {
+  const log2 = opts.log ?? ((m) => console.log(m));
+  const kind = opts.crawler ?? "auto";
+  if (!CRAWLER_KINDS.includes(kind)) {
+    throw new Error(`unknown crawler '${kind}' (use auto, crawlee, playwright or puppeteer)`);
+  }
+  const maxPages = opts.maxPages && opts.maxPages > 0 ? Math.floor(opts.maxPages) : 999;
+  const depth = typeof opts.depth === "number" && Number.isFinite(opts.depth) && opts.depth >= 0 ? Math.floor(opts.depth) : 5;
+  const maxLen = opts.maxLen ?? 1200;
+  const overlap = opts.overlap ?? 180;
+  const { docs, skipped } = await crawlWebsite(url, {
+    crawler: kind,
+    maxPages,
+    depth,
+    maxLen,
+    overlap,
+    tags: opts.tags,
+    log: log2
+  });
+  for (const s of skipped) {
+    log2(`skipped ${s.path}: ${s.reason}`);
+  }
+  const progress = opts.progress ?? new FeedProgress({ total: docs.length, rejected: skipped.length });
+  const summary = await feedDocs(ks, docs, {
+    replace: opts.replace,
+    dryRun: opts.dryRun,
+    batchSize: opts.batchSize,
+    tags: opts.tags,
+    metadata: opts.metadata,
+    log: log2,
+    progress
+  });
+  return { ...summary, skippedFiles: [...summary.skippedFiles, ...skipped] };
+}
+try {
+  if (typeof log?.setLevel === "function" && log.LEVELS) {
+    log.setLevel(log.LEVELS.ERROR);
+  }
+} catch {
+}
 
 // src/check.ts
 init_config();
@@ -754,7 +1284,7 @@ async function checkOneModel(id, model, baseUrl, names, ks, probe) {
   }
   return { id, status: "ok", detail: `model '${model}' available at ${baseUrl}${scope}` };
 }
-async function runChecks(opts = {}) {
+async function runChecks(opts) {
   const results = [];
   const major = Number(process.versions.node.split(".")[0]);
   results.push({
@@ -876,44 +1406,35 @@ async function runChecks(opts = {}) {
   results.push(
     "error" in defaultTags ? { id: "ollama", status: "fail", detail: `Ollama not reachable at ${defaultUrl} \u2014 ${defaultTags.error}` } : { id: "ollama", status: "ok", detail: `Ollama reachable at ${defaultUrl} (${defaultTags.names.length} model(s))` }
   );
-  if (opts.model) {
-    const tags = await getTags(defaultUrl);
-    if ("error" in tags) {
-      results.push({
-        id: "model",
-        status: "skip",
-        detail: `Ollama unreachable at ${defaultUrl} \u2014 cannot verify model '${opts.model}'`
-      });
-    } else {
-      results.push(await checkOneModel("model", opts.model, defaultUrl, tags.names, void 0, opts.probe ?? false));
-    }
+  const tags = await getTags(defaultUrl);
+  if ("error" in tags) {
+    results.push({
+      id: "model",
+      status: "skip",
+      detail: `Ollama unreachable at ${defaultUrl} \u2014 cannot verify model '${opts.model}'`
+    });
+  } else {
+    results.push(await checkOneModel("model", opts.model, defaultUrl, tags.names, void 0, opts.probe ?? false));
   }
   for (const ks of kss) {
-    const tags = await getTags(ks.ollama_url);
-    if ("error" in tags) {
+    const tags2 = await getTags(ks.ollama_url);
+    if ("error" in tags2) {
       results.push({
         id: `model:${ks.name}`,
         status: "fail",
-        detail: `Ollama not reachable at ${ks.ollama_url} (required by ks '${ks.name}') \u2014 ${tags.error}`
+        detail: `Ollama not reachable at ${ks.ollama_url} (required by ks '${ks.name}') \u2014 ${tags2.error}`
       });
       continue;
     }
     results.push(
-      await checkOneModel(`model:${ks.name}`, ks.embedding_model, ks.ollama_url, tags.names, ks, opts.probe ?? false)
+      await checkOneModel(`model:${ks.name}`, ks.embedding_model, ks.ollama_url, tags2.names, ks, opts.probe ?? false)
     );
   }
-  if (!opts.model && kss.length === 0) {
-    results.push({
-      id: "model",
-      status: "warn",
-      detail: "no embedding model to verify \u2014 pass --model <model> or create a knowledge source"
-    });
-  }
   return results;
 }
 
 // src/version.ts
-var VERSION = "1.0.0";
+var VERSION = "1.1.0";
 
 // src/cli.ts
 async function run(fn) {
@@ -966,7 +1487,7 @@ program.command("init").description("Initialize the database schema (extension,
     console.log("database ready");
   })
 );
-program.command("check").description("Verify environment requirements (Node, Postgres, pgvector, schema, Ollama models)").option("--model <model>", "Verify this Ollama model is available (e.g. nomic-embed-text)").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--probe", "Embed a test string with each model to verify dimensions (slower)").option("--json", "Output as JSON").action(
+program.command("check").description("Verify environment requirements (Node, Postgres, pgvector, schema, Ollama models)").requiredOption("--model <model>", "Verify this Ollama model is available (e.g. nomic-embed-text)").option("--url <url>", "Ollama base URL (default http://localhost:11434)").option("--probe", "Embed a test string with each model to verify dimensions (slower)").option("--json", "Output as JSON").action(
   (opts) => run(async () => {
     const results = await runChecks({ model: opts.model, url: opts.url, probe: opts.probe });
     if (opts.json) {
@@ -1044,6 +1565,34 @@ ksCmd.command("list").description("List knowledge sources").option("--json", "Ou
     }
   })
 );
+ksCmd.command("stats <name>").description("Show statistics for a knowledge source (entries, sources, content volume)").option("--json", "Output as JSON").action(
+  (name, opts) => run(async () => {
+    await initDb();
+    const ks = requireKs(await getKs(name), name);
+    const stats = await ksStats(ks);
+    if (opts.json) {
+      console.log(JSON.stringify(stats, null, 2));
+      return;
+    }
+    console.log(
+      `Knowledge source '${stats.ks.name}' (model=${stats.ks.embedding_model}@${stats.ks.ollama_url}, dim=${stats.ks.embedding_dim})`
+    );
+    if (stats.ks.description) console.log(`  description: ${stats.ks.description}`);
+    console.log(`  created:      ${stats.ks.created_at.toISOString()}`);
+    console.log(`  last fed:     ${stats.last_fed_at ? stats.last_fed_at.toISOString() : "never"}`);
+    console.log("");
+    if (stats.entries === 0) {
+      console.log(`  no entries yet; feed it with: gresmcp feed ${stats.ks.name}`);
+      return;
+    }
+    const size = (n) => n === null ? "-" : n.toLocaleString("en-US");
+    console.log(`  entries (chunks): ${stats.entries.toLocaleString("en-US")}`);
+    console.log(`  distinct sources: ${stats.sources.toLocaleString("en-US")}`);
+    console.log(
+      `  content:          ${stats.chars.toLocaleString("en-US")} chars (avg ${size(stats.avg_chunk_chars)}, min ${size(stats.min_chunk_chars)}, max ${size(stats.max_chunk_chars)} per chunk)`
+    );
+  })
+);
 ksCmd.command("delete <name>").description("Delete a knowledge source and all of its entries").option("--yes", "Do not ask for confirmation").action(
   (name, opts) => run(async () => {
     await initDb();
@@ -1060,15 +1609,13 @@ ksCmd.command("delete <name>").description("Delete a knowledge source and all of
     console.log(`Deleted knowledge source '${name}'`);
   })
 );
-program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--url <url>", "Ollama base URL override for this run").option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
+program.command("feed <ks>").description("Feed data into a knowledge source (from a file/folder via --path, a website via --url, or inline via --text/--stdin)").option("--path <path>", "File or folder to ingest (text, markdown, code, html)").option("--url <url>", "Website URL to scrape and ingest (crawls same-host pages linked from the seed)").option("--crawler <crawler>", "Crawler backend for --url: auto, crawlee, playwright, puppeteer (default auto; playwright/puppeteer must be installed globally)", ((v) => v.toLowerCase())).option("--max-pages <n>", "Max pages to fetch when crawling --url (default 999)", parseInt).option("--depth <n>", "Max link depth from the seed URL (default 5; 0 = seed page only)", parseInt).option("--text <text>", "Inline text entry").option("--stdin", "Read entry text from stdin").option("--title <title>", "Title for manual entries").option("--source-name <name>", "Source name for manual entries (default 'manual')").option("--tags <tags>", "Comma-separated tags").option("--metadata <key=value...>", "Custom metadata entries", ((v, prev) => [...prev, v]), []).option("--replace", "Replace existing chunks of the same source instead of deduplicating").option("--dry-run", "Parse and chunk only; do not embed or write").option("--chunk-size <n>", "Max chunk length in characters", parseInt).option("--overlap <n>", "Chunk overlap in characters", parseInt).action(
   (name, opts) => run(async () => {
     await initDb();
     const ks = requireKs(await getKs(name), name);
-    const modes = [opts.path, opts.text, opts.stdin].filter(Boolean).length;
-    if (modes !== 1) throw new Error("provide exactly one of --path, --text or --stdin");
+    const modes = [opts.path, opts.url, opts.text, opts.stdin].filter(Boolean).length;
+    if (modes !== 1) throw new Error("provide exactly one of --path, --url, --text or --stdin");
     const common = {
-      title: opts.title,
-      url: opts.url,
       replace: opts.replace,
       dryRun: opts.dryRun,
       batchSize: 32,
@@ -1078,12 +1625,19 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       metadata: parseKeyValue(opts.metadata ?? [])
     };
     let summary;
+    const since = await dbClockTimestamp();
     if (opts.path) {
       summary = await feedTarget(ks, opts.path, common);
+    } else if (opts.url) {
+      const crawler = opts.crawler;
+      if (crawler && !CRAWLER_KINDS.includes(crawler)) {
+        throw new Error(`unknown crawler '${crawler}' (use auto, crawlee, playwright or puppeteer)`);
+      }
+      summary = await feedUrl(ks, opts.url, { ...common, crawler, maxPages: opts.maxPages, depth: opts.depth });
     } else {
       const text = opts.stdin ? await readStdin() : opts.text;
       if (!text.trim()) throw new Error("empty input text");
-      summary = await feedText(ks, text, { ...common, sourceName: opts.sourceName });
+      summary = await feedText(ks, text, { ...common, title: opts.title, sourceName: opts.sourceName });
     }
     if (summary.dryRun) {
       console.log(`dry run: ${summary.documents} document(s), ${summary.chunks} chunk(s), ${summary.skippedFiles.length} file(s) skipped`);
@@ -1091,9 +1645,22 @@ program.command("feed <ks>").description("Feed data into a knowledge source (fro
       console.log(
         `fed '${ks.name}': ${summary.inserted} chunk(s) inserted` + (summary.skippedExisting ? `, ${summary.skippedExisting} unchanged chunk(s) skipped` : "") + (summary.deletedForReplace ? `, ${summary.deletedForReplace} old chunk(s) replaced` : "") + `, ${summary.skippedFiles.length} file(s) skipped`
       );
+      await printRunStats(ks, since);
     }
   })
 );
+async function printRunStats(ks, since) {
+  const stats = await ksStats(ks, since);
+  if (stats.entries === 0) return;
+  const size = (n) => n === null ? "-" : n.toLocaleString("en-US");
+  console.log("");
+  console.log(
+    `this run: ${stats.entries.toLocaleString("en-US")} chunk(s) across ${stats.sources.toLocaleString("en-US")} source(s)`
+  );
+  console.log(
+    `  content:          ${stats.chars.toLocaleString("en-US")} chars (avg ${size(stats.avg_chunk_chars)}, min ${size(stats.min_chunk_chars)}, max ${size(stats.max_chunk_chars)} per chunk)`
+  );
+}
 async function readStdin() {
   const chunks = [];
   for await (const chunk of stdin) chunks.push(chunk);