@snap-agent/rag-web 0.1.0

package/dist/index.mjs

@@ -0,0 +1,2107 @@
+// src/WebRAGPlugin.ts
+import { MongoClient } from "mongodb";
+import OpenAI from "openai";
+import * as cheerio from "cheerio";
+import * as fs from "fs";
+import * as path from "path";
+var WebRAGPlugin = class _WebRAGPlugin {
+  name = "web-rag";
+  type = "rag";
+  priority;
+  config;
+  client = null;
+  db = null;
+  openai;
+  // Embedding cache
+  embeddingCache = /* @__PURE__ */ new Map();
+  cacheStats = { hits: 0, misses: 0 };
+  constructor(config) {
+    this.config = {
+      collection: "web_content",
+      embeddingModel: "text-embedding-3-small",
+      vectorIndexName: "web_vector_index",
+      numCandidates: 100,
+      limit: 10,
+      minScore: 0.7,
+      filterableFields: ["type"],
+      ...config
+    };
+    this.priority = config.priority ?? 100;
+    this.openai = new OpenAI({ apiKey: config.openaiApiKey });
+  }
+  // ============================================================================
+  // MongoDB Connection
+  // ============================================================================
+  async getCollection() {
+    if (!this.client) {
+      this.client = new MongoClient(this.config.mongoUri);
+      await this.client.connect();
+      this.db = this.client.db(this.config.dbName);
+    }
+    return this.db.collection(this.config.collection);
+  }
+  async getLedgerCollection() {
+    if (!this.client) {
+      this.client = new MongoClient(this.config.mongoUri);
+      await this.client.connect();
+      this.db = this.client.db(this.config.dbName);
+    }
+    const name = this.config.crawlLedger?.collection ?? "web_crawl_ledger";
+    return this.db.collection(name);
+  }
+  /**
+   * List recent crawl ledger rows (for dashboards / pagination in the front).
+   */
+  async listCrawlLedger(options = {}) {
+    const col = await this.getLedgerCollection();
+    const filter = { tenantId: this.config.tenantId };
+    filter.agentId = options.agentId ?? "shared";
+    if (options.domain) filter.domain = options.domain;
+    if (options.status) filter.lastStatus = options.status;
+    const limit = Math.min(Math.max(options.limit ?? 50, 1), 500);
+    const skip = Math.max(options.skip ?? 0, 0);
+    return col.find(filter).sort({ lastCrawledAt: -1 }).skip(skip).limit(limit).toArray();
+  }
+  resolveCrawlLedgerOptions(config) {
+    const plugin = this.config.crawlLedger;
+    const per = config.crawlLedger;
+    const enabled = per?.enabled ?? plugin?.enabled ?? false;
+    if (!enabled) return null;
+    const ttlMsFailure = per?.ttlMsFailure ?? plugin?.ttlMsFailure ?? 60 * 60 * 1e3;
+    return {
+      ttlMsIndexed: per?.ttlMsIndexed ?? plugin?.ttlMsIndexed ?? 7 * 24 * 60 * 60 * 1e3,
+      ttlMsFailure,
+      ttlMsRenderError: per?.ttlMsRenderError ?? plugin?.ttlMsRenderError ?? 5 * 60 * 1e3,
+      maxPageStatuses: per?.maxPageStatuses ?? 500,
+      stripQuery: config.stripQueryParams ?? false
+    };
+  }
+  normalizeLedgerUrl(url, stripQuery) {
+    return this.normalizeWebsiteUrl(url, stripQuery);
+  }
+  shouldSkipLedger(entry, ttlMsIndexed, ttlMsFailure, ttlMsRenderError, forceRecrawl) {
+    if (forceRecrawl || !entry) return false;
+    const t = entry.lastCrawledAt instanceof Date ? entry.lastCrawledAt.getTime() : new Date(entry.lastCrawledAt).getTime();
+    const age = Date.now() - t;
+    if (entry.lastStatus === "indexed" && age < ttlMsIndexed) return true;
+    if (entry.lastStatus === "error" && age < ttlMsRenderError) return true;
+    if (entry.lastStatus !== "indexed" && entry.lastStatus !== "error" && age < ttlMsFailure) {
+      return true;
+    }
+    return false;
+  }
+  async findLedgerEntry(urlNormalized, agentId) {
+    const col = await this.getLedgerCollection();
+    return col.findOne({
+      tenantId: this.config.tenantId,
+      agentId,
+      urlNormalized
+    });
+  }
+  toLedgerStatus(doc, diag) {
+    if (doc) return "indexed";
+    if (diag?.reason === "non_html") return "non_html";
+    if (diag?.reason === "blocked_suspected") return "blocked_suspected";
+    if (diag?.reason === "render_error") return "error";
+    return "too_small";
+  }
+  async upsertLedgerRecord(params) {
+    const col = await this.getLedgerCollection();
+    let domain = "";
+    try {
+      domain = new URL(params.url).hostname;
+    } catch {
+      domain = "";
+    }
+    const now = /* @__PURE__ */ new Date();
+    const errMsg = params.errorMessage ?? params.diag?.errorMessage;
+    const $set = {
+      tenantId: this.config.tenantId,
+      agentId: params.agentId,
+      urlNormalized: params.urlNormalized,
+      url: params.url,
+      domain,
+      lastStatus: params.status,
+      lastCrawledAt: now,
+      updatedAt: now
+    };
+    if (errMsg !== void 0) {
+      $set.errorMessage = errMsg;
+    } else if (params.status === "indexed" && params.doc) {
+      $set.errorMessage = null;
+    }
+    if (params.doc) {
+      $set.modeUsed = params.diag?.modeUsed;
+      $set.contentLength = params.doc.content.length;
+      $set.title = params.doc.metadata?.title;
+      $set.docId = params.doc.id;
+    } else {
+      $set.modeUsed = params.diag?.modeUsed;
+      $set.contentLength = null;
+      $set.title = null;
+      $set.docId = null;
+    }
+    await col.updateOne(
+      {
+        tenantId: this.config.tenantId,
+        agentId: params.agentId,
+        urlNormalized: params.urlNormalized
+      },
+      { $set },
+      { upsert: true }
+    );
+  }
+  pushPageStatus(list, max, entry) {
+    list.push(entry);
+    while (list.length > max) list.shift();
+  }
+  async disconnect() {
+    if (this.client) {
+      await this.client.close();
+      this.client = null;
+      this.db = null;
+    }
+  }
+  // ============================================================================
+  // RAG Plugin Interface
+  // ============================================================================
+  /**
+   * Retrieve contextual content for a message
+   */
+  async retrieveContext(message, options = {}) {
+    const queryVector = await this.generateEmbedding(message);
+    const hardFilters = {
+      tenantId: this.config.tenantId,
+      ...options.filters
+    };
+    if (options.agentId) {
+      hardFilters.agentId = { $in: ["shared", options.agentId] };
+    }
+    const results = await this.vectorSearch({
+      queryVector,
+      hardFilters
+    });
+    let scoredResults = results;
+    if (this.config.typeBoosts) {
+      scoredResults = results.map((doc) => ({
+        ...doc,
+        score: doc.score * (this.config.typeBoosts[doc.metadata.type] ?? 1)
+      }));
+    }
+    if (this.config.recencyBoost?.enabled) {
+      const { field, decayDays, maxBoost = 1.2 } = this.config.recencyBoost;
+      const now = Date.now();
+      const decayMs = decayDays * 24 * 60 * 60 * 1e3;
+      scoredResults = scoredResults.map((doc) => {
+        const dateValue = doc.metadata[field];
+        if (!dateValue) return doc;
+        const docDate = new Date(dateValue).getTime();
+        const age = now - docDate;
+        const freshness = Math.max(0, 1 - age / decayMs);
+        const boost = 1 + (maxBoost - 1) * freshness;
+        return { ...doc, score: doc.score * boost };
+      });
+    }
+    scoredResults.sort((a, b) => b.score - a.score);
+    scoredResults = scoredResults.slice(0, this.config.limit);
+    const content = this.formatResultsToContext(scoredResults);
+    return {
+      content,
+      metadata: {
+        plugin: this.name,
+        contentCount: scoredResults.length,
+        types: [...new Set(scoredResults.map((d) => d.metadata.type))],
+        topResults: scoredResults.slice(0, 5).map((doc) => ({
+          id: doc.id,
+          type: doc.metadata.type,
+          title: doc.metadata.title,
+          url: doc.metadata.url,
+          score: doc.score
+        }))
+      }
+    };
+  }
+  /**
+   * Format retrieved content for LLM context
+   */
+  formatResultsToContext(docs) {
+    if (docs.length === 0) {
+      return "No relevant content found.";
+    }
+    const sections = ["## Relevant Content\n"];
+    for (const doc of docs) {
+      const meta = doc.metadata;
+      const header = meta.title || `${meta.type} (${doc.id})`;
+      sections.push(`### ${header}`);
+      if (meta.type) sections.push(`**Type:** ${meta.type}`);
+      if (meta.url) sections.push(`**URL:** ${meta.url}`);
+      const skipFields = ["type", "title", "url", "sourceUrl", "fetchedAt"];
+      const extraMeta = Object.entries(meta).filter(([key]) => !skipFields.includes(key)).map(([key, value]) => `**${this.formatFieldName(key)}:** ${this.formatFieldValue(value)}`);
+      if (extraMeta.length > 0) {
+        sections.push(extraMeta.join("\n"));
+      }
+      sections.push("");
+      sections.push(doc.content);
+      sections.push("");
+    }
+    return sections.join("\n");
+  }
+  formatFieldName(key) {
+    return key.replace(/([A-Z])/g, " $1").replace(/^./, (s) => s.toUpperCase());
+  }
+  formatFieldValue(value) {
+    if (Array.isArray(value)) return value.join(", ");
+    if (value instanceof Date) return value.toLocaleDateString();
+    if (typeof value === "object") return JSON.stringify(value);
+    return String(value);
+  }
+  // ============================================================================
+  // Vector Search
+  // ============================================================================
+  async vectorSearch(options) {
+    const collection = await this.getCollection();
+    const pipeline = [
+      {
+        $vectorSearch: {
+          index: this.config.vectorIndexName,
+          path: "embedding",
+          queryVector: options.queryVector,
+          numCandidates: this.config.numCandidates,
+          limit: this.config.limit * 2,
+          // Fetch more for post-filtering
+          filter: options.hardFilters
+        }
+      },
+      {
+        $addFields: {
+          score: { $meta: "vectorSearchScore" }
+        }
+      }
+    ];
+    if (this.config.minScore) {
+      pipeline.push({
+        $match: { score: { $gte: this.config.minScore } }
+      });
+    }
+    pipeline.push({ $limit: this.config.limit * 2 });
+    const results = await collection.aggregate(pipeline).toArray();
+    return results;
+  }
+  // ============================================================================
+  // Embedding Generation
+  // ============================================================================
+  async generateEmbedding(text) {
+    const cacheConfig = this.config.cache?.embeddings;
+    if (cacheConfig?.enabled) {
+      const cached = this.embeddingCache.get(text);
+      const ttl = cacheConfig.ttl ?? 36e5;
+      if (cached && Date.now() - cached.timestamp < ttl) {
+        this.cacheStats.hits++;
+        return cached.value;
+      }
+    }
+    this.cacheStats.misses++;
+    const response = await this.openai.embeddings.create({
+      model: this.config.embeddingModel,
+      input: text
+    });
+    const embedding = response.data[0].embedding;
+    if (cacheConfig?.enabled) {
+      const maxSize = cacheConfig.maxSize ?? 1e3;
+      if (this.embeddingCache.size >= maxSize) {
+        const firstKey = this.embeddingCache.keys().next().value;
+        if (firstKey) this.embeddingCache.delete(firstKey);
+      }
+      this.embeddingCache.set(text, { value: embedding, timestamp: Date.now() });
+    }
+    return embedding;
+  }
+  async generateEmbeddingsBatch(texts) {
+    const embeddings = [];
+    for (const text of texts) {
+      const embedding = await this.generateEmbedding(text);
+      embeddings.push(embedding);
+    }
+    return embeddings;
+  }
+  // ============================================================================
+  // Document Ingestion
+  // ============================================================================
+  /**
+   * Ingest documents into the CMS RAG system
+   */
+  async ingest(documents, options) {
+    const collection = await this.getCollection();
+    let indexed = 0;
+    const errors = [];
+    const batchSize = options?.batchSize ?? 10;
+    for (let i = 0; i < documents.length; i += batchSize) {
+      const batch = documents.slice(i, i + batchSize);
+      const embeddings = await this.generateEmbeddingsBatch(
+        batch.map((doc) => doc.content)
+      );
+      const docsToStore = batch.map((doc, idx) => ({
+        id: doc.id,
+        content: doc.content,
+        metadata: {
+          type: doc.metadata?.type || "content",
+          ...doc.metadata
+        },
+        tenantId: this.config.tenantId,
+        // Use 'shared' marker for tenant-wide content, specific agentId for agent-only
+        agentId: options?.agentId || "shared",
+        embedding: embeddings[idx]
+      }));
+      for (const doc of docsToStore) {
+        try {
+          const filter = {
+            tenantId: this.config.tenantId,
+            id: doc.id,
+            // Match by agentId ('shared' for tenant-wide, specific for agent-only)
+            agentId: options?.agentId || "shared"
+          };
+          await collection.updateOne(
+            filter,
+            {
+              $set: { ...doc, updatedAt: /* @__PURE__ */ new Date() },
+              $setOnInsert: { createdAt: /* @__PURE__ */ new Date() }
+            },
+            { upsert: true }
+          );
+          indexed++;
+        } catch (error) {
+          errors.push({
+            id: doc.id,
+            error: error instanceof Error ? error.message : "Unknown error"
+          });
+        }
+      }
+    }
+    return {
+      success: errors.length === 0,
+      indexed,
+      failed: errors.length,
+      errors: errors.length > 0 ? errors : void 0,
+      metadata: {
+        tenantId: this.config.tenantId,
+        collection: this.config.collection
+      }
+    };
+  }
+  /**
+   * Update a single document
+   */
+  async update(id, document, options) {
+    const collection = await this.getCollection();
+    const update = { updatedAt: /* @__PURE__ */ new Date() };
+    if (document.content) {
+      const embedding = await this.generateEmbedding(document.content);
+      update.content = document.content;
+      update.embedding = embedding;
+    }
+    if (document.metadata) {
+      for (const [key, value] of Object.entries(document.metadata)) {
+        update[`metadata.${key}`] = value;
+      }
+    }
+    const filter = {
+      tenantId: this.config.tenantId,
+      id,
+      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
+      agentId: options?.agentId || "shared"
+    };
+    await collection.updateOne(filter, { $set: update });
+  }
+  /**
+   * Delete document(s) by ID
+   */
+  async delete(ids, options) {
+    const collection = await this.getCollection();
+    const idArray = Array.isArray(ids) ? ids : [ids];
+    const filter = {
+      tenantId: this.config.tenantId,
+      id: { $in: idArray },
+      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
+      agentId: options?.agentId || "shared"
+    };
+    const result = await collection.deleteMany(filter);
+    return result.deletedCount;
+  }
+  /**
+   * Bulk operations
+   */
+  async bulk(operations, options) {
+    let inserted = 0;
+    let updated = 0;
+    let deleted = 0;
+    let failed = 0;
+    const errors = [];
+    for (const op of operations) {
+      try {
+        switch (op.type) {
+          case "insert":
+            if (op.document) {
+              await this.ingest([op.document], options);
+              inserted++;
+            }
+            break;
+          case "update":
+            if (op.document) {
+              await this.update(op.id, op.document, options);
+              updated++;
+            }
+            break;
+          case "delete":
+            const count = await this.delete(op.id, options);
+            deleted += count;
+            break;
+        }
+      } catch (error) {
+        failed++;
+        errors.push({
+          id: op.id,
+          operation: op.type,
+          error: error.message || "Unknown error"
+        });
+      }
+    }
+    return {
+      success: failed === 0,
+      inserted,
+      updated,
+      deleted,
+      failed,
+      errors: errors.length > 0 ? errors : void 0
+    };
+  }
+  // ============================================================================
+  // URL Ingestion
+  // ============================================================================
+  /**
+   * Ingest content from a URL (JSON, CSV, XML, or API)
+   */
+  async ingestFromUrl(source, options) {
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), source.timeout || 3e4);
+      const response = await fetch(source.url, {
+        headers: {
+          ...source.headers,
+          ...source.auth && this.buildAuthHeaders(source.auth)
+        },
+        signal: controller.signal
+      });
+      clearTimeout(timeoutId);
+      if (!response.ok) {
+        throw new Error(`HTTP error: ${response.status} ${response.statusText}`);
+      }
+      let documents;
+      if (source.type === "json" || source.type === "api") {
+        const data = await response.json();
+        documents = this.transformJsonToDocuments(data, source.transform);
+      } else if (source.type === "csv") {
+        const data = await response.text();
+        documents = this.transformCsvToDocuments(data, source.transform);
+      } else if (source.type === "xml") {
+        const data = await response.text();
+        documents = this.transformXmlToDocuments(data, source.transform);
+      } else {
+        throw new Error(`Unsupported source type: ${source.type}`);
+      }
+      documents = documents.map((doc) => ({
+        ...doc,
+        metadata: {
+          ...doc.metadata,
+          ...source.metadata,
+          sourceUrl: source.url,
+          fetchedAt: (/* @__PURE__ */ new Date()).toISOString()
+        }
+      }));
+      const ingestResult = await this.ingest(documents, options);
+      return {
+        ...ingestResult,
+        sourceUrl: source.url,
+        fetchedAt: /* @__PURE__ */ new Date(),
+        documentsFetched: documents.length
+      };
+    } catch (error) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        sourceUrl: source.url,
+        fetchedAt: /* @__PURE__ */ new Date(),
+        documentsFetched: 0,
+        errors: [{
+          id: "fetch",
+          error: error instanceof Error ? error.message : "Unknown error"
+        }]
+      };
+    }
+  }
+  buildAuthHeaders(auth) {
+    if (!auth) return {};
+    switch (auth.type) {
+      case "bearer":
+        return auth.token ? { Authorization: `Bearer ${auth.token}` } : {};
+      case "basic":
+        if (auth.username && auth.password) {
+          const encoded = Buffer.from(`${auth.username}:${auth.password}`).toString("base64");
+          return { Authorization: `Basic ${encoded}` };
+        }
+        return {};
+      case "api-key":
+        return auth.header && auth.key ? { [auth.header]: auth.key } : {};
+      case "custom":
+        return auth.headers || {};
+      default:
+        return {};
+    }
+  }
+  transformJsonToDocuments(data, transform) {
+    let items = data;
+    if (transform?.documentPath) {
+      items = this.extractByPath(data, transform.documentPath);
+    }
+    if (!Array.isArray(items)) {
+      items = [items];
+    }
+    const fieldMapping = transform?.fieldMapping || {};
+    return items.map((item, index) => {
+      const metadata = {};
+      for (const [targetField, sourcePath] of Object.entries(fieldMapping)) {
+        if (targetField === "id" || targetField === "content") continue;
+        if (typeof sourcePath === "function") {
+          metadata[targetField] = sourcePath();
+        } else if (sourcePath) {
+          metadata[targetField] = this.extractField(item, sourcePath);
+        }
+      }
+      if (!metadata.type) {
+        metadata.type = "content";
+      }
+      return {
+        id: this.extractField(item, fieldMapping.id || "id") || `doc-${index}`,
+        content: this.extractField(item, fieldMapping.content || "content") || JSON.stringify(item),
+        metadata
+      };
+    });
+  }
+  transformCsvToDocuments(csvData, transform) {
+    const lines = csvData.trim().split("\n");
+    if (lines.length < 2) return [];
+    const headers = this.parseCsvLine(lines[0]);
+    return lines.slice(1).map((line, index) => {
+      const values = this.parseCsvLine(line);
+      const item = headers.reduce((acc, header, i) => {
+        acc[header] = values[i] || "";
+        return acc;
+      }, {});
+      return this.transformJsonToDocuments([item], transform)[0];
+    });
+  }
+  parseCsvLine(line) {
+    const result = [];
+    let current = "";
+    let inQuotes = false;
+    for (const char of line) {
+      if (char === '"') {
+        inQuotes = !inQuotes;
+      } else if (char === "," && !inQuotes) {
+        result.push(current.trim());
+        current = "";
+      } else {
+        current += char;
+      }
+    }
+    result.push(current.trim());
+    return result;
+  }
+  transformXmlToDocuments(xmlData, transform) {
+    const items = [];
+    const itemPath = transform?.documentPath || "item";
+    const itemRegex = new RegExp(`<${itemPath}[^>]*>([\\s\\S]*?)<\\/${itemPath}>`, "gi");
+    let match;
+    while ((match = itemRegex.exec(xmlData)) !== null) {
+      const itemXml = match[1];
+      const item = {};
+      const tagRegex = /<(\w+)[^>]*>([^<]*)<\/\1>/g;
+      let tagMatch;
+      while ((tagMatch = tagRegex.exec(itemXml)) !== null) {
+        item[tagMatch[1]] = tagMatch[2].trim();
+      }
+      items.push(item);
+    }
+    return this.transformJsonToDocuments(items, transform);
+  }
+  extractByPath(obj, path2) {
+    const parts = path2.split(".");
+    let current = obj;
+    for (const part of parts) {
+      if (current == null) return void 0;
+      const arrayMatch = part.match(/^(\w+)\[(\d+)\]$/);
+      if (arrayMatch) {
+        current = current[arrayMatch[1]]?.[parseInt(arrayMatch[2])];
+      } else {
+        current = current[part];
+      }
+    }
+    return current;
+  }
+  extractField(item, path2) {
+    return this.extractByPath(item, path2);
+  }
+  // ============================================================================
+  // Drupal JSON:API Integration
+  // ============================================================================
+  /**
+   * Ingest content from a Drupal site using JSON:API
+   */
+  async ingestFromDrupal(config, options) {
+    const results = [];
+    for (const contentType of config.contentTypes) {
+      const url = `${config.baseUrl}/jsonapi/node/${contentType}`;
+      const mapping = config.mappings?.[contentType];
+      const result = await this.ingestFromUrl(
+        {
+          url,
+          type: "json",
+          auth: config.auth,
+          transform: {
+            documentPath: "data",
+            fieldMapping: {
+              id: "id",
+              content: mapping?.content || "attributes.body.processed",
+              type: () => contentType,
+              title: "attributes.title",
+              url: "attributes.path.alias",
+              ...mapping?.fields
+            }
+          }
+        },
+        options
+      );
+      results.push(result);
+    }
+    return results;
+  }
+  /**
+   * Parse Drupal JSON:API node type (e.g., 'node--project' → 'project')
+   */
+  static parseDrupalType(type) {
+    return type.replace(/^node--/, "");
+  }
+  // ============================================================================
+  // WordPress REST API Integration
+  // ============================================================================
+  /**
+   * Ingest content from a WordPress site using REST API
+   * 
+   * @example
+   * ```typescript
+   * await plugin.ingestFromWordPress({
+   *   baseUrl: 'https://myblog.com',
+   *   postTypes: ['posts', 'pages'],
+   *   perPage: 100,
+   * });
+   * ```
+   */
+  async ingestFromWordPress(config, options) {
+    const results = [];
+    const postTypes = config.postTypes || ["posts", "pages"];
+    const perPage = config.perPage || 100;
+    const maxPages = config.maxPages || 10;
+    for (const postType of postTypes) {
+      let page = 1;
+      let hasMore = true;
+      while (hasMore && page <= maxPages) {
+        const url = `${config.baseUrl}/wp-json/wp/v2/${postType}?per_page=${perPage}&page=${page}&_embed`;
+        const mapping = config.mappings?.[postType];
+        try {
+          const result = await this.ingestFromUrl(
+            {
+              url,
+              type: "json",
+              auth: config.auth,
+              transform: {
+                fieldMapping: {
+                  id: "id",
+                  content: mapping?.content || "content.rendered",
+                  type: () => this.normalizeWordPressType(postType),
+                  title: "title.rendered",
+                  url: "link",
+                  slug: "slug",
+                  publishedAt: "date",
+                  modifiedAt: "modified",
+                  author: "_embedded.author.0.name",
+                  featuredImage: "_embedded.wp:featuredmedia.0.source_url",
+                  excerpt: "excerpt.rendered",
+                  categories: "_embedded.wp:term.0",
+                  tags: "_embedded.wp:term.1",
+                  ...mapping?.fields
+                }
+              }
+            },
+            options
+          );
+          results.push(result);
+          hasMore = result.documentsFetched === perPage;
+          page++;
+        } catch (error) {
+          hasMore = false;
+        }
+      }
+    }
+    return results;
+  }
+  /**
+   * Normalize WordPress post type to a cleaner name
+   */
+  normalizeWordPressType(postType) {
+    if (postType.endsWith("s")) {
+      return postType.slice(0, -1);
+    }
+    return postType;
+  }
+  // ============================================================================
+  // Sanity.io Integration
+  // ============================================================================
+  /**
+   * Ingest content from a Sanity.io project using GROQ queries
+   * 
+   * @example
+   * ```typescript
+   * await plugin.ingestFromSanity({
+   *   projectId: 'abc123',
+   *   dataset: 'production',
+   *   queries: {
+   *     post: {
+   *       query: '*[_type == "post" && !(_id in path("drafts.**"))]',
+   *       content: 'body',
+   *       fields: {
+   *         author: 'author->name',
+   *         categories: 'categories[]->title',
+   *       },
+   *     },
+   *   },
+   * });
+   * ```
+   */
+  async ingestFromSanity(config, options) {
+    const results = [];
+    const apiVersion = config.apiVersion || "v2024-01-01";
+    const useCdn = config.useCdn !== false;
+    const baseUrl = useCdn ? `https://${config.projectId}.apicdn.sanity.io/${apiVersion}` : `https://${config.projectId}.api.sanity.io/${apiVersion}`;
+    for (const [contentType, queryConfig] of Object.entries(config.queries)) {
+      const encodedQuery = encodeURIComponent(queryConfig.query);
+      const url = `${baseUrl}/data/query/${config.dataset}?query=${encodedQuery}`;
+      const headers = {};
+      if (config.token) {
+        headers["Authorization"] = `Bearer ${config.token}`;
+      }
+      const result = await this.ingestFromUrl(
+        {
+          url,
+          type: "json",
+          headers,
+          transform: {
+            documentPath: "result",
+            fieldMapping: {
+              id: "_id",
+              content: queryConfig.content,
+              type: () => contentType,
+              title: "title",
+              slug: "slug.current",
+              publishedAt: "publishedAt",
+              updatedAt: "_updatedAt",
+              ...queryConfig.fields
+            }
+          }
+        },
+        options
+      );
+      results.push(result);
+    }
+    return results;
+  }
+  /**
+   * Convert Sanity Portable Text blocks to plain text
+   * Useful for extracting content from rich text fields
+   */
+  static sanityBlocksToText(blocks) {
+    if (!Array.isArray(blocks)) return "";
+    return blocks.filter((block) => block._type === "block").map((block) => {
+      if (!block.children) return "";
+      return block.children.map((child) => child.text || "").join("");
+    }).join("\n\n");
+  }
+  // ============================================================================
+  // Strapi Integration
+  // ============================================================================
+  /**
+   * Ingest content from a Strapi CMS (v4 by default)
+   * 
+   * @example
+   * ```typescript
+   * await plugin.ingestFromStrapi({
+   *   baseUrl: 'https://my-strapi.com',
+   *   apiToken: process.env.STRAPI_TOKEN,
+   *   contentTypes: ['articles', 'pages'],
+   *   mappings: {
+   *     articles: {
+   *       content: 'attributes.content',
+   *       fields: {
+   *         author: 'attributes.author.data.attributes.name',
+   *         category: 'attributes.category.data.attributes.name',
+   *       },
+   *     },
+   *   },
+   * });
+   * ```
+   */
+  async ingestFromStrapi(config, options) {
+    const results = [];
+    const pageSize = config.pageSize || 100;
+    const maxPages = config.maxPages || 10;
+    for (const contentType of config.contentTypes) {
+      let page = 1;
+      let hasMore = true;
+      const mapping = config.mappings?.[contentType];
+      const useAttributes = mapping?.useAttributes !== false;
+      while (hasMore && page <= maxPages) {
+        const url = `${config.baseUrl}/api/${contentType}?pagination[page]=${page}&pagination[pageSize]=${pageSize}&populate=*`;
+        const headers = {};
+        if (config.apiToken) {
+          headers["Authorization"] = `Bearer ${config.apiToken}`;
+        }
+        try {
+          const result = await this.ingestFromUrl(
+            {
+              url,
+              type: "json",
+              headers,
+              transform: {
+                documentPath: "data",
+                fieldMapping: useAttributes ? {
+                  // Strapi v4 format (with attributes)
+                  id: "id",
+                  content: mapping?.content || "attributes.content",
+                  type: () => this.normalizeStrapiType(contentType),
+                  title: "attributes.title",
+                  slug: "attributes.slug",
+                  publishedAt: "attributes.publishedAt",
+                  updatedAt: "attributes.updatedAt",
+                  ...mapping?.fields
+                } : {
+                  // Strapi v3 format (flat)
+                  id: "id",
+                  content: mapping?.content || "content",
+                  type: () => this.normalizeStrapiType(contentType),
+                  title: "title",
+                  slug: "slug",
+                  publishedAt: "published_at",
+                  updatedAt: "updated_at",
+                  ...mapping?.fields
+                }
+              }
+            },
+            options
+          );
+          results.push(result);
+          hasMore = result.documentsFetched === pageSize;
+          page++;
+        } catch (error) {
+          hasMore = false;
+        }
+      }
+    }
+    return results;
+  }
+  /**
+   * Normalize Strapi collection type to singular form
+   */
+  normalizeStrapiType(collectionType) {
+    if (collectionType.endsWith("s")) {
+      return collectionType.slice(0, -1);
+    }
+    return collectionType;
+  }
+  // ============================================================================
+  // Web Crawling - Zero Setup for Non-Technical Clients
+  // ============================================================================
+  /**
+   * Ingest content by crawling a website's sitemap
+   * Perfect for non-technical clients - just provide the sitemap URL
+   * 
+   * @example
+   * ```typescript
+   * // Simple usage - just provide the sitemap
+   * await plugin.ingestFromSitemap({
+   *   sitemapUrl: 'https://my-site/sitemap.xml',
+   * });
+   * 
+   * // Or auto-discover sitemap from base URL
+   * await plugin.ingestFromSitemap({
+   *   baseUrl: 'https://my-site',
+   * });
+   * 
+   * // With content selectors and type inference
+   * await plugin.ingestFromSitemap({
+   *   sitemapUrl: 'https://my-site/sitemap.xml',
+   *   contentSelector: 'article, .main-content',
+   *   excludePatterns: ['/cart', '/checkout', '/admin'],
+   *   typeFromUrl: {
+   *     '/projects/': 'project',
+   *     '/perspectives/': 'blog',
+   *     '/people/': 'team',
+   *   },
+   * });
+   * ```
+   */
+  async ingestFromSitemap(config, options) {
+    const maxPages = config.maxPages ?? 100;
+    const concurrency = config.concurrency ?? 3;
+    const delayMs = config.delayMs ?? 500;
+    let sitemapUrl = config.sitemapUrl;
+    if (!sitemapUrl && config.baseUrl) {
+      sitemapUrl = `${config.baseUrl.replace(/\/$/, "")}/sitemap.xml`;
+    }
+    if (!sitemapUrl) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        urlsCrawled: 0,
+        urlsSkipped: 0,
+        urlsFailed: 0,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: [{ id: "config", error: "Either sitemapUrl or baseUrl is required" }]
+      };
+    }
+    const urls = await this.parseSitemap(sitemapUrl, config);
+    let filteredUrls = urls;
+    if (config.includePatterns?.length) {
+      filteredUrls = filteredUrls.filter(
+        (url) => config.includePatterns.some((pattern) => url.includes(pattern))
+      );
+    }
+    if (config.excludePatterns?.length) {
+      filteredUrls = filteredUrls.filter(
+        (url) => !config.excludePatterns.some((pattern) => url.includes(pattern))
+      );
+    }
+    const urlsToCrawl = filteredUrls.slice(0, maxPages);
+    const urlsSkipped = filteredUrls.length - urlsToCrawl.length;
+    const result = await this.crawlUrls(urlsToCrawl, {
+      ...config,
+      concurrency,
+      delayMs
+    }, options);
+    return {
+      ...result,
+      urlsSkipped,
+      crawledAt: /* @__PURE__ */ new Date()
+    };
+  }
+  /**
+   * Ingest content from a website that has no sitemap (or sitemap is incomplete).
+   * Discovers internal links from `baseUrl` (BFS) and then crawls the discovered URLs.
+   *
+   * This uses the same extraction pipeline as `ingestFromSitemap()` (via `crawlPage()`).
+   */
+  async ingestFromWebsite(config, options) {
+    const maxPages = config.maxPages ?? 100;
+    const maxDepth = config.maxDepth ?? 3;
+    const concurrency = config.concurrency ?? 3;
+    const delayMs = config.delayMs ?? 500;
+    const timeout = config.timeout ?? 3e4;
+    const stripQueryParams = config.stripQueryParams ?? true;
+    if (!config.baseUrl) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        urlsCrawled: 0,
+        urlsSkipped: 0,
+        urlsFailed: 0,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: [{ id: "config", error: "baseUrl is required" }]
+      };
+    }
+    const dbg = this.createDebugCollector(config.debug);
+    const base = this.normalizeWebsiteUrl(config.baseUrl, stripQueryParams);
+    if (!base) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        urlsCrawled: 0,
+        urlsSkipped: 0,
+        urlsFailed: 0,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: [{ id: "config", error: "Invalid baseUrl" }]
+      };
+    }
+    const discoveredSitemaps = await this.discoverSitemaps(base, timeout, dbg);
+    dbg.log("discovery.sitemaps", { baseUrl: base, sitemaps: discoveredSitemaps });
+    let urlsToCrawl = [];
+    let urlsSkipped = 0;
+    for (const sm of discoveredSitemaps) {
+      const urls = await this.parseSitemap(sm, {
+        sitemapUrl: sm,
+        timeout
+      });
+      if (urls.length > 0) {
+        dbg.log("discovery.sitemapParsed", { sitemapUrl: sm, urlCount: urls.length });
+        let filteredUrls = urls;
+        if (config.includePatterns?.length) {
+          filteredUrls = filteredUrls.filter((u) => config.includePatterns.some((p) => u.includes(p)));
+        }
+        if (config.excludePatterns?.length) {
+          filteredUrls = filteredUrls.filter((u) => !config.excludePatterns.some((p) => u.includes(p)));
+        }
+        urlsToCrawl = filteredUrls.slice(0, maxPages);
+        urlsSkipped = Math.max(0, filteredUrls.length - urlsToCrawl.length);
+        break;
+      }
+    }
+    if (urlsToCrawl.length === 0) {
+      dbg.log("discovery.fallback", { reason: "no_sitemap_urls", method: "link_lookup" });
+      const discovery = await this.discoverInternalUrls({
+        baseUrl: base,
+        maxPages,
+        maxDepth,
+        concurrency,
+        delayMs,
+        timeout,
+        includePatterns: config.includePatterns,
+        excludePatterns: config.excludePatterns,
+        stripQueryParams
+      });
+      urlsToCrawl = discovery.urls;
+      urlsSkipped = discovery.skipped;
+      dbg.log("discovery.linkLookup", { discovered: urlsToCrawl.length, skipped: urlsSkipped });
+    }
+    const result = await this.crawlUrls(urlsToCrawl, {
+      contentSelector: config.contentSelector,
+      titleSelector: config.titleSelector,
+      removeSelectors: config.removeSelectors,
+      concurrency,
+      delayMs,
+      timeout,
+      typeFromUrl: config.typeFromUrl,
+      defaultType: config.defaultType ?? "page",
+      metadata: config.metadata,
+      includePatterns: config.includePatterns,
+      excludePatterns: config.excludePatterns,
+      stripQueryParams,
+      render: config.render,
+      renderOptions: config.renderOptions,
+      debug: config.debug,
+      crawlLedger: config.crawlLedger
+    }, options);
+    return {
+      ...result,
+      urlsSkipped,
+      crawledAt: /* @__PURE__ */ new Date(),
+      metadata: {
+        ...result.metadata || {},
+        discoveryDebug: dbg.summary()
+      }
+    };
+  }
+  /**
+   * Parse sitemap XML and extract URLs
+   */
+  async parseSitemap(sitemapUrl, config) {
+    const urls = [];
+    try {
+      const response = await fetch(sitemapUrl, {
+        headers: { "User-Agent": "SnapAgent-CMS-Crawler/1.0" },
+        signal: AbortSignal.timeout(config.timeout || 3e4)
+      });
+      if (!response.ok) {
+        console.error(`Failed to fetch sitemap: ${response.status}`);
+        return urls;
+      }
+      const xml = await response.text();
+      if (xml.includes("<sitemapindex")) {
+        const sitemapUrls = this.extractUrlsFromXml(xml, "sitemap", "loc");
+        for (const subSitemapUrl of sitemapUrls.slice(0, 10)) {
+          const subUrls = await this.parseSitemap(subSitemapUrl, config);
+          urls.push(...subUrls);
+        }
+      } else {
+        const pageUrls = this.extractUrlsFromXml(xml, "url", "loc");
+        urls.push(...pageUrls);
+      }
+    } catch (error) {
+      console.error(`Error parsing sitemap ${sitemapUrl}:`, error);
+    }
+    return urls;
+  }
+  /**
+   * Extract URLs from sitemap XML
+   */
+  extractUrlsFromXml(xml, parentTag, urlTag) {
+    const urls = [];
+    const regex = new RegExp(`<${parentTag}[^>]*>[\\s\\S]*?<${urlTag}>([^<]+)<\\/${urlTag}>[\\s\\S]*?<\\/${parentTag}>`, "gi");
+    let match;
+    while ((match = regex.exec(xml)) !== null) {
+      const url = match[1].trim();
+      if (url.startsWith("http")) {
+        urls.push(url);
+      }
+    }
+    return urls;
+  }
+  async discoverInternalUrls(input) {
+    const start = this.normalizeWebsiteUrl(input.baseUrl, input.stripQueryParams);
+    if (!start) return { urls: [], skipped: 0 };
+    const startUrl = new URL(start);
+    const visited = /* @__PURE__ */ new Set();
+    const queue = [{ url: startUrl.toString(), depth: 0 }];
+    const discovered = [];
+    let skipped = 0;
+    while (queue.length > 0 && discovered.length < input.maxPages) {
+      const batch = queue.splice(0, input.concurrency);
+      const results = await Promise.allSettled(
+        batch.map(async ({ url, depth }) => {
+          if (visited.has(url)) return { url, depth, links: [] };
+          visited.add(url);
+          if (depth > input.maxDepth) return { url, depth, links: [] };
+          if (input.includePatterns?.length && !input.includePatterns.some((p) => url.includes(p))) {
+            skipped++;
+            return { url, depth, links: [] };
+          }
+          if (input.excludePatterns?.length && input.excludePatterns.some((p) => url.includes(p))) {
+            skipped++;
+            return { url, depth, links: [] };
+          }
+          discovered.push(url);
+          if (discovered.length >= input.maxPages) return { url, depth, links: [] };
+          try {
+            const html = await this.fetchHtml(url, input.timeout);
+            if (!html) return { url, depth, links: [] };
+            const links = this.extractInternalLinks(html, startUrl, input.stripQueryParams);
+            return { url, depth, links };
+          } catch {
+            return { url, depth, links: [] };
+          }
+        })
+      );
+      for (const r of results) {
+        if (r.status !== "fulfilled") continue;
+        const { depth, links } = r.value;
+        const nextDepth = depth + 1;
+        if (nextDepth > input.maxDepth) continue;
+        for (const link of links) {
+          if (discovered.length + queue.length >= input.maxPages * 3) continue;
+          if (visited.has(link)) continue;
+          queue.push({ url: link, depth: nextDepth });
+        }
+      }
+      if (queue.length > 0 && discovered.length < input.maxPages) {
+        await this.delay(input.delayMs);
+      }
+    }
+    if (discovered.length >= input.maxPages) {
+      skipped += queue.length;
+    }
+    return { urls: discovered.slice(0, input.maxPages), skipped };
+  }
+  normalizeWebsiteUrl(inputUrl, stripQueryParams) {
+    try {
+      const u = new URL(inputUrl);
+      u.hash = "";
+      if (stripQueryParams) u.search = "";
+      return u.toString();
+    } catch {
+      return null;
+    }
+  }
+  async fetchHtml(url, timeout) {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "SnapAgent-CMS-Crawler/1.0",
+        "Accept": "text/html,application/xhtml+xml"
+      },
+      signal: AbortSignal.timeout(timeout)
+    });
+    if (!response.ok) return null;
+    const contentType = response.headers.get("content-type") || "";
+    if (!contentType.includes("text/html")) return null;
+    return await response.text();
+  }
+  extractInternalLinks(html, base, stripQueryParams) {
+    const $ = cheerio.load(html);
+    const links = /* @__PURE__ */ new Set();
+    $("a[href]").each((_, el) => {
+      const href = ($(el).attr("href") || "").trim();
+      if (!href) return;
+      if (href.startsWith("mailto:") || href.startsWith("tel:") || href.startsWith("javascript:")) return;
+      try {
+        const u = new URL(href, base);
+        if (u.origin !== base.origin) return;
+        u.hash = "";
+        if (stripQueryParams) u.search = "";
+        links.add(u.toString());
+      } catch {
+      }
+    });
+    return Array.from(links);
+  }
+  /**
+   * Ingest content from a list of URLs
+   * 
+   * @example
+   * ```typescript
+   * await plugin.ingestFromUrls([
+   *   'https://example.com/about',
+   *   'https://example.com/services',
+   *   'https://example.com/contact',
+   * ], {
+   *   contentSelector: '.page-content',
+   *   type: 'page',
+   * });
+   * ```
+   */
+  async ingestFromUrls(urls, config = {}, options) {
+    return this.crawlUrls(urls, {
+      contentSelector: config.contentSelector,
+      titleSelector: config.titleSelector,
+      removeSelectors: config.removeSelectors,
+      concurrency: config.concurrency ?? 3,
+      delayMs: config.delayMs ?? 500,
+      timeout: config.timeout ?? 3e4,
+      typeFromUrl: config.typeFromUrl,
+      defaultType: config.type || "page",
+      metadata: config.metadata,
+      stripQueryParams: config.stripQueryParams ?? false,
+      render: config.render,
+      renderOptions: config.renderOptions,
+      debug: config.debug,
+      crawlLedger: config.crawlLedger
+    }, options);
+  }
+  /**
+   * Ingest a single page from a URL (no sitemap discovery, no link lookup).
+   * Uses the same crawl pipeline (static/render/auto) as other web ingestion methods.
+   */
+  async ingestSinglePageFromUrl(config, options) {
+    if (!config?.url) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 0,
+        urlsCrawled: 0,
+        urlsSkipped: 0,
+        urlsFailed: 0,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: [{ id: "config", error: "url is required" }]
+      };
+    }
+    return this.crawlUrls([config.url], {
+      contentSelector: config.contentSelector,
+      titleSelector: config.titleSelector,
+      removeSelectors: config.removeSelectors,
+      concurrency: 1,
+      delayMs: 0,
+      timeout: config.timeout ?? 3e4,
+      typeFromUrl: config.typeFromUrl,
+      defaultType: config.type || "page",
+      metadata: config.metadata,
+      stripQueryParams: config.stripQueryParams ?? true,
+      render: config.render,
+      renderOptions: config.renderOptions,
+      debug: config.debug,
+      crawlLedger: config.crawlLedger
+    }, options);
+  }
+  /**
+   * Crawl a list of URLs and ingest their content
+   */
+  async crawlUrls(urls, config, options) {
+    const concurrency = config.concurrency ?? 3;
+    const delayMs = config.delayMs ?? 500;
+    const timeout = config.timeout ?? 3e4;
+    const renderMode = config.render ?? false;
+    const renderOptions = config.renderOptions || {};
+    const minContentLength = renderOptions.minContentLength ?? 200;
+    const dbg = this.createDebugCollector(config.debug);
+    const ledgerOpts = this.resolveCrawlLedgerOptions(config);
+    const forceRecrawl = !!(options && options.forceRecrawl);
+    const agentId = options?.agentId ?? "shared";
+    const stripQ = config.stripQueryParams ?? false;
+    const urlByNorm = /* @__PURE__ */ new Map();
+    for (const u of urls) {
+      const norm = this.normalizeLedgerUrl(u, stripQ) || u;
+      if (!urlByNorm.has(norm)) urlByNorm.set(norm, u);
+    }
+    const uniqueUrls = Array.from(urlByNorm.values());
+    const counters = {
+      staticOk: 0,
+      renderOk: 0,
+      renderFallbacks: 0,
+      nonHtml: 0,
+      tooSmall: 0,
+      blockedSuspected: 0,
+      renderErrors: 0,
+      ledgerSkipped: 0
+    };
+    let indexed = 0;
+    let urlsCrawled = 0;
+    let urlsFailed = 0;
+    const errors = [];
+    const documents = [];
+    const pageStatuses = [];
+    const maxStatuses = ledgerOpts?.maxPageStatuses ?? 500;
+    for (let i = 0; i < uniqueUrls.length; i += concurrency) {
+      const batch = uniqueUrls.slice(i, i + concurrency);
+      const results = await Promise.allSettled(
+        batch.map(async (url) => {
+          const urlNormalized = this.normalizeLedgerUrl(url, stripQ) || url;
+          if (ledgerOpts && !forceRecrawl) {
+            const entry = await this.findLedgerEntry(urlNormalized, agentId);
+            if (this.shouldSkipLedger(
+              entry,
+              ledgerOpts.ttlMsIndexed,
+              ledgerOpts.ttlMsFailure,
+              ledgerOpts.ttlMsRenderError,
+              false
+            )) {
+              counters.ledgerSkipped++;
+              this.pushPageStatus(pageStatuses, maxStatuses, {
+                url,
+                urlNormalized,
+                status: "skipped_ledger",
+                skippedReason: `fresh:${entry?.lastStatus}`,
+                contentLength: entry?.contentLength,
+                title: entry?.title,
+                docId: entry?.docId
+              });
+              dbg.log("crawl.ledgerSkip", { url, urlNormalized, lastStatus: entry?.lastStatus });
+              return { kind: "ledger_skip", url };
+            }
+          }
+          try {
+            const { doc, diag, bodyTextLengthHint } = await this.crawlPageSmart(url, config, timeout, {
+              renderMode,
+              renderOptions,
+              minContentLength,
+              dbg
+            });
+            if (diag?.modeUsed === "static_ok") counters.staticOk++;
+            if (diag?.modeUsed === "render_ok") counters.renderOk++;
+            if (diag?.modeUsed === "render_fallback_ok") counters.renderFallbacks++;
+            if (diag?.reason === "non_html") counters.nonHtml++;
+            if (diag?.reason === "too_small") counters.tooSmall++;
+            if (diag?.reason === "blocked_suspected") counters.blockedSuspected++;
+            if (diag?.reason === "render_error") counters.renderErrors++;
+            const crawlSt = this.toLedgerStatus(doc, diag);
+            if (ledgerOpts) {
+              await this.upsertLedgerRecord({
+                url,
+                urlNormalized,
+                agentId,
+                status: crawlSt,
+                doc,
+                diag
+              });
+            }
+            this.pushPageStatus(pageStatuses, maxStatuses, {
+              url,
+              urlNormalized,
+              status: crawlSt,
+              modeUsed: diag?.modeUsed,
+              contentLength: doc?.content?.length,
+              bodyTextLengthHint,
+              title: doc?.metadata?.title,
+              docId: doc?.id,
+              error: diag?.errorMessage
+            });
+            return { kind: "doc", doc, url };
+          } catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            if (ledgerOpts) {
+              await this.upsertLedgerRecord({
+                url,
+                urlNormalized,
+                agentId,
+                status: "error",
+                errorMessage: msg
+              });
+            }
+            this.pushPageStatus(pageStatuses, maxStatuses, {
+              url,
+              urlNormalized,
+              status: "error",
+              error: msg
+            });
+            throw { url, error };
+          }
+        })
+      );
+      for (const result of results) {
+        if (result.status === "fulfilled") {
+          const v = result.value;
+          if (v && typeof v === "object" && "kind" in v && v.kind === "ledger_skip") {
+            continue;
+          }
+          if (v && typeof v === "object" && "kind" in v && v.kind === "doc" && v.doc) {
+            documents.push(v.doc);
+            urlsCrawled++;
+          }
+        } else if (result.status === "rejected") {
+          urlsFailed++;
+          errors.push({
+            id: result.reason.url || "unknown",
+            error: result.reason.error?.message || "Failed to crawl"
+          });
+        }
+      }
+      if (i + concurrency < uniqueUrls.length) {
+        await this.delay(delayMs);
+      }
+    }
+    if (documents.length > 0) {
+      const ingestResult = await this.ingest(documents, options);
+      indexed = ingestResult.indexed;
+      if (ingestResult.errors) {
+        errors.push(...ingestResult.errors);
+      }
+    }
+    return {
+      success: errors.length === 0,
+      indexed,
+      failed: errors.length,
+      urlsCrawled,
+      urlsSkipped: 0,
+      urlsFailed,
+      crawledAt: /* @__PURE__ */ new Date(),
+      errors: errors.length > 0 ? errors : void 0,
+      metadata: {
+        counters,
+        pageStatuses,
+        debug: dbg.summary()
+      }
+    };
+  }
+  /**
+   * Crawl a single page and extract content
+   */
+  async crawlPage(url, config, timeout) {
+    const response = await fetch(url, {
+      headers: {
+        "User-Agent": "SnapAgent-CMS-Crawler/1.0",
+        "Accept": "text/html,application/xhtml+xml"
+      },
+      signal: AbortSignal.timeout(timeout)
+    });
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}`);
+    }
+    const contentType = response.headers.get("content-type") || "";
+    if (!contentType.includes("text/html")) {
+      return null;
+    }
+    const html = await response.text();
+    return this.extractDocumentFromHtml(url, html, config);
+  }
+  /**
+   * Default chain works for many WordPress / Elementor / block themes where `.first()`
+   * would otherwise hit an empty wrapper.
+   */
+  static DEFAULT_CONTENT_SELECTOR = 'article, main, [role="main"], #content, #primary, #main, .content, .post-content, .entry-content, .elementor-location-content, .elementor-widget-theme-post-content, .wp-block-group, .site-content, .ast-single-post, .ast-page';
+  stripNoiseFromDom($, config) {
+    const removeSelectors = config.removeSelectors || [
+      "script",
+      "style",
+      "nav",
+      "header",
+      "footer",
+      ".sidebar",
+      ".navigation",
+      ".menu",
+      ".comments",
+      '[role="navigation"]',
+      '[role="banner"]'
+    ];
+    removeSelectors.forEach((selector) => $(selector).remove());
+  }
+  /** Longest cleaned text among selector matches and full body (after noise strip). */
+  extractBestContentText($, config) {
+    const contentSelector = config.contentSelector || _WebRAGPlugin.DEFAULT_CONTENT_SELECTOR;
+    const selectors = contentSelector.split(",").map((s) => s.trim()).filter(Boolean);
+    let best = "";
+    for (const sel of selectors) {
+      $(sel).each((_, el) => {
+        const t = this.cleanContent($(el).text().trim());
+        if (t.length > best.length) best = t;
+      });
+    }
+    const bodyText = this.cleanContent($("body").text().trim());
+    if (bodyText.length > best.length) best = bodyText;
+    return best;
+  }
+  bodyTextLengthHint(html, config) {
+    const $ = cheerio.load(html);
+    this.stripNoiseFromDom($, config);
+    return this.cleanContent($("body").text().trim()).length;
+  }
+  extractDocumentFromHtml(url, html, config) {
+    const $ = cheerio.load(html);
+    this.stripNoiseFromDom($, config);
+    const titleSelector = config.titleSelector || "h1, title";
+    let title = $(titleSelector).first().text().trim();
+    if (!title) {
+      title = $("title").text().trim();
+    }
+    const content = this.extractBestContentText($, config);
+    const minChars = config.minExtractedContentLength ?? 50;
+    if (!content || content.length < minChars) return null;
+    let type = config.defaultType || "page";
+    if (config.typeFromUrl) {
+      for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
+        if (url.includes(pattern)) {
+          type = typeName;
+          break;
+        }
+      }
+    }
+    const id = this.urlToId(url);
+    return {
+      id,
+      content,
+      metadata: {
+        type,
+        title,
+        url,
+        ...config.metadata
+      }
+    };
+  }
+  looksLikeDynamicShell(html) {
+    const lower = html.toLowerCase();
+    const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
+    const body = bodyMatch?.[1] ?? html;
+    const textOnly = body.replace(/<script[\s\S]*?<\/script>/gi, "").replace(/<style[\s\S]*?<\/style>/gi, "").replace(/<noscript[\s\S]*?<\/noscript>/gi, "").replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+    const scriptCount = (body.match(/<script\b/gi) ?? []).length;
+    const hasEmptyAppMountNode = /<(div|main)[^>]+id=["'](__next|root|app)["'][^>]*>\s*<\/\1>/i.test(body);
+    const hasHydrationData = lower.includes("__next_data__") || lower.includes("__next_f") || lower.includes("window.__initial_state__") || lower.includes("window.__apollo_state__") || lower.includes("data-reactroot");
+    const asksForJavascript = lower.includes("please enable javascript") || lower.includes("enable javascript to run this app") || lower.includes("you need to enable javascript");
+    const hasLoadingHints = /\b(loading|please wait|spinner|initializing|fetching)\b/i.test(lower);
+    const textLength = textOnly.length;
+    const htmlLength = lower.length;
+    const contentDensity = textLength / Math.max(htmlLength, 1);
+    const isMostlyScripts = scriptCount >= 5 && textLength < 500;
+    const isSmallShellLike = htmlLength < 5e4 && textLength < 500 && contentDensity < 0.02;
+    return asksForJavascript || hasEmptyAppMountNode || hasHydrationData || isMostlyScripts || isSmallShellLike || hasLoadingHints && textLength < 1e3 && contentDensity < 0.05;
+  }
+  diagFromRenderedAttempt(doc, bodyTextLengthHint, renderFailure, blockedSuspected, modeOk, modeFailed) {
+    if (blockedSuspected) {
+      return {
+        doc: null,
+        diag: { modeUsed: modeFailed, reason: "blocked_suspected" }
+      };
+    }
+    if (renderFailure) {
+      return {
+        doc: null,
+        diag: { modeUsed: modeFailed, reason: "render_error", errorMessage: renderFailure }
+      };
+    }
+    return {
+      doc,
+      diag: doc ? { modeUsed: modeOk } : { modeUsed: modeFailed, reason: "too_small" },
+      bodyTextLengthHint: doc ? void 0 : bodyTextLengthHint
+    };
+  }
+  async crawlPageSmart(url, config, timeout, ctx) {
+    if (ctx.renderMode === true) {
+      const { doc, bodyTextLengthHint, renderFailure, blockedSuspected } = await this.crawlPageRendered(
+        url,
+        config,
+        timeout,
+        ctx.renderOptions,
+        ctx.dbg
+      );
+      return this.diagFromRenderedAttempt(
+        doc,
+        bodyTextLengthHint,
+        renderFailure,
+        blockedSuspected,
+        "render_ok",
+        "render_failed"
+      );
+    }
+    try {
+      const response = await fetch(url, {
+        headers: {
+          "User-Agent": "SnapAgent-CMS-Crawler/1.0",
+          "Accept": "text/html,application/xhtml+xml"
+        },
+        signal: AbortSignal.timeout(timeout)
+      });
+      if (!response.ok) {
+        const status = response.status;
+        if (status === 403 || status === 429 || status === 503) {
+          ctx.dbg.log("crawl.blocked", { url, status });
+          return { doc: null, diag: { modeUsed: "static_failed", reason: "blocked_suspected" } };
+        }
+        throw new Error(`HTTP ${status}`);
+      }
+      const contentType = response.headers.get("content-type") || "";
+      if (!contentType.includes("text/html")) {
+        return { doc: null, diag: { modeUsed: "static_failed", reason: "non_html" } };
+      }
+      const html = await response.text();
+      const doc = this.extractDocumentFromHtml(url, html, config);
+      const staticHint = !doc ? this.bodyTextLengthHint(html, config) : void 0;
+      if (doc && doc.content.length >= ctx.minContentLength) {
+        return { doc, diag: { modeUsed: "static_ok" } };
+      }
+      if (ctx.renderMode === "auto") {
+        const shouldRender = this.looksLikeDynamicShell(html) || !doc || doc.content.length < ctx.minContentLength;
+        if (shouldRender) {
+          ctx.dbg.log("crawl.renderFallback", {
+            url,
+            reason: !doc ? "no_doc" : "too_small",
+            staticLength: doc?.content?.length ?? 0
+          });
+          const {
+            doc: rendered,
+            bodyTextLengthHint: rHint,
+            renderFailure,
+            blockedSuspected
+          } = await this.crawlPageRendered(
+            url,
+            config,
+            timeout,
+            ctx.renderOptions,
+            ctx.dbg
+          );
+          const mergedHint = rHint ?? staticHint;
+          const fb = this.diagFromRenderedAttempt(
+            rendered,
+            mergedHint,
+            renderFailure,
+            blockedSuspected,
+            "render_fallback_ok",
+            "render_fallback_failed"
+          );
+          if (!rendered && (renderFailure || blockedSuspected)) {
+            fb.bodyTextLengthHint = staticHint ?? rHint;
+          }
+          return fb;
+        }
+      }
+      return {
+        doc: null,
+        diag: { modeUsed: "static_failed", reason: "too_small" },
+        bodyTextLengthHint: staticHint
+      };
+    } catch (e) {
+      throw e;
+    }
+  }
+  async crawlPageRendered(url, config, timeout, renderOptions, dbg) {
+    let playwright;
+    try {
+      playwright = await Function('return import("playwright")')();
+    } catch (e) {
+      dbg.log("render.missingDependency", { url, error: "playwright_not_installed" });
+      throw new Error("playwright is not installed. Add it to dependencies to use crawlPageRendered().");
+    }
+    const waitUntil = renderOptions.waitUntil || "domcontentloaded";
+    const waitForSelector = renderOptions.waitForSelector;
+    const scrollCfg = renderOptions.scroll || {};
+    const doScroll = scrollCfg.enabled ?? false;
+    const maxScrolls = scrollCfg.maxScrolls ?? 10;
+    const scrollDelayMs = scrollCfg.scrollDelayMs ?? 750;
+    const stableIterations = scrollCfg.stableIterations ?? 2;
+    const postRenderDelayMs = renderOptions.postRenderDelayMs ?? 0;
+    const browser = await playwright.chromium.launch({ headless: true });
+    try {
+      const page = await browser.newPage();
+      await page.goto(url, { waitUntil, timeout });
+      if (waitForSelector) {
+        await page.waitForSelector(waitForSelector, { timeout });
+      }
+      if (postRenderDelayMs > 0) {
+        await page.waitForTimeout(postRenderDelayMs);
+      }
+      if (doScroll) {
+        let stable = 0;
+        let lastLen = 0;
+        for (let i = 0; i < maxScrolls; i++) {
+          const len = await page.evaluate("(document.body?.innerText || '').length");
+          if (len <= lastLen + 20) stable++;
+          else stable = 0;
+          lastLen = len;
+          if (stable >= stableIterations) break;
+          await page.evaluate("window.scrollTo(0, document.body.scrollHeight)");
+          await page.waitForTimeout(scrollDelayMs);
+        }
+      }
+      const html = await page.content();
+      const bodyTextLengthHint = this.bodyTextLengthHint(html, config);
+      const doc = this.extractDocumentFromHtml(url, html, config);
+      if (config.debug?.saveDir && config.debug?.enabled) {
+        try {
+          const saveDir = config.debug.saveDir;
+          const safeId = this.urlToId(url) || "page";
+          const outDir = path.join(saveDir, safeId);
+          fs.mkdirSync(outDir, { recursive: true });
+          fs.writeFileSync(path.join(outDir, "rendered.html"), html, "utf8");
+          fs.writeFileSync(path.join(outDir, "extracted.txt"), doc?.content || "", "utf8");
+          fs.writeFileSync(path.join(outDir, "meta.json"), JSON.stringify(doc?.metadata || {}, null, 2), "utf8");
+        } catch (e) {
+          dbg.log("debug.saveFailed", { url, error: e instanceof Error ? e.message : "save_failed" });
+        }
+      }
+      return { doc, bodyTextLengthHint };
+    } catch (e) {
+      const msg = String(e?.message || e || "render_failed");
+      const lower = msg.toLowerCase();
+      if (lower.includes("captcha") || lower.includes("access denied")) {
+        dbg.log("render.blocked", { url, error: msg });
+        return { doc: null, bodyTextLengthHint: 0, blockedSuspected: true };
+      }
+      dbg.log("render.error", { url, error: msg });
+      return { doc: null, bodyTextLengthHint: 0, renderFailure: msg };
+    } finally {
+      await browser.close();
+    }
+  }
+  async discoverSitemaps(baseUrl, timeout, dbg) {
+    const base = new URL(baseUrl);
+    const robotsUrl = new URL("/robots.txt", base).toString();
+    const found = /* @__PURE__ */ new Set();
+    try {
+      const res = await fetch(robotsUrl, {
+        headers: { "User-Agent": "SnapAgent-CMS-Crawler/1.0" },
+        signal: AbortSignal.timeout(timeout)
+      });
+      if (res.ok) {
+        const txt = await res.text();
+        const rx = /^sitemap:\s*(\S+)/gim;
+        let m;
+        while ((m = rx.exec(txt)) !== null) {
+          const sm = m[1].trim();
+          if (sm.startsWith("http")) found.add(sm);
+        }
+        dbg.log("discovery.robots", { robotsUrl, ok: true, sitemapCount: found.size });
+      } else {
+        dbg.log("discovery.robots", { robotsUrl, ok: false, status: res.status });
+      }
+    } catch (e) {
+      dbg.log("discovery.robots", { robotsUrl, ok: false, error: e instanceof Error ? e.message : "failed" });
+    }
+    if (found.size === 0) {
+      const candidates = [
+        "/sitemap.xml",
+        "/sitemap_index.xml",
+        "/sitemap-index.xml",
+        "/wp-sitemap.xml"
+      ].map((p) => new URL(p, base).toString());
+      candidates.forEach((c) => found.add(c));
+      dbg.log("discovery.sitemapCandidates", { count: candidates.length });
+    }
+    return Array.from(found);
+  }
+  createDebugCollector(debug) {
+    const enabled = !!debug?.enabled;
+    const level = debug?.level || "summary";
+    const maxPerUrlLogs = debug?.maxPerUrlLogs ?? 200;
+    const entries = [];
+    return {
+      log: (event, data) => {
+        if (!enabled) return;
+        if (level === "summary" && !event.startsWith("discovery.") && !event.startsWith("crawl.")) return;
+        if (entries.length >= maxPerUrlLogs) return;
+        entries.push({ ts: (/* @__PURE__ */ new Date()).toISOString(), event, data });
+      },
+      summary: () => enabled ? { enabled, level, entries } : void 0
+    };
+  }
+  /**
+   * Clean extracted text content
+   */
+  cleanContent(text) {
+    return text.replace(/\s+/g, " ").replace(/\n\s*\n/g, "\n\n").replace(/\t/g, " ").trim();
+  }
+  /**
+   * Convert URL to a stable document ID
+   */
+  urlToId(url) {
+    return url.replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").replace(/-+/g, "-").replace(/^-|-$/g, "").substring(0, 100);
+  }
+  /**
+   * Delay helper
+   */
+  delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+  // ============================================================================
+  // RSS/Atom Feed Ingestion
+  // ============================================================================
+  /**
+   * Ingest content from an RSS or Atom feed
+   * 
+   * @example
+   * ```typescript
+   * // Simple RSS ingestion
+   * await plugin.ingestFromRSS({
+   *   feedUrl: 'https://myblog.com/feed/',
+   * });
+   * 
+   * // Fetch full page content for each item
+   * await plugin.ingestFromRSS({
+   *   feedUrl: 'https://myblog.com/feed/',
+   *   fetchFullContent: true,
+   *   contentSelector: 'article',
+   * });
+   * ```
+   */
+  async ingestFromRSS(config, options) {
+    try {
+      const response = await fetch(config.feedUrl, {
+        headers: { "User-Agent": "SnapAgent-CMS-Crawler/1.0" },
+        signal: AbortSignal.timeout(3e4)
+      });
+      if (!response.ok) {
+        return {
+          success: false,
+          indexed: 0,
+          failed: 1,
+          urlsCrawled: 0,
+          urlsSkipped: 0,
+          urlsFailed: 1,
+          crawledAt: /* @__PURE__ */ new Date(),
+          errors: [{ id: config.feedUrl, error: `HTTP ${response.status}` }]
+        };
+      }
+      const xml = await response.text();
+      const items = this.parseRSSFeed(xml);
+      if (items.length === 0) {
+        return {
+          success: true,
+          indexed: 0,
+          failed: 0,
+          urlsCrawled: 0,
+          urlsSkipped: 0,
+          urlsFailed: 0,
+          crawledAt: /* @__PURE__ */ new Date()
+        };
+      }
+      const documents = [];
+      const type = config.type || "post";
+      let urlsCrawled = 0;
+      let urlsFailed = 0;
+      const errors = [];
+      for (const item of items) {
+        try {
+          let content = item.content || item.description || "";
+          if (config.fetchFullContent && item.link) {
+            try {
+              const doc = await this.crawlPage(item.link, {
+                contentSelector: config.contentSelector,
+                defaultType: type
+              }, 3e4);
+              if (doc) {
+                content = doc.content;
+              }
+              urlsCrawled++;
+            } catch (error) {
+              urlsFailed++;
+            }
+          }
+          content = this.stripHtml(content);
+          if (content.length < 50) continue;
+          documents.push({
+            id: this.urlToId(item.link || item.guid || `rss-${documents.length}`),
+            content,
+            metadata: {
+              type,
+              title: item.title,
+              url: item.link,
+              publishedAt: item.pubDate,
+              author: item.author,
+              categories: item.categories,
+              ...config.metadata
+            }
+          });
+        } catch (error) {
+          errors.push({
+            id: item.link || "unknown",
+            error: error instanceof Error ? error.message : "Unknown error"
+          });
+        }
+      }
+      let indexed = 0;
+      if (documents.length > 0) {
+        const ingestResult = await this.ingest(documents, options);
+        indexed = ingestResult.indexed;
+      }
+      return {
+        success: errors.length === 0,
+        indexed,
+        failed: errors.length,
+        urlsCrawled,
+        urlsSkipped: 0,
+        urlsFailed,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: errors.length > 0 ? errors : void 0
+      };
+    } catch (error) {
+      return {
+        success: false,
+        indexed: 0,
+        failed: 1,
+        urlsCrawled: 0,
+        urlsSkipped: 0,
+        urlsFailed: 0,
+        crawledAt: /* @__PURE__ */ new Date(),
+        errors: [{
+          id: config.feedUrl,
+          error: error instanceof Error ? error.message : "Unknown error"
+        }]
+      };
+    }
+  }
+  /**
+   * Parse RSS/Atom feed XML
+   */
+  parseRSSFeed(xml) {
+    const items = [];
+    const isAtom = xml.includes("<feed") && xml.includes('xmlns="http://www.w3.org/2005/Atom"');
+    if (isAtom) {
+      const entryRegex = /<entry>([\s\S]*?)<\/entry>/gi;
+      let match;
+      while ((match = entryRegex.exec(xml)) !== null) {
+        const entry = match[1];
+        items.push({
+          title: this.extractXmlValue(entry, "title"),
+          link: this.extractAtomLink(entry),
+          guid: this.extractXmlValue(entry, "id"),
+          content: this.extractXmlValue(entry, "content") || this.extractXmlValue(entry, "summary"),
+          pubDate: this.extractXmlValue(entry, "published") || this.extractXmlValue(entry, "updated"),
+          author: this.extractXmlValue(entry, "name"),
+          // Inside <author>
+          categories: this.extractXmlValues(entry, "category", "term")
+        });
+      }
+    } else {
+      const itemRegex = /<item>([\s\S]*?)<\/item>/gi;
+      let match;
+      while ((match = itemRegex.exec(xml)) !== null) {
+        const item = match[1];
+        items.push({
+          title: this.extractXmlValue(item, "title"),
+          link: this.extractXmlValue(item, "link"),
+          guid: this.extractXmlValue(item, "guid"),
+          description: this.extractXmlValue(item, "description"),
+          content: this.extractXmlValue(item, "content:encoded") || this.extractXmlValue(item, "content"),
+          pubDate: this.extractXmlValue(item, "pubDate"),
+          author: this.extractXmlValue(item, "author") || this.extractXmlValue(item, "dc:creator"),
+          categories: this.extractXmlValues(item, "category")
+        });
+      }
+    }
+    return items;
+  }
+  /**
+   * Extract a single value from XML
+   */
+  extractXmlValue(xml, tag) {
+    const cdataRegex = new RegExp(`<${tag}[^>]*><!\\[CDATA\\[([\\s\\S]*?)\\]\\]><\\/${tag}>`, "i");
+    const cdataMatch = xml.match(cdataRegex);
+    if (cdataMatch) {
+      return cdataMatch[1].trim();
+    }
+    const regex = new RegExp(`<${tag}[^>]*>([^<]*)<\\/${tag}>`, "i");
+    const match = xml.match(regex);
+    return match ? match[1].trim() : void 0;
+  }
+  /**
+   * Extract multiple values from XML
+   */
+  extractXmlValues(xml, tag, attr) {
+    const values = [];
+    if (attr) {
+      const regex = new RegExp(`<${tag}[^>]*${attr}="([^"]*)"[^>]*/?>`, "gi");
+      let match;
+      while ((match = regex.exec(xml)) !== null) {
+        values.push(match[1]);
+      }
+    } else {
+      const regex = new RegExp(`<${tag}[^>]*>([^<]*)<\\/${tag}>`, "gi");
+      let match;
+      while ((match = regex.exec(xml)) !== null) {
+        values.push(match[1].trim());
+      }
+    }
+    return values;
+  }
+  /**
+   * Extract link from Atom entry
+   */
+  extractAtomLink(entry) {
+    const alternateMatch = entry.match(/<link[^>]*rel="alternate"[^>]*href="([^"]+)"/i);
+    if (alternateMatch) return alternateMatch[1];
+    const linkMatch = entry.match(/<link[^>]*href="([^"]+)"/i);
+    return linkMatch ? linkMatch[1] : void 0;
+  }
+  /**
+   * Strip HTML tags from content
+   */
+  stripHtml(html) {
+    return html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "").replace(/<[^>]+>/g, " ").replace(/&nbsp;/g, " ").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/\s+/g, " ").trim();
+  }
+  // ============================================================================
+  // Utility Methods
+  // ============================================================================
+  /**
+   * Get cache statistics
+   */
+  getCacheStats() {
+    const total = this.cacheStats.hits + this.cacheStats.misses;
+    const hitRate = total > 0 ? (this.cacheStats.hits / total).toFixed(3) : "0.000";
+    return { ...this.cacheStats, hitRate };
+  }
+  /**
+   * Clear embedding cache
+   */
+  clearCache() {
+    this.embeddingCache.clear();
+    this.cacheStats = { hits: 0, misses: 0 };
+  }
+  /**
+   * Get plugin configuration (for persistence)
+   */
+  getConfig() {
+    return {
+      name: this.name,
+      mongoUri: "${MONGODB_URI}",
+      // Reference env var
+      dbName: this.config.dbName,
+      collection: this.config.collection,
+      openaiApiKey: "${OPENAI_API_KEY}",
+      // Reference env var
+      embeddingModel: this.config.embeddingModel,
+      tenantId: this.config.tenantId,
+      vectorIndexName: this.config.vectorIndexName,
+      numCandidates: this.config.numCandidates,
+      limit: this.config.limit,
+      minScore: this.config.minScore,
+      filterableFields: this.config.filterableFields,
+      typeBoosts: this.config.typeBoosts,
+      recencyBoost: this.config.recencyBoost,
+      priority: this.priority
+    };
+  }
+};
+export {
+  WebRAGPlugin
+};