@snap-agent/rag-web 0.1.0 → 0.1.2

package/dist/index.d.mts

@@ -26,6 +26,8 @@ interface WebDocument {
 interface StoredWebDocument extends WebDocument {
     tenantId: string;
     agentId?: string;
+    documentId?: string;
+    chunkIndex?: number;
     embedding: number[];
     createdAt: Date;
     updatedAt?: Date;
@@ -52,6 +54,8 @@ interface WebRAGConfig {
         decayDays: number;
         maxBoost?: number;
     };
+    maxChunkSize?: number;
+    chunkOverlap?: number;
     cache?: {
         embeddings?: {
             enabled: boolean;
@@ -456,16 +460,23 @@ declare class WebRAGPlugin implements RAGPlugin {
     private vectorSearch;
     private generateEmbedding;
     private generateEmbeddingsBatch;
+    /**
+     * Split content into chunks by paragraph boundaries, respecting maxChunkSize.
+     * Returns the original content as a single chunk when chunking is disabled
+     * (maxChunkSize === 0) or the content fits within maxChunkSize.
+     */
+    private chunkContent;
     /**
      * Ingest documents into the CMS RAG system
      */
     ingest(documents: RAGDocument[], options?: IngestOptions): Promise<IngestResult>;
     /**
-     * Update a single document
+     * Update a single document.
+     * When content changes the document is re-chunked (old chunks removed, new ones inserted).
      */
     update(id: string, document: Partial<RAGDocument>, options?: IngestOptions): Promise<void>;
     /**
-     * Delete document(s) by ID
+     * Delete document(s) by ID — also removes any chunks belonging to the document.
      */
     delete(ids: string | string[], options?: IngestOptions): Promise<number>;
     /**

package/dist/index.d.ts

@@ -26,6 +26,8 @@ interface WebDocument {
 interface StoredWebDocument extends WebDocument {
     tenantId: string;
     agentId?: string;
+    documentId?: string;
+    chunkIndex?: number;
     embedding: number[];
     createdAt: Date;
     updatedAt?: Date;
@@ -52,6 +54,8 @@ interface WebRAGConfig {
         decayDays: number;
         maxBoost?: number;
     };
+    maxChunkSize?: number;
+    chunkOverlap?: number;
     cache?: {
         embeddings?: {
             enabled: boolean;
@@ -456,16 +460,23 @@ declare class WebRAGPlugin implements RAGPlugin {
     private vectorSearch;
     private generateEmbedding;
     private generateEmbeddingsBatch;
+    /**
+     * Split content into chunks by paragraph boundaries, respecting maxChunkSize.
+     * Returns the original content as a single chunk when chunking is disabled
+     * (maxChunkSize === 0) or the content fits within maxChunkSize.
+     */
+    private chunkContent;
     /**
      * Ingest documents into the CMS RAG system
      */
     ingest(documents: RAGDocument[], options?: IngestOptions): Promise<IngestResult>;
     /**
-     * Update a single document
+     * Update a single document.
+     * When content changes the document is re-chunked (old chunks removed, new ones inserted).
      */
     update(id: string, document: Partial<RAGDocument>, options?: IngestOptions): Promise<void>;
     /**
-     * Delete document(s) by ID
+     * Delete document(s) by ID — also removes any chunks belonging to the document.
      */
     delete(ids: string | string[], options?: IngestOptions): Promise<number>;
     /**

package/dist/index.js

@@ -60,6 +60,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
       limit: 10,
       minScore: 0.7,
       filterableFields: ["type"],
+      maxChunkSize: 1500,
+      chunkOverlap: 200,
       ...config
     };
     this.priority = config.priority ?? 100;
@@ -361,6 +363,52 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return embeddings;
   }
   // ============================================================================
+  // Chunking
+  // ============================================================================
+  /**
+   * Split content into chunks by paragraph boundaries, respecting maxChunkSize.
+   * Returns the original content as a single chunk when chunking is disabled
+   * (maxChunkSize === 0) or the content fits within maxChunkSize.
+   */
+  chunkContent(content) {
+    const maxSize = this.config.maxChunkSize ?? 1500;
+    if (maxSize === 0 || content.length <= maxSize) {
+      return [content];
+    }
+    const overlap = this.config.chunkOverlap ?? 200;
+    const paragraphs = content.split(/\n\n+/);
+    const chunks = [];
+    let current = "";
+    for (const para of paragraphs) {
+      const trimmed = para.trim();
+      if (!trimmed) continue;
+      if (trimmed.length > maxSize) {
+        if (current.trim()) {
+          chunks.push(current.trim());
+          current = "";
+        }
+        for (let i = 0; i < trimmed.length; i += maxSize - overlap) {
+          const slice = trimmed.slice(i, i + maxSize);
+          if (slice.trim()) chunks.push(slice.trim());
+        }
+        continue;
+      }
+      const candidate = current ? current + "\n\n" + trimmed : trimmed;
+      if (candidate.length > maxSize) {
+        if (current.trim()) {
+          chunks.push(current.trim());
+        }
+        current = trimmed;
+      } else {
+        current = candidate;
+      }
+    }
+    if (current.trim()) {
+      chunks.push(current.trim());
+    }
+    return chunks.length > 0 ? chunks : [content];
+  }
+  // ============================================================================
   // Document Ingestion
   // ============================================================================
   /**
@@ -370,47 +418,51 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const collection = await this.getCollection();
     let indexed = 0;
     const errors = [];
-    const batchSize = options?.batchSize ?? 10;
-    for (let i = 0; i < documents.length; i += batchSize) {
-      const batch = documents.slice(i, i + batchSize);
-      const embeddings = await this.generateEmbeddingsBatch(
-        batch.map((doc) => doc.content)
-      );
-      const docsToStore = batch.map((doc, idx) => ({
-        id: doc.id,
-        content: doc.content,
-        metadata: {
-          type: doc.metadata?.type || "content",
-          ...doc.metadata
-        },
-        tenantId: this.config.tenantId,
-        // Use 'shared' marker for tenant-wide content, specific agentId for agent-only
-        agentId: options?.agentId || "shared",
-        embedding: embeddings[idx]
-      }));
-      for (const doc of docsToStore) {
-        try {
-          const filter = {
+    const agentId = options?.agentId || "shared";
+    for (const doc of documents) {
+      try {
+        const chunks = this.chunkContent(doc.content);
+        const isChunked = chunks.length > 1;
+        if (isChunked) {
+          await collection.deleteMany({
+            tenantId: this.config.tenantId,
+            documentId: doc.id,
+            agentId
+          });
+        }
+        for (let i = 0; i < chunks.length; i++) {
+          const chunkId = isChunked ? `chunk-${doc.id}-${i}` : doc.id;
+          const embedding = await this.generateEmbedding(chunks[i]);
+          const storedDoc = {
+            id: chunkId,
+            content: chunks[i],
+            metadata: {
+              type: doc.metadata?.type || "content",
+              ...doc.metadata
+            },
             tenantId: this.config.tenantId,
-            id: doc.id,
-            // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-            agentId: options?.agentId || "shared"
+            agentId,
+            embedding
           };
+          if (isChunked) {
+            storedDoc.documentId = doc.id;
+            storedDoc.chunkIndex = i;
+          }
           await collection.updateOne(
-            filter,
+            { tenantId: this.config.tenantId, id: chunkId, agentId },
             {
-              $set: { ...doc, updatedAt: /* @__PURE__ */ new Date() },
+              $set: { ...storedDoc, updatedAt: /* @__PURE__ */ new Date() },
               $setOnInsert: { createdAt: /* @__PURE__ */ new Date() }
             },
             { upsert: true }
           );
-          indexed++;
-        } catch (error) {
-          errors.push({
-            id: doc.id,
-            error: error instanceof Error ? error.message : "Unknown error"
-          });
         }
+        indexed++;
+      } catch (error) {
+        errors.push({
+          id: doc.id,
+          error: error instanceof Error ? error.message : "Unknown error"
+        });
       }
     }
     return {
@@ -425,40 +477,51 @@ var WebRAGPlugin = class _WebRAGPlugin {
     };
   }
   /**
-   * Update a single document
+   * Update a single document.
+   * When content changes the document is re-chunked (old chunks removed, new ones inserted).
    */
   async update(id, document, options) {
-    const collection = await this.getCollection();
-    const update = { updatedAt: /* @__PURE__ */ new Date() };
+    const agentId = options?.agentId || "shared";
     if (document.content) {
-      const embedding = await this.generateEmbedding(document.content);
-      update.content = document.content;
-      update.embedding = embedding;
+      const fullDoc = {
+        id,
+        content: document.content,
+        metadata: document.metadata ?? { type: "content" }
+      };
+      await this.delete(id, options);
+      await this.ingest([fullDoc], options);
+      return;
     }
+    const collection = await this.getCollection();
+    const metaUpdate = { updatedAt: /* @__PURE__ */ new Date() };
     if (document.metadata) {
       for (const [key, value] of Object.entries(document.metadata)) {
-        update[`metadata.${key}`] = value;
+        metaUpdate[`metadata.${key}`] = value;
       }
     }
-    const filter = {
-      tenantId: this.config.tenantId,
-      id,
-      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-      agentId: options?.agentId || "shared"
-    };
-    await collection.updateOne(filter, { $set: update });
+    await collection.updateMany(
+      {
+        tenantId: this.config.tenantId,
+        agentId,
+        $or: [{ id }, { documentId: id }]
+      },
+      { $set: metaUpdate }
+    );
   }
   /**
-   * Delete document(s) by ID
+   * Delete document(s) by ID — also removes any chunks belonging to the document.
    */
   async delete(ids, options) {
     const collection = await this.getCollection();
     const idArray = Array.isArray(ids) ? ids : [ids];
     const filter = {
       tenantId: this.config.tenantId,
-      id: { $in: idArray },
-      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-      agentId: options?.agentId || "shared"
+      agentId: options?.agentId || "shared",
+      // Match the document itself (id) OR any chunks that belong to it (documentId)
+      $or: [
+        { id: { $in: idArray } },
+        { documentId: { $in: idArray } }
+      ]
     };
     const result = await collection.deleteMany(filter);
     return result.deletedCount;
@@ -1596,6 +1659,15 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    let imageUrl;
+    if (image) {
+      try {
+        imageUrl = new URL(image, url).href;
+      } catch {
+        imageUrl = image;
+      }
+    }
     let type = config.defaultType || "page";
     if (config.typeFromUrl) {
       for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
@@ -1613,6 +1685,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         type,
         title,
         url,
+        ...imageUrl ? { imageUrl } : {},
         ...config.metadata
       }
     };

package/dist/index.mjs

@@ -24,6 +24,8 @@ var WebRAGPlugin = class _WebRAGPlugin {
       limit: 10,
       minScore: 0.7,
       filterableFields: ["type"],
+      maxChunkSize: 1500,
+      chunkOverlap: 200,
       ...config
     };
     this.priority = config.priority ?? 100;
@@ -325,6 +327,52 @@ var WebRAGPlugin = class _WebRAGPlugin {
     return embeddings;
   }
   // ============================================================================
+  // Chunking
+  // ============================================================================
+  /**
+   * Split content into chunks by paragraph boundaries, respecting maxChunkSize.
+   * Returns the original content as a single chunk when chunking is disabled
+   * (maxChunkSize === 0) or the content fits within maxChunkSize.
+   */
+  chunkContent(content) {
+    const maxSize = this.config.maxChunkSize ?? 1500;
+    if (maxSize === 0 || content.length <= maxSize) {
+      return [content];
+    }
+    const overlap = this.config.chunkOverlap ?? 200;
+    const paragraphs = content.split(/\n\n+/);
+    const chunks = [];
+    let current = "";
+    for (const para of paragraphs) {
+      const trimmed = para.trim();
+      if (!trimmed) continue;
+      if (trimmed.length > maxSize) {
+        if (current.trim()) {
+          chunks.push(current.trim());
+          current = "";
+        }
+        for (let i = 0; i < trimmed.length; i += maxSize - overlap) {
+          const slice = trimmed.slice(i, i + maxSize);
+          if (slice.trim()) chunks.push(slice.trim());
+        }
+        continue;
+      }
+      const candidate = current ? current + "\n\n" + trimmed : trimmed;
+      if (candidate.length > maxSize) {
+        if (current.trim()) {
+          chunks.push(current.trim());
+        }
+        current = trimmed;
+      } else {
+        current = candidate;
+      }
+    }
+    if (current.trim()) {
+      chunks.push(current.trim());
+    }
+    return chunks.length > 0 ? chunks : [content];
+  }
+  // ============================================================================
   // Document Ingestion
   // ============================================================================
   /**
@@ -334,47 +382,51 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const collection = await this.getCollection();
     let indexed = 0;
     const errors = [];
-    const batchSize = options?.batchSize ?? 10;
-    for (let i = 0; i < documents.length; i += batchSize) {
-      const batch = documents.slice(i, i + batchSize);
-      const embeddings = await this.generateEmbeddingsBatch(
-        batch.map((doc) => doc.content)
-      );
-      const docsToStore = batch.map((doc, idx) => ({
-        id: doc.id,
-        content: doc.content,
-        metadata: {
-          type: doc.metadata?.type || "content",
-          ...doc.metadata
-        },
-        tenantId: this.config.tenantId,
-        // Use 'shared' marker for tenant-wide content, specific agentId for agent-only
-        agentId: options?.agentId || "shared",
-        embedding: embeddings[idx]
-      }));
-      for (const doc of docsToStore) {
-        try {
-          const filter = {
+    const agentId = options?.agentId || "shared";
+    for (const doc of documents) {
+      try {
+        const chunks = this.chunkContent(doc.content);
+        const isChunked = chunks.length > 1;
+        if (isChunked) {
+          await collection.deleteMany({
+            tenantId: this.config.tenantId,
+            documentId: doc.id,
+            agentId
+          });
+        }
+        for (let i = 0; i < chunks.length; i++) {
+          const chunkId = isChunked ? `chunk-${doc.id}-${i}` : doc.id;
+          const embedding = await this.generateEmbedding(chunks[i]);
+          const storedDoc = {
+            id: chunkId,
+            content: chunks[i],
+            metadata: {
+              type: doc.metadata?.type || "content",
+              ...doc.metadata
+            },
             tenantId: this.config.tenantId,
-            id: doc.id,
-            // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-            agentId: options?.agentId || "shared"
+            agentId,
+            embedding
           };
+          if (isChunked) {
+            storedDoc.documentId = doc.id;
+            storedDoc.chunkIndex = i;
+          }
           await collection.updateOne(
-            filter,
+            { tenantId: this.config.tenantId, id: chunkId, agentId },
             {
-              $set: { ...doc, updatedAt: /* @__PURE__ */ new Date() },
+              $set: { ...storedDoc, updatedAt: /* @__PURE__ */ new Date() },
               $setOnInsert: { createdAt: /* @__PURE__ */ new Date() }
             },
             { upsert: true }
           );
-          indexed++;
-        } catch (error) {
-          errors.push({
-            id: doc.id,
-            error: error instanceof Error ? error.message : "Unknown error"
-          });
         }
+        indexed++;
+      } catch (error) {
+        errors.push({
+          id: doc.id,
+          error: error instanceof Error ? error.message : "Unknown error"
+        });
       }
     }
     return {
@@ -389,40 +441,51 @@ var WebRAGPlugin = class _WebRAGPlugin {
     };
   }
   /**
-   * Update a single document
+   * Update a single document.
+   * When content changes the document is re-chunked (old chunks removed, new ones inserted).
    */
   async update(id, document, options) {
-    const collection = await this.getCollection();
-    const update = { updatedAt: /* @__PURE__ */ new Date() };
+    const agentId = options?.agentId || "shared";
     if (document.content) {
-      const embedding = await this.generateEmbedding(document.content);
-      update.content = document.content;
-      update.embedding = embedding;
+      const fullDoc = {
+        id,
+        content: document.content,
+        metadata: document.metadata ?? { type: "content" }
+      };
+      await this.delete(id, options);
+      await this.ingest([fullDoc], options);
+      return;
     }
+    const collection = await this.getCollection();
+    const metaUpdate = { updatedAt: /* @__PURE__ */ new Date() };
     if (document.metadata) {
       for (const [key, value] of Object.entries(document.metadata)) {
-        update[`metadata.${key}`] = value;
+        metaUpdate[`metadata.${key}`] = value;
       }
     }
-    const filter = {
-      tenantId: this.config.tenantId,
-      id,
-      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-      agentId: options?.agentId || "shared"
-    };
-    await collection.updateOne(filter, { $set: update });
+    await collection.updateMany(
+      {
+        tenantId: this.config.tenantId,
+        agentId,
+        $or: [{ id }, { documentId: id }]
+      },
+      { $set: metaUpdate }
+    );
   }
   /**
-   * Delete document(s) by ID
+   * Delete document(s) by ID — also removes any chunks belonging to the document.
    */
   async delete(ids, options) {
     const collection = await this.getCollection();
     const idArray = Array.isArray(ids) ? ids : [ids];
     const filter = {
       tenantId: this.config.tenantId,
-      id: { $in: idArray },
-      // Match by agentId ('shared' for tenant-wide, specific for agent-only)
-      agentId: options?.agentId || "shared"
+      agentId: options?.agentId || "shared",
+      // Match the document itself (id) OR any chunks that belong to it (documentId)
+      $or: [
+        { id: { $in: idArray } },
+        { documentId: { $in: idArray } }
+      ]
     };
     const result = await collection.deleteMany(filter);
     return result.deletedCount;
@@ -1560,6 +1623,15 @@ var WebRAGPlugin = class _WebRAGPlugin {
     const content = this.extractBestContentText($, config);
     const minChars = config.minExtractedContentLength ?? 50;
     if (!content || content.length < minChars) return null;
+    const image = $('meta[property="og:image"]').attr("content") || $('meta[name="twitter:image"]').attr("content") || $('meta[property="product:image"]').attr("content") || $('[itemtype*="schema.org/Product"] img, .product img, .product-image img, #product-image img').first().attr("src") || void 0;
+    let imageUrl;
+    if (image) {
+      try {
+        imageUrl = new URL(image, url).href;
+      } catch {
+        imageUrl = image;
+      }
+    }
     let type = config.defaultType || "page";
     if (config.typeFromUrl) {
       for (const [pattern, typeName] of Object.entries(config.typeFromUrl)) {
@@ -1577,6 +1649,7 @@ var WebRAGPlugin = class _WebRAGPlugin {
         type,
         title,
         url,
+        ...imageUrl ? { imageUrl } : {},
         ...config.metadata
       }
     };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@snap-agent/rag-web",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Web RAG plugin for SnapAgent SDK - Schema-agnostic content search via web crawling, CMS APIs, sitemaps, and RSS feeds",
   "main": "./dist/index.js",
   "module": "./dist/index.mjs",