@elizaos/plugin-knowledge 1.0.0-beta.73 → 1.0.0-beta.76

package/dist/index.js

@@ -1,9 +1,10 @@
 import {
   convertPdfToTextFromBuffer,
   extractTextFromFileBuffer,
+  fetchUrlContent,
   isBinaryContentType,
   loadDocsFromPath
-} from "./chunk-LHJERZV7.js";
+} from "./chunk-HUTF5APY.js";
 
 // src/index.ts
 import { logger as logger6 } from "@elizaos/core";
@@ -2973,83 +2974,163 @@ async function uploadKnowledgeHandler(req, res, runtime) {
   if (!service) {
     return sendError(res, 500, "SERVICE_NOT_FOUND", "KnowledgeService not found");
   }
-  const files = req.files;
-  if (!files || files.length === 0) {
-    return sendError(res, 400, "NO_FILES", "No files uploaded");
+  const isMultipartRequest = req.files && Object.keys(req.files).length > 0;
+  const isJsonRequest = !isMultipartRequest && req.body && (req.body.fileUrl || req.body.fileUrls);
+  if (!isMultipartRequest && !isJsonRequest) {
+    return sendError(res, 400, "INVALID_REQUEST", "Request must contain either files or URLs");
   }
   try {
-    const processingPromises = files.map(async (file, index) => {
-      let knowledgeId;
-      const originalFilename = file.originalname;
-      const worldId = req.body.worldId || runtime.agentId;
-      const filePath = file.path;
-      knowledgeId = req.body?.documentIds && req.body.documentIds[index] || req.body?.documentId || createUniqueUuid3(runtime, `knowledge-${originalFilename}-${Date.now()}`);
-      try {
-        const fileBuffer = await fs3.promises.readFile(filePath);
-        const fileExt = file.originalname.split(".").pop()?.toLowerCase() || "";
-        const filename = file.originalname;
-        const title = filename.replace(`.${fileExt}`, "");
-        const base64Content = fileBuffer.toString("base64");
-        const knowledgeItem = {
-          id: knowledgeId,
-          content: {
-            text: base64Content
-          },
-          metadata: {
-            type: MemoryType4.DOCUMENT,
-            timestamp: Date.now(),
-            source: "upload",
-            filename,
-            fileExt,
-            title,
-            path: originalFilename,
-            fileType: file.mimetype,
-            fileSize: file.size
-          }
-        };
-        const addKnowledgeOpts = {
-          clientDocumentId: knowledgeId,
-          // This is knowledgeItem.id
-          contentType: file.mimetype,
-          // Directly from multer file object
-          originalFilename,
-          // Directly from multer file object
-          content: base64Content,
-          // The base64 string of the file
-          worldId,
-          roomId: runtime.agentId,
-          // Or a more specific room ID if available
-          entityId: runtime.agentId
-        };
-        await service.addKnowledge(addKnowledgeOpts);
-        cleanupFile(filePath);
-        return {
-          id: knowledgeId,
-          filename: originalFilename,
-          type: file.mimetype,
-          size: file.size,
-          uploadedAt: Date.now(),
-          status: "success"
-        };
-      } catch (fileError) {
-        logger5.error(
-          `[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.originalname}: ${fileError}`
-        );
-        cleanupFile(filePath);
-        return {
-          id: knowledgeId,
-          filename: originalFilename,
-          status: "error_processing",
-          error: fileError.message
-        };
+    if (isMultipartRequest) {
+      const files = req.files;
+      if (!files || files.length === 0) {
+        return sendError(res, 400, "NO_FILES", "No files uploaded");
       }
-    });
-    const results = await Promise.all(processingPromises);
-    sendSuccess(res, results);
+      const processingPromises = files.map(async (file, index) => {
+        let knowledgeId;
+        const originalFilename = file.originalname;
+        const worldId = req.body.worldId || runtime.agentId;
+        const filePath = file.path;
+        knowledgeId = req.body?.documentIds && req.body.documentIds[index] || req.body?.documentId || createUniqueUuid3(runtime, `knowledge-${originalFilename}-${Date.now()}`);
+        try {
+          const fileBuffer = await fs3.promises.readFile(filePath);
+          const fileExt = file.originalname.split(".").pop()?.toLowerCase() || "";
+          const filename = file.originalname;
+          const title = filename.replace(`.${fileExt}`, "");
+          const base64Content = fileBuffer.toString("base64");
+          const knowledgeItem = {
+            id: knowledgeId,
+            content: {
+              text: base64Content
+            },
+            metadata: {
+              type: MemoryType4.DOCUMENT,
+              timestamp: Date.now(),
+              source: "upload",
+              filename,
+              fileExt,
+              title,
+              path: originalFilename,
+              fileType: file.mimetype,
+              fileSize: file.size
+            }
+          };
+          const addKnowledgeOpts = {
+            clientDocumentId: knowledgeId,
+            // This is knowledgeItem.id
+            contentType: file.mimetype,
+            // Directly from multer file object
+            originalFilename,
+            // Directly from multer file object
+            content: base64Content,
+            // The base64 string of the file
+            worldId,
+            roomId: runtime.agentId,
+            // Or a more specific room ID if available
+            entityId: runtime.agentId
+          };
+          await service.addKnowledge(addKnowledgeOpts);
+          cleanupFile(filePath);
+          return {
+            id: knowledgeId,
+            filename: originalFilename,
+            type: file.mimetype,
+            size: file.size,
+            uploadedAt: Date.now(),
+            status: "success"
+          };
+        } catch (fileError) {
+          logger5.error(
+            `[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.originalname}: ${fileError}`
+          );
+          cleanupFile(filePath);
+          return {
+            id: knowledgeId,
+            filename: originalFilename,
+            status: "error_processing",
+            error: fileError.message
+          };
+        }
+      });
+      const results = await Promise.all(processingPromises);
+      sendSuccess(res, results);
+    } else if (isJsonRequest) {
+      const fileUrls = Array.isArray(req.body.fileUrls) ? req.body.fileUrls : req.body.fileUrl ? [req.body.fileUrl] : [];
+      if (fileUrls.length === 0) {
+        return sendError(res, 400, "MISSING_URL", "File URL is required");
+      }
+      const processingPromises = fileUrls.map(async (fileUrl) => {
+        try {
+          const knowledgeId = createUniqueUuid3(runtime, fileUrl);
+          const urlObject = new URL(fileUrl);
+          const pathSegments = urlObject.pathname.split("/");
+          const encodedFilename = pathSegments[pathSegments.length - 1] || "document.pdf";
+          const originalFilename = decodeURIComponent(encodedFilename);
+          logger5.info(`[KNOWLEDGE URL HANDLER] Fetching content from URL: ${fileUrl}`);
+          const { content, contentType: fetchedContentType } = await fetchUrlContent(fileUrl);
+          let contentType = fetchedContentType;
+          if (contentType === "application/octet-stream") {
+            const fileExtension = originalFilename.split(".").pop()?.toLowerCase();
+            if (fileExtension) {
+              if (["pdf"].includes(fileExtension)) {
+                contentType = "application/pdf";
+              } else if (["txt", "text"].includes(fileExtension)) {
+                contentType = "text/plain";
+              } else if (["md", "markdown"].includes(fileExtension)) {
+                contentType = "text/markdown";
+              } else if (["doc", "docx"].includes(fileExtension)) {
+                contentType = "application/msword";
+              } else if (["html", "htm"].includes(fileExtension)) {
+                contentType = "text/html";
+              } else if (["json"].includes(fileExtension)) {
+                contentType = "application/json";
+              } else if (["xml"].includes(fileExtension)) {
+                contentType = "application/xml";
+              }
+            }
+          }
+          const addKnowledgeOpts = {
+            clientDocumentId: knowledgeId,
+            contentType,
+            originalFilename,
+            content,
+            // Use the base64 encoded content from the URL
+            worldId: runtime.agentId,
+            roomId: runtime.agentId,
+            entityId: runtime.agentId,
+            // Store the source URL in metadata
+            metadata: {
+              url: fileUrl
+            }
+          };
+          logger5.debug(`[KNOWLEDGE URL HANDLER] Processing knowledge from URL: ${fileUrl} (type: ${contentType})`);
+          const result = await service.addKnowledge(addKnowledgeOpts);
+          return {
+            id: result.clientDocumentId,
+            fileUrl,
+            filename: originalFilename,
+            message: "Knowledge created successfully",
+            createdAt: Date.now(),
+            fragmentCount: result.fragmentCount,
+            status: "success"
+          };
+        } catch (urlError) {
+          logger5.error(`[KNOWLEDGE URL HANDLER] Error processing URL ${fileUrl}: ${urlError}`);
+          return {
+            fileUrl,
+            status: "error_processing",
+            error: urlError.message
+          };
+        }
+      });
+      const results = await Promise.all(processingPromises);
+      sendSuccess(res, results);
+    }
   } catch (error) {
-    logger5.error("[KNOWLEDGE UPLOAD HANDLER] Error uploading knowledge:", error);
-    cleanupFiles(files);
-    sendError(res, 500, "UPLOAD_ERROR", "Failed to upload knowledge", error.message);
+    logger5.error("[KNOWLEDGE HANDLER] Error processing knowledge:", error);
+    if (isMultipartRequest) {
+      cleanupFiles(req.files);
+    }
+    sendError(res, 500, "PROCESSING_ERROR", "Failed to process knowledge", error.message);
   }
 }
 async function getKnowledgeDocumentsHandler(req, res, runtime) {
@@ -3066,22 +3147,42 @@ async function getKnowledgeDocumentsHandler(req, res, runtime) {
     const limit = req.query.limit ? Number.parseInt(req.query.limit, 10) : 20;
     const before = req.query.before ? Number.parseInt(req.query.before, 10) : Date.now();
     const includeEmbedding = req.query.includeEmbedding === "true";
+    const fileUrls = req.query.fileUrls ? typeof req.query.fileUrls === "string" && req.query.fileUrls.includes(",") ? req.query.fileUrls.split(",") : [req.query.fileUrls] : null;
     const memories = await service.getMemories({
       tableName: "documents",
       count: limit,
       end: before
     });
-    const cleanMemories = includeEmbedding ? memories : memories.map((memory) => ({
+    let filteredMemories = memories;
+    if (fileUrls && fileUrls.length > 0) {
+      const urlBasedIds = fileUrls.map((url) => createUniqueUuid3(runtime, url));
+      filteredMemories = memories.filter(
+        (memory) => urlBasedIds.includes(memory.id) || // If the ID corresponds directly
+        // Or if the URL is stored in the metadata (check if it exists)
+        memory.metadata && "url" in memory.metadata && typeof memory.metadata.url === "string" && fileUrls.includes(memory.metadata.url)
+      );
+      logger5.debug(`[KNOWLEDGE GET HANDLER] Filtered documents by URLs: ${fileUrls.length} URLs, found ${filteredMemories.length} matching documents`);
+    }
+    const cleanMemories = includeEmbedding ? filteredMemories : filteredMemories.map((memory) => ({
       ...memory,
       embedding: void 0
     }));
-    sendSuccess(res, { memories: cleanMemories });
+    sendSuccess(res, {
+      memories: cleanMemories,
+      urlFiltered: fileUrls ? true : false,
+      totalFound: cleanMemories.length,
+      totalRequested: fileUrls ? fileUrls.length : 0
+    });
   } catch (error) {
     logger5.error("[KNOWLEDGE GET HANDLER] Error retrieving documents:", error);
     sendError(res, 500, "RETRIEVAL_ERROR", "Failed to retrieve documents", error.message);
   }
 }
 async function deleteKnowledgeDocumentHandler(req, res, runtime) {
+  logger5.debug(`[KNOWLEDGE DELETE HANDLER] Received DELETE request:
+    - path: ${req.path}
+    - params: ${JSON.stringify(req.params)}
+  `);
   const service = runtime.getService(KnowledgeService.serviceType);
   if (!service) {
     return sendError(
@@ -3091,27 +3192,104 @@ async function deleteKnowledgeDocumentHandler(req, res, runtime) {
       "KnowledgeService not found for deleteKnowledgeDocumentHandler"
     );
   }
-  const knowledgeId = req.path.split("/documents/")[1];
+  const knowledgeId = req.params.knowledgeId;
   if (!knowledgeId || knowledgeId.length < 36) {
+    logger5.error(`[KNOWLEDGE DELETE HANDLER] Invalid knowledge ID format: ${knowledgeId}`);
     return sendError(res, 400, "INVALID_ID", "Invalid Knowledge ID format");
   }
   try {
-    await service.deleteMemory(knowledgeId);
+    const typedKnowledgeId = knowledgeId;
+    logger5.debug(`[KNOWLEDGE DELETE HANDLER] Attempting to delete document with ID: ${typedKnowledgeId}`);
+    await service.deleteMemory(typedKnowledgeId);
+    logger5.info(`[KNOWLEDGE DELETE HANDLER] Successfully deleted document with ID: ${typedKnowledgeId}`);
     sendSuccess(res, null, 204);
   } catch (error) {
     logger5.error(`[KNOWLEDGE DELETE HANDLER] Error deleting document ${knowledgeId}:`, error);
     sendError(res, 500, "DELETE_ERROR", "Failed to delete document", error.message);
   }
 }
+async function getKnowledgeByIdHandler(req, res, runtime) {
+  logger5.debug(`[KNOWLEDGE GET BY ID HANDLER] Received GET request:
+    - path: ${req.path}
+    - params: ${JSON.stringify(req.params)}
+  `);
+  const service = runtime.getService(KnowledgeService.serviceType);
+  if (!service) {
+    return sendError(
+      res,
+      500,
+      "SERVICE_NOT_FOUND",
+      "KnowledgeService not found for getKnowledgeByIdHandler"
+    );
+  }
+  const knowledgeId = req.params.knowledgeId;
+  if (!knowledgeId || knowledgeId.length < 36) {
+    logger5.error(`[KNOWLEDGE GET BY ID HANDLER] Invalid knowledge ID format: ${knowledgeId}`);
+    return sendError(res, 400, "INVALID_ID", "Invalid Knowledge ID format");
+  }
+  try {
+    logger5.debug(`[KNOWLEDGE GET BY ID HANDLER] Retrieving document with ID: ${knowledgeId}`);
+    const memories = await service.getMemories({
+      tableName: "documents",
+      count: 1e3
+    });
+    const typedKnowledgeId = knowledgeId;
+    const document = memories.find((memory) => memory.id === typedKnowledgeId);
+    if (!document) {
+      return sendError(res, 404, "NOT_FOUND", `Knowledge with ID ${typedKnowledgeId} not found`);
+    }
+    const cleanDocument = {
+      ...document,
+      embedding: void 0
+    };
+    sendSuccess(res, { document: cleanDocument });
+  } catch (error) {
+    logger5.error(`[KNOWLEDGE GET BY ID HANDLER] Error retrieving document ${knowledgeId}:`, error);
+    sendError(res, 500, "RETRIEVAL_ERROR", "Failed to retrieve document", error.message);
+  }
+}
 async function knowledgePanelHandler(req, res, runtime) {
+  const agentId = runtime.agentId;
   try {
     const currentDir = path3.dirname(new URL(import.meta.url).pathname);
     const frontendPath = path3.join(currentDir, "../dist/index.html");
     if (fs3.existsSync(frontendPath)) {
       const html = await fs3.promises.readFile(frontendPath, "utf8");
+      const injectedHtml = html.replace(
+        "<head>",
+        `<head>
+          <script>
+            window.ELIZA_CONFIG = {
+              agentId: '${agentId}',
+              apiBase: '/api/agents/${agentId}/plugins/knowledge'
+            };
+          </script>`
+      );
       res.writeHead(200, { "Content-Type": "text/html" });
-      res.end(html);
+      res.end(injectedHtml);
     } else {
+      let cssFile = "index.css";
+      let jsFile = "index.js";
+      const manifestPath = path3.join(currentDir, "../dist/manifest.json");
+      if (fs3.existsSync(manifestPath)) {
+        try {
+          const manifestContent = await fs3.promises.readFile(manifestPath, "utf8");
+          const manifest = JSON.parse(manifestContent);
+          for (const [key, value] of Object.entries(manifest)) {
+            if (typeof value === "object" && value !== null) {
+              if (key.endsWith(".css") || value.file?.endsWith(".css")) {
+                cssFile = value.file || key;
+              }
+              if (key.endsWith(".js") || value.file?.endsWith(".js")) {
+                jsFile = value.file || key;
+              }
+            }
+          }
+        } catch (manifestError) {
+          logger5.error("[KNOWLEDGE PANEL] Error reading manifest:", manifestError);
+        }
+      }
+      logger5.debug(`[KNOWLEDGE PANEL] Using fallback with CSS: ${cssFile}, JS: ${jsFile}`);
       const html = `
 <!DOCTYPE html>
 <html lang="en">
@@ -3119,7 +3297,13 @@ async function knowledgePanelHandler(req, res, runtime) {
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>Knowledge</title>
-    <link rel="stylesheet" href="./assets/index-BlRATUqY.css">
+    <script>
+      window.ELIZA_CONFIG = {
+        agentId: '${agentId}',
+        apiBase: '/api/agents/${agentId}/plugins/knowledge'
+      };
+    </script>
+    <link rel="stylesheet" href="./assets/${cssFile}">
     <style>
         body { font-family: system-ui, -apple-system, sans-serif; margin: 0; padding: 20px; }
         .container { max-width: 1200px; margin: 0 auto; }
@@ -3128,11 +3312,11 @@ async function knowledgePanelHandler(req, res, runtime) {
 </head>
 <body>
     <div class="container">
-        <div id="knowledge-root">
+        <div id="root">
             <div class="loading">Loading Knowledge Library...</div>
         </div>
     </div>
-    <script type="module" src="./assets/index-7riujMow.js"></script>
+    <script type="module" src="./assets/${jsFile}"></script>
 </body>
 </html>`;
       res.writeHead(200, { "Content-Type": "text/html" });
@@ -3184,6 +3368,26 @@ async function frontendAssetHandler(req, res, runtime) {
     sendError(res, 500, "ASSET_ERROR", `Failed to load asset ${req.url}`, error.message);
   }
 }
+async function getKnowledgeChunksHandler(req, res, runtime) {
+  const service = runtime.getService(KnowledgeService.serviceType);
+  if (!service) {
+    return sendError(res, 500, "SERVICE_NOT_FOUND", "KnowledgeService not found");
+  }
+  try {
+    const limit = req.query.limit ? Number.parseInt(req.query.limit, 10) : 100;
+    const before = req.query.before ? Number.parseInt(req.query.before, 10) : Date.now();
+    const chunks = await service.getMemories({
+      tableName: "knowledge",
+      // or whatever table stores the chunks
+      count: limit,
+      end: before
+    });
+    sendSuccess(res, { chunks });
+  } catch (error) {
+    logger5.error("[KNOWLEDGE CHUNKS GET HANDLER] Error retrieving chunks:", error);
+    sendError(res, 500, "RETRIEVAL_ERROR", "Failed to retrieve knowledge chunks", error.message);
+  }
+}
 var knowledgeRoutes = [
   {
     type: "GET",
@@ -3199,7 +3403,7 @@ var knowledgeRoutes = [
   },
   {
     type: "POST",
-    path: "/upload",
+    path: "/documents",
     handler: uploadKnowledgeHandler,
     isMultipart: true
   },
@@ -3208,10 +3412,20 @@ var knowledgeRoutes = [
     path: "/documents",
     handler: getKnowledgeDocumentsHandler
   },
+  {
+    type: "GET",
+    path: "/documents/:knowledgeId",
+    handler: getKnowledgeByIdHandler
+  },
   {
     type: "DELETE",
-    path: "/documents/*",
+    path: "/documents/:knowledgeId",
     handler: deleteKnowledgeDocumentHandler
+  },
+  {
+    type: "GET",
+    path: "/knowledges",
+    handler: getKnowledgeChunksHandler
   }
 ];
 
@@ -3260,7 +3474,7 @@ var knowledgePlugin = {
             try {
               const service = runtime.getService(KnowledgeService.serviceType);
               if (service instanceof KnowledgeService) {
-                const { loadDocsFromPath: loadDocsFromPath2 } = await import("./docs-loader-25N4HXDV.js");
+                const { loadDocsFromPath: loadDocsFromPath2 } = await import("./docs-loader-2S6H7F4L.js");
                 const result = await loadDocsFromPath2(service, runtime.agentId);
                 if (result.successful > 0) {
                   logger6.info(`Loaded ${result.successful} documents from docs folder on startup`);