@elizaos/plugin-knowledge 1.0.7 → 1.0.8

package/README.md

@@ -232,6 +232,44 @@ const searchResults = await knowledgeService.searchKnowledge({
 - **Contextual mode = better understanding** (but slower processing)
 - **Batch document uploads** rather than one-by-one for better performance
 
+### File Upload Configuration
+
+#### Automatic MIME Type Correction
+The frontend automatically corrects MIME types for code and document files to ensure proper processing:
+
+```typescript
+const getCorrectMimeType = (file: File): string => {
+  const filename = file.name.toLowerCase();
+  const ext = filename.split('.').pop() || '';
+
+  // Map common text file extensions to text/plain
+  const textExtensions = [
+    'ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs',
+    'py', 'pyw', 'pyi', 'java', 'c', 'cpp', 'cc', 'cxx', 'h', 'hpp',
+    'cs', 'php', 'rb', 'go', 'rs', 'swift', 'kt', 'kts', 'scala',
+    // ... and many more
+  ];
+
+  if (textExtensions.includes(ext)) {
+    return 'text/plain';
+  } else if (['md', 'markdown'].includes(ext)) {
+    return 'text/markdown';
+  } else if (ext === 'json') {
+    return 'application/json';
+  }
+  // ... additional mappings
+
+  return file.type || 'application/octet-stream';
+};
+```
+
+#### Supported File Types
+
+- **Documents**: PDF, DOC, DOCX
+- **Code**: JS, TS, PY, JAVA, C, CPP, etc.
+- **Text**: TXT, MD, CSV, JSON, XML, HTML
+- **Configuration**: ENV, CFG, INI, YAML, YML
+
 </details>
 
 ## 📝 License

package/dist/index.js

@@ -3270,41 +3270,6 @@ var knowledgeActions = [processKnowledgeAction, searchKnowledgeAction];
 import { createUniqueUuid as createUniqueUuid2, logger as logger6, ModelType as ModelType4 } from "@elizaos/core";
 import fs3 from "fs";
 import path3 from "path";
-import multer from "multer";
-var createUploadMiddleware = (runtime) => {
-  const uploadDir = runtime.getSetting("KNOWLEDGE_UPLOAD_DIR") || "/tmp/uploads/";
-  const maxFileSize = parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILE_SIZE") || "52428800");
-  const maxFiles = parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILES") || "10");
-  const allowedMimeTypes = runtime.getSetting("KNOWLEDGE_ALLOWED_MIME_TYPES")?.split(",") || [
-    "text/plain",
-    "text/markdown",
-    "application/pdf",
-    "application/msword",
-    "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
-    "text/html",
-    "application/json",
-    "application/xml",
-    "text/csv"
-  ];
-  return multer({
-    dest: uploadDir,
-    limits: {
-      fileSize: maxFileSize,
-      files: maxFiles
-    },
-    fileFilter: (req, file, cb) => {
-      if (allowedMimeTypes.includes(file.mimetype)) {
-        cb(null, true);
-      } else {
-        cb(
-          new Error(
-            `File type ${file.mimetype} not allowed. Allowed types: ${allowedMimeTypes.join(", ")}`
-          )
-        );
-      }
-    }
-  });
-};
 function sendSuccess(res, data, status = 200) {
   res.writeHead(status, { "Content-Type": "application/json" });
   res.end(JSON.stringify({ success: true, data }));
@@ -3324,7 +3289,11 @@ var cleanupFile = (filePath) => {
 };
 var cleanupFiles = (files) => {
   if (files) {
-    files.forEach((file) => cleanupFile(file.path));
+    files.forEach((file) => {
+      if (file.tempFilePath) {
+        cleanupFile(file.tempFilePath);
+      }
+    });
   }
 };
 async function uploadKnowledgeHandler(req, res, runtime) {
@@ -3332,20 +3301,56 @@ async function uploadKnowledgeHandler(req, res, runtime) {
   if (!service) {
     return sendError(res, 500, "SERVICE_NOT_FOUND", "KnowledgeService not found");
   }
-  const hasUploadedFiles = req.files && req.files.length > 0;
+  const hasUploadedFiles = req.files && Object.keys(req.files).length > 0;
   const isJsonRequest = !hasUploadedFiles && req.body && (req.body.fileUrl || req.body.fileUrls);
   if (!hasUploadedFiles && !isJsonRequest) {
     return sendError(res, 400, "INVALID_REQUEST", "Request must contain either files or URLs");
   }
   try {
     if (hasUploadedFiles) {
-      const files = req.files;
+      let files = [];
+      if (req.files.files) {
+        if (Array.isArray(req.files.files)) {
+          files = req.files.files;
+        } else {
+          files = [req.files.files];
+        }
+      } else if (req.files.file) {
+        files = [req.files.file];
+      } else {
+        files = Object.values(req.files).flat();
+      }
       if (!files || files.length === 0) {
         return sendError(res, 400, "NO_FILES", "No files uploaded");
       }
+      const invalidFiles = files.filter((file) => {
+        if (file.truncated) {
+          logger6.warn(`File ${file.name} was truncated during upload`);
+          return true;
+        }
+        if (file.size === 0) {
+          logger6.warn(`File ${file.name} is empty`);
+          return true;
+        }
+        if (!file.name || file.name.trim() === "") {
+          logger6.warn(`File has no name`);
+          return true;
+        }
+        if (!file.data && !file.tempFilePath) {
+          logger6.warn(`File ${file.name} has no data or temp file path`);
+          return true;
+        }
+        return false;
+      });
+      if (invalidFiles.length > 0) {
+        cleanupFiles(files);
+        const invalidFileNames = invalidFiles.map((f) => f.name || "unnamed").join(", ");
+        return sendError(res, 400, "INVALID_FILES", `Invalid or corrupted files: ${invalidFileNames}`);
+      }
       const agentId = req.body.agentId || req.query.agentId;
       if (!agentId) {
         logger6.error("[KNOWLEDGE UPLOAD HANDLER] No agent ID provided in request");
+        cleanupFiles(files);
         return sendError(
           res,
           400,
@@ -3357,24 +3362,50 @@ async function uploadKnowledgeHandler(req, res, runtime) {
       logger6.info(`[KNOWLEDGE UPLOAD HANDLER] Processing upload for agent: ${agentId}`);
       const processingPromises = files.map(async (file, index) => {
         let knowledgeId;
-        const originalFilename = file.originalname;
-        const filePath = file.path;
+        const originalFilename = file.name;
+        const filePath = file.tempFilePath;
         knowledgeId = req.body?.documentIds && req.body.documentIds[index] || req.body?.documentId || createUniqueUuid2(runtime, `knowledge-${originalFilename}-${Date.now()}`);
         logger6.debug(
           `[KNOWLEDGE UPLOAD HANDLER] File: ${originalFilename}, Agent ID: ${agentId}, World ID: ${worldId}, Knowledge ID: ${knowledgeId}`
         );
         try {
-          const fileBuffer = await fs3.promises.readFile(filePath);
+          let fileBuffer;
+          if (filePath && fs3.existsSync(filePath)) {
+            try {
+              const stats = await fs3.promises.stat(filePath);
+              if (stats.size === 0) {
+                throw new Error("Temporary file is empty");
+              }
+              fileBuffer = await fs3.promises.readFile(filePath);
+              logger6.debug(`[KNOWLEDGE UPLOAD] Read ${fileBuffer.length} bytes from temp file: ${filePath}`);
+            } catch (fsError) {
+              throw new Error(`Failed to read temporary file: ${fsError.message}`);
+            }
+          } else if (file.data && Buffer.isBuffer(file.data)) {
+            fileBuffer = file.data;
+            logger6.debug(`[KNOWLEDGE UPLOAD] Using in-memory buffer of ${fileBuffer.length} bytes`);
+          } else {
+            throw new Error("No file data available - neither temp file nor buffer found");
+          }
+          if (!Buffer.isBuffer(fileBuffer) || fileBuffer.length === 0) {
+            throw new Error("Invalid or empty file buffer");
+          }
+          if (fileBuffer.length !== file.size) {
+            logger6.warn(`File size mismatch for ${originalFilename}: expected ${file.size}, got ${fileBuffer.length}`);
+          }
           const base64Content = fileBuffer.toString("base64");
+          if (!base64Content || base64Content.length === 0) {
+            throw new Error("Failed to convert file to base64");
+          }
           const addKnowledgeOpts = {
             agentId,
             // Pass the agent ID from frontend
             clientDocumentId: knowledgeId,
             // This is knowledgeItem.id
             contentType: file.mimetype,
-            // Directly from multer file object
+            // Directly from express-fileupload file object
             originalFilename,
-            // Directly from multer file object
+            // Directly from express-fileupload file object
             content: base64Content,
             // The base64 string of the file
             worldId,
@@ -3384,7 +3415,9 @@ async function uploadKnowledgeHandler(req, res, runtime) {
             // Use the correct agent ID
           };
           await service.addKnowledge(addKnowledgeOpts);
-          cleanupFile(filePath);
+          if (filePath) {
+            cleanupFile(filePath);
+          }
           return {
             id: knowledgeId,
             filename: originalFilename,
@@ -3395,9 +3428,11 @@ async function uploadKnowledgeHandler(req, res, runtime) {
           };
         } catch (fileError) {
           logger6.error(
-            `[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.originalname}: ${fileError}`
+            `[KNOWLEDGE UPLOAD HANDLER] Error processing file ${file.name}: ${fileError}`
           );
-          cleanupFile(filePath);
+          if (filePath) {
+            cleanupFile(filePath);
+          }
           return {
             id: knowledgeId,
             filename: originalFilename,
@@ -3498,8 +3533,9 @@ async function uploadKnowledgeHandler(req, res, runtime) {
     }
   } catch (error) {
     logger6.error("[KNOWLEDGE HANDLER] Error processing knowledge:", error);
-    if (hasUploadedFiles) {
-      cleanupFiles(req.files);
+    if (hasUploadedFiles && req.files) {
+      const allFiles = Object.values(req.files).flat();
+      cleanupFiles(allFiles);
     }
     sendError(res, 500, "PROCESSING_ERROR", "Failed to process knowledge", error.message);
   }
@@ -3857,19 +3893,25 @@ async function searchKnowledgeHandler(req, res, runtime) {
     sendError(res, 500, "SEARCH_ERROR", "Failed to search knowledge", error.message);
   }
 }
-async function uploadKnowledgeWithMulter(req, res, runtime) {
-  const upload = createUploadMiddleware(runtime);
-  const uploadArray = upload.array(
-    "files",
-    parseInt(runtime.getSetting("KNOWLEDGE_MAX_FILES") || "10")
-  );
-  uploadArray(req, res, (err) => {
-    if (err) {
-      logger6.error("[KNOWLEDGE UPLOAD] Multer error:", err);
-      return sendError(res, 400, "UPLOAD_ERROR", err.message);
-    }
-    uploadKnowledgeHandler(req, res, runtime);
+async function handleKnowledgeUpload(req, res, runtime) {
+  logger6.debug("[KNOWLEDGE UPLOAD] Starting upload handler");
+  logger6.debug("[KNOWLEDGE UPLOAD] Request details:", {
+    method: req.method,
+    url: req.url,
+    contentType: req.headers["content-type"],
+    contentLength: req.headers["content-length"],
+    hasFiles: req.files ? Object.keys(req.files).length : 0,
+    hasBody: req.body ? Object.keys(req.body).length : 0
   });
+  try {
+    logger6.debug("[KNOWLEDGE UPLOAD] Using files parsed by global middleware");
+    await uploadKnowledgeHandler(req, res, runtime);
+  } catch (handlerError) {
+    logger6.error("[KNOWLEDGE UPLOAD] Handler error:", handlerError);
+    if (!res.headersSent) {
+      sendError(res, 500, "HANDLER_ERROR", "Failed to process upload");
+    }
+  }
 }
 var knowledgeRoutes = [
   {
@@ -3887,7 +3929,7 @@ var knowledgeRoutes = [
   {
     type: "POST",
     path: "/documents",
-    handler: uploadKnowledgeWithMulter
+    handler: handleKnowledgeUpload
   },
   {
     type: "GET",