memorylake-openclaw 1.0.2-beta.3 → 1.0.2-beta.5

package/index.ts

@@ -780,6 +780,15 @@ const memoryLakeConfigSchema = {
 // Plugin Definition
 // ============================================================================
 
+/** Shared type for the upload / uploadAuto function signature */
+type UploadFn = (opts: {
+  host: string;
+  apiKey: string;
+  projectId: string;
+  filePath: string;
+  fileName: string;
+}) => Promise<unknown>;
+
 const memoryPlugin = {
   id: "memorylake-openclaw",
   name: "Memory (MemoryLake)",
@@ -1711,9 +1720,13 @@ const memoryPlugin = {
               console.error("No project ID configured. Use --project-id or set up agent/workspace config.");
               return;
             }
+            if (!effectiveCfg.host || !effectiveCfg.apiKey) {
+              console.error("Missing host or apiKey in config. Check your MemoryLake configuration.");
+              return;
+            }
 
             // Lazy import upload.mjs (use uploadAuto to support archives)
-            let uploadFn: (o: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>;
+            let uploadFn: UploadFn;
             try {
               const uploadModule = await import(
                 /* webpackIgnore: true */
@@ -1816,7 +1829,7 @@ const memoryPlugin = {
     // Auto-upload: upload inbound files to MemoryLake before prompt build
     if (cfg.autoUpload) {
       // Lazy-load upload function from upload.mjs
-      let uploadAutoFn: ((opts: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>) | undefined;
+      let uploadAutoFn: UploadFn | undefined;
 
       api.on("before_prompt_build", (event, ctx) => {
         if ((ctx as any)?.trigger !== "user") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memorylake-openclaw",
-  "version": "1.0.2-beta.3",
+  "version": "1.0.2-beta.5",
   "type": "module",
   "description": "MemoryLake memory backend for OpenClaw",
   "license": "MIT",
@@ -17,7 +17,10 @@
   ],
   "dependencies": {
     "@sinclair/typebox": "0.34.47",
-    "got": "^14.0.0"
+    "7zip-min": "^3.0.1",
+    "adm-zip": "^0.5.17",
+    "got": "^14.0.0",
+    "tar": "^7.5.13"
   },
   "openclaw": {
     "extensions": [

package/skills/memorylake-upload/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: memorylake-upload
-description: Use when the user wants to upload files, documents, PDFs, archives, directories, or other data to MemoryLake. Supports single files, archives (zip, tar.gz, tgz, tar.bz2, tar, rar, 7z, etc.) which are extracted then uploaded, and directories which are recursively uploaded.
+description: Use when the user wants to upload files, documents, PDFs, archives, directories, or other data to MemoryLake. Supports single files, archives (zip, tar.gz, tgz, tar, gz, 7z, tar.bz2, bz2) which are extracted then uploaded, and directories which are recursively uploaded.
 ---
 
 # MemoryLake File Upload
@@ -13,7 +13,7 @@ Upload local files, archives, or directories to MemoryLake using the multipart u
 
 - User wants to upload a file (PDF, DOCX, image, etc.) to MemoryLake
 - User wants to add a local document to a MemoryLake project
-- User wants to upload an archive (zip, tar.gz, tgz, tar.bz2, tbz2, tar.xz, txz, tar, gz, bz2, xz, rar, 7z) — files inside will be extracted and uploaded one by one
+- User wants to upload an archive (zip, tar.gz, tgz, tar, gz, 7z, tar.bz2, bz2) — files inside will be extracted and uploaded one by one
 - User wants to upload an entire directory/folder — all files will be recursively collected and uploaded
 
 ## Step 1 -- Read MemoryLake Config

package/skills/memorylake-upload/scripts/upload.mjs

@@ -21,7 +21,11 @@ import path from 'path';
 import https from 'https';
 import http from 'http';
 import os from 'os';
-import { execFileSync } from 'child_process';
+import { createGunzip } from 'zlib';
+import { pipeline } from 'stream/promises';
+import AdmZip from 'adm-zip';
+import { extract as tarExtract } from 'tar';
+import sevenZip from '7zip-min';
 
 // API base path
 const API_BASE = '/openapi/memorylake';
@@ -234,11 +238,20 @@ export async function upload({ host, apiKey, projectId, filePath, fileName }) {
 // Archive support
 // ============================================================================
 
-const ARCHIVE_EXTENSIONS = [
-  '.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz',
-  '.tar', '.zip', '.gz', '.bz2', '.xz', '.rar', '.7z',
+/** Formats we can extract with npm packages (adm-zip, tar, zlib, 7zip-min) */
+const SUPPORTED_ARCHIVE_EXTENSIONS = [
+  '.tar.gz', '.tgz', '.tar.bz2', '.tbz2',
+  '.tar', '.zip', '.gz', '.bz2', '.7z',
 ];
 
+/** Recognized but unsupported — detected so we give a clear error instead of uploading raw */
+const UNSUPPORTED_ARCHIVE_EXTENSIONS = [
+  '.tar.xz', '.txz', '.xz', '.rar',
+];
+
+/** All recognized archive extensions (compound first for matching priority) */
+const ALL_ARCHIVE_EXTENSIONS = [...SUPPORTED_ARCHIVE_EXTENSIONS, ...UNSUPPORTED_ARCHIVE_EXTENSIONS];
+
 /**
  * Check if a file is an archive based on its extension.
  * Returns the matched extension or null.
@@ -246,100 +259,78 @@ const ARCHIVE_EXTENSIONS = [
 function getArchiveExtension(filePath) {
   const lower = filePath.toLowerCase();
   // Check compound extensions first (e.g. .tar.gz before .gz)
-  for (const ext of ARCHIVE_EXTENSIONS) {
+  for (const ext of ALL_ARCHIVE_EXTENSIONS) {
     if (lower.endsWith(ext)) return ext;
   }
   return null;
 }
 
-/** Max buffer size for single-file decompression (gz/bz2/xz) stdout capture: 512 MB */
-const DECOMPRESS_MAX_BUFFER = 512 * 1024 * 1024;
+/**
+ * Promisified wrapper around 7zip-min's callback-based unpack API.
+ */
+function sevenZipExtract(archivePath, destDir) {
+  return new Promise((resolve, reject) => {
+    sevenZip.unpack(archivePath, destDir, (err) => {
+      if (err) reject(err);
+      else resolve();
+    });
+  });
+}
 
 /**
  * Extract an archive to the given destination directory.
+ * Uses npm packages (adm-zip, tar, zlib, 7zip-min) for cross-platform compatibility.
  */
-const TOOL_INSTALL_HINTS = {
-  unzip:   'Install via: sudo apt install unzip (Linux) / brew install unzip (macOS) / choco install unzip (Windows)',
-  tar:     'tar should be pre-installed on most systems (Linux/macOS/Windows 10+)',
-  gunzip:  'Install via: sudo apt install gzip (Linux) / brew install gzip (macOS) / choco install gzip (Windows)',
-  bunzip2: 'Install via: sudo apt install bzip2 (Linux) / brew install bzip2 (macOS) / choco install bzip2 (Windows)',
-  xz:      'Install via: sudo apt install xz-utils (Linux) / brew install xz (macOS) / choco install xz (Windows)',
-  unrar:   'Install via: sudo apt install unrar (Linux) / brew install unrar (macOS) / choco install unrar (Windows)',
-  '7z':    'Install via: sudo apt install p7zip-full (Linux) / brew install 7zip (macOS) / choco install 7zip (Windows)',
-};
-
-function runExtractCommand(tool, args) {
-  try {
-    execFileSync(tool, args, { stdio: 'pipe' });
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      const hint = TOOL_INSTALL_HINTS[tool] || '';
-      throw new Error(`"${tool}" not found. ${hint}`);
-    }
-    throw err;
+async function extractArchive(archivePath, destDir) {
+  const ext = getArchiveExtension(archivePath);
+
+  // Reject recognized-but-unsupported formats early with a clear message
+  if (UNSUPPORTED_ARCHIVE_EXTENSIONS.includes(ext)) {
+    throw new Error(`Unsupported archive format: ${ext}. Please convert to .zip, .tar.gz or .7z`);
   }
-}
 
-function extractArchive(archivePath, destDir) {
-  const ext = getArchiveExtension(archivePath);
+  console.log(`  Format: ${ext}`);
 
   switch (ext) {
-    case '.zip':
-      runExtractCommand('unzip', ['-o', archivePath, '-d', destDir]);
+    case '.zip': {
+      const zip = new AdmZip(archivePath);
+      zip.extractAllTo(destDir, true);
       break;
+    }
     case '.tar.gz':
     case '.tgz':
-      runExtractCommand('tar', ['-xzf', archivePath, '-C', destDir]);
-      break;
-    case '.tar.bz2':
-    case '.tbz2':
-      runExtractCommand('tar', ['-xjf', archivePath, '-C', destDir]);
-      break;
-    case '.tar.xz':
-    case '.txz':
-      runExtractCommand('tar', ['-xJf', archivePath, '-C', destDir]);
-      break;
     case '.tar':
-      runExtractCommand('tar', ['-xf', archivePath, '-C', destDir]);
+      await tarExtract({ file: archivePath, cwd: destDir });
       break;
     case '.gz': {
       const baseName = path.basename(archivePath).slice(0, -3);
-      try {
-        const output = execFileSync('gunzip', ['-c', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
-        fs.writeFileSync(path.join(destDir, baseName), output);
-      } catch (err) {
-        if (err.code === 'ENOENT') throw new Error(`"gunzip" not found. ${TOOL_INSTALL_HINTS.gunzip}`);
-        throw err;
-      }
-      break;
-    }
-    case '.bz2': {
-      const baseName = path.basename(archivePath).slice(0, -4);
-      try {
-        const output = execFileSync('bunzip2', ['-c', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
-        fs.writeFileSync(path.join(destDir, baseName), output);
-      } catch (err) {
-        if (err.code === 'ENOENT') throw new Error(`"bunzip2" not found. ${TOOL_INSTALL_HINTS.bunzip2}`);
-        throw err;
-      }
+      const destPath = path.join(destDir, baseName);
+      await pipeline(
+        fs.createReadStream(archivePath),
+        createGunzip(),
+        fs.createWriteStream(destPath),
+      );
       break;
     }
-    case '.xz': {
-      const baseName = path.basename(archivePath).slice(0, -3);
+    case '.tar.bz2':
+    case '.tbz2': {
+      // 7za only strips the bz2 layer, producing a .tar — need a second pass
+      const tmpBz2 = fs.mkdtempSync(path.join(os.tmpdir(), 'memorylake-bz2-'));
       try {
-        const output = execFileSync('xz', ['-dc', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
-        fs.writeFileSync(path.join(destDir, baseName), output);
-      } catch (err) {
-        if (err.code === 'ENOENT') throw new Error(`"xz" not found. ${TOOL_INSTALL_HINTS.xz}`);
-        throw err;
+        await sevenZipExtract(archivePath, tmpBz2);
+        // Find the resulting .tar inside tmpBz2
+        const inner = fs.readdirSync(tmpBz2).find(f => f.endsWith('.tar'));
+        if (inner) {
+          await tarExtract({ file: path.join(tmpBz2, inner), cwd: destDir });
+        }
+      } finally {
+        fs.rmSync(tmpBz2, { recursive: true, force: true });
       }
       break;
     }
-    case '.rar':
-      runExtractCommand('unrar', ['x', '-o+', archivePath, destDir + '/']);
-      break;
+    case '.bz2':
     case '.7z':
-      runExtractCommand('7z', ['x', archivePath, `-o${destDir}`, '-y']);
+      await sevenZipExtract(archivePath, destDir);
       break;
     default:
       throw new Error(`Unsupported archive format: ${path.basename(archivePath)}`);
@@ -352,18 +343,26 @@ function extractArchive(archivePath, destDir) {
 const IGNORED_NAMES = new Set(['.DS_Store', 'Thumbs.db', 'desktop.ini']);
 const IGNORED_DIRS = new Set(['__MACOSX', '.Spotlight-V100', '.Trashes']);
 
+/** Sanity limit to prevent accidentally uploading huge trees (e.g. node_modules) */
+const MAX_COLLECT_FILES = 500;
+
 /**
  * Recursively collect all files in a directory, skipping hidden/system junk.
+ * Throws if the file count exceeds MAX_COLLECT_FILES.
  */
-function collectFiles(dir) {
+function collectFiles(dir, _counter = { value: 0 }) {
   const files = [];
   for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
     if (IGNORED_NAMES.has(entry.name)) continue;
     const full = path.join(dir, entry.name);
     if (entry.isDirectory()) {
       if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
-      files.push(...collectFiles(full));
+      files.push(...collectFiles(full, _counter));
     } else if (entry.isFile()) {
+      _counter.value++;
+      if (_counter.value > MAX_COLLECT_FILES) {
+        throw new Error(`Too many files (>${MAX_COLLECT_FILES}). Please upload a smaller directory or archive.`);
+      }
       files.push(full);
     }
   }
@@ -438,7 +437,7 @@ export async function uploadAuto({ host, apiKey, projectId, filePath, fileName }
 
     const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memorylake-upload-'));
     try {
-      extractArchive(filePath, tmpDir);
+      await extractArchive(filePath, tmpDir);
       const files = collectFiles(tmpDir);
 
       if (files.length === 0) {