memorylake-openclaw 1.0.2-beta.1 → 1.0.2-beta.3

package/index.ts

@@ -7,7 +7,7 @@
  * - 9 tools: memory_search, memory_list, memory_store, memory_get, memory_forget, document_search, document_download, advanced_web_search, open_data_search
  * - Auto-recall: injects relevant memories and document excerpts before each agent turn
  * - Auto-capture: stores key facts scoped to the current session after each agent turn
- * - CLI: openclaw memorylake search, openclaw memorylake stats
+ * - CLI: openclaw memorylake search, openclaw memorylake stats, openclaw memorylake upload
  */
 
 import fs from "node:fs";
@@ -1325,7 +1325,7 @@ const memoryPlugin = {
         name: "document_download",
         label: "Document Download",
         description:
-          "Download a document (image, PDF, etc.) from MemoryLake and send it to the user. Returns a temporary download URL. Use document_search first to find the document_id.",
+          "Download a document (image, PDF, etc.) from MemoryLake and get a temporary download URL. After calling this tool, you MUST call the `message` tool with action='send' and media=<the returned URL> to deliver the file to the user.",
         parameters: Type.Object({
           documentId: Type.String({
             description:
@@ -1345,13 +1345,10 @@ const memoryPlugin = {
               content: [
                 {
                   type: "text",
-                  text: `Document ${documentId} is ready. Tell the user you are sending the file now.\nMEDIA: ${downloadUrl}`,
+                  text: `Document ${documentId} ready. Download URL (expires in ~20 minutes):\n${downloadUrl}\n\nYou MUST now call the message tool with action="send" and media set to this URL to deliver the file to the user.`,
                 },
               ],
-              details: {
-                media: { mediaUrl: downloadUrl },
-                documentId,
-              },
+              details: { documentId },
             };
           } catch (err) {
             return {
@@ -1684,6 +1681,65 @@ const memoryPlugin = {
             }
           });
 
+        memorylake
+          .command("upload")
+          .description("Upload files or directories to MemoryLake")
+          .argument("<path>", "File or directory path to upload")
+          .option("--agent <id>", "Agent ID (resolves workspace and per-agent projectId)")
+          .option("--project-id <id>", "Override project ID (takes precedence over --agent)")
+          .action(async (targetPath: string, opts: { agent?: string; projectId?: string }) => {
+            // Resolve effective config: --project-id > agent workspace config > global config
+            let effectiveCfg = cfg;
+            if (opts.agent) {
+              try {
+                const openclawPath = path.join(os.homedir(), ".openclaw", "openclaw.json");
+                const openclaw = JSON.parse(fs.readFileSync(openclawPath, "utf-8"));
+                const agents = openclaw?.agents;
+                const agentEntry = agents?.list?.find((a: any) => a.id === opts.agent);
+                const workspace = agentEntry?.workspace || agents?.defaults?.workspace;
+                if (workspace) {
+                  effectiveCfg = resolveConfig({ workspaceDir: workspace });
+                } else {
+                  console.warn(`Warning: no workspace found for agent "${opts.agent}", using global config.`);
+                }
+              } catch (err) {
+                console.warn(`Warning: failed to resolve agent config: ${String(err)}, using global config.`);
+              }
+            }
+            const effectiveProjectId = opts.projectId || effectiveCfg.projectId;
+            if (!effectiveProjectId) {
+              console.error("No project ID configured. Use --project-id or set up agent/workspace config.");
+              return;
+            }
+
+            // Lazy import upload.mjs (use uploadAuto to support archives)
+            let uploadFn: (o: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>;
+            try {
+              const uploadModule = await import(
+                /* webpackIgnore: true */
+                new URL("./skills/memorylake-upload/scripts/upload.mjs", import.meta.url).href
+              );
+              uploadFn = uploadModule.uploadAuto;
+            } catch (err) {
+              console.error(`Failed to load upload module: ${String(err)}`);
+              return;
+            }
+
+            const absPath = path.resolve(targetPath);
+
+            try {
+              await uploadFn({
+                host: effectiveCfg.host,
+                apiKey: effectiveCfg.apiKey,
+                projectId: effectiveProjectId,
+                filePath: absPath,
+                fileName: path.basename(absPath),
+              });
+            } catch (err) {
+              console.error(`Upload failed: ${String(err)}`);
+            }
+          });
+
         memorylake
           .command("stats")
           .description("Show memory statistics from MemoryLake")
@@ -1760,7 +1816,7 @@ const memoryPlugin = {
     // Auto-upload: upload inbound files to MemoryLake before prompt build
     if (cfg.autoUpload) {
       // Lazy-load upload function from upload.mjs
-      let uploadFn: ((opts: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>) | undefined;
+      let uploadAutoFn: ((opts: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>) | undefined;
 
       api.on("before_prompt_build", (event, ctx) => {
         if ((ctx as any)?.trigger !== "user") {
@@ -1785,17 +1841,17 @@ const memoryPlugin = {
         // Fire-and-forget: upload asynchronously without blocking
         (async () => {
           // Lazy import upload.mjs
-          if (!uploadFn) {
+          if (!uploadAutoFn) {
             const uploadModule = await import(
               /* webpackIgnore: true */
               new URL("./skills/memorylake-upload/scripts/upload.mjs", import.meta.url).href
             );
-            uploadFn = uploadModule.upload;
+            uploadAutoFn = uploadModule.uploadAuto;
           }
 
           for (const { filePath, stat } of filesToUpload) {
             try {
-              await uploadFn!({
+              await uploadAutoFn!({
                 host: effectiveCfg.host,
                 apiKey: effectiveCfg.apiKey,
                 projectId: effectiveCfg.projectId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memorylake-openclaw",
-  "version": "1.0.2-beta.1",
+  "version": "1.0.2-beta.3",
   "type": "module",
   "description": "MemoryLake memory backend for OpenClaw",
   "license": "MIT",

package/skills/memorylake-upload/SKILL.md

@@ -1,18 +1,20 @@
 ---
 name: memorylake-upload
-description: Use when the user wants to upload files, documents, PDFs, or other data files to MemoryLake and associate them with a project.
+description: Use when the user wants to upload files, documents, PDFs, archives, directories, or other data to MemoryLake. Supports single files, archives (zip, tar.gz, tgz, tar.bz2, tar, rar, 7z, etc.) which are extracted then uploaded, and directories which are recursively uploaded.
 ---
 
 # MemoryLake File Upload
 
 ## Overview
 
-Upload local files to MemoryLake using the multipart upload API, then associate them with a project.
+Upload local files, archives, or directories to MemoryLake using the multipart upload API, then associate them with a project. Archives are automatically detected, extracted, and each file inside is uploaded individually. Directories are recursively traversed and all files inside are uploaded.
 
 ## When to Use
 
 - User wants to upload a file (PDF, DOCX, image, etc.) to MemoryLake
 - User wants to add a local document to a MemoryLake project
+- User wants to upload an archive (zip, tar.gz, tgz, tar.bz2, tbz2, tar.xz, txz, tar, gz, bz2, xz, rar, 7z) — files inside will be extracted and uploaded one by one
+- User wants to upload an entire directory/folder — all files will be recursively collected and uploaded
 
 ## Step 1 -- Read MemoryLake Config
 
@@ -30,22 +32,39 @@ If the script exits with an error, stop and inform the user.
 
 The upload script is at `scripts/upload.mjs` relative to **this skill's SKILL.md**.
 
+The script automatically detects the input type and handles it accordingly — plain files are uploaded directly, archives are extracted first, and directories are recursively traversed.
+
 ```bash
+# Single file
 node {path-to-this-skill}/scripts/upload.mjs \
   --host {host} \
   --api-key {apiKey} \
   --project-id {projectId} \
   --file-name {fileName} \
   /path/to/file
+
+# Archive (auto-detected, extracted, each file uploaded)
+node {path-to-this-skill}/scripts/upload.mjs \
+  --host {host} \
+  --api-key {apiKey} \
+  --project-id {projectId} \
+  /path/to/archive.zip
+
+# Directory (recursively uploads all files inside)
+node {path-to-this-skill}/scripts/upload.mjs \
+  --host {host} \
+  --api-key {apiKey} \
+  --project-id {projectId} \
+  /path/to/my-folder/
 ```
 
-`--file-name` is the original file name as provided by the user (e.g., `report-Q1.pdf`). This is required because the local file path may be a temp path or renamed file that doesn't reflect the real name.
+`--file-name` is the original file name as provided by the user (e.g., `report-Q1.pdf`). This is required because the local file path may be a temp path or renamed file that doesn't reflect the real name. For archives and directories, `--file-name` is not needed since individual file names are taken from the contents.
 
 ## Step 3 -- Handle Output
 
-The script prints progress for each step (create upload, upload parts, complete, add to project).
+The script prints progress for each step (create upload, upload parts, complete, add to project). For archives and directories, it additionally prints a summary of succeeded/failed uploads.
 
-- **Success**: Report the document ID and file name to the user
+- **Success**: Report the document ID and file name to the user. For archives/directories, report the total count and list each uploaded file
 - **Failure**: The script prints the specific error (file not found, auth failed, API error). Read the error message and relay it to the user — don't guess the cause
 
 ## Common Mistakes

package/skills/memorylake-upload/scripts/upload.mjs

@@ -20,6 +20,8 @@ import fs from 'fs';
 import path from 'path';
 import https from 'https';
 import http from 'http';
+import os from 'os';
+import { execFileSync } from 'child_process';
 
 // API base path
 const API_BASE = '/openapi/memorylake';
@@ -228,6 +230,233 @@ export async function upload({ host, apiKey, projectId, filePath, fileName }) {
   return doc;
 }
 
+// ============================================================================
+// Archive support
+// ============================================================================
+
+const ARCHIVE_EXTENSIONS = [
+  '.tar.gz', '.tgz', '.tar.bz2', '.tbz2', '.tar.xz', '.txz',
+  '.tar', '.zip', '.gz', '.bz2', '.xz', '.rar', '.7z',
+];
+
+/**
+ * Check if a file is an archive based on its extension.
+ * Returns the matched extension or null.
+ */
+function getArchiveExtension(filePath) {
+  const lower = filePath.toLowerCase();
+  // Check compound extensions first (e.g. .tar.gz before .gz)
+  for (const ext of ARCHIVE_EXTENSIONS) {
+    if (lower.endsWith(ext)) return ext;
+  }
+  return null;
+}
+
+/** Max buffer size for single-file decompression (gz/bz2/xz) stdout capture: 512 MB */
+const DECOMPRESS_MAX_BUFFER = 512 * 1024 * 1024;
+
+/**
+ * Extract an archive to the given destination directory.
+ */
+const TOOL_INSTALL_HINTS = {
+  unzip:   'Install via: sudo apt install unzip (Linux) / brew install unzip (macOS) / choco install unzip (Windows)',
+  tar:     'tar should be pre-installed on most systems (Linux/macOS/Windows 10+)',
+  gunzip:  'Install via: sudo apt install gzip (Linux) / brew install gzip (macOS) / choco install gzip (Windows)',
+  bunzip2: 'Install via: sudo apt install bzip2 (Linux) / brew install bzip2 (macOS) / choco install bzip2 (Windows)',
+  xz:      'Install via: sudo apt install xz-utils (Linux) / brew install xz (macOS) / choco install xz (Windows)',
+  unrar:   'Install via: sudo apt install unrar (Linux) / brew install unrar (macOS) / choco install unrar (Windows)',
+  '7z':    'Install via: sudo apt install p7zip-full (Linux) / brew install 7zip (macOS) / choco install 7zip (Windows)',
+};
+
+function runExtractCommand(tool, args) {
+  try {
+    execFileSync(tool, args, { stdio: 'pipe' });
+  } catch (err) {
+    if (err.code === 'ENOENT') {
+      const hint = TOOL_INSTALL_HINTS[tool] || '';
+      throw new Error(`"${tool}" not found. ${hint}`);
+    }
+    throw err;
+  }
+}
+
+function extractArchive(archivePath, destDir) {
+  const ext = getArchiveExtension(archivePath);
+
+  switch (ext) {
+    case '.zip':
+      runExtractCommand('unzip', ['-o', archivePath, '-d', destDir]);
+      break;
+    case '.tar.gz':
+    case '.tgz':
+      runExtractCommand('tar', ['-xzf', archivePath, '-C', destDir]);
+      break;
+    case '.tar.bz2':
+    case '.tbz2':
+      runExtractCommand('tar', ['-xjf', archivePath, '-C', destDir]);
+      break;
+    case '.tar.xz':
+    case '.txz':
+      runExtractCommand('tar', ['-xJf', archivePath, '-C', destDir]);
+      break;
+    case '.tar':
+      runExtractCommand('tar', ['-xf', archivePath, '-C', destDir]);
+      break;
+    case '.gz': {
+      const baseName = path.basename(archivePath).slice(0, -3);
+      try {
+        const output = execFileSync('gunzip', ['-c', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
+        fs.writeFileSync(path.join(destDir, baseName), output);
+      } catch (err) {
+        if (err.code === 'ENOENT') throw new Error(`"gunzip" not found. ${TOOL_INSTALL_HINTS.gunzip}`);
+        throw err;
+      }
+      break;
+    }
+    case '.bz2': {
+      const baseName = path.basename(archivePath).slice(0, -4);
+      try {
+        const output = execFileSync('bunzip2', ['-c', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
+        fs.writeFileSync(path.join(destDir, baseName), output);
+      } catch (err) {
+        if (err.code === 'ENOENT') throw new Error(`"bunzip2" not found. ${TOOL_INSTALL_HINTS.bunzip2}`);
+        throw err;
+      }
+      break;
+    }
+    case '.xz': {
+      const baseName = path.basename(archivePath).slice(0, -3);
+      try {
+        const output = execFileSync('xz', ['-dc', archivePath], { maxBuffer: DECOMPRESS_MAX_BUFFER });
+        fs.writeFileSync(path.join(destDir, baseName), output);
+      } catch (err) {
+        if (err.code === 'ENOENT') throw new Error(`"xz" not found. ${TOOL_INSTALL_HINTS.xz}`);
+        throw err;
+      }
+      break;
+    }
+    case '.rar':
+      runExtractCommand('unrar', ['x', '-o+', archivePath, destDir + '/']);
+      break;
+    case '.7z':
+      runExtractCommand('7z', ['x', archivePath, `-o${destDir}`, '-y']);
+      break;
+    default:
+      throw new Error(`Unsupported archive format: ${path.basename(archivePath)}`);
+  }
+}
+
+/**
+ * Names and directories to skip when collecting files.
+ */
+const IGNORED_NAMES = new Set(['.DS_Store', 'Thumbs.db', 'desktop.ini']);
+const IGNORED_DIRS = new Set(['__MACOSX', '.Spotlight-V100', '.Trashes']);
+
+/**
+ * Recursively collect all files in a directory, skipping hidden/system junk.
+ */
+function collectFiles(dir) {
+  const files = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (IGNORED_NAMES.has(entry.name)) continue;
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
+      files.push(...collectFiles(full));
+    } else if (entry.isFile()) {
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/**
+ * Upload multiple files with progress summary.
+ * @param {Array<string>} files - absolute paths of files to upload
+ * @param {string} label - label for the summary line (e.g. "Archive" / "Directory")
+ */
+async function uploadMany({ host, apiKey, projectId, files, label }) {
+  const results = [];
+  let success = 0;
+  let failed = 0;
+
+  for (const f of files) {
+    try {
+      const doc = await upload({
+        host, apiKey, projectId,
+        filePath: f,
+        fileName: path.basename(f),
+      });
+      results.push(doc);
+      success++;
+    } catch (err) {
+      console.error(`Failed to upload ${path.basename(f)}: ${err.message}`);
+      failed++;
+    }
+  }
+
+  console.log(`\n${label} upload complete: ${success} succeeded, ${failed} failed.`);
+  return results;
+}
+
+/**
+ * Smart upload: automatically handles plain files, archives, and directories.
+ *
+ * - Plain file  → upload directly
+ * - Archive     → extract to temp dir, upload each extracted file
+ * - Directory   → recursively collect files, upload each
+ *
+ * Returns a single doc object for plain files, or an array for multi-file cases.
+ */
+export async function uploadAuto({ host, apiKey, projectId, filePath, fileName }) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Path not found: ${filePath}`);
+  }
+
+  const stat = fs.statSync(filePath);
+
+  // --- Directory ---
+  if (stat.isDirectory()) {
+    const displayName = fileName || path.basename(filePath);
+    console.log(`\nUploading directory: ${displayName}`);
+
+    const files = collectFiles(filePath);
+    if (files.length === 0) {
+      console.log('Directory is empty, nothing to upload.');
+      return [];
+    }
+    console.log(`Found ${files.length} file(s)\n`);
+
+    return uploadMany({ host, apiKey, projectId, files, label: 'Directory' });
+  }
+
+  // --- Archive ---
+  if (getArchiveExtension(filePath)) {
+    const displayName = fileName || path.basename(filePath);
+    console.log(`\nDetected archive: ${displayName}`);
+    console.log('Extracting...');
+
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memorylake-upload-'));
+    try {
+      extractArchive(filePath, tmpDir);
+      const files = collectFiles(tmpDir);
+
+      if (files.length === 0) {
+        console.log('Archive is empty, nothing to upload.');
+        return [];
+      }
+      console.log(`Extracted ${files.length} file(s)\n`);
+
+      return await uploadMany({ host, apiKey, projectId, files, label: 'Archive' });
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+  }
+
+  // --- Plain file ---
+  return upload({ host, apiKey, projectId, filePath, fileName });
+}
+
 // CLI entry point
 async function main() {
   const args = process.argv.slice(2);
@@ -237,17 +466,21 @@ async function main() {
 MemoryLake File Upload
 
 Usage:
-  node upload.mjs --host <url> --api-key <key> --project-id <id> <file_path>
+  node upload.mjs --host <url> --api-key <key> --project-id <id> <path>
 
 Arguments:
   --host       Base URL (e.g., http://10.71.10.71:3002)
   --api-key    API key for authentication
   --project-id Project ID to associate the document with (required)
-  --file-name  Custom file name (default: basename of file_path)
-  file_path    Path to the file to upload
+  --file-name  Custom file name (default: basename of path)
+  path         Path to a file, archive, or directory to upload
+
+Archives are auto-detected and extracted; directories are recursively uploaded.
 
 Examples:
   node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 document.pdf
+  node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 docs.zip
+  node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 ./my-folder/
 `);
     process.exit(0);
   }
@@ -262,7 +495,7 @@ Examples:
   }
 
   try {
-    await upload({ host, apiKey, projectId, filePath, fileName });
+    await uploadAuto({ host, apiKey, projectId, filePath, fileName });
     console.log('\nDone!\n');
   } catch (err) {
     console.error(`\nError: ${err.message}\n`);