memorylake-openclaw 1.0.2-beta.4 → 1.0.2-beta.6

package/index.ts

@@ -5,9 +5,9 @@
  *
  * Features:
  * - 9 tools: memory_search, memory_list, memory_store, memory_get, memory_forget, document_search, document_download, advanced_web_search, open_data_search
- * - Auto-recall: injects memory-search instructions (+ open-data categories) into the system prompt before each agent turn
+ * - Auto-recall: injects relevant memories and document excerpts before each agent turn
  * - Auto-capture: stores key facts scoped to the current session after each agent turn
- * - CLI: openclaw memorylake search, openclaw memorylake stats
+ * - CLI: openclaw memorylake search, openclaw memorylake stats, openclaw memorylake upload
  */
 
 import fs from "node:fs";
@@ -15,6 +15,7 @@ import fsPromises from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import got from "got";
+import { pipeline } from "node:stream/promises";
 import { Type } from "@sinclair/typebox";
 import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
 import { loadCoreAgentDeps } from "./core-bridge";
@@ -780,6 +781,15 @@ const memoryLakeConfigSchema = {
 // Plugin Definition
 // ============================================================================
 
+/** Shared type for the upload / uploadAuto function signature */
+type UploadFn = (opts: {
+  host: string;
+  apiKey: string;
+  projectId: string;
+  filePath: string;
+  fileName: string;
+}) => Promise<unknown>;
+
 const memoryPlugin = {
   id: "memorylake-openclaw",
   name: "Memory (MemoryLake)",
@@ -994,23 +1004,6 @@ const memoryPlugin = {
               content: r.content,
               created_at: r.created_at,
             }));
-
-            // Check for unresolved conflicts among returned memories
-            const conflictMemoryIds = results
-              .filter((r) => r.has_unresolved_conflict)
-              .map((r) => r.id);
-            if (conflictMemoryIds.length > 0) {
-              try {
-                const effectiveUserId = userId ?? effectiveCfg.userId;
-                const conflicts = await effectiveProvider.listConflicts(conflictMemoryIds, effectiveUserId);
-                if (conflicts.length > 0) {
-                  const conflictText = buildConflictContext(conflicts);
-                  sections.push(`## Memory Conflicts\nThe following memories have unresolved conflicts. Review and help the user resolve them if relevant:\n\n${conflictText}`);
-                }
-              } catch (err) {
-                sections.push(`## Memory Conflicts\nFailed to fetch conflicts: ${String(err)}`);
-              }
-            }
           } else if (memoryResult.status === "rejected") {
             sections.push(`## Memories\nMemory search failed: ${String(memoryResult.reason)}`);
           }
@@ -1028,7 +1021,7 @@ const memoryPlugin = {
               content: [
                 { type: "text", text: "No relevant memories or documents found." },
               ],
-              details: { memoryCount: 0, documentCount: 0, memories: [] },
+              details: { count: 0 },
             };
           }
 
@@ -1337,35 +1330,76 @@ const memoryPlugin = {
       { name: "document_search" },
     );
 
+    /**
+     * Try to extract a usable filename from a pre-signed download URL.
+     * Returns null if the URL doesn't contain a recognizable filename.
+     */
+    function fileNameFromUrl(urlStr: string): string | null {
+      try {
+        const p = new URL(urlStr).pathname;
+        const base = path.posix.basename(p);
+        if (base && /\.\w{1,10}$/.test(base)) return decodeURIComponent(base);
+      } catch { /* ignore */ }
+      return null;
+    }
+
     api.registerTool(
       (ctx) => ({
         name: "document_download",
         label: "Document Download",
         description:
-          "Download a document (image, PDF, etc.) from MemoryLake and get a temporary download URL. After calling this tool, you MUST call the `message` tool with action='send' and media=<the returned URL> to deliver the file to the user.",
+          "Download a document (image, PDF, etc.) from MemoryLake to local disk. After calling this tool, you MUST call the `message` tool with action='send' and media=<the returned local file path> to deliver the file to the user.",
         parameters: Type.Object({
           documentId: Type.String({
             description:
               "The document ID to download (from document_search results or document listing)",
           }),
+          fileName: Type.Optional(
+            Type.String({
+              description:
+                "Original file name for saving locally (e.g. 'report.pdf'). Falls back to the name in the download URL or the document ID.",
+            }),
+          ),
         }),
         async execute(_toolCallId, params) {
           const effectiveCfg = resolveConfig(ctx);
           const effectiveProvider = getProvider(effectiveCfg);
-          const { documentId } = params as { documentId: string };
+          const { documentId, fileName } = params as {
+            documentId: string;
+            fileName?: string;
+          };
 
           try {
+            // 1. Get pre-signed download URL
             const downloadUrl =
               await effectiveProvider.getDocumentDownloadUrl(documentId);
 
+            // 2. Determine local save directory (cross-platform)
+            const workspaceDir = (ctx as any)?.workspaceDir;
+            const downloadDir = workspaceDir
+              ? path.join(workspaceDir, ".memorylake", "downloads")
+              : path.join(os.tmpdir(), "memorylake-downloads");
+            fs.mkdirSync(downloadDir, { recursive: true });
+
+            // 3. Determine filename: explicit param > URL-derived > documentId
+            const finalName =
+              fileName || fileNameFromUrl(downloadUrl) || documentId;
+            const localPath = path.join(downloadDir, finalName);
+
+            // 4. Stream download to local file
+            await pipeline(
+              got.stream(downloadUrl),
+              fs.createWriteStream(localPath),
+            );
+
             return {
               content: [
                 {
                   type: "text",
-                  text: `Document ${documentId} ready. Download URL (expires in ~20 minutes):\n${downloadUrl}\n\nYou MUST now call the message tool with action="send" and media set to this URL to deliver the file to the user.`,
+                  text: `Document ${documentId} downloaded to:\n${localPath}\n\nYou MUST now call the message tool with action="send" and media set to this local path to deliver the file to the user.`,
                 },
               ],
-              details: { documentId },
+              details: { documentId, localPath },
             };
           } catch (err) {
             return {
@@ -1698,6 +1732,69 @@ const memoryPlugin = {
             }
           });
 
+        memorylake
+          .command("upload")
+          .description("Upload files or directories to MemoryLake")
+          .argument("<path>", "File or directory path to upload")
+          .option("--agent <id>", "Agent ID (resolves workspace and per-agent projectId)")
+          .option("--project-id <id>", "Override project ID (takes precedence over --agent)")
+          .action(async (targetPath: string, opts: { agent?: string; projectId?: string }) => {
+            // Resolve effective config: --project-id > agent workspace config > global config
+            let effectiveCfg = cfg;
+            if (opts.agent) {
+              try {
+                const openclawPath = path.join(os.homedir(), ".openclaw", "openclaw.json");
+                const openclaw = JSON.parse(fs.readFileSync(openclawPath, "utf-8"));
+                const agents = openclaw?.agents;
+                const agentEntry = agents?.list?.find((a: any) => a.id === opts.agent);
+                const workspace = agentEntry?.workspace || agents?.defaults?.workspace;
+                if (workspace) {
+                  effectiveCfg = resolveConfig({ workspaceDir: workspace });
+                } else {
+                  console.warn(`Warning: no workspace found for agent "${opts.agent}", using global config.`);
+                }
+              } catch (err) {
+                console.warn(`Warning: failed to resolve agent config: ${String(err)}, using global config.`);
+              }
+            }
+            const effectiveProjectId = opts.projectId || effectiveCfg.projectId;
+            if (!effectiveProjectId) {
+              console.error("No project ID configured. Use --project-id or set up agent/workspace config.");
+              return;
+            }
+            if (!effectiveCfg.host || !effectiveCfg.apiKey) {
+              console.error("Missing host or apiKey in config. Check your MemoryLake configuration.");
+              return;
+            }
+
+            // Lazy import upload.mjs (use uploadAuto to support archives)
+            let uploadFn: UploadFn;
+            try {
+              const uploadModule = await import(
+                /* webpackIgnore: true */
+                new URL("./skills/memorylake-upload/scripts/upload.mjs", import.meta.url).href
+              );
+              uploadFn = uploadModule.uploadAuto;
+            } catch (err) {
+              console.error(`Failed to load upload module: ${String(err)}`);
+              return;
+            }
+
+            const absPath = path.resolve(targetPath);
+
+            try {
+              await uploadFn({
+                host: effectiveCfg.host,
+                apiKey: effectiveCfg.apiKey,
+                projectId: effectiveProjectId,
+                filePath: absPath,
+                fileName: path.basename(absPath),
+              });
+            } catch (err) {
+              console.error(`Upload failed: ${String(err)}`);
+            }
+          });
+
         memorylake
           .command("stats")
           .description("Show memory statistics from MemoryLake")
@@ -1774,7 +1871,7 @@ const memoryPlugin = {
     // Auto-upload: upload inbound files to MemoryLake before prompt build
     if (cfg.autoUpload) {
       // Lazy-load upload function from upload.mjs
-      let uploadFn: ((opts: { host: string; apiKey: string; projectId: string; filePath: string; fileName: string }) => Promise<unknown>) | undefined;
+      let uploadAutoFn: UploadFn | undefined;
 
       api.on("before_prompt_build", (event, ctx) => {
         if ((ctx as any)?.trigger !== "user") {
@@ -1799,17 +1896,17 @@ const memoryPlugin = {
         // Fire-and-forget: upload asynchronously without blocking
         (async () => {
           // Lazy import upload.mjs
-          if (!uploadFn) {
+          if (!uploadAutoFn) {
             const uploadModule = await import(
               /* webpackIgnore: true */
               new URL("./skills/memorylake-upload/scripts/upload.mjs", import.meta.url).href
             );
-            uploadFn = uploadModule.upload;
+            uploadAutoFn = uploadModule.uploadAuto;
           }
 
           for (const { filePath, stat } of filesToUpload) {
             try {
-              await uploadFn!({
+              await uploadAutoFn!({
                 host: effectiveCfg.host,
                 apiKey: effectiveCfg.apiKey,
                 projectId: effectiveCfg.projectId,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memorylake-openclaw",
-  "version": "1.0.2-beta.4",
+  "version": "1.0.2-beta.6",
   "type": "module",
   "description": "MemoryLake memory backend for OpenClaw",
   "license": "MIT",
@@ -17,7 +17,10 @@
   ],
   "dependencies": {
     "@sinclair/typebox": "0.34.47",
-    "got": "^14.0.0"
+    "7zip-min": "^3.0.1",
+    "adm-zip": "^0.5.17",
+    "got": "^14.0.0",
+    "tar": "^7.5.13"
   },
   "openclaw": {
     "extensions": [

package/skills/memorylake-upload/SKILL.md

@@ -1,18 +1,20 @@
 ---
 name: memorylake-upload
-description: Use when the user wants to upload files, documents, PDFs, or other data files to MemoryLake and associate them with a project.
+description: Use when the user wants to upload files, documents, PDFs, archives, directories, or other data to MemoryLake. Supports single files, archives (zip, tar.gz, tgz, tar, gz, 7z, tar.bz2, bz2) which are extracted then uploaded, and directories which are recursively uploaded.
 ---
 
 # MemoryLake File Upload
 
 ## Overview
 
-Upload local files to MemoryLake using the multipart upload API, then associate them with a project.
+Upload local files, archives, or directories to MemoryLake using the multipart upload API, then associate them with a project. Archives are automatically detected, extracted, and each file inside is uploaded individually. Directories are recursively traversed and all files inside are uploaded.
 
 ## When to Use
 
 - User wants to upload a file (PDF, DOCX, image, etc.) to MemoryLake
 - User wants to add a local document to a MemoryLake project
+- User wants to upload an archive (zip, tar.gz, tgz, tar, gz, 7z, tar.bz2, bz2) — files inside will be extracted and uploaded one by one
+- User wants to upload an entire directory/folder — all files will be recursively collected and uploaded
 
 ## Step 1 -- Read MemoryLake Config
 
@@ -30,22 +32,39 @@ If the script exits with an error, stop and inform the user.
 
 The upload script is at `scripts/upload.mjs` relative to **this skill's SKILL.md**.
 
+The script automatically detects the input type and handles it accordingly — plain files are uploaded directly, archives are extracted first, and directories are recursively traversed.
+
 ```bash
+# Single file
 node {path-to-this-skill}/scripts/upload.mjs \
   --host {host} \
   --api-key {apiKey} \
   --project-id {projectId} \
   --file-name {fileName} \
   /path/to/file
+
+# Archive (auto-detected, extracted, each file uploaded)
+node {path-to-this-skill}/scripts/upload.mjs \
+  --host {host} \
+  --api-key {apiKey} \
+  --project-id {projectId} \
+  /path/to/archive.zip
+
+# Directory (recursively uploads all files inside)
+node {path-to-this-skill}/scripts/upload.mjs \
+  --host {host} \
+  --api-key {apiKey} \
+  --project-id {projectId} \
+  /path/to/my-folder/
 ```
 
-`--file-name` is the original file name as provided by the user (e.g., `report-Q1.pdf`). This is required because the local file path may be a temp path or renamed file that doesn't reflect the real name.
+`--file-name` is the original file name as provided by the user (e.g., `report-Q1.pdf`). This is required because the local file path may be a temp path or renamed file that doesn't reflect the real name. For archives and directories, `--file-name` is not needed since individual file names are taken from the contents.
 
 ## Step 3 -- Handle Output
 
-The script prints progress for each step (create upload, upload parts, complete, add to project).
+The script prints progress for each step (create upload, upload parts, complete, add to project). For archives and directories, it additionally prints a summary of succeeded/failed uploads.
 
-- **Success**: Report the document ID and file name to the user
+- **Success**: Report the document ID and file name to the user. For archives/directories, report the total count and list each uploaded file
 - **Failure**: The script prints the specific error (file not found, auth failed, API error). Read the error message and relay it to the user — don't guess the cause
 
 ## Common Mistakes

package/skills/memorylake-upload/scripts/upload.mjs

@@ -20,6 +20,12 @@ import fs from 'fs';
 import path from 'path';
 import https from 'https';
 import http from 'http';
+import os from 'os';
+import { createGunzip } from 'zlib';
+import { pipeline } from 'stream/promises';
+import AdmZip from 'adm-zip';
+import { extract as tarExtract } from 'tar';
+import sevenZip from '7zip-min';
 
 // API base path
 const API_BASE = '/openapi/memorylake';
@@ -228,6 +234,243 @@ export async function upload({ host, apiKey, projectId, filePath, fileName }) {
   return doc;
 }
 
+// ============================================================================
+// Archive support
+// ============================================================================
+
+/** Formats we can extract with npm packages (adm-zip, tar, zlib, 7zip-min) */
+const SUPPORTED_ARCHIVE_EXTENSIONS = [
+  '.tar.gz', '.tgz', '.tar.bz2', '.tbz2',
+  '.tar', '.zip', '.gz', '.bz2', '.7z',
+];
+
+/** Recognized but unsupported — detected so we give a clear error instead of uploading raw */
+const UNSUPPORTED_ARCHIVE_EXTENSIONS = [
+  '.tar.xz', '.txz', '.xz', '.rar',
+];
+
+/** All recognized archive extensions (compound first for matching priority) */
+const ALL_ARCHIVE_EXTENSIONS = [...SUPPORTED_ARCHIVE_EXTENSIONS, ...UNSUPPORTED_ARCHIVE_EXTENSIONS];
+
+/**
+ * Check if a file is an archive based on its extension.
+ * Returns the matched extension or null.
+ */
+function getArchiveExtension(filePath) {
+  const lower = filePath.toLowerCase();
+  // Check compound extensions first (e.g. .tar.gz before .gz)
+  for (const ext of ALL_ARCHIVE_EXTENSIONS) {
+    if (lower.endsWith(ext)) return ext;
+  }
+  return null;
+}
+
+/**
+ * Promisified wrapper around 7zip-min's callback-based unpack API.
+ */
+function sevenZipExtract(archivePath, destDir) {
+  return new Promise((resolve, reject) => {
+    sevenZip.unpack(archivePath, destDir, (err) => {
+      if (err) reject(err);
+      else resolve();
+    });
+  });
+}
+
+/**
+ * Extract an archive to the given destination directory.
+ * Uses npm packages (adm-zip, tar, zlib, 7zip-min) for cross-platform compatibility.
+ */
+async function extractArchive(archivePath, destDir) {
+  const ext = getArchiveExtension(archivePath);
+
+  // Reject recognized-but-unsupported formats early with a clear message
+  if (UNSUPPORTED_ARCHIVE_EXTENSIONS.includes(ext)) {
+    throw new Error(`Unsupported archive format: ${ext}. Please convert to .zip, .tar.gz or .7z`);
+  }
+
+  console.log(`  Format: ${ext}`);
+
+  switch (ext) {
+    case '.zip': {
+      const zip = new AdmZip(archivePath);
+      zip.extractAllTo(destDir, true);
+      break;
+    }
+    case '.tar.gz':
+    case '.tgz':
+    case '.tar':
+      await tarExtract({ file: archivePath, cwd: destDir });
+      break;
+    case '.gz': {
+      const baseName = path.basename(archivePath).slice(0, -3);
+      const destPath = path.join(destDir, baseName);
+      await pipeline(
+        fs.createReadStream(archivePath),
+        createGunzip(),
+        fs.createWriteStream(destPath),
+      );
+      break;
+    }
+    case '.tar.bz2':
+    case '.tbz2': {
+      // 7za only strips the bz2 layer, producing a .tar — need a second pass
+      const tmpBz2 = fs.mkdtempSync(path.join(os.tmpdir(), 'memorylake-bz2-'));
+      try {
+        await sevenZipExtract(archivePath, tmpBz2);
+        // Find the resulting .tar inside tmpBz2
+        const inner = fs.readdirSync(tmpBz2).find(f => f.endsWith('.tar'));
+        if (inner) {
+          await tarExtract({ file: path.join(tmpBz2, inner), cwd: destDir });
+        }
+      } finally {
+        fs.rmSync(tmpBz2, { recursive: true, force: true });
+      }
+      break;
+    }
+    case '.bz2':
+    case '.7z':
+      await sevenZipExtract(archivePath, destDir);
+      break;
+    default:
+      throw new Error(`Unsupported archive format: ${path.basename(archivePath)}`);
+  }
+}
+
+/**
+ * Names and directories to skip when collecting files.
+ */
+const IGNORED_NAMES = new Set(['.DS_Store', 'Thumbs.db', 'desktop.ini']);
+const IGNORED_DIRS = new Set(['__MACOSX', '.Spotlight-V100', '.Trashes']);
+
+/** Sanity limit to prevent accidentally uploading huge trees (e.g. node_modules) */
+const MAX_COLLECT_FILES = 500;
+
+/**
+ * Recursively collect all files in a directory, skipping hidden/system junk.
+ * Throws if the file count exceeds MAX_COLLECT_FILES.
+ */
+function collectFiles(dir, _counter = { value: 0 }) {
+  const files = [];
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    if (IGNORED_NAMES.has(entry.name)) continue;
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (IGNORED_DIRS.has(entry.name) || entry.name.startsWith('.')) continue;
+      files.push(...collectFiles(full, _counter));
+    } else if (entry.isFile()) {
+      _counter.value++;
+      if (_counter.value > MAX_COLLECT_FILES) {
+        throw new Error(`Too many files (>${MAX_COLLECT_FILES}). Please upload a smaller directory or archive.`);
+      }
+      files.push(full);
+    }
+  }
+  return files;
+}
+
+/** Max number of concurrent uploads */
+const UPLOAD_CONCURRENCY = 10;
+
+/**
+ * Upload multiple files with a sliding-window concurrency pool.
+ * Up to UPLOAD_CONCURRENCY uploads run simultaneously; as soon as one
+ * finishes the next file starts immediately — no waiting for a full batch.
+ * @param {Array<string>} files - absolute paths of files to upload
+ * @param {string} label - label for the summary line (e.g. "Archive" / "Directory")
+ */
+async function uploadMany({ host, apiKey, projectId, files, label }) {
+  const results = [];
+  let success = 0;
+  let failed = 0;
+  let nextIndex = 0;
+
+  function runNext() {
+    if (nextIndex >= files.length) return null;
+    const i = nextIndex++;
+    const f = files[i];
+    return upload({
+      host, apiKey, projectId,
+      filePath: f,
+      fileName: path.basename(f),
+    }).then(doc => {
+      results.push(doc);
+      success++;
+    }).catch(err => {
+      console.error(`Failed to upload ${path.basename(f)}: ${err.message}`);
+      failed++;
+    }).then(() => runNext());   // slot freed → pick up next file immediately
+  }
+
+  // Kick off the initial pool of workers
+  const workers = [];
+  for (let w = 0; w < Math.min(UPLOAD_CONCURRENCY, files.length); w++) {
+    workers.push(runNext());
+  }
+  await Promise.all(workers);
+
+  console.log(`\n${label} upload complete: ${success} succeeded, ${failed} failed.`);
+  return results;
+}
+
+/**
+ * Smart upload: automatically handles plain files, archives, and directories.
+ *
+ * - Plain file  → upload directly
+ * - Archive     → extract to temp dir, upload each extracted file
+ * - Directory   → recursively collect files, upload each
+ *
+ * Returns a single doc object for plain files, or an array for multi-file cases.
+ */
+export async function uploadAuto({ host, apiKey, projectId, filePath, fileName }) {
+  if (!fs.existsSync(filePath)) {
+    throw new Error(`Path not found: ${filePath}`);
+  }
+
+  const stat = fs.statSync(filePath);
+
+  // --- Directory ---
+  if (stat.isDirectory()) {
+    const displayName = fileName || path.basename(filePath);
+    console.log(`\nUploading directory: ${displayName}`);
+
+    const files = collectFiles(filePath);
+    if (files.length === 0) {
+      console.log('Directory is empty, nothing to upload.');
+      return [];
+    }
+    console.log(`Found ${files.length} file(s)\n`);
+
+    return uploadMany({ host, apiKey, projectId, files, label: 'Directory' });
+  }
+
+  // --- Archive ---
+  if (getArchiveExtension(filePath)) {
+    const displayName = fileName || path.basename(filePath);
+    console.log(`\nDetected archive: ${displayName}`);
+    console.log('Extracting...');
+
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memorylake-upload-'));
+    try {
+      await extractArchive(filePath, tmpDir);
+      const files = collectFiles(tmpDir);
+
+      if (files.length === 0) {
+        console.log('Archive is empty, nothing to upload.');
+        return [];
+      }
+      console.log(`Extracted ${files.length} file(s)\n`);
+
+      return await uploadMany({ host, apiKey, projectId, files, label: 'Archive' });
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+  }
+
+  // --- Plain file ---
+  return upload({ host, apiKey, projectId, filePath, fileName });
+}
+
 // CLI entry point
 async function main() {
   const args = process.argv.slice(2);
@@ -237,17 +480,21 @@ async function main() {
 MemoryLake File Upload
 
 Usage:
-  node upload.mjs --host <url> --api-key <key> --project-id <id> <file_path>
+  node upload.mjs --host <url> --api-key <key> --project-id <id> <path>
 
 Arguments:
   --host       Base URL (e.g., http://10.71.10.71:3002)
   --api-key    API key for authentication
   --project-id Project ID to associate the document with (required)
-  --file-name  Custom file name (default: basename of file_path)
-  file_path    Path to the file to upload
+  --file-name  Custom file name (default: basename of path)
+  path         Path to a file, archive, or directory to upload
+
+Archives are auto-detected and extracted; directories are recursively uploaded.
 
 Examples:
   node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 document.pdf
+  node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 docs.zip
+  node upload.mjs --host http://10.71.10.71:3002 --api-key sk-xxx --project-id proj-abc123 ./my-folder/
 `);
     process.exit(0);
   }
@@ -262,7 +509,7 @@ Examples:
   }
 
   try {
-    await upload({ host, apiKey, projectId, filePath, fileName });
+    await uploadAuto({ host, apiKey, projectId, filePath, fileName });
     console.log('\nDone!\n');
   } catch (err) {
     console.error(`\nError: ${err.message}\n`);