memorylake-openclaw 1.1.0 → 1.1.2

package/CHANGELOG.md

@@ -0,0 +1,55 @@
+# Changelog
+
+## v1.1.0 — 2026-04-21
+
+Major release introducing **model-driven memory recall**, **cross-platform archive upload**, and a full modular refactor of the plugin.
+
+### Features
+
+#### Memory & retrieval
+- **Model-driven recall** (#27): replaced server-side prefetch/inject with instructions that prompt the model to call `retrieve_context` itself — eliminates query-rewrite + search latency on every turn.
+- **Renamed** `memory_search` → `retrieve_context`; the tool now searches **both memories and documents in a single call**, running provider searches in parallel via `Promise.allSettled`.
+- **Memory conflicts** are surfaced in `retrieve_context` results (previously only in auto-recall path).
+- **Per-turn `[MEMORYLAKE REMINDER]`** prepended to each user turn to keep retrieval enforcement alive in long conversations where the system prompt is far away.
+
+#### Upload & documents
+- New **`openclaw memorylake upload`** CLI command (`--agent`, `--project-id`).
+- **Auto-detect files / archives / directories** — a single path can be a plain file, an archive, or a directory tree.
+- **Cross-platform archive extraction via npm packages** (no external tools required):
+  - zip, tar, tar.gz, tgz — `adm-zip` + `tar`
+  - 7z, bz2, tar.bz2 — `7zip-min` (bundled 7za binaries)
+  - rar — `node-unrar-js` (WASM)
+  - xz, tar.xz, txz — `xz-decompress` (streaming, OOM-safe)
+- New **`document_download`** tool: streams file to `{workspaceDir}/.memorylake/downloads/`, extracts filename from `Content-Disposition` (supports RFC 5987 / Unicode).
+- Sliding-window concurrency pool: 10 uploads in flight, next file starts as soon as one finishes.
+
+#### Config & install
+- **Hot-reload global config** (#35): each tool/hook call re-reads `~/.openclaw/openclaw.json`, so `apiKey` / `projectId` / `host` / `topK` changes take effect without gateway restart.
+- **Installer auto-sets `tools.profile = "full"`** (#34), prompting before overwriting a non-`full` value.
+- **Installer `--host` / `-MLHost` / `MEMORYLAKE_HOST`** (#36) for overriding the MemoryLake host URL.
+- Plugin version now included in memory metadata.
+
+### Refactor
+- Monolithic `index.ts` (~2190 lines) split into focused modules under `lib/`:
+  - `types.ts`, `config.ts`, `provider.ts`, `plugin-context.ts`, `core-bridge.ts`
+  - `tools/` (memory-tools, document-tools, search-tools)
+  - `hooks/` (auto-recall, auto-capture, auto-upload)
+  - `cli/`, `prompt/`, `helpers/`, `utils/`
+- `index.ts` is now ~60 lines — plugin skeleton only.
+
+### Removed
+- `document_search` tool — redundant with `retrieve_context`.
+- `memory_get` tool — rarely needed.
+
+### Fixes
+- Strip `[MEMORYLAKE REMINDER]` prefix before auto-capturing user messages so reminders don't leak into stored memories.
+- `execSync` → `execFileSync` to prevent command injection via malicious filenames in archive extraction.
+- Filter junk files (`.DS_Store`, `__MACOSX`, `Thumbs.db`, dotfiles) when collecting from archives/directories.
+- Await `uploadMany` in archive handler so temp dir isn't deleted before uploads finish.
+- Make mandatory `retrieve_context` language conditional on `autoRecall` being enabled.
+- 500-file guard on directory collection to avoid accidental `node_modules` uploads.
+- Plugin manifest: added `name`, `description`, `configSchema`; added `compat` / `build` fields for ClawHub publishing.
+
+### Manifest
+- `package.json`: `1.0.1` → `1.1.0`.
+- Tag: [`v1.1.0`](https://github.com/memorylake-ai/memorylake-openclaw/releases/tag/v1.1.0).

package/lib/cli/register-cli.ts

@@ -1,10 +1,10 @@
-import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import type { PluginContext } from "../plugin-context";
 import type { MemoryLakeConfig, UploadFn } from "../types";
 import { getProvider } from "../provider";
 import { buildSearchOptions } from "../utils/builders";
+import { readJson5ConfigFile } from "../utils/config-parser";
 
 export function registerCli(pctx: PluginContext, cfg: MemoryLakeConfig): void {
   const { api, resolveConfig } = pctx;
@@ -58,7 +58,7 @@ export function registerCli(pctx: PluginContext, cfg: MemoryLakeConfig): void {
           if (opts.agent) {
             try {
               const openclawPath = path.join(os.homedir(), ".openclaw", "openclaw.json");
-              const openclaw = JSON.parse(fs.readFileSync(openclawPath, "utf-8"));
+              const openclaw = readJson5ConfigFile(openclawPath) as any;
               const agents = openclaw?.agents;
               const agentEntry = agents?.list?.find((a: any) => a.id === opts.agent);
               const workspace = agentEntry?.workspace || agents?.defaults?.workspace;

package/lib/plugin-context.ts

@@ -4,6 +4,7 @@ import os from "node:os";
 import type { OpenClawPluginApi } from "openclaw/plugin-sdk";
 import type { MemoryLakeConfig } from "./types";
 import { ALLOWED_KEYS, memoryLakeConfigSchema } from "./config";
+import { readJson5ConfigFile } from "./utils/config-parser";
 
 const PLUGIN_ID = "memorylake-openclaw";
 const GLOBAL_CONFIG_PATH = path.join(os.homedir(), ".openclaw", "openclaw.json");
@@ -15,7 +16,7 @@ const GLOBAL_CONFIG_PATH = path.join(os.homedir(), ".openclaw", "openclaw.json")
  */
 function readGlobalConfig(logger: OpenClawPluginApi["logger"]): MemoryLakeConfig | null {
   try {
-    const raw = JSON.parse(fs.readFileSync(GLOBAL_CONFIG_PATH, "utf-8"));
+    const raw = readJson5ConfigFile(GLOBAL_CONFIG_PATH) as any;
     const pluginCfg = raw?.plugins?.entries?.[PLUGIN_ID]?.config;
     if (!pluginCfg) {
       logger.info(`memorylake-openclaw: no plugin config found in global config (path: ${GLOBAL_CONFIG_PATH}, pluginId: ${PLUGIN_ID})`);

package/lib/types.ts

@@ -123,6 +123,7 @@ export interface DocumentSearchResult {
   title?: string;
   footnote?: string;
   sheet_name?: string;
+  semantic_sheet_name?: string;
   figure_id?: number;
 }
 

package/lib/utils/builders.ts

@@ -21,7 +21,9 @@ export function buildDocumentContext(
 
     if (result.type === "table") {
       const title = result.title || "Untitled Table";
-      parts.push(`### Table: ${title} (from ${source}, doc_id: ${docId})`);
+      const sheetLabel = result.semantic_sheet_name || result.sheet_name;
+      const sheetPart = sheetLabel ? `, sheet: ${sheetLabel}` : "";
+      parts.push(`### Table: ${title} (from ${source}${sheetPart}, doc_id: ${docId})`);
       if (result.footnote) parts.push(`Note: ${result.footnote}`);
 
       for (const innerTable of highlight?.inner_tables ?? []) {

package/lib/utils/config-parser.ts

@@ -0,0 +1,14 @@
+import fs from "node:fs";
+import JSON5 from "json5";
+
+/**
+ * Read and parse a JSON5-compatible config file.
+ */
+export function readJson5ConfigFile(filePath: string): unknown {
+  const source = fs.readFileSync(filePath, "utf-8");
+  try {
+    return JSON5.parse(source);
+  } catch (err) {
+    throw new Error(`Failed to parse JSON5 config file "${filePath}": ${String(err)}`);
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memorylake-openclaw",
-  "version": "1.1.0",
+  "version": "1.1.2",
   "type": "module",
   "description": "MemoryLake memory backend for OpenClaw",
   "license": "MIT",
@@ -20,6 +20,7 @@
     "7zip-min": "^3.0.1",
     "adm-zip": "^0.5.17",
     "got": "^14.0.0",
+    "json5": "^2.2.3",
     "node-unrar-js": "^2.0.2",
     "tar": "^7.5.13",
     "xz-decompress": "^0.2.3"

package/skills/common/get-config.mjs

@@ -3,6 +3,7 @@
 import { readFileSync, existsSync } from "node:fs";
 import { join } from "node:path";
 import { homedir } from "node:os";
+import JSON5 from "json5";
 
 // Parse --agent
 const args = process.argv.slice(2);
@@ -15,7 +16,13 @@ const agentId = args[agentIdx + 1];
 
 // Read global config
 const openclawPath = join(homedir(), ".openclaw", "openclaw.json");
-const openclaw = JSON.parse(readFileSync(openclawPath, "utf-8"));
+let openclaw;
+try {
+  openclaw = JSON5.parse(readFileSync(openclawPath, "utf-8"));
+} catch (err) {
+  console.error(`Error: failed to parse JSON5 config file "${openclawPath}": ${String(err)}`);
+  process.exit(1);
+}
 const globalCfg = openclaw?.plugins?.entries?.["memorylake-openclaw"]?.config;
 if (!globalCfg) {
   console.error("Error: memorylake-openclaw plugin config not found");
@@ -35,9 +42,14 @@ if (!workspace) {
 const merged = { ...globalCfg };
 const localPath = join(workspace, ".memorylake", "config.json");
 if (existsSync(localPath)) {
-  const raw = JSON.parse(readFileSync(localPath, "utf-8"));
-  if (raw && typeof raw === "object" && !Array.isArray(raw)) {
-    Object.assign(merged, raw);
+  try {
+    const raw = JSON.parse(readFileSync(localPath, "utf-8"));
+    if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+      Object.assign(merged, raw);
+    }
+  } catch (err) {
+    console.error(`Error: failed to parse workspace config at ${localPath}: ${String(err)}`);
+    process.exit(1);
   }
 }
 merged.host = merged.host || "https://app.memorylake.ai";

package/test/json5_config_smoke.test.mjs

@@ -0,0 +1,104 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const repoRoot = resolve(process.cwd());
+const getConfigScript = join(repoRoot, "skills/common/get-config.mjs");
+const pluginContextSource = join(repoRoot, "lib/plugin-context.ts");
+const registerCliSource = join(repoRoot, "lib/cli/register-cli.ts");
+
+function runGetConfig(homeDir, agentId = "a1") {
+  return spawnSync("node", [getConfigScript, "--agent", agentId], {
+    env: { ...process.env, HOME: homeDir },
+    encoding: "utf8",
+  });
+}
+
+describe("json5 config smoke", () => {
+  it("accepts JSON5 openclaw.json in get-config.mjs", () => {
+    const root = mkdtempSync(join(tmpdir(), "ml-json5-ok-"));
+    const home = root;
+    const workspace = join(root, "workspace");
+    mkdirSync(join(home, ".openclaw"), { recursive: true });
+    mkdirSync(join(workspace, ".memorylake"), { recursive: true });
+
+    writeFileSync(
+      join(home, ".openclaw", "openclaw.json"),
+      `{
+  // allow comments
+  plugins: {
+    entries: {
+      "memorylake-openclaw": {
+        config: {
+          apiKey: "k",
+          projectId: "p",
+          host: "https://app.memorylake.ai",
+        },
+      },
+    },
+  },
+  agents: {
+    list: [{ id: "a1", workspace: "${workspace.replaceAll("\\", "\\\\")}" }],
+  },
+}
+`,
+    );
+    writeFileSync(join(workspace, ".memorylake", "config.json"), JSON.stringify({ topK: 5 }));
+
+    const result = runGetConfig(home);
+    assert.equal(result.status, 0, result.stderr);
+    const parsed = JSON.parse(result.stdout);
+    assert.equal(parsed.projectId, "p");
+    assert.equal(parsed.workspace, workspace);
+    assert.equal(parsed.topK, 5);
+  });
+
+  it("returns clear non-zero error for malformed JSON5 global config", () => {
+    const root = mkdtempSync(join(tmpdir(), "ml-json5-bad-global-"));
+    const home = root;
+    mkdirSync(join(home, ".openclaw"), { recursive: true });
+    writeFileSync(join(home, ".openclaw", "openclaw.json"), "{ invalid json5 }");
+
+    const result = runGetConfig(home);
+    assert.notEqual(result.status, 0);
+    assert.match(result.stderr, /failed to parse JSON5 config file/);
+  });
+
+  it("keeps workspace override as strict JSON", () => {
+    const root = mkdtempSync(join(tmpdir(), "ml-json5-bad-local-"));
+    const home = root;
+    const workspace = join(root, "workspace");
+    mkdirSync(join(home, ".openclaw"), { recursive: true });
+    mkdirSync(join(workspace, ".memorylake"), { recursive: true });
+
+    writeFileSync(
+      join(home, ".openclaw", "openclaw.json"),
+      JSON.stringify({
+        plugins: {
+          entries: {
+            "memorylake-openclaw": {
+              config: { apiKey: "k", projectId: "p", host: "https://app.memorylake.ai" },
+            },
+          },
+        },
+        agents: { list: [{ id: "a1", workspace }] },
+      }),
+    );
+    writeFileSync(join(workspace, ".memorylake", "config.json"), "{ trailing: 1, }");
+
+    const result = runGetConfig(home);
+    assert.notEqual(result.status, 0);
+    assert.match(result.stderr, /failed to parse workspace config/);
+  });
+
+  it("ensures plugin and CLI global config paths use shared JSON5 parser", () => {
+    const pluginContext = readFileSync(pluginContextSource, "utf8");
+    const registerCli = readFileSync(registerCliSource, "utf8");
+
+    assert.match(pluginContext, /readJson5ConfigFile\(GLOBAL_CONFIG_PATH\)/);
+    assert.match(registerCli, /readJson5ConfigFile\(openclawPath\)/);
+  });
+});