memory-lancedb-pro 1.0.7 → 1.0.10

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 1.0.10
+
+- Fix: avoid blocking OpenClaw gateway startup on external network calls by running startup self-checks in the background with timeouts.
+
+## 1.0.9
+
+- Change: update default `retrieval.rerankModel` to `jina-reranker-v3` (still fully configurable).
+
+## 1.0.8
+
+- Add: JSONL distill extractor supports optional agent allowlist via env var `OPENCLAW_JSONL_DISTILL_ALLOWED_AGENT_IDS` (default off / compatible).
+
 ## 1.0.7
 
 - Fix: resolve `agentId` from hook context (`ctx?.agentId`) for `before_agent_start` and `agent_end`, restoring per-agent scope isolation when using multi-agent setups.

package/README.md

@@ -161,6 +161,32 @@ Filters out low-quality content at both auto-capture and tool-store stages:
 
 ## Installation
 
+### AI-safe install notes (anti-hallucination)
+
+If you are following this README using an AI assistant, **do not assume defaults**. Always run these commands first and use the real output:
+
+```bash
+openclaw config get agents.defaults.workspace
+openclaw config get plugins.load.paths
+openclaw config get plugins.slots.memory
+openclaw config get plugins.entries.memory-lancedb-pro
+```
+
+Recommendations:
+- Prefer **absolute paths** in `plugins.load.paths` unless you have confirmed the active workspace.
+- If you use `${JINA_API_KEY}` (or any `${...}` variable) in config, ensure the **Gateway service process** has that environment variable (system services often do **not** inherit your interactive shell env).
+- After changing plugin config, run `openclaw gateway restart`.
+
+### Jina API keys (embedding + rerank)
+
+- **Embedding**: set `embedding.apiKey` to your Jina key (recommended: use an env var like `${JINA_API_KEY}`).
+- **Rerank** (when `retrieval.rerankProvider: "jina"`): you can typically use the **same** Jina key for `retrieval.rerankApiKey`.
+- If you use a different rerank provider (`siliconflow`, `pinecone`, etc.), `retrieval.rerankApiKey` should be that provider’s key.
+
+Key storage guidance:
+- Avoid committing secrets into git.
+- Using `${...}` env vars is fine, but make sure the **Gateway service process** has those env vars (system services often do not inherit your interactive shell environment).
+
 ### What is the “OpenClaw workspace”?
 
 In OpenClaw, the **agent workspace** is the agent’s working directory (default: `~/.openclaw/workspace`).
@@ -168,7 +194,9 @@ According to the docs, the workspace is the **default cwd**, and **relative path
 
 > Note: OpenClaw configuration typically lives under `~/.openclaw/openclaw.json` (separate from the workspace).
 
-**Common mistake:** cloning the plugin somewhere else, while keeping `plugins.load.paths: ["plugins/memory-lancedb-pro"]` (a **relative path**). In that case OpenClaw will look for `plugins/memory-lancedb-pro` under your **workspace** and fail to load it.
+**Common mistake:** cloning the plugin somewhere else, while keeping a **relative path** like `plugins.load.paths: ["plugins/memory-lancedb-pro"]`. Relative paths can be resolved against different working directories depending on how the Gateway is started.
+
+To avoid ambiguity, use an **absolute path** (Option B) or clone into `<workspace>/plugins/` (Option A) and keep your config consistent.
 
 ### Option A (recommended): clone into `plugins/` under your workspace
 
@@ -285,8 +313,8 @@ openclaw config get plugins.slots.memory
     "bm25Weight": 0.3,
     "minScore": 0.3,
     "rerank": "cross-encoder",
-    "rerankApiKey": "jina_xxx",
-    "rerankModel": "jina-reranker-v2-base-multilingual",
+    "rerankApiKey": "${JINA_API_KEY}",
+    "rerankModel": "jina-reranker-v3",
     "rerankEndpoint": "https://api.jina.ai/v1/rerank",
     "rerankProvider": "jina",
     "candidatePoolSize": 20,
@@ -334,7 +362,7 @@ Cross-encoder reranking supports multiple providers via `rerankProvider`:
 
 | Provider | `rerankProvider` | Endpoint | Example Model |
 |----------|-----------------|----------|---------------|
-| **Jina** (default) | `jina` | `https://api.jina.ai/v1/rerank` | `jina-reranker-v2-base-multilingual` |
+| **Jina** (default) | `jina` | `https://api.jina.ai/v1/rerank` | `jina-reranker-v3` |
 | **SiliconFlow** (free tier available) | `siliconflow` | `https://api.siliconflow.com/v1/rerank` | `BAAI/bge-reranker-v2-m3`, `Qwen/Qwen3-Reranker-8B` |
 | **Pinecone** | `pinecone` | `https://api.pinecone.io/rerank` | `bge-reranker-v2-m3` |
 
@@ -382,7 +410,21 @@ OpenClaw already persists **full session transcripts** as JSONL files:
 
 This plugin focuses on **high-quality long-term memory**. If you dump raw transcripts into LanceDB, retrieval quality quickly degrades.
 
-Instead, you can run an **hourly distiller** that:
+Instead, **recommended (2026-02+)** is a **non-blocking `/new` pipeline**:
+
+- Trigger: `command:new` (you type `/new`)
+- Hook: enqueue a tiny JSON task file (fast; no LLM calls inside the hook)
+- Worker: a user-level systemd service watches the inbox and runs **Gemini Map-Reduce** on the session JSONL transcript
+- Store: writes **0–20** high-signal, atomic lessons into LanceDB Pro via `openclaw memory-pro import`
+- Keywords: each memory includes `Keywords (zh)` with a simple taxonomy (Entity + Action + Symptom). Entity keywords must be copied verbatim from the transcript (no hallucinated project names).
+- Notify: optional Telegram/Discord notification (even if 0 lessons)
+
+See the self-contained example files in:
+- `examples/new-session-distill/`
+
+---
+
+Legacy option: an **hourly distiller** cron that:
 
 1) Incrementally reads only the **newly appended tail** of each session JSONL (byte-offset cursor)
 2) Filters noise (tool output, injected `<relevant-memories>`, logs, boilerplate)
@@ -414,6 +456,19 @@ The script is **safe**: it never modifies session logs.
 
 By default it skips historical reset snapshots (`*.reset.*`) and excludes the distiller agent itself (`memory-distiller`) to prevent self-ingestion loops.
 
+### Optional: restrict distillation sources (allowlist)
+
+By default, the extractor scans **all agents** (except `memory-distiller`).
+
+If you want higher signal (e.g., only distill from your main assistant + coding bot), set:
+
+```bash
+export OPENCLAW_JSONL_DISTILL_ALLOWED_AGENT_IDS="main,code-agent"
+```
+
+- Unset / empty / `*` / `all` → allow all agents (default)
+- Comma-separated list → only those agents are scanned
+
 ### Recommended setup (dedicated distiller agent)
 
 #### 1) Create a dedicated agent

package/README_CN.md

@@ -112,7 +112,7 @@ Query → BM25 FTS ─────┘
 
 ### 2. 跨编码器 Rerank
 
-- **Jina Reranker API**: `jina-reranker-v2-base-multilingual`（5s 超时保护）
+- **Jina Reranker API**: `jina-reranker-v3`（5s 超时保护）
 - **混合评分**: 60% cross-encoder score + 40% 原始融合分
 - **降级策略**: API 失败时回退到 cosine similarity rerank
 
@@ -162,6 +162,32 @@ Query → BM25 FTS ─────┘
 
 ## 安装
 
+### AI 安装指引（防幻觉版）
+
+如果你是用 AI 按 README 操作，**不要假设任何默认值**。请先运行以下命令，并以真实输出为准：
+
+```bash
+openclaw config get agents.defaults.workspace
+openclaw config get plugins.load.paths
+openclaw config get plugins.slots.memory
+openclaw config get plugins.entries.memory-lancedb-pro
+```
+
+建议：
+- `plugins.load.paths` 建议优先用**绝对路径**（除非你已确认当前 workspace）。
+- 如果配置里使用 `${JINA_API_KEY}`（或任何 `${...}` 变量），务必确保运行 Gateway 的**服务进程环境**里真的有这些变量（systemd/launchd/docker 通常不会继承你终端的 export）。
+- 修改插件配置后，运行 `openclaw gateway restart` 使其生效。
+
+### Jina API Key（Embedding + Rerank）如何填写
+
+- **Embedding**：将 `embedding.apiKey` 设置为你的 Jina key（推荐用环境变量 `${JINA_API_KEY}`）。
+- **Rerank**（当 `retrieval.rerankProvider: "jina"`）：通常可以直接复用同一个 Jina key，填到 `retrieval.rerankApiKey`。
+- 如果你选择了其它 rerank provider（如 `siliconflow` / `pinecone`），则 `retrieval.rerankApiKey` 应填写对应提供商的 key。
+
+Key 存储建议：
+- 不要把 key 提交到 git。
+- 使用 `${...}` 环境变量没问题，但务必确保运行 Gateway 的**服务进程环境**里真的有该变量（systemd/launchd/docker 往往不会继承你终端的 export）。
+
 ### 什么是 “OpenClaw workspace”？
 
 在 OpenClaw 中，**agent workspace（工作区）** 是 Agent 的工作目录（默认：`~/.openclaw/workspace`）。
@@ -169,7 +195,9 @@ Query → BM25 FTS ─────┘
 
 > 说明：OpenClaw 的配置文件通常在 `~/.openclaw/openclaw.json`，与 workspace 是分开的。
 
-**最常见的安装错误：** 把插件 clone 到别的目录，但在配置里仍然写 `"paths": ["plugins/memory-lancedb-pro"]`（这是**相对路径**）。OpenClaw 会去 workspace 下找 `plugins/memory-lancedb-pro`，导致加载失败，于是出现“安装位置不对”的反馈。
+**最常见的安装错误：** 把插件 clone 到别的目录，但在配置里仍然写类似 `"paths": ["plugins/memory-lancedb-pro"]` 的**相对路径**。相对路径的解析基准会受 Gateway 启动方式/工作目录影响，容易指向错误位置。
+
+为避免歧义：建议用**绝对路径**（方案 B），或把插件放在 `<workspace>/plugins/`（方案 A）并保持配置一致。
 
 ### 方案 A（推荐）：克隆到 workspace 的 `plugins/` 目录下
 
@@ -286,8 +314,8 @@ openclaw config get plugins.slots.memory
     "bm25Weight": 0.3,
     "minScore": 0.3,
     "rerank": "cross-encoder",
-    "rerankApiKey": "jina_xxx",
-    "rerankModel": "jina-reranker-v2-base-multilingual",
+    "rerankApiKey": "${JINA_API_KEY}",
+    "rerankModel": "jina-reranker-v3",
     "candidatePoolSize": 20,
     "recencyHalfLifeDays": 14,
     "recencyWeight": 0.1,
@@ -337,7 +365,21 @@ OpenClaw 会把每个 Agent 的完整会话自动落盘为 JSONL：
 
 但 JSONL 含大量噪声（tool 输出、系统块、重复回调等），**不建议直接把原文塞进 LanceDB**。
 
-本插件提供一个安全的 extractor 脚本 `scripts/jsonl_distill.py`，配合 OpenClaw 的 `cron` + 独立 distiller agent，实现“增量蒸馏 → 高质量记忆入库”：
+**推荐方案（2026-02+）**：使用 **/new 非阻塞沉淀管线**（Hooks + systemd worker），在你执行 `/new` 时异步提取高价值经验并写入 LanceDB Pro：
+
+- 触发：`command:new`（你在聊天里发送 `/new`）
+- Hook：只投递一个很小的 task.json（毫秒级，不调用 LLM，不阻塞 `/new`）
+- Worker：systemd 常驻进程监听队列，读取 session `.jsonl`，用 Gemini **Map-Reduce** 抽取 0～20 条高信噪比记忆
+- 写入：通过 `openclaw memory-pro import` 写入 LanceDB Pro（插件内部仍会 embedding + 查重）
+- 中文关键词：每条记忆包含 `Keywords (zh)`，并遵循三要素（实体/动作/症状）。其中“实体关键词”必须从 transcript 原文逐字拷贝（禁止编造项目名）。
+- 通知：可选（可做到即使 0 条也通知）
+
+示例文件：
+- `examples/new-session-distill/`
+
+---
+
+Legacy 方案：本插件也提供一个安全的 extractor 脚本 `scripts/jsonl_distill.py`，配合 OpenClaw 的 `cron` + 独立 distiller agent，实现“增量蒸馏 → 高质量记忆入库”：（适合不依赖 `/new` 的全自动场景）
 
 - 只读取每个 JSONL 文件**新增尾巴**（byte offset cursor），避免重复和 token 浪费
 - 生成一个小型 batch JSON
@@ -357,6 +399,19 @@ OpenClaw 会把每个 Agent 的完整会话自动落盘为 JSONL：
 
 > 脚本只读 session JSONL，不会修改原始日志。
 
+### （可选）启用 Agent 来源白名单（提高信噪比）
+
+默认情况下，extractor 会扫描 **所有 Agent**（但会排除 `memory-distiller` 自身，防止自我吞噬）。
+
+如果你只想从某些 Agent 蒸馏（例如只蒸馏 `main` + `code-agent`），可以设置环境变量：
+
+```bash
+export OPENCLAW_JSONL_DISTILL_ALLOWED_AGENT_IDS="main,code-agent"
+```
+
+- 不设置 / 空 / `*` / `all`：扫描全部（默认）
+- 逗号分隔列表：只扫描列表内 agentId
+
 ### 推荐部署（独立 distiller agent）
 
 #### 1）创建 distiller agent（示例用 gpt-5.2）

package/examples/new-session-distill/README.md

@@ -0,0 +1,19 @@
+# New Session Distillation (Recommended)
+
+This example shows a **non-blocking /new distillation pipeline**:
+
+- Trigger: `command:new` (when you type `/new`)
+- Hook: enqueue a small JSON task file (fast, no LLM calls)
+- Worker: a user-level systemd service watches the inbox and runs **Gemini Map-Reduce** over the session JSONL transcript
+- Storage: write high-signal, atomic lessons into LanceDB Pro via `openclaw memory-pro import`
+- Notify: send a notification message (optional)
+
+Files included:
+- `hook/enqueue-lesson-extract/` — OpenClaw workspace hook
+- `worker/lesson-extract-worker.mjs` — Map-Reduce extractor + importer + notifier
+- `worker/systemd/lesson-extract-worker.service` — user systemd unit
+
+You must provide:
+- `GEMINI_API_KEY` in an env file loaded by systemd
+
+Install steps are documented in the main repo README.

package/examples/new-session-distill/hook/enqueue-lesson-extract/HOOK.md

@@ -0,0 +1,21 @@
+---
+name: enqueue-lesson-extract
+description: "Enqueue a lesson-extraction task on /new (async Map-Reduce → LanceDB Pro)"
+metadata:
+  {
+    "openclaw": {
+      "emoji": "🧾",
+      "events": ["command:new"]
+    }
+  }
+---
+
+# Enqueue Lesson Extract Hook
+
+Writes a small JSON task file to a queue directory when `/new` is issued.
+
+This is intentionally fast and non-blocking. A separate systemd worker consumes tasks and:
+- reads the session JSONL transcript
+- runs Map-Reduce extraction with Gemini Flash
+- writes high-signal, deduped lessons into LanceDB Pro
+- sends a notification (optional)

package/examples/new-session-distill/hook/enqueue-lesson-extract/handler.ts

@@ -0,0 +1,102 @@
+import type { HookHandler } from "../../src/hooks/hooks.js";
+import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
+import { basename, dirname, join } from "node:path";
+import { createHash } from "node:crypto";
+import { fileURLToPath } from "node:url";
+
+function findSessionFile(event: any): string | null {
+  const prev = event.context?.previousSessionEntry;
+  const curr = event.context?.sessionEntry;
+
+  const candidates = [prev?.sessionFile, curr?.sessionFile].filter(Boolean);
+
+  for (const file of candidates) {
+    if (file && existsSync(file)) return file;
+  }
+
+  // If the file was renamed to .reset.*, look for the latest one
+  for (const file of candidates) {
+    if (!file) continue;
+    const dir = dirname(file);
+    const base = basename(file);
+    const resetPrefix = `${base}.reset.`;
+    try {
+      const resetFiles = readdirSync(dir)
+        .filter((name: string) => name.startsWith(resetPrefix))
+        .sort();
+      if (resetFiles.length > 0) return join(dir, resetFiles[resetFiles.length - 1]);
+    } catch {
+      // ignore
+    }
+  }
+
+  return null;
+}
+
+function sha1(input: string): string {
+  return createHash("sha1").update(input).digest("hex");
+}
+
+function repoRoot(): string {
+  // examples/new-session-distill/hook/enqueue-lesson-extract/handler.ts -> up to repo root
+  const here = dirname(fileURLToPath(import.meta.url));
+  return join(here, "..", "..", "..", "..", "..");
+}
+
+const handler: HookHandler = async (event) => {
+  if (event.type !== "command" || event.action !== "new") return;
+
+  try {
+    const sessionKey = event.sessionKey || "unknown";
+    const ctxEntry = (event.context?.previousSessionEntry || event.context?.sessionEntry) as any;
+    const sessionId = (ctxEntry?.sessionId as string) || "unknown";
+    const source = (event.context?.commandSource as string) || "unknown";
+
+    const sessionFile = findSessionFile(event);
+    if (!sessionFile) {
+      console.error("[enqueue-lesson-extract] No session file found; skipping");
+      return;
+    }
+
+    const root = repoRoot();
+    const queueDir = join(root, "workspaces", "main", "tasks", "lesson-extract", "inbox");
+    mkdirSync(queueDir, { recursive: true });
+
+    const ts = new Date(event.timestamp || Date.now()).toISOString();
+    const taskId = sha1([sessionKey, sessionId, sessionFile, ts].join("|"));
+
+    const task = {
+      taskId,
+      agentId: "main",
+      scope: "agent:main",
+      event: {
+        type: "command:new",
+        timestamp: ts,
+        sessionKey,
+        source,
+      },
+      session: {
+        sessionId,
+        sessionFile,
+      },
+      extract: {
+        maxFinal: 20,
+        mapChunkChars: 12000,
+        mapOverlapMsgs: 10,
+      },
+    };
+
+    const safeTs = ts.replace(/[:.]/g, "-");
+    const filename = `${safeTs}-${taskId.slice(0, 8)}.json`;
+    const outPath = join(queueDir, filename);
+
+    writeFileSync(outPath, JSON.stringify(task, null, 2) + "\n", "utf-8");
+  } catch (err) {
+    console.error(
+      "[enqueue-lesson-extract] Error:",
+      err instanceof Error ? err.message : String(err)
+    );
+  }
+};
+
+export default handler;

package/examples/new-session-distill/worker/lesson-extract-worker.mjs

@@ -0,0 +1,412 @@
+#!/usr/bin/env node
+
+/**
+ * lesson-extract-worker (example)
+ *
+ * Watches a repo-backed inbox for lesson extraction tasks and processes them asynchronously:
+ * - reads session JSONL transcript (streaming)
+ * - Map: per-chunk extraction via Gemini (native API)
+ * - Reduce: merge/dedupe/score -> 0..20 lessons
+ * - writes to LanceDB Pro via `openclaw memory-pro import`
+ * - sends Telegram notification via `openclaw message send` (optional)
+ */
+
+import fs from "node:fs";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { spawn } from "node:child_process";
+import readline from "node:readline";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// In your deployment, set LESSON_QUEUE_ROOT to your workspace queue.
+// By default we assume repo layout similar to OpenClaw-Memory.
+const REPO_ROOT = path.resolve(__dirname, "..", "..", "..", "..", "..");
+const QUEUE_ROOT = process.env.LESSON_QUEUE_ROOT || path.join(REPO_ROOT, "workspaces", "main", "tasks", "lesson-extract");
+
+const INBOX = path.join(QUEUE_ROOT, "inbox");
+const PROCESSING = path.join(QUEUE_ROOT, "processing");
+const DONE = path.join(QUEUE_ROOT, "done");
+const ERROR = path.join(QUEUE_ROOT, "error");
+
+const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
+const GEMINI_MODEL = process.env.GEMINI_MODEL || "gemini-3-flash-preview";
+
+const ONCE = process.argv.includes("--once");
+
+function ensureDirs() {
+  for (const d of [INBOX, PROCESSING, DONE, ERROR]) {
+    fs.mkdirSync(d, { recursive: true });
+  }
+}
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function run(cmd, args, opts = {}) {
+  return new Promise((resolve) => {
+    const child = spawn(cmd, args, { ...opts, stdio: ["ignore", "pipe", "pipe"] });
+    let out = "";
+    let err = "";
+    child.stdout.on("data", (d) => (out += d.toString("utf-8")));
+    child.stderr.on("data", (d) => (err += d.toString("utf-8")));
+    child.on("close", (code) => resolve({ code: code ?? 0, out, err }));
+  });
+}
+
+function safeJsonParse(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return null;
+  }
+}
+
+function normalizeText(s) {
+  return (s || "")
+    .trim()
+    .replace(/\s+/g, " ")
+    .replace(/[“”]/g, '"')
+    .replace(/[‘’]/g, "'")
+    .toLowerCase();
+}
+
+function detectLang(text) {
+  const s = text || "";
+  const cjk = (s.match(/[\u4e00-\u9fff]/g) || []).length;
+  const latin = (s.match(/[A-Za-z]/g) || []).length;
+  if (cjk > latin * 0.8) return "zh";
+  if (latin > cjk * 0.8) return "en";
+  return "mixed";
+}
+
+async function* iterJsonlMessages(sessionFile) {
+  const stream = fs.createReadStream(sessionFile, { encoding: "utf-8" });
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+
+  let id = 0;
+  for await (const line of rl) {
+    if (!line) continue;
+    const obj = safeJsonParse(line);
+    if (!obj || obj.type !== "message") continue;
+    const m = obj.message;
+    if (!m || (m.role !== "user" && m.role !== "assistant")) continue;
+
+    let text = "";
+    if (typeof m.content === "string") {
+      text = m.content;
+    } else if (Array.isArray(m.content)) {
+      text = m.content
+        .filter((c) => c && c.type === "text" && c.text)
+        .map((c) => c.text)
+        .join("\n");
+    }
+    text = (text || "").trim();
+    if (!text) continue;
+
+    id++;
+    yield {
+      id,
+      role: m.role,
+      timestamp: obj.timestamp || "",
+      text,
+    };
+  }
+}
+
+async function buildChunksFromJsonl(sessionFile, { maxChars = 12000, overlapMsgs = 10, maxChunks = 200 } = {}) {
+  const chunks = [];
+  let chunk = [];
+  let size = 0;
+
+  const sampleTexts = [];
+
+  for await (const m of iterJsonlMessages(sessionFile)) {
+    if (sampleTexts.length < 200) sampleTexts.push(m.text);
+
+    const line = `[${m.role === "user" ? "U" : "A"}${m.id}] ${m.text}\n`;
+
+    if (size + line.length > maxChars && chunk.length > 0) {
+      chunks.push(chunk);
+      if (chunks.length >= maxChunks) break;
+
+      chunk = chunk.slice(Math.max(0, chunk.length - overlapMsgs));
+      size = chunk.reduce((acc, mm) => acc + (`[${mm.role === "user" ? "U" : "A"}${mm.id}] ${mm.text}\n`).length, 0);
+    }
+
+    chunk.push(m);
+    size += line.length;
+  }
+
+  if (chunk.length > 0 && chunks.length < maxChunks) chunks.push(chunk);
+
+  const lang = detectLang(sampleTexts.join("\n"));
+  const messageCount = chunk.length === 0 && chunks.length === 0 ? 0 : chunks[chunks.length - 1][chunks[chunks.length - 1].length - 1].id;
+
+  return { chunks, lang, messageCount };
+}
+
+function buildMapPrompt({ lang, chunk }) {
+  const langInstr = lang === "zh" ? "请用中文输出 lessons。" : lang === "en" ? "Output lessons in English." : "Follow the dominant language of the transcript.";
+
+  return `You are extracting high-signal technical lessons from a chat transcript chunk.\n\nRules:\n- Output STRICT JSON only. No markdown, no backticks.\n- If nothing valuable, output: {\"lessons\":[]}\n- Max 8 lessons.\n- Each lesson.text must be <= 480 characters.\n- Categories: fact | decision | preference | other (use fact/decision primarily).\n- importance: number 0..1 (high-signal: 0.8-0.95).\n- evidence MUST quote exact short snippets from the chunk and include message_ids.\n- Do NOT include secrets/tokens/credentials.\n- Add Keywords (zh) inside each lesson:\n  - Include >=1 Entity keyword that appears verbatim in the chunk (project/library/tool/service/config key/error code).\n  - Include >=1 Action keyword (e.g., 修复/回滚/重启/迁移/去重/限流).\n  - Include >=1 Symptom keyword (e.g., OOM/超时/429/重复/命中率差).\n  - Do NOT invent entity names; copy entity keywords from the chunk.\n\n${langInstr}\n\nChunk:\n${chunk.map((m) => `[${m.role === "user" ? "U" : "A"}${m.id}] ${m.text}`).join("\n\n")}\n\nReturn JSON schema:\n{\n  \"lessons\": [\n    {\n      \"category\": \"fact\",\n      \"importance\": 0.8,\n      \"text\": \"Pitfall: ... Cause: ... Fix: ... Prevention: ...\",\n      \"evidence\": [\n        {\"message_ids\":[12,13],\"quote\":\"...\"}\n      ],\n      \"tags\": [\"optional\"]\n    }\n  ]\n}`;
+}
+
+async function geminiGenerateJson(prompt) {
+  if (!GEMINI_API_KEY) throw new Error("GEMINI_API_KEY is not set");
+
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/${GEMINI_MODEL}:generateContent?key=${GEMINI_API_KEY}`;
+
+  const body = {
+    contents: [{ role: "user", parts: [{ text: prompt }] }],
+    generationConfig: {
+      temperature: 0.2,
+      maxOutputTokens: 4096,
+    },
+  };
+
+  const res = await fetch(url, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+  });
+
+  const json = await res.json();
+  if (!res.ok) {
+    throw new Error(`Gemini error ${res.status}: ${JSON.stringify(json).slice(0, 500)}`);
+  }
+
+  const text = json?.candidates?.[0]?.content?.parts?.map((p) => p.text).join("") || "";
+  return text;
+}
+
+function coerceLessons(obj) {
+  const lessons = Array.isArray(obj?.lessons) ? obj.lessons : [];
+  return lessons
+    .filter((l) => l && typeof l.text === "string" && l.text.trim().length >= 10)
+    .map((l) => ({
+      category: ["fact", "decision", "preference", "other"].includes(l.category) ? l.category : "other",
+      importance: typeof l.importance === "number" ? l.importance : 0.7,
+      text: l.text.trim().slice(0, 480),
+      evidence: Array.isArray(l.evidence) ? l.evidence : [],
+      tags: Array.isArray(l.tags) ? l.tags : [],
+    }));
+}
+
+function scoreLesson(l) {
+  let s = 0;
+  const t = l.text || "";
+  if (/pitfall\s*:|cause\s*:|fix\s*:|prevention\s*:/i.test(t)) s += 2;
+  if (/decision principle|trigger\s*:|action\s*:/i.test(t)) s += 2;
+  if (/\b(openclaw|docker|systemd|ssh|git|api|json|yaml|config)\b/i.test(t)) s += 1;
+  if (t.length < 120) s += 0.5;
+  if (l.evidence?.length >= 1) s += 1;
+  if (l.evidence?.length >= 2) s += 0.5;
+  const imp = Math.max(0, Math.min(1, l.importance ?? 0.7));
+  s += imp;
+  return s;
+}
+
+function reduceLessons(allLessons, maxFinal = 20) {
+  const seen = new Set();
+  const merged = [];
+
+  for (const l of allLessons) {
+    const key = normalizeText(l.text);
+    if (!key) continue;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    merged.push(l);
+  }
+
+  merged.sort((a, b) => scoreLesson(b) - scoreLesson(a));
+
+  const filtered = merged.filter((l) => {
+    if (!l.evidence || l.evidence.length === 0) return false;
+    const t = normalizeText(l.text);
+    if (t.length < 20) return false;
+    if (/(be careful|best practice|should|建议|注意)/.test(t) && !/(cause|fix|prevention|trigger|action|原因|修复|预防|触发)/.test(t)) {
+      return false;
+    }
+    return true;
+  });
+
+  return filtered.slice(0, maxFinal);
+}
+
+async function importToLanceDb({ lessons, scope }) {
+  const importFile = path.join("/tmp", `lesson-import-${Date.now()}.json`);
+  const payload = {
+    memories: lessons.map((l) => ({
+      text: l.text,
+      importance: Math.max(0.0, Math.min(1.0, l.importance ?? 0.7)),
+      category: l.category,
+    })),
+  };
+  await fsp.writeFile(importFile, JSON.stringify(payload), "utf-8");
+
+  const { code, out, err } = await run("openclaw", ["memory-pro", "import", importFile, "--scope", scope], { cwd: REPO_ROOT });
+  await fsp.unlink(importFile).catch(() => {});
+
+  return { code, out, err };
+}
+
+async function notifyTelegram(chatId, message) {
+  const args = ["message", "send", "--channel", "telegram", "--target", String(chatId), "--message", message];
+  await run("openclaw", args, { cwd: REPO_ROOT });
+}
+
+async function processTaskFile(taskPath) {
+  const started = Date.now();
+  const taskRaw = await fsp.readFile(taskPath, "utf-8");
+  const task = JSON.parse(taskRaw);
+
+  const baseName = path.basename(taskPath);
+  const processingPath = path.join(PROCESSING, baseName);
+  await fsp.rename(taskPath, processingPath);
+
+  const result = {
+    taskId: task.taskId,
+    startedAt: nowIso(),
+    finishedAt: null,
+    ok: false,
+    sessionId: task.session?.sessionId,
+    sessionFile: task.session?.sessionFile,
+    stats: {},
+    error: null,
+  };
+
+  try {
+    const sessionFile = task.session?.sessionFile;
+    if (!sessionFile || !fs.existsSync(sessionFile)) {
+      throw new Error(`sessionFile missing or not found: ${sessionFile}`);
+    }
+
+    const { chunks, lang, messageCount } = await buildChunksFromJsonl(sessionFile, {
+      maxChars: task.extract?.mapChunkChars ?? 12000,
+      overlapMsgs: task.extract?.mapOverlapMsgs ?? 10,
+      maxChunks: 200,
+    });
+
+    const allLessons = [];
+    let mapErrors = 0;
+
+    for (let idx = 0; idx < chunks.length; idx++) {
+      const prompt = buildMapPrompt({ lang, chunk: chunks[idx] });
+      try {
+        const text = await geminiGenerateJson(prompt);
+        const obj = safeJsonParse(text);
+        if (!obj) {
+          mapErrors++;
+          continue;
+        }
+        const lessons = coerceLessons(obj);
+        for (const l of lessons) allLessons.push(l);
+      } catch {
+        mapErrors++;
+      }
+    }
+
+    const reduced = reduceLessons(allLessons, task.extract?.maxFinal ?? 20);
+
+    const scope = task.scope || "agent:main";
+    const importRes = await importToLanceDb({ lessons: reduced, scope });
+
+    const durationMs = Date.now() - started;
+    result.ok = importRes.code === 0;
+    result.finishedAt = nowIso();
+    result.stats = {
+      lang,
+      messages: messageCount,
+      chunks: chunks.length,
+      mapCandidates: allLessons.length,
+      mapErrors,
+      reduced: reduced.length,
+      importCode: importRes.code,
+      durationMs,
+    };
+
+    const notifyChatId = task.notify?.telegramChatId;
+    if (notifyChatId) {
+      const text = [
+        `Lesson Extract ✅ (${task.agentId || "main"})`,
+        `taskId: ${task.taskId?.slice(0, 8) || "unknown"}`,
+        `sessionId: ${task.session?.sessionId || "unknown"}`,
+        `lang: ${lang}`,
+        `messages: ${messageCount}, chunks: ${chunks.length}`,
+        `candidates: ${allLessons.length}, reduced: ${reduced.length}`,
+        `import: code=${importRes.code}`,
+        `time: ${(durationMs / 1000).toFixed(1)}s`,
+      ].join("\n");
+      await notifyTelegram(notifyChatId, text);
+    }
+
+    const donePath = path.join(DONE, `${task.taskId}.json`);
+    await fsp.writeFile(donePath, JSON.stringify(result, null, 2) + "\n", "utf-8");
+    await fsp.unlink(processingPath).catch(() => {});
+  } catch (err) {
+    result.ok = false;
+    result.finishedAt = nowIso();
+    result.error = err instanceof Error ? err.message : String(err);
+
+    const durationMs = Date.now() - started;
+    result.stats.durationMs = durationMs;
+
+    const notifyChatId = task.notify?.telegramChatId;
+    if (notifyChatId) {
+      await notifyTelegram(
+        notifyChatId,
+        `Lesson Extract ❌ (${task.agentId || "main"})\n` +
+          `taskId: ${task.taskId?.slice(0, 8) || "unknown"}\n` +
+          `error: ${result.error}\n` +
+          `time: ${(durationMs / 1000).toFixed(1)}s`
+      );
+    }
+
+    const errPath = path.join(ERROR, `${task.taskId}.json`);
+    await fsp.writeFile(errPath, JSON.stringify(result, null, 2) + "\n", "utf-8");
+    await fsp.unlink(processingPath).catch(() => {});
+  }
+}
+
+async function drainInboxOnce() {
+  ensureDirs();
+  const files = (await fsp.readdir(INBOX)).filter((f) => f.endsWith(".json")).sort();
+  for (const f of files) {
+    await processTaskFile(path.join(INBOX, f));
+  }
+}
+
+async function main() {
+  ensureDirs();
+  await drainInboxOnce();
+  if (ONCE) return;
+
+  const watcher = fs.watch(INBOX, async (_eventType, filename) => {
+    if (!filename || !filename.endsWith(".json")) return;
+    const full = path.join(INBOX, filename);
+    setTimeout(() => {
+      processTaskFile(full).catch(() => {});
+    }, 150);
+  });
+
+  let alive = true;
+  const shutdown = () => {
+    alive = false;
+    watcher.close();
+  };
+  process.on("SIGINT", shutdown);
+  process.on("SIGTERM", shutdown);
+
+  while (alive) {
+    await new Promise((r) => setTimeout(r, 5_000));
+  }
+}
+
+main().catch((err) => {
+  console.error(String(err));
+  process.exit(1);
+});

package/examples/new-session-distill/worker/systemd/lesson-extract-worker.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=OpenClaw Lesson Extract Worker (Gemini Map-Reduce -> LanceDB Pro)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory=/home/ubuntu/clawd
+EnvironmentFile=%h/.config/lesson-extract-worker.env
+Environment=NODE_ENV=production
+Environment=LESSON_QUEUE_ROOT=/home/ubuntu/clawd/workspaces/main/tasks/lesson-extract
+Environment=GEMINI_MODEL=gemini-3-flash-preview
+ExecStart=/usr/bin/env node /home/ubuntu/clawd/scripts/lesson-extract-worker.mjs
+Restart=always
+RestartSec=2
+NoNewPrivileges=true
+PrivateTmp=true
+
+[Install]
+WantedBy=default.target

package/index.ts

@@ -639,32 +639,53 @@ const memoryLanceDBProPlugin = {
     api.registerService({
       id: "memory-lancedb-pro",
       start: async () => {
-        try {
-          // Test components
-          const embedTest = await embedder.test();
-          const retrievalTest = await retriever.test();
-
-          api.logger.info(
-            `memory-lancedb-pro: initialized successfully ` +
-            `(embedding: ${embedTest.success ? 'OK' : 'FAIL'}, ` +
-            `retrieval: ${retrievalTest.success ? 'OK' : 'FAIL'}, ` +
-            `mode: ${retrievalTest.mode}, ` +
-            `FTS: ${retrievalTest.hasFtsSupport ? 'enabled' : 'disabled'})`
-          );
-
-          if (!embedTest.success) {
-            api.logger.warn(`memory-lancedb-pro: embedding test failed: ${embedTest.error}`);
+        // IMPORTANT: Do not block gateway startup on external network calls.
+        // If embedding/retrieval tests hang (bad network / slow provider), the gateway
+        // may never bind its HTTP port, causing restart timeouts.
+
+        const withTimeout = async <T>(p: Promise<T>, ms: number, label: string): Promise<T> => {
+          let timeout: ReturnType<typeof setTimeout> | undefined;
+          const timeoutPromise = new Promise<never>((_, reject) => {
+            timeout = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
+          });
+          try {
+            return await Promise.race([p, timeoutPromise]);
+          } finally {
+            if (timeout) clearTimeout(timeout);
           }
-          if (!retrievalTest.success) {
-            api.logger.warn(`memory-lancedb-pro: retrieval test failed: ${retrievalTest.error}`);
+        };
+
+        const runStartupChecks = async () => {
+          try {
+            // Test components (bounded time)
+            const embedTest = await withTimeout(embedder.test(), 8_000, "embedder.test()");
+            const retrievalTest = await withTimeout(retriever.test(), 8_000, "retriever.test()");
+
+            api.logger.info(
+              `memory-lancedb-pro: initialized successfully ` +
+                `(embedding: ${embedTest.success ? "OK" : "FAIL"}, ` +
+                `retrieval: ${retrievalTest.success ? "OK" : "FAIL"}, ` +
+                `mode: ${retrievalTest.mode}, ` +
+                `FTS: ${retrievalTest.hasFtsSupport ? "enabled" : "disabled"})`
+            );
+
+            if (!embedTest.success) {
+              api.logger.warn(`memory-lancedb-pro: embedding test failed: ${embedTest.error}`);
+            }
+            if (!retrievalTest.success) {
+              api.logger.warn(`memory-lancedb-pro: retrieval test failed: ${retrievalTest.error}`);
+            }
+          } catch (error) {
+            api.logger.warn(`memory-lancedb-pro: startup checks failed: ${String(error)}`);
           }
+        };
 
-          // Run initial backup after a short delay, then schedule daily
-          setTimeout(() => runBackup(), 60_000); // 1 min after start
-          backupTimer = setInterval(() => runBackup(), BACKUP_INTERVAL_MS);
-        } catch (error) {
-          api.logger.warn(`memory-lancedb-pro: startup test failed: ${String(error)}`);
-        }
+        // Fire-and-forget: allow gateway to start serving immediately.
+        setTimeout(() => void runStartupChecks(), 0);
+
+        // Run initial backup after a short delay, then schedule daily
+        setTimeout(() => void runBackup(), 60_000); // 1 min after start
+        backupTimer = setInterval(() => void runBackup(), BACKUP_INTERVAL_MS);
       },
       stop: () => {
         if (backupTimer) {

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "memory-lancedb-pro",
   "name": "Memory (LanceDB Pro)",
   "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, and management CLI",
-  "version": "1.0.7",
+  "version": "1.0.10",
   "kind": "memory",
   "configSchema": {
     "type": "object",
@@ -108,7 +108,7 @@
           },
           "rerankModel": {
             "type": "string",
-            "default": "jina-reranker-v2-base-multilingual",
+            "default": "jina-reranker-v3",
             "description": "Reranker model name"
           },
           "rerankEndpoint": {
@@ -118,7 +118,11 @@
           },
           "rerankProvider": {
             "type": "string",
-            "enum": ["jina", "siliconflow", "pinecone"],
+            "enum": [
+              "jina",
+              "siliconflow",
+              "pinecone"
+            ],
             "default": "jina",
             "description": "Reranker provider format. Determines request/response shape and auth header."
           },
@@ -300,8 +304,8 @@
     },
     "retrieval.rerankModel": {
       "label": "Reranker Model",
-      "placeholder": "jina-reranker-v2-base-multilingual",
-      "help": "Reranker model name (e.g. jina-reranker-v2-base-multilingual, BAAI/bge-reranker-v2-m3)",
+      "placeholder": "jina-reranker-v3",
+      "help": "Reranker model name (e.g. jina-reranker-v3, BAAI/bge-reranker-v2-m3)",
       "advanced": true
     },
     "retrieval.rerankEndpoint": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-lancedb-pro",
-  "version": "1.0.7",
+  "version": "1.0.10",
   "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, and management CLI",
   "type": "module",
   "main": "index.ts",

package/scripts/jsonl_distill.py

@@ -36,6 +36,22 @@ EXCLUDED_AGENT_IDS = {
     "memory-distiller",
 }
 
+# Source allowlist (optional quality control).
+# Default (env unset): allow all agents (except EXCLUDED_AGENT_IDS).
+# If set: only distill from the listed agent IDs.
+# Example:
+#   OPENCLAW_JSONL_DISTILL_ALLOWED_AGENT_IDS=main,code-agent
+ENV_ALLOWED_AGENT_IDS = "OPENCLAW_JSONL_DISTILL_ALLOWED_AGENT_IDS"
+
+
+def _get_allowed_agent_ids() -> Optional[set[str]]:
+    raw = os.environ.get(ENV_ALLOWED_AGENT_IDS, "").strip()
+    if not raw or raw in ("*", "all"):
+        return None
+    parts = [p.strip() for p in raw.split(",") if p.strip()]
+    return set(parts) if parts else None
+
+
 
 NOISE_PREFIXES = (
     "✅ New session started",
@@ -175,12 +191,16 @@ def _list_session_files(agents_dir: Path) -> List[Tuple[str, Path]]:
     if not agents_dir.exists():
         return results
 
+    allowed_agent_ids = _get_allowed_agent_ids()
+
     for agent_dir in sorted(agents_dir.iterdir()):
         if not agent_dir.is_dir():
             continue
         agent_id = agent_dir.name
         if agent_id in EXCLUDED_AGENT_IDS:
             continue
+        if allowed_agent_ids is not None and agent_id not in allowed_agent_ids:
+            continue
         sessions_dir = agent_dir / "sessions"
         if not sessions_dir.exists():
             continue

package/src/retriever.ts

@@ -26,7 +26,7 @@ export interface RetrievalConfig {
   filterNoise: boolean;
   /** Reranker API key (enables cross-encoder reranking) */
   rerankApiKey?: string;
-  /** Reranker model (default: jina-reranker-v2-base-multilingual) */
+  /** Reranker model (default: jina-reranker-v3) */
   rerankModel?: string;
   /** Reranker API endpoint (default: https://api.jina.ai/v1/rerank). */
   rerankEndpoint?: string;
@@ -90,7 +90,7 @@ export const DEFAULT_RETRIEVAL_CONFIG: RetrievalConfig = {
   recencyHalfLifeDays: 14,
   recencyWeight: 0.10,
   filterNoise: true,
-  rerankModel: "jina-reranker-v2-base-multilingual",
+  rerankModel: "jina-reranker-v3",
   rerankEndpoint: "https://api.jina.ai/v1/rerank",
   lengthNormAnchor: 500,
   hardMinScore: 0.35,
@@ -427,7 +427,7 @@ export class MemoryRetriever {
     if (this.config.rerank === "cross-encoder" && this.config.rerankApiKey) {
       try {
         const provider = this.config.rerankProvider || "jina";
-        const model = this.config.rerankModel || "jina-reranker-v2-base-multilingual";
+        const model = this.config.rerankModel || "jina-reranker-v3";
         const endpoint = this.config.rerankEndpoint || "https://api.jina.ai/v1/rerank";
         const documents = results.map(r => r.entry.text);