openclaw-memory-alibaba-local 0.1.0

package/README.md

@@ -0,0 +1,88 @@
+# openclaw-memory-alibaba-local
+
+OpenClaw 记忆插件：**本地 LanceDB** 向量存储。支持用户记忆三分类（`user_memory_fact` / `user_memory_preference` / `user_memory_decision`）、可选 **全文按条** 落库（`full_context_*`）、自进化记忆、LLM 或正则抽取、可选 LLM 去重/冲突处理、可选召回时间衰减。
+
+- **自动召回**：`before_prompt_build`（不含 `full_context_*`，仅用户记忆 + 自进化）。
+- **自动写入**：仅在 **`agent_end`**（本轮成功结束）时处理：按 **消息 role（source）** 维护游标，计算各 source 的 **delta** → 对 `full_context_*` **立即 embedding 并写入 LanceDB**（同一轮共用 `batchId`，管理端全文 Tab **按 batch 可折叠**）→ **`Promise.all` 并行**：仅 **user** 的 delta 走 **用户记忆** 抽取链路；**user + assistant** 的 delta 拼成上下文走 **自进化** 链路。游标文件：`dbPath` 下 `memory-alibaba-local-agent-end-cursors.json`（若存在旧版 `memory-alibaba-local-full-context-cursor.json` 会在首次加载时尝试改名为新文件并迁移为按 role 计数）。
+
+**Embedding** 支持 **`local`**（默认，本机 `llama-embedding` 从 **stdin** 读入文本）与 **`remote`**（OpenAI 兼容 `/v1/embeddings`）。**LLM** 仍可用 DashScope 兼容接口。相似度与官方 **memory-lancedb** 一致：`score = 1 / (1 + L2_distance)`。
+
+**长文本**：按**空行分段**；每段用 **约 `maxToken` 个 token（字数/4）** 为上限，超长段再切分。每段单独算向量并**单独占一行**（不向量化合并）；召回时对查询各段分别搜向量，再按 **`category` + 正文** 合并取最高分。
+
+## 与官方 memory-lancedb 共用目录时的表名
+
+默认 **`dbPath`** 与官方插件相同：`~/.openclaw/memory/lancedb`。本插件使用独立表 **`openclaw_memories_alibaba_local`**（常量 `LANCEDB_TABLE_NAME`），不与官方的 `memories` 表混用。
+
+### 表字段与重建（`chunkIndex`）
+
+设计上每条向量行带有 **`seqInBatch`**（如 `agent_end` 同一 `batchId` 内的消息序）与 **`chunkIndex`**（同一条逻辑记忆因长文本切分产生的多段向量序号，从 0 递增）。**新建表**会自动带上这些列。
+
+若你曾在无 `chunkIndex` 的旧版本下建过表，管理端列表可能出现 *No field named "chunkIndex"*。测试或升级时可**删掉该 LanceDB 表后重启**（插件会按新 schema 建表），例如在本机 `dbPath` 目录下删除名为 `openclaw_memories_alibaba_local` 的表数据（具体文件布局以 LanceDB 版本为准），或临时改用新的 `dbPath`。**注意：会清空本插件表内全部记忆行。**
+
+## 最简配置（默认本机 embedding）
+
+```json
+{
+  "plugins": {
+    "slots": { "memory": "openclaw-memory-alibaba-local" },
+    "entries": {
+      "openclaw-memory-alibaba-local": {
+        "enabled": true,
+        "config": {
+          "embedding": { "mode": "local" },
+          "llm": {
+            "apiKey": "${DASHSCOPE_API_KEY}",
+            "model": "qwen-plus",
+            "baseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
+可选 **`local`** 字段：`commandPrefix`（默认使用 `llama-embedding -m ~/.openclaw/embedding_model/embeddinggemma-300M-Q8_0.gguf -f /dev/stdin --embd-output-format json`）、`dimensions`（默认 **768**）、`maxToken`（默认 **2048**）。命令需从 **stdin** 读入待嵌入文本；推荐使用 **`--embd-output-format json`**（与 OpenAI 列表格式兼容）。
+
+## 远程 embedding（OpenAI 兼容）
+
+必须显式 **`"mode": "remote"`**，并填写 **`apiKey` / `model` / `baseUrl` / `dimensions` / `maxToken`**（不在此写默认值；配置错误通常在**第一次 embed** 时失败，启动阶段不校验远程连通性）。
+
+```json
+"embedding": {
+  "mode": "remote",
+  "apiKey": "${DASHSCOPE_API_KEY}",
+  "model": "text-embedding-v3",
+  "baseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+  "dimensions": 1024,
+  "maxToken": 2048
+}
+```
+
+默认 **`memoryExtractionMethod` 为 `llm`**，因此需要可用的 **`llm.apiKey`** 与 **`llm.model`**（可在插件里写全，或见下节从主机配置补齐）。若改为 **`regex`**，可不配 `llm`（除非开启 **`memory_duplication_conflict_process`**）。
+
+## LLM 默认值（`openclaw.json`）
+
+当需要 LLM 时，若插件 **`config.llm`** 未写某字段，会按顺序从主机 OpenClaw 配置文件读取（路径：`OPENCLAW_CONFIG_PATH`，否则为 `OPENCLAW_STATE_DIR/openclaw.json`，再否则 `~/.openclaw/openclaw.json`）：
+
+| 插件字段 | 来源 |
+|----------|------|
+| `apiKey` | `models.providers.bailian.apiKey`（支持字符串或 `SecretRef`，如 `{ "source": "env", "provider": "default", "id": "DASHSCOPE_API_KEY" }`） |
+| `baseUrl` | `models.providers.bailian.baseUrl`（缺省仍为 DashScope 兼容地址） |
+| `model` | `agents.defaults.model`：可为字符串，或对象里的 **`primary`**（OpenClaw 常用 `bailian/qwen-plus` 形式） |
+
+**模型名格式**：若 `primary` 为 `bailian/xxx` 或 `dashscope/xxx`，发给 Chat Completions 时会自动去掉 provider 前缀，只保留 **`xxx`**，与 DashScope 兼容接口一致。
+
+插件里显式写的 `llm` 字段始终优先于上述默认值。若既未在插件中提供完整 `apiKey`+`model`，主机配置也无法补齐，启动会报错。
+
+## 依赖
+
+- Node 环境下需能加载 **`@lancedb/lancedb`** 原生绑定（部分平台与官方 memory-lancedb 相同限制）。
+
+## 源码与发布
+
+本包为可独立安装的 OpenClaw 插件；远程仓库地址以 **`package.json`** 中 **`repository`** 字段为准。
+
+## 许可证
+
+MIT

package/bm25-recall.ts

@@ -0,0 +1,71 @@
+/**
+ * Okapi BM25 over an in-memory corpus (supplement to vector recall).
+ * Tokenization: Unicode letters + numbers (works for Latin; CJK often single-char tokens).
+ */
+
+export function tokenizeForBm25(text: string): string[] {
+  const m = text.toLowerCase().match(/[\p{L}\p{N}]+/gu);
+  return m ?? [];
+}
+
+export type Bm25DocInput = { id: string; text: string };
+
+/** Returns docs sorted by BM25 score descending. */
+export function scoreDocumentsBm25(query: string, docs: Bm25DocInput[]): Array<{ id: string; score: number }> {
+  if (docs.length === 0) {
+    return [];
+  }
+  const k1 = 1.2;
+  const b = 0.75;
+
+  type Prepared = { id: string; freqs: Map<string, number>; len: number };
+  const prepared: Prepared[] = docs.map((d) => {
+    const terms = tokenizeForBm25(d.text);
+    const freqs = new Map<string, number>();
+    for (const t of terms) {
+      freqs.set(t, (freqs.get(t) ?? 0) + 1);
+    }
+    return { id: d.id, freqs, len: terms.length };
+  });
+
+  const N = prepared.length;
+  const avgdl = prepared.reduce((s, p) => s + p.len, 0) / Math.max(1, N);
+
+  const df = new Map<string, number>();
+  for (const p of prepared) {
+    for (const term of p.freqs.keys()) {
+      df.set(term, (df.get(term) ?? 0) + 1);
+    }
+  }
+
+  const qTerms = tokenizeForBm25(query);
+  if (qTerms.length === 0) {
+    return prepared.map((p) => ({ id: p.id, score: 0 }));
+  }
+
+  const scores = new Map<string, number>();
+  for (const p of prepared) {
+    scores.set(p.id, 0);
+  }
+
+  for (const term of qTerms) {
+    const dfi = df.get(term) ?? 0;
+    if (dfi === 0) {
+      continue;
+    }
+    const idf = Math.log((N - dfi + 0.5) / (dfi + 0.5) + 1);
+    for (const p of prepared) {
+      const f = p.freqs.get(term) ?? 0;
+      if (f === 0) {
+        continue;
+      }
+      const denom = f + k1 * (1 - b + (b * p.len) / avgdl);
+      const add = (idf * (f * (k1 + 1))) / denom;
+      scores.set(p.id, (scores.get(p.id) ?? 0) + add);
+    }
+  }
+
+  return [...scores.entries()]
+    .map(([id, score]) => ({ id, score }))
+    .sort((a, b) => b.score - a.score);
+}

package/capture-state.ts

@@ -0,0 +1,206 @@
+/**
+ * Persisted cursors for agent_end capture: per-message-role counts (not flat transcript index).
+ * File: memory-alibaba-local-agent-end-cursors.json next to LanceDB dir.
+ * Migrates legacy memory-alibaba-local-full-context-cursor.json (lastEndExclusive) on first use.
+ */
+
+import { readFileSync, writeFileSync, mkdirSync, existsSync, renameSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+const CURSOR_FILENAME_V2 = "memory-alibaba-local-agent-end-cursors.json";
+const LEGACY_CURSOR_FILENAME = "memory-alibaba-local-full-context-cursor.json";
+
+/** v2: how many messages per role have been fully processed (including empty-body turns). */
+export type AgentEndCursorEntryV2 = {
+  version: 2;
+  roleCounts: Record<string, number>;
+  lastMessagesLength: number;
+};
+
+export type LegacyCursorEntry = {
+  lastEndExclusive: number;
+};
+
+export type CursorFileEntry = AgentEndCursorEntryV2 | LegacyCursorEntry;
+
+function cursorPathV2(lancedbDir: string): string {
+  return join(lancedbDir, CURSOR_FILENAME_V2);
+}
+
+function legacyCursorPath(lancedbDir: string): string {
+  return join(lancedbDir, LEGACY_CURSOR_FILENAME);
+}
+
+/**
+ * Infer agentId when ctx.agentId is missing. OpenClaw canonical keys are `agent:<agentId>:<rest>` (≥3 segments).
+ * Plugin fallback `session:<sessionId>` must not use the literal prefix `session` as agentId (would hide rows in UI when Agent defaults to `main`).
+ */
+export function parseAgentIdFromSessionKey(sessionKey: string): string {
+  const parts = sessionKey.split(":").filter(Boolean);
+  if (parts.length >= 2 && parts[0] === "agent") {
+    return parts[1] || "main";
+  }
+  const head = (parts[0] ?? "").toLowerCase();
+  if (head === "session") {
+    return "main";
+  }
+  return parts[0] || "main";
+}
+
+export function getFullContextCursorKey(agentId: string, sessionKey: string): string {
+  return `${agentId}\n${sessionKey}`;
+}
+
+function isV2Entry(v: unknown): v is AgentEndCursorEntryV2 {
+  return (
+    !!v &&
+    typeof v === "object" &&
+    !Array.isArray(v) &&
+    (v as AgentEndCursorEntryV2).version === 2 &&
+    typeof (v as AgentEndCursorEntryV2).roleCounts === "object" &&
+    (v as AgentEndCursorEntryV2).roleCounts !== null &&
+    !Array.isArray((v as AgentEndCursorEntryV2).roleCounts)
+  );
+}
+
+function isLegacyEntry(v: unknown): v is LegacyCursorEntry {
+  return (
+    !!v &&
+    typeof v === "object" &&
+    !Array.isArray(v) &&
+    typeof (v as LegacyCursorEntry).lastEndExclusive === "number" &&
+    Number.isFinite((v as LegacyCursorEntry).lastEndExclusive)
+  );
+}
+
+/** Load all cursor entries (v2 and/or legacy shapes). Migrates legacy file into v2 path once if needed. */
+export function loadAgentEndCursorMap(lancedbDir: string): Record<string, CursorFileEntry> {
+  const v2Path = cursorPathV2(lancedbDir);
+  const legPath = legacyCursorPath(lancedbDir);
+
+  if (!existsSync(v2Path) && existsSync(legPath)) {
+    try {
+      renameSync(legPath, v2Path);
+    } catch {
+      // if rename fails, read legacy below
+    }
+  }
+
+  const readPath = existsSync(v2Path) ? v2Path : existsSync(legPath) ? legPath : v2Path;
+  try {
+    const raw = readFileSync(readPath, "utf8");
+    const parsed = JSON.parse(raw) as unknown;
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      return {};
+    }
+    const out: Record<string, CursorFileEntry> = {};
+    for (const [k, v] of Object.entries(parsed as Record<string, unknown>)) {
+      if (isV2Entry(v)) {
+        out[k] = {
+          version: 2,
+          roleCounts: { ...v.roleCounts },
+          lastMessagesLength:
+            typeof v.lastMessagesLength === "number" && Number.isFinite(v.lastMessagesLength)
+              ? Math.floor(v.lastMessagesLength)
+              : 0,
+        };
+      } else if (isLegacyEntry(v)) {
+        out[k] = { lastEndExclusive: Math.max(0, Math.floor(v.lastEndExclusive)) };
+      }
+    }
+    return out;
+  } catch {
+    return {};
+  }
+}
+
+export function saveAgentEndCursorMap(
+  lancedbDir: string,
+  map: Record<string, CursorFileEntry>,
+): void {
+  const p = cursorPathV2(lancedbDir);
+  try {
+    mkdirSync(dirname(p), { recursive: true });
+  } catch {
+    // ignore
+  }
+  writeFileSync(p, JSON.stringify(map, null, 0), "utf8");
+}
+
+/** Role key stable for counting (aligns with full_context source roles). */
+export function normalizeRoleForCursor(role: string): string {
+  const t = (role ?? "").trim();
+  if (!t) {
+    return "others";
+  }
+  if (t === "developer") {
+    return "system";
+  }
+  if (t === "toolResult" || t === "tool_result") {
+    return "tool_result";
+  }
+  return t;
+}
+
+export function countRolesInMessagesPrefix(messages: unknown[], endExclusive: number): Record<string, number> {
+  const counts: Record<string, number> = {};
+  const end = Math.max(0, Math.min(Math.floor(endExclusive), messages.length));
+  for (let i = 0; i < end; i++) {
+    const role = normalizeRoleForCursor(getMessageRoleRaw(messages[i]));
+    counts[role] = (counts[role] ?? 0) + 1;
+  }
+  return counts;
+}
+
+function getMessageRoleRaw(msg: unknown): string {
+  if (!msg || typeof msg !== "object") {
+    return "";
+  }
+  const r = (msg as Record<string, unknown>).role;
+  return typeof r === "string" ? r : "";
+}
+
+/**
+ * Resolve saved role counts for this session key; migrates legacy lastEndExclusive using current transcript.
+ */
+export function resolveRoleCountsForSession(
+  entry: CursorFileEntry | undefined,
+  messages: unknown[],
+  log: { info: (m: string) => void },
+): { roleCounts: Record<string, number>; lastMessagesLength: number } {
+  if (isV2Entry(entry)) {
+    return {
+      roleCounts: { ...entry.roleCounts },
+      lastMessagesLength: entry.lastMessagesLength ?? 0,
+    };
+  }
+  if (isLegacyEntry(entry)) {
+    const end = Math.min(Math.max(0, entry.lastEndExclusive), messages.length);
+    log.info("openclaw-memory-alibaba-local: migrated legacy full-context cursor to per-role counts");
+    return {
+      roleCounts: countRolesInMessagesPrefix(messages, end),
+      lastMessagesLength: end,
+    };
+  }
+  return { roleCounts: {}, lastMessagesLength: 0 };
+}
+
+/** @deprecated use loadAgentEndCursorMap */
+export function loadFullContextCursors(lancedbDir: string): Record<string, { lastEndExclusive: number }> {
+  const map = loadAgentEndCursorMap(lancedbDir);
+  const out: Record<string, { lastEndExclusive: number }> = {};
+  for (const [k, v] of Object.entries(map)) {
+    if (isLegacyEntry(v)) {
+      out[k] = { lastEndExclusive: v.lastEndExclusive };
+    }
+  }
+  return out;
+}
+
+/** @deprecated no-op for v2; kept for API compatibility */
+export function saveFullContextCursors(
+  _lancedbDir: string,
+  _cursors: Record<string, { lastEndExclusive: number }>,
+): void {
+  // v2 uses saveAgentEndCursorMap only
+}

package/categories.ts

@@ -0,0 +1,106 @@
+/**
+ * Memory category constants.
+ * user_memory is subdivided into fact / preference / decision for storage and recall.
+ */
+
+/** User memory sub-categories (user-side facts, preferences, decisions) */
+export const USER_MEMORY_FACT = "user_memory_fact" as const;
+export const USER_MEMORY_PREFERENCE = "user_memory_preference" as const;
+export const USER_MEMORY_DECISION = "user_memory_decision" as const;
+
+/** All user memory category values (for recall: treat as one logical "user_memory" set) */
+export const USER_MEMORY_CATEGORIES = [
+  USER_MEMORY_FACT,
+  USER_MEMORY_PREFERENCE,
+  USER_MEMORY_DECISION,
+] as const;
+
+export type UserMemoryCategory =
+  | typeof USER_MEMORY_FACT
+  | typeof USER_MEMORY_PREFERENCE
+  | typeof USER_MEMORY_DECISION;
+
+/** Full context (conversation audit). Legacy single category. */
+export const FULL_CONTEXT_MEMORY = "full_context_memory" as const;
+
+/** Full context by source (user / assistant / system / tool / tool_result / others) */
+export const FULL_CONTEXT_USER = "full_context_user" as const;
+export const FULL_CONTEXT_ASSISTANT = "full_context_assistant" as const;
+export const FULL_CONTEXT_SYSTEM = "full_context_system" as const;
+export const FULL_CONTEXT_TOOL = "full_context_tool" as const;
+export const FULL_CONTEXT_TOOL_RESULT = "full_context_tool_result" as const;
+export const FULL_CONTEXT_OTHERS = "full_context_others" as const;
+
+export const FULL_CONTEXT_SOURCE_CATEGORIES = [
+  FULL_CONTEXT_USER,
+  FULL_CONTEXT_ASSISTANT,
+  FULL_CONTEXT_SYSTEM,
+  FULL_CONTEXT_TOOL,
+  FULL_CONTEXT_TOOL_RESULT,
+  FULL_CONTEXT_OTHERS,
+] as const;
+
+export type FullContextSourceCategory = (typeof FULL_CONTEXT_SOURCE_CATEGORIES)[number];
+
+export function isFullContextSourceCategory(cat: string): cat is FullContextSourceCategory {
+  return FULL_CONTEXT_SOURCE_CATEGORIES.includes(cat as FullContextSourceCategory);
+}
+
+/** Self-improving memory sub-categories */
+export const SELF_IMPROVING_LEARNINGS = "self_improving_learnings" as const;
+export const SELF_IMPROVING_ERRORS = "self_improving_errors" as const;
+export const SELF_IMPROVING_FEATURE_REQUESTS = "self_improving_feature_requests" as const;
+
+export const SELF_IMPROVING_CATEGORIES = [
+  SELF_IMPROVING_LEARNINGS,
+  SELF_IMPROVING_ERRORS,
+  SELF_IMPROVING_FEATURE_REQUESTS,
+] as const;
+
+export type SelfImprovingCategory = (typeof SELF_IMPROVING_CATEGORIES)[number];
+
+/** All allowed category values */
+export type MemoryCategory =
+  | UserMemoryCategory
+  | typeof FULL_CONTEXT_MEMORY
+  | FullContextSourceCategory
+  | SelfImprovingCategory;
+
+export const ALL_CATEGORIES: readonly MemoryCategory[] = [
+  ...USER_MEMORY_CATEGORIES,
+  FULL_CONTEXT_MEMORY,
+  ...FULL_CONTEXT_SOURCE_CATEGORIES,
+  ...SELF_IMPROVING_CATEGORIES,
+];
+
+export function isUserMemoryCategory(cat: string): cat is UserMemoryCategory {
+  return USER_MEMORY_CATEGORIES.includes(cat as UserMemoryCategory);
+}
+
+export function isSelfImprovingCategory(cat: string): cat is SelfImprovingCategory {
+  return SELF_IMPROVING_CATEGORIES.includes(cat as SelfImprovingCategory);
+}
+
+/** Session id for rows inserted from the admin UI (manual add). */
+export const MANUAL_INSERT_SESSION = "manual_insert" as const;
+
+/** Chinese labels for admin UI and APIs (fixed mapping). */
+export const MEMORY_CATEGORY_LABEL_ZH: Readonly<Record<MemoryCategory, string>> = {
+  [USER_MEMORY_FACT]: "用户事实",
+  [USER_MEMORY_PREFERENCE]: "用户偏好",
+  [USER_MEMORY_DECISION]: "用户决策",
+  [FULL_CONTEXT_MEMORY]: "全文记忆",
+  [FULL_CONTEXT_USER]: "全文 · 用户消息",
+  [FULL_CONTEXT_ASSISTANT]: "全文 · AI助手消息",
+  [FULL_CONTEXT_SYSTEM]: "全文 · 系统消息",
+  [FULL_CONTEXT_TOOL]: "全文 · 工具调用",
+  [FULL_CONTEXT_TOOL_RESULT]: "全文 · 工具结果",
+  [FULL_CONTEXT_OTHERS]: "全文 · 其他消息",
+  [SELF_IMPROVING_LEARNINGS]: "最佳实践",
+  [SELF_IMPROVING_ERRORS]: "错误经验",
+  [SELF_IMPROVING_FEATURE_REQUESTS]: "行为诉求",
+};
+
+export function categoryLabelZh(category: string): string {
+  return MEMORY_CATEGORY_LABEL_ZH[category as MemoryCategory] ?? category;
+}