memory-lancedb-pro 1.0.11 → 1.0.13

package/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 1.0.13
+
+- Fix: Force `encoding_format: "float"` for OpenAI-compatible embedding requests to avoid base64/float ambiguity and dimension mismatch issues with some providers/gateways.
+- Feat: Add Voyage AI (`voyage`) as a supported rerank provider, using `top_k` and `Authorization: Bearer` header.
+- Refactor: Harden rerank response parser to accept both `results[]`/`data[]` payload shapes and `relevance_score`/`score` field names across all providers.
+
+## 1.0.12
+
+- Fix: ghost memories stuck in autoRecall after deletion (#15). BM25-only results from stale FTS index are now validated via `store.hasId()` before inclusion in fused results. Removed the BM25-only floor score of 0.5 that allowed deleted entries to survive `hardMinScore` filtering.
+- Fix: HEARTBEAT pattern now matches anywhere in the prompt (not just at start), preventing autoRecall from triggering on prefixed HEARTBEAT messages.
+- Add: `autoRecallMinLength` config option to set a custom minimum prompt length for autoRecall (default: 15 chars English, 6 CJK). Prompts shorter than this threshold are skipped.
+- Add: `ping`, `pong`, `test`, `debug` added to skip patterns in adaptive retrieval.
+
 ## 1.0.11
 
 - Change: set `autoRecall` default to `false` to avoid the model echoing injected `<relevant-memories>` blocks.

package/README.md

@@ -393,8 +393,13 @@ Cross-encoder reranking supports multiple providers via `rerankProvider`:
 |----------|-----------------|----------|---------------|
 | **Jina** (default) | `jina` | `https://api.jina.ai/v1/rerank` | `jina-reranker-v3` |
 | **SiliconFlow** (free tier available) | `siliconflow` | `https://api.siliconflow.com/v1/rerank` | `BAAI/bge-reranker-v2-m3`, `Qwen/Qwen3-Reranker-8B` |
+| **Voyage AI** | `voyage` | `https://api.voyageai.com/v1/rerank` | `rerank-2.5` |
 | **Pinecone** | `pinecone` | `https://api.pinecone.io/rerank` | `bge-reranker-v2-m3` |
 
+Notes:
+- `voyage` sends `{ model, query, documents }` without `top_n`.
+- Voyage responses are parsed from `data[].relevance_score`.
+
 <details>
 <summary><strong>SiliconFlow Example</strong></summary>
 
@@ -412,6 +417,23 @@ Cross-encoder reranking supports multiple providers via `rerankProvider`:
 
 </details>
 
+<details>
+<summary><strong>Voyage Example</strong></summary>
+
+```json
+{
+  "retrieval": {
+    "rerank": "cross-encoder",
+    "rerankProvider": "voyage",
+    "rerankEndpoint": "https://api.voyageai.com/v1/rerank",
+    "rerankApiKey": "${VOYAGE_API_KEY}",
+    "rerankModel": "rerank-2.5"
+  }
+}
+```
+
+</details>
+
 <details>
 <summary><strong>Pinecone Example</strong></summary>
 

package/index.ts

@@ -37,6 +37,7 @@ interface PluginConfig {
   dbPath?: string;
   autoCapture?: boolean;
   autoRecall?: boolean;
+  autoRecallMinLength?: number;
   captureAssistant?: boolean;
   retrieval?: {
     mode?: "hybrid" | "vector";
@@ -48,7 +49,7 @@ interface PluginConfig {
     rerankApiKey?: string;
     rerankModel?: string;
     rerankEndpoint?: string;
-    rerankProvider?: "jina" | "siliconflow" | "pinecone";
+    rerankProvider?: "jina" | "siliconflow" | "voyage" | "pinecone";
     recencyHalfLifeDays?: number;
     recencyWeight?: number;
     filterNoise?: boolean;
@@ -200,7 +201,7 @@ async function readSessionMessages(filePath: string, messageCount: number): Prom
             }
           }
         }
-      } catch {}
+      } catch { }
     }
 
     if (messages.length === 0) return null;
@@ -225,7 +226,7 @@ async function readSessionContentWithResetFallback(sessionFilePath: string, mess
       const latestResetPath = join(dir, resetCandidates[resetCandidates.length - 1]);
       return await readSessionMessages(latestResetPath, messageCount);
     }
-  } catch {}
+  } catch { }
 
   return primary;
 }
@@ -264,7 +265,7 @@ async function findPreviousSessionFile(sessionsDir: string, currentSessionFile?:
         .sort().reverse();
       if (nonReset.length > 0) return join(sessionsDir, nonReset[0]);
     }
-  } catch {}
+  } catch { }
 }
 
 // ============================================================================
@@ -367,7 +368,7 @@ const memoryLanceDBProPlugin = {
     // Default is OFF to prevent the model from accidentally echoing injected context.
     if (config.autoRecall === true) {
       api.on("before_agent_start", async (event, ctx) => {
-        if (!event.prompt || shouldSkipRetrieval(event.prompt)) {
+        if (!event.prompt || shouldSkipRetrieval(event.prompt, config.autoRecallMinLength)) {
           return;
         }
 
@@ -623,7 +624,7 @@ const memoryLanceDBProPlugin = {
         if (files.length > 7) {
           const { unlink } = await import("node:fs/promises");
           for (const old of files.slice(0, files.length - 7)) {
-            await unlink(join(backupDir, old)).catch(() => {});
+            await unlink(join(backupDir, old)).catch(() => { });
           }
         }
 
@@ -664,10 +665,10 @@ const memoryLanceDBProPlugin = {
 
             api.logger.info(
               `memory-lancedb-pro: initialized successfully ` +
-                `(embedding: ${embedTest.success ? "OK" : "FAIL"}, ` +
-                `retrieval: ${retrievalTest.success ? "OK" : "FAIL"}, ` +
-                `mode: ${retrievalTest.mode}, ` +
-                `FTS: ${retrievalTest.hasFtsSupport ? "enabled" : "disabled"})`
+              `(embedding: ${embedTest.success ? "OK" : "FAIL"}, ` +
+              `retrieval: ${retrievalTest.success ? "OK" : "FAIL"}, ` +
+              `mode: ${retrievalTest.mode}, ` +
+              `FTS: ${retrievalTest.hasFtsSupport ? "enabled" : "disabled"})`
             );
 
             if (!embedTest.success) {
@@ -701,54 +702,55 @@ const memoryLanceDBProPlugin = {
 };
 
 function parsePluginConfig(value: unknown): PluginConfig {
-    if (!value || typeof value !== "object" || Array.isArray(value)) {
-      throw new Error("memory-lancedb-pro config required");
-    }
-    const cfg = value as Record<string, unknown>;
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("memory-lancedb-pro config required");
+  }
+  const cfg = value as Record<string, unknown>;
 
-    const embedding = cfg.embedding as Record<string, unknown> | undefined;
-    if (!embedding) {
-      throw new Error("embedding config is required");
-    }
+  const embedding = cfg.embedding as Record<string, unknown> | undefined;
+  if (!embedding) {
+    throw new Error("embedding config is required");
+  }
 
-    const apiKey = typeof embedding.apiKey === "string"
-      ? embedding.apiKey
-      : process.env.OPENAI_API_KEY || "";
+  const apiKey = typeof embedding.apiKey === "string"
+    ? embedding.apiKey
+    : process.env.OPENAI_API_KEY || "";
 
-    if (!apiKey) {
-      throw new Error("embedding.apiKey is required (set directly or via OPENAI_API_KEY env var)");
-    }
+  if (!apiKey) {
+    throw new Error("embedding.apiKey is required (set directly or via OPENAI_API_KEY env var)");
+  }
 
-    return {
-      embedding: {
-        provider: "openai-compatible",
-        apiKey,
-        model: typeof embedding.model === "string" ? embedding.model : "text-embedding-3-small",
-        baseURL: typeof embedding.baseURL === "string" ? resolveEnvVars(embedding.baseURL) : undefined,
-        // Accept number, numeric string, or env-var string (e.g. "${EMBED_DIM}").
-        // Also accept legacy top-level `dimensions` for convenience.
-        dimensions: parsePositiveInt(embedding.dimensions ?? cfg.dimensions),
-        taskQuery: typeof embedding.taskQuery === "string" ? embedding.taskQuery : undefined,
-        taskPassage: typeof embedding.taskPassage === "string" ? embedding.taskPassage : undefined,
-        normalized: typeof embedding.normalized === "boolean" ? embedding.normalized : undefined,
-      },
-      dbPath: typeof cfg.dbPath === "string" ? cfg.dbPath : undefined,
-      autoCapture: cfg.autoCapture !== false,
-      // Default OFF: only enable when explicitly set to true.
-      autoRecall: cfg.autoRecall === true,
-      captureAssistant: cfg.captureAssistant === true,
-      retrieval: typeof cfg.retrieval === "object" && cfg.retrieval !== null ? cfg.retrieval as any : undefined,
-      scopes: typeof cfg.scopes === "object" && cfg.scopes !== null ? cfg.scopes as any : undefined,
-      enableManagementTools: cfg.enableManagementTools === true,
-      sessionMemory: typeof cfg.sessionMemory === "object" && cfg.sessionMemory !== null
-        ? {
-            enabled: (cfg.sessionMemory as Record<string, unknown>).enabled !== false,
-            messageCount: typeof (cfg.sessionMemory as Record<string, unknown>).messageCount === "number"
-              ? (cfg.sessionMemory as Record<string, unknown>).messageCount as number
-              : undefined,
-          }
-        : undefined,
-    };
+  return {
+    embedding: {
+      provider: "openai-compatible",
+      apiKey,
+      model: typeof embedding.model === "string" ? embedding.model : "text-embedding-3-small",
+      baseURL: typeof embedding.baseURL === "string" ? resolveEnvVars(embedding.baseURL) : undefined,
+      // Accept number, numeric string, or env-var string (e.g. "${EMBED_DIM}").
+      // Also accept legacy top-level `dimensions` for convenience.
+      dimensions: parsePositiveInt(embedding.dimensions ?? cfg.dimensions),
+      taskQuery: typeof embedding.taskQuery === "string" ? embedding.taskQuery : undefined,
+      taskPassage: typeof embedding.taskPassage === "string" ? embedding.taskPassage : undefined,
+      normalized: typeof embedding.normalized === "boolean" ? embedding.normalized : undefined,
+    },
+    dbPath: typeof cfg.dbPath === "string" ? cfg.dbPath : undefined,
+    autoCapture: cfg.autoCapture !== false,
+    // Default OFF: only enable when explicitly set to true.
+    autoRecall: cfg.autoRecall === true,
+    autoRecallMinLength: parsePositiveInt(cfg.autoRecallMinLength),
+    captureAssistant: cfg.captureAssistant === true,
+    retrieval: typeof cfg.retrieval === "object" && cfg.retrieval !== null ? cfg.retrieval as any : undefined,
+    scopes: typeof cfg.scopes === "object" && cfg.scopes !== null ? cfg.scopes as any : undefined,
+    enableManagementTools: cfg.enableManagementTools === true,
+    sessionMemory: typeof cfg.sessionMemory === "object" && cfg.sessionMemory !== null
+      ? {
+        enabled: (cfg.sessionMemory as Record<string, unknown>).enabled !== false,
+        messageCount: typeof (cfg.sessionMemory as Record<string, unknown>).messageCount === "number"
+          ? (cfg.sessionMemory as Record<string, unknown>).messageCount as number
+          : undefined,
+      }
+      : undefined,
+  };
 }
 
-export default memoryLanceDBProPlugin;
+export default memoryLanceDBProPlugin;

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "memory-lancedb-pro",
   "name": "Memory (LanceDB Pro)",
   "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, and management CLI",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "kind": "memory",
   "configSchema": {
     "type": "object",
@@ -61,6 +61,13 @@
         "type": "boolean",
         "default": false
       },
+      "autoRecallMinLength": {
+        "type": "integer",
+        "minimum": 1,
+        "maximum": 200,
+        "default": 15,
+        "description": "Minimum prompt length (in characters) to trigger auto-recall. Prompts shorter than this are skipped. Default: 15 for English, 6 for CJK."
+      },
       "captureAssistant": {
         "type": "boolean"
       },
@@ -122,6 +129,7 @@
             "enum": [
               "jina",
               "siliconflow",
+              "voyage",
               "pinecone"
             ],
             "default": "jina",
@@ -266,6 +274,11 @@
       "label": "Auto-Recall",
       "help": "Automatically inject relevant memories into context"
     },
+    "autoRecallMinLength": {
+      "label": "Auto-Recall Min Length",
+      "help": "Minimum prompt length to trigger auto-recall (shorter prompts are skipped). Default: 15 chars for English, 6 for CJK.",
+      "advanced": true
+    },
     "captureAssistant": {
       "label": "Capture Assistant Messages",
       "help": "Also auto-capture assistant messages (default false to reduce memory pollution)",
@@ -317,7 +330,7 @@
     },
     "retrieval.rerankProvider": {
       "label": "Reranker Provider",
-      "help": "Provider format: jina (default), siliconflow, or pinecone",
+      "help": "Provider format: jina (default), siliconflow, voyage, or pinecone",
       "advanced": true
     },
     "retrieval.candidatePoolSize": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-lancedb-pro",
-  "version": "1.0.11",
+  "version": "1.0.13",
   "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, and management CLI",
   "type": "module",
   "main": "index.ts",
@@ -40,4 +40,4 @@
     "jiti": "^2.6.0",
     "typescript": "^5.9.3"
   }
-}
+}

package/src/adaptive-retrieval.ts

@@ -18,9 +18,11 @@ const SKIP_PATTERNS = [
   /^(go ahead|continue|proceed|do it|start|begin|next|实施|开始|继续|好的|可以|行)\s*[.!]?$/i,
   // Pure emoji
   /^[\p{Emoji}\s]+$/u,
-  // Heartbeat/system
-  /^HEARTBEAT/i,
+  // Heartbeat/system (match anywhere, not just at start, to handle prefixed formats)
+  /HEARTBEAT/i,
   /^\[System/i,
+  // Single-word utility pings
+  /^(ping|pong|test|debug)\s*[.!?]?$/i,
 ];
 
 // Queries that SHOULD trigger retrieval even if short
@@ -61,8 +63,10 @@ function normalizeQuery(query: string): string {
 /**
  * Determine if a query should skip memory retrieval.
  * Returns true if retrieval should be skipped.
+ * @param query The raw prompt text
+ * @param minLength Optional minimum length override (if set, overrides built-in thresholds)
  */
-export function shouldSkipRetrieval(query: string): boolean {
+export function shouldSkipRetrieval(query: string, minLength?: number): boolean {
   const trimmed = normalizeQuery(query);
 
   // Force retrieve if query has memory-related intent (checked FIRST,
@@ -75,11 +79,17 @@ export function shouldSkipRetrieval(query: string): boolean {
   // Skip if matches any skip pattern
   if (SKIP_PATTERNS.some(p => p.test(trimmed))) return true;
 
+  // If caller provides a custom minimum length, use it
+  if (minLength !== undefined && minLength > 0) {
+    if (trimmed.length < minLength && !trimmed.includes('?') && !trimmed.includes('？')) return true;
+    return false;
+  }
+
   // Skip very short non-question messages (likely commands or affirmations)
   // CJK characters carry more meaning per character, so use a lower threshold
   const hasCJK = /[\u4e00-\u9fff\u3040-\u309f\u30a0-\u30ff\uac00-\ud7af]/.test(trimmed);
-  const minLength = hasCJK ? 6 : 15;
-  if (trimmed.length < minLength && !trimmed.includes('?') && !trimmed.includes('？')) return true;
+  const defaultMinLength = hasCJK ? 6 : 15;
+  if (trimmed.length < defaultMinLength && !trimmed.includes('?') && !trimmed.includes('？')) return true;
 
   // Default: do retrieve
   return false;

package/src/embedder.ts

@@ -236,6 +236,8 @@ export class Embedder {
     const payload: any = {
       model: this.model,
       input,
+      // Force float output to avoid SDK default base64 decoding path.
+      encoding_format: "float",
     };
 
     if (task) payload.task = task;

package/src/retriever.ts

@@ -33,8 +33,9 @@ export interface RetrievalConfig {
   /** Reranker provider format. Determines request/response shape and auth header.
    *  - "jina" (default): Authorization: Bearer, string[] documents, results[].relevance_score
    *  - "siliconflow": same format as jina (alias, for clarity)
+   *  - "voyage": Authorization: Bearer, string[] documents, data[].relevance_score
    *  - "pinecone": Api-Key header, {text}[] documents, data[].score */
-  rerankProvider?: "jina" | "siliconflow" | "pinecone";
+  rerankProvider?: "jina" | "siliconflow" | "voyage" | "pinecone";
   /**
    * Length normalization: penalize long entries that dominate via sheer keyword
    * density. Formula: score *= 1 / (1 + log2(charLen / anchor)).
@@ -115,7 +116,7 @@ function clamp01(value: number, fallback: number): number {
 // Rerank Provider Adapters
 // ============================================================================
 
-type RerankProvider = "jina" | "siliconflow" | "pinecone";
+type RerankProvider = "jina" | "siliconflow" | "voyage" | "pinecone";
 
 interface RerankItem { index: number; score: number }
 
@@ -144,6 +145,20 @@ function buildRerankRequest(
           rank_fields: ["text"],
         },
       };
+    case "voyage":
+      return {
+        headers: {
+          "Content-Type": "application/json",
+          "Authorization": `Bearer ${apiKey}`,
+        },
+        body: {
+          model,
+          query,
+          documents,
+          // Voyage uses top_k (not top_n) to limit reranked outputs.
+          top_k: topN,
+        },
+      };
     case "siliconflow":
     case "jina":
     default:
@@ -167,20 +182,56 @@ function parseRerankResponse(
   provider: RerankProvider,
   data: Record<string, unknown>,
 ): RerankItem[] | null {
+  const parseItems = (
+    items: unknown,
+    scoreKeys: Array<"score" | "relevance_score">,
+  ): RerankItem[] | null => {
+    if (!Array.isArray(items)) return null;
+    const parsed: RerankItem[] = [];
+    for (const raw of items as Array<Record<string, unknown>>) {
+      const index = typeof raw?.index === "number" ? raw.index : Number(raw?.index);
+      if (!Number.isFinite(index)) continue;
+      let score: number | null = null;
+      for (const key of scoreKeys) {
+        const value = raw?.[key];
+        const n = typeof value === "number" ? value : Number(value);
+        if (Number.isFinite(n)) {
+          score = n;
+          break;
+        }
+      }
+      if (score === null) continue;
+      parsed.push({ index, score });
+    }
+    return parsed.length > 0 ? parsed : null;
+  };
+
   switch (provider) {
     case "pinecone": {
-      // Pinecone: { data: [{ index, score, document }] }
-      const items = data.data as Array<{ index: number; score: number }> | undefined;
-      if (!Array.isArray(items)) return null;
-      return items.map(r => ({ index: r.index, score: r.score }));
+      // Pinecone: usually { data: [{ index, score, ... }] }
+      // Also tolerate results[] with score/relevance_score for robustness.
+      return (
+        parseItems(data.data, ["score", "relevance_score"]) ??
+        parseItems(data.results, ["score", "relevance_score"])
+      );
+    }
+    case "voyage": {
+      // Voyage: usually { data: [{ index, relevance_score }] }
+      // Also tolerate results[] for compatibility across gateways.
+      return (
+        parseItems(data.data, ["relevance_score", "score"]) ??
+        parseItems(data.results, ["relevance_score", "score"])
+      );
     }
     case "siliconflow":
     case "jina":
     default: {
-      // Jina / SiliconFlow: { results: [{ index, relevance_score }] }
-      const items = data.results as Array<{ index: number; relevance_score: number }> | undefined;
-      if (!Array.isArray(items)) return null;
-      return items.map(r => ({ index: r.index, score: r.relevance_score }));
+      // Jina / SiliconFlow: usually { results: [{ index, relevance_score }] }
+      // Also tolerate data[] for compatibility across gateways.
+      return (
+        parseItems(data.results, ["relevance_score", "score"]) ??
+        parseItems(data.data, ["relevance_score", "score"])
+      );
     }
   }
 }
@@ -282,8 +333,8 @@ export class MemoryRetriever {
       this.runBM25Search(query, candidatePoolSize, scopeFilter, category),
     ]);
 
-    // Fuse results using RRF
-    const fusedResults = this.fuseResults(vectorResults, bm25Results);
+    // Fuse results using RRF (async: validates BM25-only entries exist in store)
+    const fusedResults = await this.fuseResults(vectorResults, bm25Results);
 
     // Apply minimum score threshold
     const filtered = fusedResults.filter(r => r.score >= this.config.minScore);
@@ -357,10 +408,10 @@ export class MemoryRetriever {
     }));
   }
 
-  private fuseResults(
+  private async fuseResults(
     vectorResults: Array<MemorySearchResult & { rank: number }>,
     bm25Results: Array<MemorySearchResult & { rank: number }>
-  ): RetrievalResult[] {
+  ): Promise<RetrievalResult[]> {
     // Create maps for quick lookup
     const vectorMap = new Map<string, MemorySearchResult & { rank: number }>();
     const bm25Map = new Map<string, MemorySearchResult & { rank: number }>();
@@ -383,6 +434,18 @@ export class MemoryRetriever {
       const vectorResult = vectorMap.get(id);
       const bm25Result = bm25Map.get(id);
 
+      // FIX(#15): BM25-only results may be "ghost" entries whose vector data was
+      // deleted but whose FTS index entry lingers until the next index rebuild.
+      // Validate that the entry actually exists in the store before including it.
+      if (!vectorResult && bm25Result) {
+        try {
+          const exists = await this.store.hasId(id);
+          if (!exists) continue; // Skip ghost entry
+        } catch {
+          // If hasId fails, keep the result (fail-open)
+        }
+      }
+
       // Use the result with more complete data (prefer vector result if both exist)
       const baseResult = vectorResult || bm25Result!;
 
@@ -392,12 +455,12 @@ export class MemoryRetriever {
       const bm25Hit = bm25Result ? 1 : 0;
 
       // Base = vector score; BM25 hit boosts by up to 15%
-      // BM25-only results use their normalized score (floor 0.5) so exact keyword
-      // matches aren't buried — e.g. searching "JINA_API_KEY" should surface even
-      // when vector distance is large.
+      // BM25-only results use their raw BM25 score so exact keyword matches
+      // (e.g. searching "JINA_API_KEY") still surface. The previous floor of 0.5
+      // was too generous and allowed ghost entries to survive hardMinScore (0.35).
       const fusedScore = vectorResult
         ? clamp01(vectorScore + (bm25Hit * 0.15 * vectorScore), 0.1)
-        : clamp01(Math.max(bm25Result!.score, 0.5), 0.1);
+        : clamp01(bm25Result!.score, 0.1);
 
       fusedResults.push({
         entry: baseResult.entry,
@@ -719,4 +782,4 @@ export function createRetriever(
 ): MemoryRetriever {
   const fullConfig = { ...DEFAULT_RETRIEVAL_CONFIG, ...config };
   return new MemoryRetriever(store, embedder, fullConfig);
-}
+}