memory-lancedb-pro 1.0.26 → 1.1.0-beta.2

package/openclaw.plugin.json

@@ -2,7 +2,7 @@
   "id": "memory-lancedb-pro",
   "name": "Memory (LanceDB Pro)",
   "description": "Enhanced LanceDB-backed long-term memory with hybrid retrieval, multi-scope isolation, long-context chunking, and management CLI",
-  "version": "1.0.26",
+  "version": "1.1.0-beta.2",
   "kind": "memory",
   "configSchema": {
     "type": "object",
@@ -263,6 +263,44 @@
             }
           }
         }
+      },
+      "smartExtraction": {
+        "type": "boolean",
+        "default": true,
+        "description": "Enable LLM-powered smart memory extraction (6-category system). Falls back to regex capture when disabled or init fails."
+      },
+      "llm": {
+        "type": "object",
+        "additionalProperties": false,
+        "properties": {
+          "apiKey": {
+            "type": "string",
+            "description": "API key for LLM extraction (defaults to embedding apiKey)"
+          },
+          "model": {
+            "type": "string",
+            "default": "gpt-4o-mini",
+            "description": "LLM model for memory extraction and dedup"
+          },
+          "baseURL": {
+            "type": "string",
+            "description": "Base URL for LLM API (defaults to embedding baseURL)"
+          }
+        }
+      },
+      "extractMinMessages": {
+        "type": "integer",
+        "minimum": 1,
+        "maximum": 50,
+        "default": 4,
+        "description": "Minimum conversation messages before smart extraction triggers"
+      },
+      "extractMaxChars": {
+        "type": "integer",
+        "minimum": 500,
+        "maximum": 100000,
+        "default": 8000,
+        "description": "Maximum characters of conversation text to process for extraction"
       }
     },
     "required": [
@@ -321,6 +359,36 @@
       "help": "Directory path for the LanceDB database files",
       "advanced": true
     },
+    "smartExtraction": {
+      "label": "Smart Extraction",
+      "help": "Enable LLM-powered 6-category memory extraction. Falls back to regex capture when off."
+    },
+    "llm.apiKey": {
+      "label": "LLM API Key",
+      "sensitive": true,
+      "help": "API key for smart extraction LLM (defaults to embedding apiKey)",
+      "advanced": true
+    },
+    "llm.model": {
+      "label": "LLM Model",
+      "help": "Model for memory extraction and dedup (default: gpt-4o-mini)",
+      "advanced": true
+    },
+    "llm.baseURL": {
+      "label": "LLM Base URL",
+      "help": "Base URL for LLM API (defaults to embedding baseURL)",
+      "advanced": true
+    },
+    "extractMinMessages": {
+      "label": "Min Messages for Extraction",
+      "help": "Minimum conversation messages before smart extraction triggers",
+      "advanced": true
+    },
+    "extractMaxChars": {
+      "label": "Max Chars for Extraction",
+      "help": "Maximum conversation characters to process for extraction",
+      "advanced": true
+    },
     "autoCapture": {
       "label": "Auto-Capture",
       "help": "Automatically capture important information from conversations"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "memory-lancedb-pro",
-  "version": "1.0.26",
+  "version": "1.1.0-beta.2", 
   "description": "OpenClaw enhanced LanceDB memory plugin with hybrid retrieval (Vector + BM25), cross-encoder rerank, multi-scope isolation, long-context chunking, and management CLI",
   "type": "module",
   "main": "index.ts",

package/src/access-tracker.ts

@@ -83,9 +83,14 @@ export function parseAccessMetadata(
 
   const obj = parsed as Record<string, unknown>;
 
-  const rawCount = typeof obj.accessCount === "number" ? obj.accessCount : 0;
+  // Support both camelCase and snake_case keys (beta smart-memory uses snake_case).
+  const rawCountAny = obj.accessCount ?? obj.access_count;
+  const rawCount =
+    typeof rawCountAny === "number" ? rawCountAny : Number(rawCountAny ?? 0);
+
+  const rawLastAny = obj.lastAccessedAt ?? obj.last_accessed_at;
   const rawLastAccessed =
-    typeof obj.lastAccessedAt === "number" ? obj.lastAccessedAt : 0;
+    typeof rawLastAny === "number" ? rawLastAny : Number(rawLastAny ?? 0);
 
   return {
     accessCount: clampAccessCount(rawCount),
@@ -126,10 +131,15 @@ export function buildUpdatedMetadata(
   const prev = parseAccessMetadata(existingMetadata);
   const newCount = clampAccessCount(prev.accessCount + accessDelta);
 
+  const now = Date.now();
+
   return JSON.stringify({
     ...existing,
+    // Write both camelCase and snake_case for compatibility.
     accessCount: newCount,
-    lastAccessedAt: Date.now(),
+    lastAccessedAt: now,
+    access_count: newCount,
+    last_accessed_at: now,
   });
 }
 

package/src/decay-engine.ts

@@ -0,0 +1,227 @@
+/**
+ * Decay Engine — Weibull stretched-exponential decay model
+ * Ported from memx-memory/src/memory/decay.ts
+ *
+ * Composite score = recencyWeight * recency + frequencyWeight * frequency + intrinsicWeight * intrinsic
+ *
+ * - Recency: Weibull decay with importance-modulated half-life and tier-specific beta
+ * - Frequency: Logarithmic saturation with time-weighted access pattern bonus
+ * - Intrinsic: importance × confidence
+ */
+
+import type { MemoryTier } from "./memory-categories.js";
+
+// ============================================================================
+// Types
+// ============================================================================
+
+const MS_PER_DAY = 86_400_000;
+
+export interface DecayConfig {
+  /** Days until recency score halves (default: 30) */
+  recencyHalfLifeDays: number;
+  /** Weight of recency in composite (default: 0.4) */
+  recencyWeight: number;
+  /** Weight of access frequency (default: 0.3) */
+  frequencyWeight: number;
+  /** Weight of importance × confidence (default: 0.3) */
+  intrinsicWeight: number;
+  /** Below this composite = stale (default: 0.3) */
+  staleThreshold: number;
+  /** Minimum search boost (default: 0.3) */
+  searchBoostMin: number;
+  /** Importance modulation coefficient for half-life (default: 1.5) */
+  importanceModulation: number;
+  /** Weibull beta for Core tier — sub-exponential (default: 0.8) */
+  betaCore: number;
+  /** Weibull beta for Working tier — standard exponential (default: 1.0) */
+  betaWorking: number;
+  /** Weibull beta for Peripheral tier — super-exponential (default: 1.3) */
+  betaPeripheral: number;
+  /** Decay floor for Core memories (default: 0.9) */
+  coreDecayFloor: number;
+  /** Decay floor for Working memories (default: 0.7) */
+  workingDecayFloor: number;
+  /** Decay floor for Peripheral memories (default: 0.5) */
+  peripheralDecayFloor: number;
+}
+
+export const DEFAULT_DECAY_CONFIG: DecayConfig = {
+  recencyHalfLifeDays: 30,
+  recencyWeight: 0.4,
+  frequencyWeight: 0.3,
+  intrinsicWeight: 0.3,
+  staleThreshold: 0.3,
+  searchBoostMin: 0.3,
+  importanceModulation: 1.5,
+  betaCore: 0.8,
+  betaWorking: 1.0,
+  betaPeripheral: 1.3,
+  coreDecayFloor: 0.9,
+  workingDecayFloor: 0.7,
+  peripheralDecayFloor: 0.5,
+};
+
+export interface DecayScore {
+  memoryId: string;
+  recency: number;
+  frequency: number;
+  intrinsic: number;
+  composite: number;
+}
+
+/** Minimal memory fields needed for decay calculation. */
+export interface DecayableMemory {
+  id: string;
+  importance: number;
+  confidence: number;
+  tier: MemoryTier;
+  accessCount: number;
+  createdAt: number;
+  lastAccessedAt: number;
+}
+
+export interface DecayEngine {
+  /** Calculate decay score for a single memory */
+  score(memory: DecayableMemory, now?: number): DecayScore;
+  /** Calculate decay scores for multiple memories */
+  scoreAll(memories: DecayableMemory[], now?: number): DecayScore[];
+  /** Apply decay boost to search results (multiplies each score by boost) */
+  applySearchBoost(
+    results: Array<{ memory: DecayableMemory; score: number }>,
+    now?: number,
+  ): void;
+  /** Find stale memories (composite below threshold) */
+  getStaleMemories(
+    memories: DecayableMemory[],
+    now?: number,
+  ): DecayScore[];
+}
+
+// ============================================================================
+// Factory
+// ============================================================================
+
+export function createDecayEngine(
+  config: DecayConfig = DEFAULT_DECAY_CONFIG,
+): DecayEngine {
+  const {
+    recencyHalfLifeDays: halfLife,
+    recencyWeight: rw,
+    frequencyWeight: fw,
+    intrinsicWeight: iw,
+    staleThreshold,
+    searchBoostMin: boostMin,
+    importanceModulation: mu,
+    betaCore,
+    betaWorking,
+    betaPeripheral,
+    coreDecayFloor,
+    workingDecayFloor,
+    peripheralDecayFloor,
+  } = config;
+
+  function getTierBeta(tier: MemoryTier): number {
+    switch (tier) {
+      case "core":
+        return betaCore;
+      case "working":
+        return betaWorking;
+      case "peripheral":
+        return betaPeripheral;
+    }
+  }
+
+  function getTierFloor(tier: MemoryTier): number {
+    switch (tier) {
+      case "core":
+        return coreDecayFloor;
+      case "working":
+        return workingDecayFloor;
+      case "peripheral":
+        return peripheralDecayFloor;
+    }
+  }
+
+  /**
+   * Recency: Weibull stretched-exponential decay with importance-modulated half-life.
+   * effectiveHL = halfLife * exp(mu * importance)
+   * lambda = ln(2) / effectiveHL
+   * recency = exp(-lambda * daysSince^beta)
+   */
+  function recency(memory: DecayableMemory, now: number): number {
+    const lastActive =
+      memory.accessCount > 0 ? memory.lastAccessedAt : memory.createdAt;
+    const daysSince = Math.max(0, (now - lastActive) / MS_PER_DAY);
+    const effectiveHL = halfLife * Math.exp(mu * memory.importance);
+    const lambda = Math.LN2 / effectiveHL;
+    const beta = getTierBeta(memory.tier);
+    return Math.exp(-lambda * Math.pow(daysSince, beta));
+  }
+
+  /**
+   * Frequency: logarithmic saturation curve with time-weighted access pattern bonus.
+   * base = 1 - exp(-accessCount / 5)
+   * For memories with >1 access, a recentness bonus is applied.
+   */
+  function frequency(memory: DecayableMemory): number {
+    const base = 1 - Math.exp(-memory.accessCount / 5);
+    if (memory.accessCount <= 1) return base;
+
+    const lastActive =
+      memory.accessCount > 0 ? memory.lastAccessedAt : memory.createdAt;
+    const accessSpanDays = Math.max(
+      1,
+      (lastActive - memory.createdAt) / MS_PER_DAY,
+    );
+    const avgGapDays = accessSpanDays / Math.max(memory.accessCount - 1, 1);
+    const recentnessBonus = Math.exp(-avgGapDays / 30);
+    return base * (0.5 + 0.5 * recentnessBonus);
+  }
+
+  /**
+   * Intrinsic value: importance × confidence.
+   */
+  function intrinsic(memory: DecayableMemory): number {
+    return memory.importance * memory.confidence;
+  }
+
+  function scoreOne(memory: DecayableMemory, now: number): DecayScore {
+    const r = recency(memory, now);
+    const f = frequency(memory);
+    const i = intrinsic(memory);
+    const composite = rw * r + fw * f + iw * i;
+
+    return {
+      memoryId: memory.id,
+      recency: r,
+      frequency: f,
+      intrinsic: i,
+      composite,
+    };
+  }
+
+  return {
+    score(memory, now = Date.now()) {
+      return scoreOne(memory, now);
+    },
+
+    scoreAll(memories, now = Date.now()) {
+      return memories.map((m) => scoreOne(m, now));
+    },
+
+    applySearchBoost(results, now = Date.now()) {
+      for (const r of results) {
+        const ds = scoreOne(r.memory, now);
+        r.score *= Math.max(getTierFloor(r.memory.tier), ds.composite);
+      }
+    },
+
+    getStaleMemories(memories, now = Date.now()) {
+      const scores = memories.map((m) => scoreOne(m, now));
+      return scores
+        .filter((s) => s.composite < staleThreshold)
+        .sort((a, b) => a.composite - b.composite);
+    },
+  };
+}

package/src/extraction-prompts.ts

@@ -0,0 +1,205 @@
+/**
+ * Prompt templates for intelligent memory extraction.
+ * Ported from epro-memory/prompts.ts (OpenViking origin).
+ *
+ * Three mandatory prompts:
+ * - buildExtractionPrompt: 6-category L0/L1/L2 extraction with few-shot
+ * - buildDedupPrompt: CREATE/MERGE/SKIP dedup decision
+ * - buildMergePrompt: Memory merge with three-level structure
+ */
+
+export function buildExtractionPrompt(
+  conversationText: string,
+  user: string,
+): string {
+  return `Analyze the following session context and extract memories worth long-term preservation.
+
+User: ${user}
+
+Target Output Language: auto (detect from recent messages)
+
+## Recent Conversation
+${conversationText}
+
+# Memory Extraction Criteria
+
+## What is worth remembering?
+- Personalized information: Information specific to this user, not general domain knowledge
+- Long-term validity: Information that will still be useful in future sessions
+- Specific and clear: Has concrete details, not vague generalizations
+
+## What is NOT worth remembering?
+- General knowledge that anyone would know
+- Temporary information: One-time questions or conversations
+- Vague information: "User has questions about a feature" (no specific details)
+- Tool output, error logs, or boilerplate
+
+# Memory Classification
+
+## Core Decision Logic
+
+| Question | Answer | Category |
+|----------|--------|----------|
+| Who is the user? | Identity, attributes | profile |
+| What does the user prefer? | Preferences, habits | preferences |
+| What is this thing? | Person, project, organization | entities |
+| What happened? | Decision, milestone | events |
+| How was it solved? | Problem + solution | cases |
+| What is the process? | Reusable steps | patterns |
+
+## Precise Definition
+
+**profile** - User identity (static attributes). Test: "User is..."
+**preferences** - User preferences (tendencies). Test: "User prefers/likes..."
+**entities** - Continuously existing nouns. Test: "XXX's state is..."
+**events** - Things that happened. Test: "XXX did/completed..."
+**cases** - Problem + solution pairs. Test: Contains "problem -> solution"
+**patterns** - Reusable processes. Test: Can be used in "similar situations"
+
+## Common Confusion
+- "Plan to do X" -> events (action, not entity)
+- "Project X status: Y" -> entities (describes entity)
+- "User prefers X" -> preferences (not profile)
+- "Encountered problem A, used solution B" -> cases (not events)
+- "General process for handling certain problems" -> patterns (not cases)
+
+# Three-Level Structure
+
+Each memory contains three levels:
+
+**abstract (L0)**: One-liner index
+- Merge types (preferences/entities/profile/patterns): \`[Merge key]: [Description]\`
+- Independent types (events/cases): Specific description
+
+**overview (L1)**: Structured Markdown summary with category-specific headings
+
+**content (L2)**: Full narrative with background and details
+
+# Few-shot Examples
+
+## profile
+\`\`\`json
+{
+  "category": "profile",
+  "abstract": "User basic info: AI development engineer, 3 years LLM experience",
+  "overview": "## Background\\n- Occupation: AI development engineer\\n- Experience: 3 years LLM development\\n- Tech stack: Python, LangChain",
+  "content": "User is an AI development engineer with 3 years of LLM application development experience."
+}
+\`\`\`
+
+## preferences
+\`\`\`json
+{
+  "category": "preferences",
+  "abstract": "Python code style: No type hints, concise and direct",
+  "overview": "## Preference Domain\\n- Language: Python\\n- Topic: Code style\\n\\n## Details\\n- No type hints\\n- Concise function comments\\n- Direct implementation",
+  "content": "User prefers Python code without type hints, with concise function comments."
+}
+\`\`\`
+
+## cases
+\`\`\`json
+{
+  "category": "cases",
+  "abstract": "LanceDB BigInt error -> Use Number() coercion before arithmetic",
+  "overview": "## Problem\\nLanceDB 0.26+ returns BigInt for numeric columns\\n\\n## Solution\\nCoerce values with Number(...) before arithmetic",
+  "content": "When LanceDB returns BigInt values, wrap them with Number() before doing arithmetic operations."
+}
+\`\`\`
+
+# Output Format
+
+Return JSON:
+{
+  "memories": [
+    {
+      "category": "profile|preferences|entities|events|cases|patterns",
+      "abstract": "One-line index",
+      "overview": "Structured Markdown summary",
+      "content": "Full narrative"
+    }
+  ]
+}
+
+Notes:
+- Output language should match the dominant language in the conversation
+- Only extract truly valuable personalized information
+- If nothing worth recording, return {"memories": []}
+- Maximum 5 memories per extraction
+- Preferences should be aggregated by topic`;
+}
+
+export function buildDedupPrompt(
+  candidateAbstract: string,
+  candidateOverview: string,
+  candidateContent: string,
+  existingMemories: string,
+): string {
+  return `Determine how to handle this candidate memory.
+
+**Candidate Memory**:
+Abstract: ${candidateAbstract}
+Overview: ${candidateOverview}
+Content: ${candidateContent}
+
+**Existing Similar Memories**:
+${existingMemories}
+
+Please decide:
+- SKIP: Candidate memory duplicates existing memories, no need to save
+- CREATE: This is completely new information, should be created
+- MERGE: Candidate memory should be merged with an existing memory
+
+IMPORTANT: "events" and "cases" categories are independent records — they do NOT support MERGE.
+For these categories, only use SKIP or CREATE.
+
+Return JSON format:
+{
+  "decision": "skip|create|merge",
+  "match_index": 1,
+  "reason": "Decision reason"
+}
+
+If decision is "merge", set "match_index" to the number of the existing memory to merge with (1-based).`;
+}
+
+export function buildMergePrompt(
+  existingAbstract: string,
+  existingOverview: string,
+  existingContent: string,
+  newAbstract: string,
+  newOverview: string,
+  newContent: string,
+  category: string,
+): string {
+  return `Merge the following memory into a single coherent record with all three levels.
+
+**Category**: ${category}
+
+**Existing Memory:**
+Abstract: ${existingAbstract}
+Overview:
+${existingOverview}
+Content:
+${existingContent}
+
+**New Information:**
+Abstract: ${newAbstract}
+Overview:
+${newOverview}
+Content:
+${newContent}
+
+Requirements:
+- Remove duplicate information
+- Keep the most up-to-date details
+- Maintain a coherent narrative
+- Keep code identifiers / URIs / model names unchanged when they are proper nouns
+
+Return JSON:
+{
+  "abstract": "Merged one-line abstract",
+  "overview": "Merged structured Markdown overview",
+  "content": "Merged full content"
+}`;
+}

package/src/llm-client.ts

@@ -0,0 +1,92 @@
+/**
+ * LLM Client for memory extraction and dedup decisions.
+ * Uses OpenAI-compatible API (reuses the embedding provider config).
+ *
+ * Inspired by epro-memory/llm.ts
+ */
+
+import OpenAI from "openai";
+
+export interface LlmClientConfig {
+  apiKey: string;
+  model: string;
+  baseURL?: string;
+  timeoutMs?: number;
+}
+
+export interface LlmClient {
+  /** Send a prompt and parse the JSON response. Returns null on failure. */
+  completeJson<T>(prompt: string): Promise<T | null>;
+}
+
+/**
+ * Extract JSON from an LLM response that may be wrapped in markdown fences
+ * or contain surrounding text.
+ */
+function extractJsonFromResponse(text: string): string | null {
+  // Try markdown code fence first (```json ... ``` or ``` ... ```)
+  const fenceMatch = text.match(/```(?:json)?\s*\n?([\s\S]*?)```/);
+  if (fenceMatch) {
+    return fenceMatch[1].trim();
+  }
+
+  // Try balanced brace extraction
+  const firstBrace = text.indexOf("{");
+  if (firstBrace === -1) return null;
+
+  let depth = 0;
+  let lastBrace = -1;
+  for (let i = firstBrace; i < text.length; i++) {
+    if (text[i] === "{") depth++;
+    else if (text[i] === "}") {
+      depth--;
+      if (depth === 0) {
+        lastBrace = i;
+        break;
+      }
+    }
+  }
+
+  if (lastBrace === -1) return null;
+  return text.substring(firstBrace, lastBrace + 1);
+}
+
+export function createLlmClient(config: LlmClientConfig): LlmClient {
+  const client = new OpenAI({
+    apiKey: config.apiKey,
+    baseURL: config.baseURL,
+    timeout: config.timeoutMs ?? 30000,
+  });
+
+  return {
+    async completeJson<T>(prompt: string): Promise<T | null> {
+      try {
+        const response = await client.chat.completions.create({
+          model: config.model,
+          messages: [
+            {
+              role: "system",
+              content:
+                "You are a memory extraction assistant. Always respond with valid JSON only.",
+            },
+            { role: "user", content: prompt },
+          ],
+          temperature: 0.1,
+        });
+
+        const raw = response.choices?.[0]?.message?.content;
+        if (!raw) return null;
+
+        const jsonStr = extractJsonFromResponse(raw);
+        if (!jsonStr) return null;
+
+        return JSON.parse(jsonStr) as T;
+      } catch (err) {
+        // Graceful degradation — return null so caller can fall back
+        return null;
+      }
+    },
+  };
+}
+
+export { extractJsonFromResponse };