@vectorize-io/hindsight-openclaw 0.5.1 → 0.6.0

package/openclaw.plugin.json

@@ -2,6 +2,14 @@
   "id": "hindsight-openclaw",
   "name": "Hindsight Memory",
   "kind": "memory",
+  "configContracts": {
+    "secretInputs": {
+      "paths": [
+        { "path": "llmApiKey", "expected": "string" },
+        { "path": "hindsightApiToken", "expected": "string" }
+      ]
+    }
+  },
   "configSchema": {
     "type": "object",
     "properties": {
@@ -27,7 +35,7 @@
       },
       "llmProvider": {
         "type": "string",
-        "description": "LLM provider for Hindsight memory (e.g. 'openai', 'anthropic', 'gemini', 'groq', 'ollama', 'openai-codex', 'claude-code'). Takes priority over auto-detection but not over HINDSIGHT_API_LLM_PROVIDER env var.",
+        "description": "LLM provider for Hindsight memory (e.g. 'openai', 'anthropic', 'gemini', 'groq', 'ollama', 'openai-codex', 'claude-code').",
         "enum": [
           "openai",
           "anthropic",
@@ -42,9 +50,13 @@
         "type": "string",
         "description": "LLM model to use (e.g. 'gpt-4o-mini', 'claude-3-5-haiku-20241022'). Used with llmProvider."
       },
-      "llmApiKeyEnv": {
+      "llmApiKey": {
+        "type": ["string", "object"],
+        "description": "API key for the LLM provider used by the Hindsight memory daemon. Set via 'openclaw config set ... --ref-source env --ref-id OPENAI_API_KEY' to reference an env var without storing plaintext."
+      },
+      "llmBaseUrl": {
         "type": "string",
-        "description": "Name of the env var holding the API key (e.g. 'MY_CUSTOM_KEY'). If not set, uses the standard env var for the chosen provider."
+        "description": "Optional base URL override for OpenAI-compatible providers (e.g. 'https://openrouter.ai/api/v1')."
       },
       "embedPackagePath": {
         "type": "string",
@@ -60,7 +72,7 @@
         "description": "External Hindsight API URL (e.g. 'https://mcp.hindsight.devcraft.team'). When set, skips local daemon and connects directly to this API."
       },
       "hindsightApiToken": {
-        "type": "string",
+        "type": ["string", "object"],
         "description": "API token for external Hindsight API authentication. Required if the external API has authentication enabled."
       },
       "dynamicBankId": {
@@ -68,10 +80,26 @@
         "description": "Enable per-user memory banks. When true, memories are isolated by user per channel (e.g., slack-U123, telegram-456789). When false, all users share a single 'openclaw' bank.",
         "default": true
       },
+      "bankId": {
+        "type": "string",
+        "description": "Static bank ID used when dynamicBankId is false."
+      },
       "bankIdPrefix": {
         "type": "string",
         "description": "Optional prefix for bank IDs (e.g., 'prod' results in 'prod-slack-U123'). Useful for separating environments."
       },
+      "retainTags": {
+        "type": "array",
+        "items": {
+          "type": "string"
+        },
+        "description": "Tags applied to every retained document (e.g. ['source_system:openclaw', 'agent:agentname'])."
+      },
+      "retainSource": {
+        "type": "string",
+        "description": "Source value written into retained document metadata. Defaults to 'openclaw'.",
+        "default": "openclaw"
+      },
       "autoRecall": {
         "type": "boolean",
         "description": "Automatically recall memories on every prompt and inject them as context. Set to false when agent has its own recall tool.",
@@ -82,7 +110,8 @@
         "items": {
           "type": "string"
         },
-        "description": "Message providers to exclude from recall and retain (e.g. ['telegram', 'discord'])"
+        "description": "Message providers to exclude from recall and retain (e.g. ['heartbeat', 'telegram', 'discord'])",
+        "default": ["heartbeat"]
       },
       "dynamicBankGranularity": {
         "type": "array",
@@ -215,6 +244,35 @@
         "type": "number",
         "description": "Interval in ms to batch retain/recall log summaries. 0 = log every event individually. Default: 300000 (5 min).",
         "default": 300000
+      },
+      "retainQueuePath": {
+        "type": "string",
+        "description": "Path to JSONL file for buffering failed retains (external API mode only). Default: ~/.openclaw/data/hindsight-retain-queue.jsonl"
+      },
+      "retainQueueMaxAgeMs": {
+        "type": "number",
+        "description": "Max age in ms for queued retain items. -1 = keep forever.",
+        "default": -1
+      },
+      "retainQueueFlushIntervalMs": {
+        "type": "number",
+        "description": "How often to attempt flushing queued retains in ms. Default: 60000 (1 min).",
+        "default": 60000
+      },
+      "ignoreSessionPatterns": {
+        "type": "array",
+        "items": { "type": "string" },
+        "description": "Session key glob patterns to skip entirely (no recall, no retain). E.g. [\"agent:main:**\", \"agent:*:cron:**\"]. * matches non-colon chars, ** matches anything."
+      },
+      "statelessSessionPatterns": {
+        "type": "array",
+        "items": { "type": "string" },
+        "description": "Session key glob patterns for read-only sessions: retain is always skipped, recall is skipped when skipStatelessSessions is true. E.g. [\"agent:*:subagent:**\", \"agent:*:heartbeat:**\"]."
+      },
+      "skipStatelessSessions": {
+        "type": "boolean",
+        "description": "When true (default), sessions matching statelessSessionPatterns also skip recall. When false, they can recall but never retain.",
+        "default": true
       }
     },
     "additionalProperties": false
@@ -244,9 +302,14 @@
       "label": "LLM Model",
       "placeholder": "e.g. gpt-4o-mini, claude-3-5-haiku-20241022"
     },
-    "llmApiKeyEnv": {
-      "label": "API Key Env Var",
-      "placeholder": "e.g. MY_CUSTOM_API_KEY (optional)"
+    "llmApiKey": {
+      "label": "LLM API Key",
+      "placeholder": "API key for the chosen LLM provider",
+      "sensitive": true
+    },
+    "llmBaseUrl": {
+      "label": "LLM Base URL",
+      "placeholder": "e.g. https://openrouter.ai/api/v1 (optional)"
     },
     "embedPackagePath": {
       "label": "Local Package Path (Dev)",
@@ -262,23 +325,36 @@
     },
     "hindsightApiToken": {
       "label": "External API Token",
-      "placeholder": "API token if external API requires authentication"
+      "placeholder": "API token if external API requires authentication",
+      "sensitive": true
     },
     "dynamicBankId": {
       "label": "Dynamic Bank IDs",
       "placeholder": "true (isolate memories per channel)"
     },
+    "bankId": {
+      "label": "Static Bank ID",
+      "placeholder": "e.g. openclaw, shared-bank (used when dynamicBankId is false)"
+    },
     "bankIdPrefix": {
       "label": "Bank ID Prefix",
       "placeholder": "e.g., prod, staging (optional)"
     },
+    "retainTags": {
+      "label": "Retain Tags",
+      "placeholder": "e.g. ['source_system:openclaw', 'agent:agentname']"
+    },
+    "retainSource": {
+      "label": "Retain Source",
+      "placeholder": "openclaw"
+    },
     "autoRecall": {
       "label": "Auto-Recall",
       "placeholder": "true (inject memories on every prompt)"
     },
     "excludeProviders": {
       "label": "Excluded Providers",
-      "placeholder": "e.g. telegram, discord"
+      "placeholder": "e.g. heartbeat, telegram, discord"
     },
     "dynamicBankGranularity": {
       "label": "Bank Granularity",
@@ -349,6 +425,30 @@
     "logSummaryIntervalMs": {
       "label": "Log Summary Interval (ms)",
       "placeholder": "300000"
+    },
+    "retainQueuePath": {
+      "label": "Retain Queue File Path",
+      "placeholder": "~/.openclaw/data/hindsight-retain-queue.jsonl"
+    },
+    "retainQueueMaxAgeMs": {
+      "label": "Retain Queue Max Age (ms)",
+      "placeholder": "-1 (forever)"
+    },
+    "retainQueueFlushIntervalMs": {
+      "label": "Retain Queue Flush Interval (ms)",
+      "placeholder": "60000"
+    },
+    "ignoreSessionPatterns": {
+      "label": "Ignore Session Patterns",
+      "placeholder": "e.g. [\"agent:main:**\", \"agent:*:cron:**\"]"
+    },
+    "statelessSessionPatterns": {
+      "label": "Stateless Session Patterns",
+      "placeholder": "e.g. [\"agent:*:subagent:**\", \"agent:*:heartbeat:**\"]"
+    },
+    "skipStatelessSessions": {
+      "label": "Skip Stateless Sessions",
+      "placeholder": "true (default)"
     }
   }
 }

package/package.json

@@ -1,9 +1,13 @@
 {
   "name": "@vectorize-io/hindsight-openclaw",
-  "version": "0.5.1",
+  "version": "0.6.0",
   "description": "Hindsight memory plugin for OpenClaw - biomimetic long-term memory with fact extraction",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "bin": {
+    "hindsight-openclaw-backfill": "dist/backfill.js",
+    "hindsight-openclaw-setup": "dist/setup.js"
+  },
   "type": "module",
   "openclaw": {
     "extensions": [
@@ -34,25 +38,29 @@
     "build": "tsc",
     "dev": "tsc --watch",
     "clean": "rm -rf dist",
+    "backfill:help": "node dist/backfill.js --help",
     "test": "vitest run src",
     "test:watch": "vitest src",
     "test:integration": "vitest run --config vitest.integration.config.ts",
     "prepublishOnly": "npm run clean && npm run build"
   },
   "dependencies": {
-    "node-fetch": "^3.3.2"
+    "@clack/prompts": "^1.2.0",
+    "@vectorize-io/hindsight-client": "^0.5.0",
+    "@vectorize-io/hindsight-all": "^0.1.0"
   },
   "devDependencies": {
     "@types/node": "^20.0.0",
-    "@vitest/ui": "^4.0.18",
+    "@vitest/ui": "^4.1.2",
     "typescript": "^5.3.0",
-    "vitest": "^4.0.18"
+    "vitest": "^4.1.2"
   },
   "engines": {
     "node": ">=22"
   },
   "overrides": {
     "rollup": "^4.59.0",
-    "picomatch": ">=2.3.2 <3.0.0 || >=4.0.4"
+    "picomatch": ">=2.3.2 <3.0.0 || >=4.0.4",
+    "vite": ">=8.0.5"
   }
 }

package/dist/client.d.ts

@@ -1,34 +0,0 @@
-import type { RetainRequest, RetainResponse, RecallRequest, RecallResponse } from './types.js';
-export interface HindsightClientOptions {
-    llmModel?: string;
-    embedVersion?: string;
-    embedPackagePath?: string;
-    apiUrl?: string;
-    apiToken?: string;
-}
-export declare class HindsightClient {
-    private bankId;
-    private llmModel?;
-    private embedVersion;
-    private embedPackagePath?;
-    private apiUrl?;
-    private apiToken?;
-    constructor(opts: HindsightClientOptions);
-    private get httpMode();
-    /**
-     * Get the command and base args to run hindsight-embed.
-     * Returns [command, ...baseArgs] for use with execFile/spawn (no shell).
-     */
-    private getEmbedCommand;
-    private httpHeaders;
-    setBankId(bankId: string): void;
-    setBankMission(mission: string): Promise<void>;
-    private setBankMissionHttp;
-    private setBankMissionSubprocess;
-    retain(request: RetainRequest): Promise<RetainResponse>;
-    private retainHttp;
-    private retainSubprocess;
-    recall(request: RecallRequest, timeoutMs?: number): Promise<RecallResponse>;
-    private recallHttp;
-    private recallSubprocess;
-}

package/dist/client.js

@@ -1,215 +0,0 @@
-import { execFile } from 'child_process';
-import { promisify } from 'util';
-import { writeFile, mkdir, rm } from 'fs/promises';
-import { tmpdir } from 'os';
-import { join } from 'path';
-import { randomBytes } from 'crypto';
-import * as log from './logger.js';
-const execFileAsync = promisify(execFile);
-const MAX_BUFFER = 5 * 1024 * 1024; // 5 MB — large transcripts can exceed default 1 MB
-const DEFAULT_TIMEOUT_MS = 15_000;
-/** Strip null bytes from strings — Node 22 rejects them in execFile() args */
-const sanitize = (s) => s.replace(/\0/g, '');
-/**
- * Sanitize a string for use as a cross-platform filename.
- * Replaces characters illegal on Windows or Unix with underscores.
- */
-function sanitizeFilename(name) {
-    // Replace characters illegal on Windows (\/:*?"<>|) and control chars
-    return name.replace(/[\\/:*?"<>|\x00-\x1f]/g, '_').slice(0, 200) || 'content';
-}
-export class HindsightClient {
-    bankId = 'default';
-    llmModel;
-    embedVersion;
-    embedPackagePath;
-    apiUrl;
-    apiToken;
-    constructor(opts) {
-        this.llmModel = opts.llmModel;
-        this.embedVersion = opts.embedVersion || 'latest';
-        this.embedPackagePath = opts.embedPackagePath;
-        this.apiUrl = opts.apiUrl?.replace(/\/$/, ''); // strip trailing slash
-        this.apiToken = opts.apiToken;
-    }
-    get httpMode() {
-        return !!this.apiUrl;
-    }
-    /**
-     * Get the command and base args to run hindsight-embed.
-     * Returns [command, ...baseArgs] for use with execFile/spawn (no shell).
-     */
-    getEmbedCommand() {
-        if (this.embedPackagePath) {
-            return ['uv', 'run', '--directory', this.embedPackagePath, 'hindsight-embed'];
-        }
-        const embedPackage = this.embedVersion ? `hindsight-embed@${this.embedVersion}` : 'hindsight-embed@latest';
-        return ['uvx', embedPackage];
-    }
-    httpHeaders() {
-        const headers = { 'Content-Type': 'application/json' };
-        if (this.apiToken) {
-            headers['Authorization'] = `Bearer ${this.apiToken}`;
-        }
-        return headers;
-    }
-    setBankId(bankId) {
-        this.bankId = bankId;
-    }
-    // --- setBankMission ---
-    async setBankMission(mission) {
-        if (!mission || mission.trim().length === 0) {
-            return;
-        }
-        if (this.httpMode) {
-            return this.setBankMissionHttp(mission);
-        }
-        return this.setBankMissionSubprocess(mission);
-    }
-    async setBankMissionHttp(mission) {
-        try {
-            const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}`;
-            const res = await fetch(url, {
-                method: 'PUT',
-                headers: this.httpHeaders(),
-                body: JSON.stringify({ mission }),
-                signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
-            });
-            if (!res.ok) {
-                const body = await res.text().catch(() => '');
-                throw new Error(`HTTP ${res.status}: ${body}`);
-            }
-            log.verbose('bank mission set via HTTP');
-        }
-        catch (error) {
-            log.warn(`could not set bank mission (bank may not exist yet): ${error}`);
-        }
-    }
-    async setBankMissionSubprocess(mission) {
-        const [cmd, ...baseArgs] = this.getEmbedCommand();
-        const args = [...baseArgs, '--profile', 'openclaw', 'bank', 'mission', this.bankId, sanitize(mission)];
-        try {
-            const { stdout } = await execFileAsync(cmd, args, { maxBuffer: MAX_BUFFER });
-            log.verbose(`bank mission set: ${stdout.trim()}`);
-        }
-        catch (error) {
-            // Don't fail if mission set fails - bank might not exist yet, will be created on first retain
-            log.warn(`could not set bank mission (bank may not exist yet): ${error}`);
-        }
-    }
-    // --- retain ---
-    async retain(request) {
-        if (this.httpMode) {
-            return this.retainHttp(request);
-        }
-        return this.retainSubprocess(request);
-    }
-    async retainHttp(request) {
-        const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}/memories`;
-        const body = {
-            items: [{
-                    content: request.content,
-                    document_id: request.document_id || 'conversation',
-                    metadata: request.metadata,
-                }],
-            async: true,
-        };
-        const res = await fetch(url, {
-            method: 'POST',
-            headers: this.httpHeaders(),
-            body: JSON.stringify(body),
-            signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
-        });
-        if (!res.ok) {
-            const text = await res.text().catch(() => '');
-            throw new Error(`Failed to retain memory (HTTP ${res.status}): ${text}`);
-        }
-        const data = await res.json();
-        log.verbose(`retained via HTTP (async): ${JSON.stringify(data).substring(0, 200)}`);
-        return {
-            message: 'Memory queued for background processing',
-            document_id: request.document_id || 'conversation',
-            memory_unit_ids: [],
-        };
-    }
-    async retainSubprocess(request) {
-        const docId = request.document_id || 'conversation';
-        // Write content to a temp file to avoid E2BIG (ARG_MAX) errors when passing
-        // large conversations as arguments.
-        const tempDir = join(tmpdir(), `hindsight_${randomBytes(8).toString('hex')}`);
-        const safeFilename = sanitizeFilename(docId);
-        const tempFile = join(tempDir, `${safeFilename}.txt`);
-        try {
-            await mkdir(tempDir, { recursive: true });
-            await writeFile(tempFile, sanitize(request.content), 'utf8');
-            const [cmd, ...baseArgs] = this.getEmbedCommand();
-            const args = [...baseArgs, '--profile', 'openclaw', 'memory', 'retain-files', this.bankId, tempFile, '--async'];
-            const { stdout } = await execFileAsync(cmd, args, { maxBuffer: MAX_BUFFER });
-            log.verbose(`retained (async): ${stdout.trim()}`);
-            return {
-                message: 'Memory queued for background processing',
-                document_id: docId,
-                memory_unit_ids: [],
-            };
-        }
-        catch (error) {
-            throw new Error(`Failed to retain memory: ${error}`, { cause: error });
-        }
-        finally {
-            await rm(tempDir, { recursive: true, force: true }).catch(() => { });
-        }
-    }
-    // --- recall ---
-    async recall(request, timeoutMs) {
-        if (this.httpMode) {
-            return this.recallHttp(request, timeoutMs);
-        }
-        return this.recallSubprocess(request, timeoutMs);
-    }
-    async recallHttp(request, timeoutMs) {
-        const url = `${this.apiUrl}/v1/default/banks/${encodeURIComponent(this.bankId)}/memories/recall`;
-        // Defense-in-depth: truncate query to stay under API's 500-token limit
-        const MAX_QUERY_CHARS = 800;
-        const query = request.query.length > MAX_QUERY_CHARS
-            ? (log.warn(`truncating recall query from ${request.query.length} to ${MAX_QUERY_CHARS} chars`),
-                request.query.substring(0, MAX_QUERY_CHARS))
-            : request.query;
-        const body = {
-            query,
-            max_tokens: request.max_tokens || 1024,
-        };
-        if (request.budget) {
-            body.budget = request.budget;
-        }
-        if (request.types) {
-            body.types = request.types;
-        }
-        const res = await fetch(url, {
-            method: 'POST',
-            headers: this.httpHeaders(),
-            body: JSON.stringify(body),
-            signal: AbortSignal.timeout(timeoutMs ?? DEFAULT_TIMEOUT_MS),
-        });
-        if (!res.ok) {
-            const text = await res.text().catch(() => '');
-            throw new Error(`Failed to recall memories (HTTP ${res.status}): ${text}`);
-        }
-        return res.json();
-    }
-    async recallSubprocess(request, timeoutMs) {
-        const query = sanitize(request.query);
-        const maxTokens = request.max_tokens || 1024;
-        const [cmd, ...baseArgs] = this.getEmbedCommand();
-        const args = [...baseArgs, '--profile', 'openclaw', 'memory', 'recall', this.bankId, query, '--output', 'json', '--max-tokens', String(maxTokens)];
-        try {
-            const { stdout } = await execFileAsync(cmd, args, {
-                maxBuffer: MAX_BUFFER,
-                timeout: timeoutMs ?? 30_000, // subprocess gets a longer default
-            });
-            return JSON.parse(stdout);
-        }
-        catch (error) {
-            throw new Error(`Failed to recall memories: ${error}`, { cause: error });
-        }
-    }
-}

package/dist/embed-manager.d.ts

@@ -1,27 +0,0 @@
-export declare class HindsightEmbedManager {
-    private process;
-    private port;
-    private baseUrl;
-    private embedDir;
-    private llmProvider;
-    private llmApiKey;
-    private llmModel?;
-    private llmBaseUrl?;
-    private daemonIdleTimeout;
-    private embedVersion;
-    private embedPackagePath?;
-    constructor(port: number, llmProvider: string, llmApiKey: string, llmModel?: string, llmBaseUrl?: string, daemonIdleTimeout?: number, // Default: never timeout
-    embedVersion?: string, // Default: latest
-    embedPackagePath?: string);
-    /**
-     * Get the command to run hindsight-embed (either local or from PyPI)
-     */
-    private getEmbedCommand;
-    start(): Promise<void>;
-    stop(): Promise<void>;
-    private waitForReady;
-    getBaseUrl(): string;
-    isRunning(): boolean;
-    checkHealth(): Promise<boolean>;
-    private configureProfile;
-}