@vectorize-io/hindsight-openclaw 0.5.0 → 0.6.0

package/README.md

@@ -5,30 +5,72 @@ Biomimetic long-term memory for [OpenClaw](https://openclaw.ai) using [Hindsight
 ## Quick Start
 
 ```bash
-# 1. Configure your LLM provider for memory extraction
-# Option A: OpenAI
-export OPENAI_API_KEY="sk-your-key"
-
-# Option B: Claude Code (no API key needed)
-export HINDSIGHT_API_LLM_PROVIDER=claude-code
-
-# Option C: OpenAI Codex (no API key needed)
-export HINDSIGHT_API_LLM_PROVIDER=openai-codex
-
-# 2. Install and enable the plugin
+# 1. Install the plugin
 openclaw plugins install @vectorize-io/hindsight-openclaw
 
+# 2. Run the interactive setup wizard
+npx --package @vectorize-io/hindsight-openclaw hindsight-openclaw-setup
+
 # 3. Start OpenClaw
 openclaw gateway
 ```
 
-That's it! The plugin will automatically start capturing and recalling memories.
+`hindsight-openclaw-setup` walks you through picking one of three modes:
+
+- **Cloud** — managed Hindsight. Pick an API token env var, done.
+- **External API** — your own running Hindsight deployment. Prompts for the URL and optional token.
+- **Embedded daemon** — spawns a local `hindsight-embed` daemon on this machine. Prompts for the LLM provider (OpenAI / Anthropic / Gemini / Groq / Claude Code / Codex / Ollama) and the env var that holds the API key.
+
+Credentials are always written as `SecretRef` objects that reference an environment variable — the key itself never ends up in plaintext on disk. `--ref-source file` and `--ref-source exec` are also supported by OpenClaw for mounted-secret and Vault-style setups; once you want to use them, set them via `openclaw config set` (see below).
+
+### Manual configuration (without the wizard)
+
+The wizard is a convenience wrapper — all of the same fields can be set directly with `openclaw config set`:
+
+```bash
+# Embedded daemon with OpenAI
+openclaw config set plugins.entries.hindsight-openclaw.config.llmProvider openai
+openclaw config set plugins.entries.hindsight-openclaw.config.llmApiKey \
+    --ref-source env --ref-provider default --ref-id OPENAI_API_KEY
+
+# Or: Claude Code (no API key needed)
+openclaw config set plugins.entries.hindsight-openclaw.config.llmProvider claude-code
+
+# Or: point at an external Hindsight API
+openclaw config set plugins.entries.hindsight-openclaw.config.hindsightApiUrl https://mcp.hindsight.example.com
+openclaw config set plugins.entries.hindsight-openclaw.config.hindsightApiToken \
+    --ref-source env --ref-id HINDSIGHT_API_TOKEN
+```
+
+## Migrating from 0.5.x
+
+0.6.0 removes all process-environment reads from the plugin. Configuration that
+previously came from shell env vars must now go through OpenClaw's plugin config
+(with SecretRef for credentials). Concrete mappings:
+
+| Old (0.5.x) | New (0.6.0) |
+|---|---|
+| `OPENAI_API_KEY=…` (auto-detected) | `openclaw config set plugins.entries.hindsight-openclaw.config.llmProvider openai` <br> `openclaw config set plugins.entries.hindsight-openclaw.config.llmApiKey --ref-source env --ref-id OPENAI_API_KEY` |
+| `HINDSIGHT_API_LLM_PROVIDER=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.llmProvider …` |
+| `HINDSIGHT_API_LLM_MODEL=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.llmModel …` |
+| `HINDSIGHT_API_LLM_API_KEY=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.llmApiKey --ref-source env --ref-id …` |
+| `HINDSIGHT_API_LLM_BASE_URL=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.llmBaseUrl …` |
+| `HINDSIGHT_EMBED_API_URL=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.hindsightApiUrl …` |
+| `HINDSIGHT_EMBED_API_TOKEN=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.hindsightApiToken --ref-source env --ref-id …` |
+| `HINDSIGHT_BANK_ID=…` | `openclaw config set plugins.entries.hindsight-openclaw.config.bankId …` |
+| `llmApiKeyEnv: "MY_KEY"` (plugin config) | `llmApiKey` configured as a SecretRef with `--ref-id MY_KEY` |
+
+If your shell already exports `OPENAI_API_KEY`, the SecretRef config above resolves
+to the same value at startup — no need to change your shell setup, just point the
+plugin at the variable explicitly. Run `openclaw config validate` after migrating
+to confirm the new shape parses cleanly.
 
 ## Features
 
 - **Auto-capture** and **auto-recall** of memories each turn, injected into system prompt space so recalled memories stay out of the visible chat transcript
 - **Memory isolation** — configurable per agent, channel, user, or provider via `dynamicBankGranularity`
-- **Retention controls** — choose which message roles to retain and toggle auto-retain on/off
+- **Historical backfill CLI** — import prior OpenClaw session history into Hindsight using the active plugin bank-routing config by default
+- **Retention controls** — choose which message roles to retain, toggle auto-retain on/off, and stamp retained documents with consistent tags/source metadata
 
 ## Configuration
 
@@ -42,13 +84,17 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
 | `embedVersion` | `"latest"` | hindsight-embed version |
 | `embedPackagePath` | — | Local path to `hindsight-embed` package for development |
 | `bankMission` | — | Agent identity/purpose stored on the memory bank. Helps the engine understand context for better fact extraction. Set once per bank — not a recall prompt. |
-| `llmProvider` | auto-detect | LLM provider override for memory extraction (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `openai-codex`, `claude-code`) |
-| `llmModel` | provider default | LLM model override used with `llmProvider` |
-| `llmApiKeyEnv` | provider standard env var | Custom env var name for the provider API key |
+| `llmProvider` | — | LLM provider for memory extraction (`openai`, `anthropic`, `gemini`, `groq`, `ollama`, `openai-codex`, `claude-code`). Required unless `hindsightApiUrl` is set. |
+| `llmModel` | provider default | LLM model used with `llmProvider` |
+| `llmApiKey` | — | API key for the LLM provider. **Sensitive** — set via `openclaw config set ... --ref-source env --ref-id OPENAI_API_KEY` to reference an env var (or `--ref-source file`/`exec` for mounted-secret/Vault sources). |
+| `llmBaseUrl` | — | Optional base URL override for OpenAI-compatible providers (e.g. `https://openrouter.ai/api/v1`) |
 | `dynamicBankId` | `true` | Enable per-context memory banks |
+| `bankId` | — | Static bank ID used when `dynamicBankId` is `false`. |
 | `bankIdPrefix` | — | Prefix for bank IDs (e.g. `"prod"`) |
+| `retainTags` | `[]` | Tags applied to every retained document, useful for cross-agent/source labeling (e.g. `source_system:openclaw`, `agent:agentname`) |
+| `retainSource` | `"openclaw"` | `source` value written into retained document metadata |
 | `dynamicBankGranularity` | `["agent", "channel", "user"]` | Fields used to derive bank ID. Options: `agent`, `channel`, `user`, `provider` |
-| `excludeProviders` | `[]` | Message providers to skip for recall/retain (e.g. `slack`, `telegram`, `discord`) |
+| `excludeProviders` | `["heartbeat"]` | Message providers to skip for recall/retain (e.g. `heartbeat`, `slack`, `telegram`, `discord`) |
 | `autoRecall` | `true` | Auto-inject memories before each turn. Set to `false` when the agent has its own recall tool. |
 | `autoRetain` | `true` | Auto-retain conversations after each turn |
 | `retainRoles` | `["user", "assistant"]` | Which message roles to retain. Options: `user`, `assistant`, `system`, `tool` |
@@ -63,7 +109,45 @@ Optional settings in `~/.openclaw/openclaw.json` under `plugins.entries.hindsigh
 | `recallMaxQueryChars` | `800` | Maximum character length for the composed recall query before calling recall. |
 | `recallPromptPreamble` | built-in string | Prompt text placed above recalled memories in the injected `<hindsight_memories>` system-context block. |
 | `hindsightApiUrl` | — | External Hindsight API URL (skips local daemon) |
-| `hindsightApiToken` | — | Auth token for external API |
+| `hindsightApiToken` | — | Auth token for external API. **Sensitive** — set via `openclaw config set ... --ref-source env --ref-id HINDSIGHT_API_TOKEN`. |
+| `ignoreSessionPatterns` | `[]` | Session key glob patterns to skip entirely — no recall, no retain (e.g. `["agent:*:cron:**"]`) |
+| `statelessSessionPatterns` | `[]` | Session key glob patterns for read-only sessions — retain is always skipped; recall is skipped when `skipStatelessSessions` is `true` (e.g. `["agent:*:subagent:**", "agent:*:heartbeat:**"]`) |
+| `skipStatelessSessions` | `true` | When `true`, sessions matching `statelessSessionPatterns` also skip recall. Set to `false` to allow recall but still skip retain. |
+
+### Session pattern filtering
+
+`ignoreSessionPatterns` and `statelessSessionPatterns` accept glob patterns matched against the session key (format: `agent:<agentId>:<type>:<uuid>`).
+
+Glob syntax:
+- `*` — matches any characters except `:` (single segment)
+- `**` — matches anything including `:` (multiple segments)
+
+| Pattern | Matches |
+|---|---|
+| `agent:*:cron:**` | All cron sessions for any agent |
+| `agent:*:subagent:**` | All subagent sessions for any agent |
+| `agent:main:**` | All sessions under the `main` agent |
+
+**Difference between the two options:**
+
+| | `ignoreSessionPatterns` | `statelessSessionPatterns` |
+|---|---|---|
+| Retain | Skipped | Always skipped |
+| Recall | Skipped | Skipped only when `skipStatelessSessions: true` |
+
+**Example config** — exclude cron jobs from memory entirely, allow subagents to read but not write memories:
+
+```json
+{
+  "ignoreSessionPatterns": ["agent:*:cron:**"],
+  "statelessSessionPatterns": ["agent:*:subagent:**"],
+  "skipStatelessSessions": false
+}
+```
+
+## Retention details
+
+Retained documents use stable session-scoped IDs like `openclaw:agent:agentname:discord:channel:123:turn:000001` (or `...:window:000002` for chunked retention), and include richer metadata such as `session_key`, `agent_id`, `provider`, `channel_id`, `thread_id`, `sender_id`, `turn_index`, and `retention_scope`.
 
 ## Documentation
 
@@ -105,6 +189,51 @@ tail -f ~/.hindsight/profiles/openclaw.log
 uvx hindsight-embed@latest profile list
 ```
 
+## Backfilling Existing OpenClaw History
+
+The package includes a config-aware backfill CLI for importing historical OpenClaw sessions into Hindsight.
+
+By default it mirrors the active plugin settings for:
+
+- `dynamicBankId`
+- `dynamicBankGranularity`
+- `bankIdPrefix`
+- local daemon vs external `hindsightApiUrl`
+
+Dry-run example:
+
+```bash
+npx --package @vectorize-io/hindsight-openclaw hindsight-openclaw-backfill \
+  --openclaw-root ~/.openclaw \
+  --dry-run
+```
+
+Direct invocation from a built checkout:
+
+```bash
+node dist/backfill.js --openclaw-root ~/.openclaw --dry-run
+```
+
+Migration-oriented overrides are explicit:
+
+```bash
+node dist/backfill.js \
+  --openclaw-root ~/.openclaw \
+  --bank-strategy agent \
+  --agent proj-run \
+  --resume \
+  --max-pending-operations 10
+```
+
+Useful options:
+
+- `--agent <id>` limit import to selected agents
+- `--exclude-archive` ignore `sessions-archive-from-migration_backup`
+- `--bank-strategy mirror-config|agent|fixed`
+- `--resume` skip only entries already finalized as completed
+- `--checkpoint <path>` store progress outside the default location
+- `--wait-until-drained` block until the touched bank queues have finished and checkpoint state can be finalized
+
 ## Links
 
 - [Hindsight Documentation](https://vectorize.io/hindsight)

package/dist/backfill-lib.d.ts

@@ -0,0 +1,63 @@
+import type { PluginConfig } from './types.js';
+export interface BackfillCliOptions {
+    openclawRoot: string;
+    includeArchive: boolean;
+    selectedAgents?: Set<string>;
+    limit?: number;
+    bankStrategy: 'mirror-config' | 'agent' | 'fixed';
+    fixedBank?: string;
+}
+export interface SessionMessage {
+    role: 'user' | 'assistant' | 'system' | 'tool';
+    content: string | Array<{
+        type?: string;
+        text?: string;
+    }>;
+}
+export interface ParsedSessionFile {
+    filePath: string;
+    agentId: string;
+    sessionId: string;
+    sessionKey?: string;
+    startedAt?: string;
+    messages: SessionMessage[];
+}
+export interface BackfillPlanEntry {
+    filePath: string;
+    agentId: string;
+    sessionId: string;
+    startedAt?: string;
+    bankId: string;
+    documentId: string;
+    transcript: string;
+    messageCount: number;
+}
+export interface BackfillCheckpointEntry {
+    status: 'enqueued' | 'completed' | 'failed';
+    bankId: string;
+    filePath: string;
+    sessionId: string;
+    updatedAt: string;
+    error?: string;
+}
+export interface BackfillCheckpoint {
+    version: 1;
+    entries: Record<string, BackfillCheckpointEntry>;
+}
+export declare function defaultOpenClawRoot(): string;
+export declare function defaultCheckpointPath(openclawRoot: string): string;
+export declare function loadPluginConfigFromOpenClawRoot(openclawRoot: string): PluginConfig;
+export declare function parseSessionFile(filePath: string, agentId: string): ParsedSessionFile;
+export declare function discoverSessionFiles(openclawRoot: string, includeArchive: boolean): Array<{
+    agentId: string;
+    filePath: string;
+}>;
+export declare function stableDocumentId(session: ParsedSessionFile, bankId: string): string;
+export declare function buildBackfillPlan(pluginConfig: PluginConfig, opts: BackfillCliOptions): {
+    entries: BackfillPlanEntry[];
+    discoveredSessions: number;
+    skippedEmpty: number;
+};
+export declare function loadCheckpoint(checkpointPath: string): BackfillCheckpoint;
+export declare function saveCheckpoint(checkpointPath: string, checkpoint: BackfillCheckpoint): void;
+export declare function checkpointKey(entry: Pick<BackfillPlanEntry, 'bankId' | 'documentId'>): string;

package/dist/backfill-lib.js

@@ -0,0 +1,201 @@
+import { homedir } from 'os';
+import { dirname, join, resolve } from 'path';
+import { readFileSync, existsSync, mkdirSync, writeFileSync, readdirSync } from 'fs';
+import { deriveBankId, prepareRetentionTranscript } from './index.js';
+const DEFAULT_PLUGIN_CONFIG = {
+    dynamicBankId: true,
+    retainRoles: ['user', 'assistant'],
+};
+export function defaultOpenClawRoot() {
+    return resolve(join(homedir(), '.openclaw'));
+}
+export function defaultCheckpointPath(openclawRoot) {
+    return join(openclawRoot, 'data', 'hindsight-backfill-checkpoint.json');
+}
+export function loadPluginConfigFromOpenClawRoot(openclawRoot) {
+    const configPath = join(openclawRoot, 'openclaw.json');
+    const raw = JSON.parse(readFileSync(configPath, 'utf8'));
+    return {
+        ...DEFAULT_PLUGIN_CONFIG,
+        ...(raw.plugins?.entries?.['hindsight-openclaw']?.config || {}),
+    };
+}
+function extractTextContent(content) {
+    if (typeof content === 'string') {
+        return content;
+    }
+    if (Array.isArray(content)) {
+        return content
+            .filter((block) => !!block && typeof block === 'object')
+            .filter((block) => block.type === 'text' && typeof block.text === 'string')
+            .map((block) => block.text || '')
+            .join('\n');
+    }
+    return '';
+}
+function readJsonLines(filePath) {
+    const content = readFileSync(filePath, 'utf8');
+    return content
+        .split('\n')
+        .map((line) => line.trim())
+        .filter(Boolean)
+        .map((line) => JSON.parse(line));
+}
+export function parseSessionFile(filePath, agentId) {
+    const records = readJsonLines(filePath);
+    let sessionId = filePath.split('/').pop()?.replace(/\.jsonl$/, '') || 'session';
+    let sessionKey;
+    let startedAt;
+    const messages = [];
+    for (const record of records) {
+        if (record.type === 'session') {
+            sessionId = typeof record.id === 'string' ? record.id : sessionId;
+            startedAt = typeof record.timestamp === 'string' ? record.timestamp : startedAt;
+            sessionKey = typeof record.sessionKey === 'string' ? record.sessionKey : sessionKey;
+            continue;
+        }
+        if (record.type !== 'message' || !record.message || typeof record.message !== 'object') {
+            continue;
+        }
+        const message = record.message;
+        const role = message.role;
+        if (role !== 'user' && role !== 'assistant' && role !== 'system' && role !== 'tool') {
+            continue;
+        }
+        const text = extractTextContent(message.content);
+        if (!text.trim()) {
+            continue;
+        }
+        messages.push({
+            role,
+            content: typeof message.content === 'string' ? message.content : [{ type: 'text', text }],
+        });
+        if (!sessionKey && typeof record.sessionKey === 'string') {
+            sessionKey = record.sessionKey;
+        }
+    }
+    return {
+        filePath,
+        agentId,
+        sessionId,
+        sessionKey,
+        startedAt,
+        messages,
+    };
+}
+function sessionDirectories(openclawRoot, includeArchive) {
+    const agentsRoot = join(openclawRoot, 'agents');
+    if (!existsSync(agentsRoot)) {
+        return [];
+    }
+    const result = [];
+    for (const entry of readdirSync(agentsRoot, { withFileTypes: true })) {
+        if (!entry.isDirectory())
+            continue;
+        const agentId = entry.name;
+        const sessionsDir = join(agentsRoot, agentId, 'sessions');
+        if (existsSync(sessionsDir)) {
+            result.push({ agentId, path: sessionsDir });
+        }
+        if (includeArchive) {
+            const archiveDir = join(agentsRoot, agentId, 'sessions-archive-from-migration_backup');
+            if (existsSync(archiveDir)) {
+                result.push({ agentId, path: archiveDir });
+            }
+        }
+    }
+    return result.sort((a, b) => a.agentId.localeCompare(b.agentId) || a.path.localeCompare(b.path));
+}
+export function discoverSessionFiles(openclawRoot, includeArchive) {
+    const sessions = [];
+    for (const dir of sessionDirectories(openclawRoot, includeArchive)) {
+        for (const entry of readdirSync(dir.path, { withFileTypes: true })) {
+            if (!entry.isFile() || !entry.name.endsWith('.jsonl'))
+                continue;
+            sessions.push({
+                agentId: dir.agentId,
+                filePath: join(dir.path, entry.name),
+            });
+        }
+    }
+    return sessions.sort((a, b) => a.agentId.localeCompare(b.agentId) || a.filePath.localeCompare(b.filePath));
+}
+function backfillContextForSession(session) {
+    return {
+        agentId: session.agentId,
+        sessionKey: session.sessionKey,
+    };
+}
+function deriveTargetBank(session, pluginConfig, bankStrategy, fixedBank) {
+    if (bankStrategy === 'agent') {
+        return session.agentId;
+    }
+    if (bankStrategy === 'fixed') {
+        if (!fixedBank) {
+            throw new Error('fixed bank strategy requires --fixed-bank');
+        }
+        return fixedBank;
+    }
+    return deriveBankId(backfillContextForSession(session), pluginConfig);
+}
+export function stableDocumentId(session, bankId) {
+    return `backfill::${bankId}::${session.agentId}::${session.sessionId}`;
+}
+export function buildBackfillPlan(pluginConfig, opts) {
+    const entries = [];
+    let discoveredSessions = 0;
+    let skippedEmpty = 0;
+    for (const candidate of discoverSessionFiles(opts.openclawRoot, opts.includeArchive)) {
+        if (opts.selectedAgents && !opts.selectedAgents.has(candidate.agentId)) {
+            continue;
+        }
+        discoveredSessions += 1;
+        const parsed = parseSessionFile(candidate.filePath, candidate.agentId);
+        const retention = prepareRetentionTranscript(parsed.messages, pluginConfig, true);
+        if (!retention) {
+            skippedEmpty += 1;
+            continue;
+        }
+        const bankId = deriveTargetBank(parsed, pluginConfig, opts.bankStrategy, opts.fixedBank);
+        entries.push({
+            filePath: parsed.filePath,
+            agentId: parsed.agentId,
+            sessionId: parsed.sessionId,
+            startedAt: parsed.startedAt,
+            bankId,
+            documentId: stableDocumentId(parsed, bankId),
+            transcript: retention.transcript,
+            messageCount: retention.messageCount,
+        });
+        if (opts.limit && entries.length >= opts.limit) {
+            break;
+        }
+    }
+    return { entries, discoveredSessions, skippedEmpty };
+}
+export function loadCheckpoint(checkpointPath) {
+    if (!existsSync(checkpointPath)) {
+        return { version: 1, entries: {} };
+    }
+    const raw = JSON.parse(readFileSync(checkpointPath, 'utf8'));
+    if (raw.version !== 1 || !raw.entries || typeof raw.entries !== 'object') {
+        return { version: 1, entries: {} };
+    }
+    return {
+        version: 1,
+        entries: Object.fromEntries(Object.entries(raw.entries).map(([key, entry]) => [
+            key,
+            {
+                ...entry,
+                status: entry.status === 'queued' ? 'enqueued' : entry.status,
+            },
+        ])),
+    };
+}
+export function saveCheckpoint(checkpointPath, checkpoint) {
+    mkdirSync(dirname(checkpointPath), { recursive: true });
+    writeFileSync(checkpointPath, JSON.stringify(checkpoint, null, 2) + '\n', 'utf8');
+}
+export function checkpointKey(entry) {
+    return `${entry.bankId}::${entry.documentId}`;
+}

package/dist/backfill.d.ts

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+import type { BankStats, PluginConfig } from './types.js';
+import { type BackfillCheckpoint, type BackfillPlanEntry } from './backfill-lib.js';
+interface BackfillRuntime {
+    apiUrl: string;
+    apiToken?: string;
+    stop(): Promise<void>;
+}
+export declare function filterEntriesForResume(entries: BackfillPlanEntry[], checkpoint: BackfillCheckpoint, resume: boolean): BackfillPlanEntry[];
+export declare function splitResumeEntries(entries: BackfillPlanEntry[], checkpoint: BackfillCheckpoint, waitUntilDrained: boolean): {
+    entriesToEnqueue: BackfillPlanEntry[];
+    alreadyEnqueuedKeys: string[];
+};
+export declare function applyDrainResults(checkpoint: BackfillCheckpoint, touchedEntriesByBank: Map<string, string[]>, finalStatsByBank: Map<string, BankStats>, initialFailedOperationsByBank: Map<string, number>): {
+    completed: number;
+    unresolved: number;
+    warnings: string[];
+};
+export declare function createBackfillRuntime(pluginConfig: PluginConfig, explicitApiUrl?: string, explicitApiToken?: string): Promise<BackfillRuntime>;
+export declare function runCli(argv?: string[]): Promise<void>;
+export declare function isDirectExecution(entrypoint?: string | undefined, moduleUrl?: string): boolean;
+export {};