@totalreclaw/totalreclaw 1.0.0

package/embedding.ts

@@ -0,0 +1,84 @@
+/**
+ * TotalReclaw Plugin - Local Embedding via @huggingface/transformers
+ *
+ * Uses the Xenova/bge-small-en-v1.5 ONNX model to generate 384-dimensional
+ * text embeddings locally. No API key needed, no data leaves the machine.
+ *
+ * This preserves the zero-knowledge guarantee: embeddings are generated
+ * CLIENT-SIDE before encryption, so no plaintext ever reaches an external API.
+ *
+ * Model details:
+ *   - Quantized (int8) ONNX model: ~33.8MB download on first use
+ *   - Cached in ~/.cache/huggingface/ after first download
+ *   - Lazy initialization: first call ~2-3s (model load), subsequent ~15ms
+ *   - Output: 384-dimensional normalized embedding vector
+ *   - For retrieval, queries should be prefixed with an instruction string
+ *     (documents/passages should NOT be prefixed)
+ *
+ * Dependencies: @huggingface/transformers (handles model download, WordPiece
+ * tokenization, ONNX inference, mean pooling, and normalization).
+ */
+
+// @ts-ignore - @huggingface/transformers types may not be perfect
+import { pipeline, type FeatureExtractionPipeline } from '@huggingface/transformers';
+
+/** ONNX-optimized bge-small-en-v1.5 from HuggingFace Hub. */
+const MODEL_ID = 'Xenova/bge-small-en-v1.5';
+
+/** Fixed output dimensionality for bge-small-en-v1.5. */
+const EMBEDDING_DIM = 384;
+
+/**
+ * Query instruction prefix for bge-small-en-v1.5 retrieval tasks.
+ *
+ * Per the BAAI model card: prepend this to short queries when searching
+ * for relevant passages. Do NOT prepend for documents/passages being stored.
+ */
+const QUERY_PREFIX = 'Represent this sentence for searching relevant passages: ';
+
+/** Lazily initialized feature extraction pipeline. */
+let extractor: FeatureExtractionPipeline | null = null;
+
+/**
+ * Generate a 384-dimensional embedding vector for the given text.
+ *
+ * On first call, downloads and loads the ONNX model (~33.8MB, cached).
+ * Subsequent calls reuse the loaded model and run in ~15ms.
+ *
+ * For bge-small-en-v1.5, queries should set `isQuery: true` to prepend the
+ * retrieval instruction prefix. Documents being stored should use the default
+ * (`isQuery: false`) so no prefix is added.
+ *
+ * @param text - The text to embed.
+ * @param options - Optional settings.
+ * @param options.isQuery - If true, prepend the BGE query instruction prefix
+ *                          for improved retrieval accuracy (default: false).
+ * @returns 384-dimensional normalized embedding as a number array.
+ */
+export async function generateEmbedding(
+  text: string,
+  options?: { isQuery?: boolean },
+): Promise<number[]> {
+  if (!extractor) {
+    extractor = await pipeline('feature-extraction', MODEL_ID, {
+      // Use quantized (int8) model for smaller download (~33.8MB vs ~67MB)
+      quantized: true,
+    });
+  }
+
+  const input = options?.isQuery ? QUERY_PREFIX + text : text;
+  const output = await extractor(input, { pooling: 'mean', normalize: true });
+  // output.data is a Float32Array; convert to plain number[]
+  return Array.from(output.data as Float32Array);
+}
+
+/**
+ * Get the embedding vector dimensionality.
+ *
+ * Always returns 384 (fixed for bge-small-en-v1.5).
+ * This is needed by downstream code (e.g. LSH hasher) to know the vector
+ * size without calling the embedding model.
+ */
+export function getEmbeddingDims(): number {
+  return EMBEDDING_DIM;
+}

package/extractor.ts

@@ -0,0 +1,210 @@
+/**
+ * TotalReclaw Plugin - Fact Extractor
+ *
+ * Uses LLM calls to extract atomic facts from conversation messages.
+ * Matches the extraction prompts described in SKILL.md.
+ */
+
+import { chatCompletion, resolveLLMConfig } from './llm-client.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export interface ExtractedFact {
+  text: string;
+  type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal';
+  importance: number; // 1-10
+}
+
+interface ContentBlock {
+  type?: string;
+  text?: string;
+  thinking?: string;
+}
+
+interface ConversationMessage {
+  role?: string;
+  content?: string | ContentBlock[];
+  text?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Extraction Prompt
+// ---------------------------------------------------------------------------
+
+const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction engine. Analyze the conversation and extract atomic facts worth remembering long-term.
+
+Rules:
+1. Each fact must be a single, atomic piece of information
+2. Focus on user-specific information: preferences, decisions, facts about them, their goals
+3. Skip generic knowledge, greetings, and small talk
+4. Skip information that is only relevant to the current conversation
+5. Score importance 1-10 (7+ = worth storing, below 7 = skip)
+6. Only extract facts with importance >= 6
+
+Types:
+- fact: Objective information about the user
+- preference: Likes, dislikes, or preferences
+- decision: Choices the user has made
+- episodic: Events or experiences
+- goal: Objectives or targets
+
+Return a JSON array (no markdown, no code fences):
+[{"text": "...", "type": "...", "importance": N}, ...]
+
+If nothing is worth extracting, return: []`;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract text content from a conversation message (handles various formats).
+ *
+ * OpenClaw AgentMessage objects use content arrays:
+ *   { role: "user", content: [{ type: "text", text: "..." }] }
+ *   { role: "assistant", content: [{ type: "text", text: "..." }, { type: "toolCall", ... }] }
+ *
+ * We also handle the simpler { role, content: "string" } format.
+ */
+function messageToText(msg: unknown): { role: string; content: string } | null {
+  if (!msg || typeof msg !== 'object') return null;
+
+  const m = msg as ConversationMessage;
+  const role = m.role ?? 'unknown';
+
+  // Only keep user and assistant messages
+  if (role !== 'user' && role !== 'assistant') return null;
+
+  let textContent: string;
+
+  if (typeof m.content === 'string') {
+    // Simple string content
+    textContent = m.content;
+  } else if (Array.isArray(m.content)) {
+    // OpenClaw AgentMessage format: array of content blocks
+    // Extract text from { type: "text", text: "..." } blocks
+    const textParts = (m.content as ContentBlock[])
+      .filter((block) => block.type === 'text' && typeof block.text === 'string')
+      .map((block) => block.text as string);
+    textContent = textParts.join('\n');
+  } else if (typeof m.text === 'string') {
+    // Fallback: { text: "..." } field
+    textContent = m.text;
+  } else {
+    return null;
+  }
+
+  if (textContent.length < 3) return null;
+
+  return { role, content: textContent };
+}
+
+/**
+ * Truncate messages to fit within a token budget (rough estimate: 4 chars per token).
+ */
+function truncateMessages(messages: Array<{ role: string; content: string }>, maxChars: number): string {
+  const lines: string[] = [];
+  let totalChars = 0;
+
+  for (const msg of messages) {
+    const line = `[${msg.role}]: ${msg.content}`;
+    if (totalChars + line.length > maxChars) break;
+    lines.push(line);
+    totalChars += line.length;
+  }
+
+  return lines.join('\n\n');
+}
+
+/**
+ * Parse the LLM response into structured facts.
+ */
+function parseFactsResponse(response: string): ExtractedFact[] {
+  // Strip markdown code fences if present
+  let cleaned = response.trim();
+  if (cleaned.startsWith('```')) {
+    cleaned = cleaned.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+  }
+
+  try {
+    const parsed = JSON.parse(cleaned);
+    if (!Array.isArray(parsed)) return [];
+
+    return parsed
+      .filter(
+        (f: unknown) =>
+          f &&
+          typeof f === 'object' &&
+          typeof (f as ExtractedFact).text === 'string' &&
+          (f as ExtractedFact).text.length >= 5,
+      )
+      .map((f: unknown) => {
+        const fact = f as Record<string, unknown>;
+        return {
+          text: String(fact.text).slice(0, 512),
+          type: (['fact', 'preference', 'decision', 'episodic', 'goal'].includes(String(fact.type))
+            ? String(fact.type)
+            : 'fact') as ExtractedFact['type'],
+          importance: Math.max(1, Math.min(10, Number(fact.importance) || 5)),
+        };
+      })
+      .filter((f) => f.importance >= 6); // Only keep important facts
+  } catch {
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main extraction function
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract facts from a list of conversation messages using LLM.
+ *
+ * @param rawMessages - The messages array from the hook event (unknown[])
+ * @param mode - 'turn' for agent_end (recent only), 'full' for compaction/reset
+ * @returns Array of extracted facts, or empty array on failure.
+ */
+export async function extractFacts(
+  rawMessages: unknown[],
+  mode: 'turn' | 'full',
+): Promise<ExtractedFact[]> {
+  const config = resolveLLMConfig();
+  if (!config) return []; // No LLM available
+
+  // Parse messages
+  const parsed = rawMessages
+    .map(messageToText)
+    .filter((m): m is { role: string; content: string } => m !== null);
+
+  if (parsed.length === 0) return [];
+
+  // For 'turn' mode, only look at last 6 messages (3 turns)
+  // For 'full' mode, use all messages but truncate to fit token budget
+  const relevantMessages = mode === 'turn' ? parsed.slice(-6) : parsed;
+
+  // Truncate to ~3000 tokens worth of text
+  const conversationText = truncateMessages(relevantMessages, 12_000);
+
+  if (conversationText.length < 20) return [];
+
+  const userPrompt =
+    mode === 'turn'
+      ? `Extract important facts from these recent conversation turns:\n\n${conversationText}`
+      : `Extract ALL valuable long-term memories from this conversation before it is lost:\n\n${conversationText}`;
+
+  try {
+    const response = await chatCompletion(config, [
+      { role: 'system', content: EXTRACTION_SYSTEM_PROMPT },
+      { role: 'user', content: userPrompt },
+    ]);
+
+    if (!response) return [];
+
+    return parseFactsResponse(response);
+  } catch {
+    return []; // Fail silently -- hooks must never break the agent
+  }
+}

package/generate-mnemonic.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env npx tsx
+/**
+ * Generate a BIP-39 12-word mnemonic for use as TOTALRECLAW_MASTER_PASSWORD.
+ *
+ * Usage: npx tsx generate-mnemonic.ts
+ */
+import { generateMnemonic } from '@scure/bip39';
+import { wordlist } from '@scure/bip39/wordlists/english.js';
+
+const mnemonic = generateMnemonic(wordlist, 128);
+console.log('\n  Your TotalReclaw master mnemonic (12 words):\n');
+console.log(`  ${mnemonic}\n`);
+console.log('  WRITE THIS DOWN. If you lose it, your memories are unrecoverable.');
+console.log('  Set it as TOTALRECLAW_MASTER_PASSWORD in your .env file.\n');

package/hot-cache-wrapper.ts

@@ -0,0 +1,126 @@
+/**
+ * Hot cache wrapper for the plugin.
+ *
+ * Self-contained AES-256-GCM encrypted cache (same implementation as
+ * client/src/cache/hot-cache.ts but without cross-package import).
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+export interface HotFact {
+  id: string;
+  text: string;
+  importance: number;
+}
+
+interface CachePayload {
+  hotFacts: HotFact[];
+  factCount: number;
+  lastSyncedBlock: number;
+  smartAccountAddress: string;
+  lastUpdatedAt?: number;        // Unix timestamp (ms) of last cache update
+  lastQueryEmbedding?: number[]; // Embedding of last search query
+}
+
+const MAX_HOT_FACTS = 30;
+const IV_LENGTH = 12;
+const TAG_LENGTH = 16;
+
+export class PluginHotCache {
+  private hotFacts: HotFact[] = [];
+  private factCount = 0;
+  private lastSyncedBlock = 0;
+  private smartAccountAddress = '';
+  private lastUpdatedAt = 0;
+  private lastQueryEmbedding: number[] | null = null;
+  private key: Buffer;
+
+  constructor(private cachePath: string, hexKey: string) {
+    this.key = Buffer.from(hexKey, 'hex');
+  }
+
+  getHotFacts(): HotFact[] { return [...this.hotFacts]; }
+  getFactCount(): number { return this.factCount; }
+  getLastSyncedBlock(): number { return this.lastSyncedBlock; }
+  getSmartAccountAddress(): string { return this.smartAccountAddress; }
+  getLastUpdatedAt(): number { return this.lastUpdatedAt; }
+  getLastQueryEmbedding(): number[] | null { return this.lastQueryEmbedding ? [...this.lastQueryEmbedding] : null; }
+
+  setHotFacts(facts: HotFact[]): void {
+    const sorted = [...facts].sort((a, b) => b.importance - a.importance);
+    this.hotFacts = sorted.slice(0, MAX_HOT_FACTS);
+    this.lastUpdatedAt = Date.now();
+  }
+
+  setFactCount(count: number): void { this.factCount = count; }
+  setLastSyncedBlock(block: number): void { this.lastSyncedBlock = block; }
+  setSmartAccountAddress(addr: string): void { this.smartAccountAddress = addr; }
+  setLastUpdatedAt(ts: number): void { this.lastUpdatedAt = ts; }
+  setLastQueryEmbedding(embedding: number[] | null): void { this.lastQueryEmbedding = embedding ? [...embedding] : null; }
+
+  /**
+   * Check if the cache is fresh (within TTL).
+   * @param ttlMs TTL in milliseconds (default: 5 minutes)
+   */
+  isFresh(ttlMs: number = 300_000): boolean {
+    if (this.lastUpdatedAt === 0) return false;
+    return (Date.now() - this.lastUpdatedAt) < ttlMs;
+  }
+
+  flush(): void {
+    const payload: CachePayload = {
+      hotFacts: this.hotFacts,
+      factCount: this.factCount,
+      lastSyncedBlock: this.lastSyncedBlock,
+      smartAccountAddress: this.smartAccountAddress,
+      lastUpdatedAt: this.lastUpdatedAt,
+      lastQueryEmbedding: this.lastQueryEmbedding,
+    };
+
+    const plaintext = Buffer.from(JSON.stringify(payload), 'utf-8');
+    const iv = crypto.randomBytes(IV_LENGTH);
+    const cipher = crypto.createCipheriv('aes-256-gcm', this.key, iv);
+    const encrypted = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+    const tag = cipher.getAuthTag();
+
+    const output = Buffer.concat([iv, tag, encrypted]);
+
+    const dir = path.dirname(this.cachePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(this.cachePath, output);
+  }
+
+  load(): void {
+    if (!fs.existsSync(this.cachePath)) return;
+
+    try {
+      const data = fs.readFileSync(this.cachePath);
+      if (data.length < IV_LENGTH + TAG_LENGTH) return;
+
+      const iv = data.subarray(0, IV_LENGTH);
+      const tag = data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
+      const ciphertext = data.subarray(IV_LENGTH + TAG_LENGTH);
+
+      const decipher = crypto.createDecipheriv('aes-256-gcm', this.key, iv);
+      decipher.setAuthTag(tag);
+      const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+
+      const payload: CachePayload = JSON.parse(decrypted.toString('utf-8'));
+      this.hotFacts = payload.hotFacts || [];
+      this.factCount = payload.factCount || 0;
+      this.lastSyncedBlock = payload.lastSyncedBlock || 0;
+      this.smartAccountAddress = payload.smartAccountAddress || '';
+      this.lastUpdatedAt = payload.lastUpdatedAt || 0;
+      this.lastQueryEmbedding = payload.lastQueryEmbedding || null;
+    } catch {
+      this.hotFacts = [];
+      this.factCount = 0;
+      this.lastSyncedBlock = 0;
+      this.smartAccountAddress = '';
+      this.lastUpdatedAt = 0;
+      this.lastQueryEmbedding = null;
+    }
+  }
+}