@totalreclaw/totalreclaw 1.0.4 → 1.1.0

package/extractor.ts

@@ -0,0 +1,237 @@
+/**
+ * TotalReclaw Plugin - Fact Extractor
+ *
+ * Uses LLM calls to extract atomic facts from conversation messages.
+ * Matches the extraction prompts described in SKILL.md.
+ */
+
+import { chatCompletion, resolveLLMConfig } from './llm-client.js';
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type ExtractionAction = 'ADD' | 'UPDATE' | 'DELETE' | 'NOOP';
+
+export interface ExtractedFact {
+  text: string;
+  type: 'fact' | 'preference' | 'decision' | 'episodic' | 'goal';
+  importance: number; // 1-10
+  action: ExtractionAction;
+  existingFactId?: string;
+}
+
+interface ContentBlock {
+  type?: string;
+  text?: string;
+  thinking?: string;
+}
+
+interface ConversationMessage {
+  role?: string;
+  content?: string | ContentBlock[];
+  text?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Extraction Prompt
+// ---------------------------------------------------------------------------
+
+const EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction engine. Analyze the conversation and extract atomic facts worth remembering long-term.
+
+Rules:
+1. Each fact must be a single, atomic piece of information
+2. Focus on user-specific information: preferences, decisions, facts about them, their goals
+3. Skip generic knowledge, greetings, and small talk
+4. Skip information that is only relevant to the current conversation
+5. Score importance 1-10 (7+ = worth storing, below 7 = skip)
+6. Only extract facts with importance >= 6
+
+Types:
+- fact: Objective information about the user
+- preference: Likes, dislikes, or preferences
+- decision: Choices the user has made
+- episodic: Events or experiences
+- goal: Objectives or targets
+
+Actions (compare against existing memories if provided):
+- ADD: New fact, no conflict with existing memories
+- UPDATE: Modifies or refines an existing memory (provide existingFactId)
+- DELETE: Contradicts an existing memory — the old one is now wrong (provide existingFactId)
+- NOOP: Already captured in existing memories or not worth storing
+
+Return a JSON array (no markdown, no code fences):
+[{"text": "...", "type": "...", "importance": N, "action": "ADD|UPDATE|DELETE|NOOP", "existingFactId": "..."}, ...]
+
+If nothing is worth extracting, return: []`;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract text content from a conversation message (handles various formats).
+ *
+ * OpenClaw AgentMessage objects use content arrays:
+ *   { role: "user", content: [{ type: "text", text: "..." }] }
+ *   { role: "assistant", content: [{ type: "text", text: "..." }, { type: "toolCall", ... }] }
+ *
+ * We also handle the simpler { role, content: "string" } format.
+ */
+function messageToText(msg: unknown): { role: string; content: string } | null {
+  if (!msg || typeof msg !== 'object') return null;
+
+  const m = msg as ConversationMessage;
+  const role = m.role ?? 'unknown';
+
+  // Only keep user and assistant messages
+  if (role !== 'user' && role !== 'assistant') return null;
+
+  let textContent: string;
+
+  if (typeof m.content === 'string') {
+    // Simple string content
+    textContent = m.content;
+  } else if (Array.isArray(m.content)) {
+    // OpenClaw AgentMessage format: array of content blocks
+    // Extract text from { type: "text", text: "..." } blocks
+    const textParts = (m.content as ContentBlock[])
+      .filter((block) => block.type === 'text' && typeof block.text === 'string')
+      .map((block) => block.text as string);
+    textContent = textParts.join('\n');
+  } else if (typeof m.text === 'string') {
+    // Fallback: { text: "..." } field
+    textContent = m.text;
+  } else {
+    return null;
+  }
+
+  if (textContent.length < 3) return null;
+
+  return { role, content: textContent };
+}
+
+/**
+ * Truncate messages to fit within a token budget (rough estimate: 4 chars per token).
+ */
+function truncateMessages(messages: Array<{ role: string; content: string }>, maxChars: number): string {
+  const lines: string[] = [];
+  let totalChars = 0;
+
+  for (const msg of messages) {
+    const line = `[${msg.role}]: ${msg.content}`;
+    if (totalChars + line.length > maxChars) break;
+    lines.push(line);
+    totalChars += line.length;
+  }
+
+  return lines.join('\n\n');
+}
+
+/**
+ * Parse the LLM response into structured facts.
+ */
+function parseFactsResponse(response: string): ExtractedFact[] {
+  // Strip markdown code fences if present
+  let cleaned = response.trim();
+  if (cleaned.startsWith('```')) {
+    cleaned = cleaned.replace(/^```(?:json)?\n?/, '').replace(/\n?```$/, '').trim();
+  }
+
+  try {
+    const parsed = JSON.parse(cleaned);
+    if (!Array.isArray(parsed)) return [];
+
+    return parsed
+      .filter(
+        (f: unknown) =>
+          f &&
+          typeof f === 'object' &&
+          typeof (f as ExtractedFact).text === 'string' &&
+          (f as ExtractedFact).text.length >= 5,
+      )
+      .map((f: unknown) => {
+        const fact = f as Record<string, unknown>;
+        const validActions: ExtractionAction[] = ['ADD', 'UPDATE', 'DELETE', 'NOOP'];
+        const action = validActions.includes(String(fact.action) as ExtractionAction)
+          ? (String(fact.action) as ExtractionAction)
+          : 'ADD'; // Default to ADD for backward compatibility
+        return {
+          text: String(fact.text).slice(0, 512),
+          type: (['fact', 'preference', 'decision', 'episodic', 'goal'].includes(String(fact.type))
+            ? String(fact.type)
+            : 'fact') as ExtractedFact['type'],
+          importance: Math.max(1, Math.min(10, Number(fact.importance) || 5)),
+          action,
+          existingFactId: typeof fact.existingFactId === 'string' ? fact.existingFactId : undefined,
+        };
+      })
+      .filter((f) => f.importance >= 6 || f.action === 'DELETE'); // DELETE actions pass regardless of importance
+  } catch {
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Main extraction function
+// ---------------------------------------------------------------------------
+
+/**
+ * Extract facts from a list of conversation messages using LLM.
+ *
+ * @param rawMessages - The messages array from the hook event (unknown[])
+ * @param mode - 'turn' for agent_end (recent only), 'full' for compaction/reset
+ * @param existingMemories - Optional list of existing memories for dedup context
+ * @returns Array of extracted facts, or empty array on failure.
+ */
+export async function extractFacts(
+  rawMessages: unknown[],
+  mode: 'turn' | 'full',
+  existingMemories?: Array<{ id: string; text: string }>,
+): Promise<ExtractedFact[]> {
+  const config = resolveLLMConfig();
+  if (!config) return []; // No LLM available
+
+  // Parse messages
+  const parsed = rawMessages
+    .map(messageToText)
+    .filter((m): m is { role: string; content: string } => m !== null);
+
+  if (parsed.length === 0) return [];
+
+  // For 'turn' mode, only look at last 6 messages (3 turns)
+  // For 'full' mode, use all messages but truncate to fit token budget
+  const relevantMessages = mode === 'turn' ? parsed.slice(-6) : parsed;
+
+  // Truncate to ~3000 tokens worth of text
+  const conversationText = truncateMessages(relevantMessages, 12_000);
+
+  if (conversationText.length < 20) return [];
+
+  // Build existing memories context if available
+  let memoriesContext = '';
+  if (existingMemories && existingMemories.length > 0) {
+    const memoriesStr = existingMemories
+      .map((m) => `[ID: ${m.id}] ${m.text}`)
+      .join('\n');
+    memoriesContext = `\n\nExisting memories (use these for dedup — classify as UPDATE/DELETE/NOOP if they conflict or overlap):\n${memoriesStr}`;
+  }
+
+  const userPrompt =
+    mode === 'turn'
+      ? `Extract important facts from these recent conversation turns:\n\n${conversationText}${memoriesContext}`
+      : `Extract ALL valuable long-term memories from this conversation before it is lost:\n\n${conversationText}${memoriesContext}`;
+
+  try {
+    const response = await chatCompletion(config, [
+      { role: 'system', content: EXTRACTION_SYSTEM_PROMPT },
+      { role: 'user', content: userPrompt },
+    ]);
+
+    if (!response) return [];
+
+    return parseFactsResponse(response);
+  } catch {
+    return []; // Fail silently -- hooks must never break the agent
+  }
+}

package/generate-mnemonic.ts

@@ -0,0 +1,14 @@
+#!/usr/bin/env npx tsx
+/**
+ * Generate a BIP-39 12-word mnemonic for use as TOTALRECLAW_MASTER_PASSWORD.
+ *
+ * Usage: npx tsx generate-mnemonic.ts
+ */
+import { generateMnemonic } from '@scure/bip39';
+import { wordlist } from '@scure/bip39/wordlists/english.js';
+
+const mnemonic = generateMnemonic(wordlist, 128);
+console.log('\n  Your TotalReclaw master mnemonic (12 words):\n');
+console.log(`  ${mnemonic}\n`);
+console.log('  WRITE THIS DOWN. If you lose it, your memories are unrecoverable.');
+console.log('  Set it as TOTALRECLAW_MASTER_PASSWORD in your .env file.\n');

package/hot-cache-wrapper.ts

@@ -0,0 +1,126 @@
+/**
+ * Hot cache wrapper for the plugin.
+ *
+ * Self-contained AES-256-GCM encrypted cache (same implementation as
+ * client/src/cache/hot-cache.ts but without cross-package import).
+ */
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+
+export interface HotFact {
+  id: string;
+  text: string;
+  importance: number;
+}
+
+interface CachePayload {
+  hotFacts: HotFact[];
+  factCount: number;
+  lastSyncedBlock: number;
+  smartAccountAddress: string;
+  lastUpdatedAt?: number;        // Unix timestamp (ms) of last cache update
+  lastQueryEmbedding?: number[]; // Embedding of last search query
+}
+
+const MAX_HOT_FACTS = 30;
+const IV_LENGTH = 12;
+const TAG_LENGTH = 16;
+
+export class PluginHotCache {
+  private hotFacts: HotFact[] = [];
+  private factCount = 0;
+  private lastSyncedBlock = 0;
+  private smartAccountAddress = '';
+  private lastUpdatedAt = 0;
+  private lastQueryEmbedding: number[] | null = null;
+  private key: Buffer;
+
+  constructor(private cachePath: string, hexKey: string) {
+    this.key = Buffer.from(hexKey, 'hex');
+  }
+
+  getHotFacts(): HotFact[] { return [...this.hotFacts]; }
+  getFactCount(): number { return this.factCount; }
+  getLastSyncedBlock(): number { return this.lastSyncedBlock; }
+  getSmartAccountAddress(): string { return this.smartAccountAddress; }
+  getLastUpdatedAt(): number { return this.lastUpdatedAt; }
+  getLastQueryEmbedding(): number[] | null { return this.lastQueryEmbedding ? [...this.lastQueryEmbedding] : null; }
+
+  setHotFacts(facts: HotFact[]): void {
+    const sorted = [...facts].sort((a, b) => b.importance - a.importance);
+    this.hotFacts = sorted.slice(0, MAX_HOT_FACTS);
+    this.lastUpdatedAt = Date.now();
+  }
+
+  setFactCount(count: number): void { this.factCount = count; }
+  setLastSyncedBlock(block: number): void { this.lastSyncedBlock = block; }
+  setSmartAccountAddress(addr: string): void { this.smartAccountAddress = addr; }
+  setLastUpdatedAt(ts: number): void { this.lastUpdatedAt = ts; }
+  setLastQueryEmbedding(embedding: number[] | null): void { this.lastQueryEmbedding = embedding ? [...embedding] : null; }
+
+  /**
+   * Check if the cache is fresh (within TTL).
+   * @param ttlMs TTL in milliseconds (default: 5 minutes)
+   */
+  isFresh(ttlMs: number = 300_000): boolean {
+    if (this.lastUpdatedAt === 0) return false;
+    return (Date.now() - this.lastUpdatedAt) < ttlMs;
+  }
+
+  flush(): void {
+    const payload: CachePayload = {
+      hotFacts: this.hotFacts,
+      factCount: this.factCount,
+      lastSyncedBlock: this.lastSyncedBlock,
+      smartAccountAddress: this.smartAccountAddress,
+      lastUpdatedAt: this.lastUpdatedAt,
+      lastQueryEmbedding: this.lastQueryEmbedding,
+    };
+
+    const plaintext = Buffer.from(JSON.stringify(payload), 'utf-8');
+    const iv = crypto.randomBytes(IV_LENGTH);
+    const cipher = crypto.createCipheriv('aes-256-gcm', this.key, iv);
+    const encrypted = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+    const tag = cipher.getAuthTag();
+
+    const output = Buffer.concat([iv, tag, encrypted]);
+
+    const dir = path.dirname(this.cachePath);
+    if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
+    fs.writeFileSync(this.cachePath, output);
+  }
+
+  load(): void {
+    if (!fs.existsSync(this.cachePath)) return;
+
+    try {
+      const data = fs.readFileSync(this.cachePath);
+      if (data.length < IV_LENGTH + TAG_LENGTH) return;
+
+      const iv = data.subarray(0, IV_LENGTH);
+      const tag = data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
+      const ciphertext = data.subarray(IV_LENGTH + TAG_LENGTH);
+
+      const decipher = crypto.createDecipheriv('aes-256-gcm', this.key, iv);
+      decipher.setAuthTag(tag);
+      const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+
+      const payload: CachePayload = JSON.parse(decrypted.toString('utf-8'));
+      this.hotFacts = payload.hotFacts || [];
+      this.factCount = payload.factCount || 0;
+      this.lastSyncedBlock = payload.lastSyncedBlock || 0;
+      this.smartAccountAddress = payload.smartAccountAddress || '';
+      this.lastUpdatedAt = payload.lastUpdatedAt || 0;
+      this.lastQueryEmbedding = payload.lastQueryEmbedding || null;
+    } catch {
+      this.hotFacts = [];
+      this.factCount = 0;
+      this.lastSyncedBlock = 0;
+      this.smartAccountAddress = '';
+      this.lastUpdatedAt = 0;
+      this.lastQueryEmbedding = null;
+    }
+  }
+}

package/import-adapters/base-adapter.ts

@@ -0,0 +1,93 @@
+import type {
+  NormalizedFact,
+  ImportSource,
+  AdapterParseResult,
+  ProgressCallback,
+} from './types.js';
+
+/**
+ * Abstract base class for import adapters.
+ *
+ * Each adapter:
+ * 1. Fetches or reads source data
+ * 2. Parses into NormalizedFact[]
+ * 3. Validates each fact
+ *
+ * The caller (import tool) handles encryption + storage.
+ */
+export abstract class BaseImportAdapter {
+  abstract readonly source: ImportSource;
+  abstract readonly displayName: string;
+
+  /**
+   * Parse source data into normalized facts.
+   *
+   * For API sources, this fetches from the API.
+   * For file sources, this parses the provided content.
+   */
+  abstract parse(
+    input: { content?: string; api_key?: string; source_user_id?: string; api_url?: string; file_path?: string },
+    onProgress?: ProgressCallback,
+  ): Promise<AdapterParseResult>;
+
+  /**
+   * Validate and clean a single fact.
+   * Returns null if the fact should be skipped.
+   */
+  protected validateFact(fact: Partial<NormalizedFact>): NormalizedFact | null {
+    // Text is required and must be non-empty
+    if (!fact.text || typeof fact.text !== 'string' || fact.text.trim().length < 3) {
+      return null;
+    }
+
+    // Truncate to 512 chars
+    const text = fact.text.trim().slice(0, 512);
+
+    // Normalize type
+    const validTypes = ['fact', 'preference', 'decision', 'episodic', 'goal'] as const;
+    const type = validTypes.includes(fact.type as typeof validTypes[number])
+      ? (fact.type as NormalizedFact['type'])
+      : 'fact';
+
+    // Normalize importance to 1-10
+    let importance = fact.importance ?? 5;
+    if (importance < 0 || importance > 1) {
+      // Already on 1-10 scale
+      importance = Math.max(1, Math.min(10, Math.round(importance)));
+    } else {
+      // 0-1 scale — convert to 1-10
+      importance = Math.max(1, Math.round(importance * 10));
+    }
+
+    return {
+      text,
+      type,
+      importance,
+      source: fact.source ?? this.source,
+      sourceId: fact.sourceId,
+      sourceTimestamp: fact.sourceTimestamp,
+      tags: fact.tags,
+    };
+  }
+
+  /**
+   * Batch-validate an array of partial facts.
+   */
+  protected validateFacts(
+    rawFacts: Partial<NormalizedFact>[],
+  ): { facts: NormalizedFact[]; invalidCount: number } {
+    const facts: NormalizedFact[] = [];
+    let invalidCount = 0;
+
+    for (const raw of rawFacts) {
+      const validated = this.validateFact(raw);
+      if (validated) {
+        facts.push(validated);
+      } else {
+        invalidCount++;
+      }
+    }
+
+    return { facts, invalidCount };
+  }
+}