memory-crystal 0.2.0

package/src/migrate.ts

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+// memory-crystal/migrate.ts — Import chunks from context-embeddings.sqlite
+// Re-embeds with configured provider (OpenAI/Ollama/Google).
+
+import { Crystal, resolveConfig } from './core.js';
+import Database from 'better-sqlite3';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+const BATCH_SIZE = 50;
+
+async function main() {
+  const args = process.argv.slice(2);
+  const dryRun = args.includes('--dry-run');
+  const providerFlag = args.find((_, i) => args[i - 1] === '--provider');
+
+  const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || '', '.openclaw');
+  const sourcePath = join(openclawHome, 'memory', 'context-embeddings.sqlite');
+
+  if (!existsSync(sourcePath)) {
+    console.error(`Source not found: ${sourcePath}`);
+    process.exit(1);
+  }
+
+  const sourceDb = new Database(sourcePath, { readonly: true });
+  sourceDb.pragma('journal_mode = WAL');
+
+  // Count existing chunks
+  const total = (sourceDb.prepare('SELECT COUNT(*) as count FROM conversation_chunks').get() as any).count;
+  console.log(`Found ${total} chunks in context-embeddings.sqlite`);
+
+  if (dryRun) {
+    // Show sample
+    const samples = sourceDb.prepare('SELECT chunk_text, role, session_key, timestamp FROM conversation_chunks ORDER BY timestamp DESC LIMIT 5').all() as any[];
+    console.log('\nSample (5 most recent):');
+    for (const s of samples) {
+      const date = s.timestamp ? new Date(s.timestamp).toISOString().slice(0, 10) : 'unknown';
+      console.log(`  [${date}] [${s.role}] ${s.chunk_text.slice(0, 80)}...`);
+    }
+    console.log(`\nRun without --dry-run to import all ${total} chunks.`);
+    sourceDb.close();
+    return;
+  }
+
+  // Initialize crystal
+  const overrides: any = {};
+  if (providerFlag) overrides.embeddingProvider = providerFlag;
+  const config = resolveConfig(overrides);
+  const crystal = new Crystal(config);
+  await crystal.init();
+
+  console.log(`Embedding provider: ${config.embeddingProvider}`);
+  console.log(`Target: ${config.dataDir}`);
+  console.log(`Migrating ${total} chunks in batches of ${BATCH_SIZE}...`);
+
+  // Fetch all chunks ordered by timestamp
+  const rows = sourceDb.prepare(`
+    SELECT chunk_text, role, agent_id, session_key, timestamp, compaction_number
+    FROM conversation_chunks
+    ORDER BY timestamp ASC
+  `).all() as any[];
+
+  let imported = 0;
+  let failed = 0;
+
+  for (let i = 0; i < rows.length; i += BATCH_SIZE) {
+    const batch = rows.slice(i, i + BATCH_SIZE);
+    const chunks = batch.map(row => ({
+      text: row.chunk_text,
+      role: (row.role || 'assistant') as 'user' | 'assistant' | 'system',
+      source_type: 'conversation' as const,
+      source_id: row.session_key || 'unknown',
+      agent_id: row.agent_id || 'main',
+      token_count: Math.ceil((row.chunk_text?.length || 0) / 4),
+      created_at: row.timestamp ? new Date(row.timestamp).toISOString() : new Date().toISOString(),
+    }));
+
+    try {
+      const count = await crystal.ingest(chunks);
+      imported += count;
+      const pct = Math.round((imported / total) * 100);
+      process.stdout.write(`\r  ${imported}/${total} (${pct}%)`);
+    } catch (err: any) {
+      failed += batch.length;
+      console.error(`\n  Batch error at ${i}: ${err.message}`);
+    }
+  }
+
+  console.log(`\n\nMigration complete:`);
+  console.log(`  Imported: ${imported}`);
+  console.log(`  Failed:   ${failed}`);
+  console.log(`  Provider: ${config.embeddingProvider}`);
+
+  const status = await crystal.status();
+  console.log(`  Total chunks in crystal: ${status.chunks}`);
+
+  crystal.close();
+  sourceDb.close();
+}
+
+main().catch(err => {
+  console.error(`Migration failed: ${err.message}`);
+  process.exit(1);
+});

package/src/mirror-sync.ts

@@ -0,0 +1,175 @@
+#!/usr/bin/env node
+// memory-crystal/mirror-sync.ts — Device-side mirror pull.
+// Pulls encrypted DB snapshot from relay Worker, verifies integrity,
+// decrypts, and replaces local read-only crystal mirror.
+//
+// Usage:
+//   node mirror-sync.js              Pull latest mirror (if available)
+//   node mirror-sync.js --status     Show mirror state
+//   node mirror-sync.js --force      Pull even if current mirror is recent
+
+import { loadRelayKey, decrypt, decryptJSON, hashBuffer, type EncryptedPayload } from './crypto.js';
+import { ldmPaths } from './ldm.js';
+import { readFileSync, writeFileSync, existsSync, mkdirSync, renameSync, unlinkSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+
+const HOME = process.env.HOME || '';
+const RELAY_URL = process.env.CRYSTAL_RELAY_URL || '';
+const RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || '';
+const OC_DIR = join(HOME, '.openclaw');
+const _ldmPaths = ldmPaths();
+const MIRROR_DIR = join(_ldmPaths.root, 'memory');
+const MIRROR_DB_PATH = _ldmPaths.crystalDb;
+const MIRROR_STATE_PATH = join(OC_DIR, 'memory', 'mirror-sync-state.json');
+
+interface MirrorState {
+  lastSync: string | null;
+  lastHash: string | null;
+  lastSize: number | null;
+}
+
+function loadState(): MirrorState {
+  try {
+    if (existsSync(MIRROR_STATE_PATH)) {
+      return JSON.parse(readFileSync(MIRROR_STATE_PATH, 'utf-8'));
+    }
+  } catch {}
+  return { lastSync: null, lastHash: null, lastSize: null };
+}
+
+function saveState(state: MirrorState): void {
+  const dir = dirname(MIRROR_STATE_PATH);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  writeFileSync(MIRROR_STATE_PATH, JSON.stringify(state, null, 2));
+}
+
+// ── Pull mirror ──
+
+async function pullMirror(force: boolean): Promise<boolean> {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error('CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set');
+  }
+
+  const relayKey = loadRelayKey();
+
+  // List available mirror blobs
+  const listResp = await fetch(`${RELAY_URL}/pickup/mirror`, {
+    headers: { 'Authorization': `Bearer ${RELAY_TOKEN}` },
+  });
+
+  if (!listResp.ok) {
+    throw new Error(`Relay list failed: ${listResp.status} ${await listResp.text()}`);
+  }
+
+  const listData = await listResp.json() as { count: number; blobs: Array<{ id: string; size: number; dropped_at: string }> };
+
+  if (listData.count === 0) {
+    process.stderr.write('[mirror-sync] no mirror available\n');
+    return false;
+  }
+
+  // Take the latest blob (last in list by drop time)
+  const latestBlob = listData.blobs[listData.blobs.length - 1];
+
+  // Fetch the encrypted mirror
+  const blobResp = await fetch(`${RELAY_URL}/pickup/mirror/${latestBlob.id}`, {
+    headers: { 'Authorization': `Bearer ${RELAY_TOKEN}` },
+  });
+
+  if (!blobResp.ok) {
+    throw new Error(`Mirror fetch failed: ${blobResp.status}`);
+  }
+
+  const encryptedText = await blobResp.text();
+  const mirrorPayload = JSON.parse(encryptedText) as { meta: EncryptedPayload; db: EncryptedPayload };
+
+  // Decrypt metadata
+  const meta = decryptJSON<{ hash: string; size: number; pushed_at: string }>(mirrorPayload.meta, relayKey);
+
+  // Check if we already have this version
+  const state = loadState();
+  if (!force && state.lastHash === meta.hash) {
+    process.stderr.write('[mirror-sync] mirror is already up to date\n');
+    return false;
+  }
+
+  // Decrypt the DB
+  const dbData = decrypt(mirrorPayload.db, relayKey);
+
+  // Verify integrity
+  const actualHash = hashBuffer(dbData);
+  if (actualHash !== meta.hash) {
+    throw new Error(
+      `Mirror integrity check failed!\n` +
+      `  Expected: ${meta.hash}\n` +
+      `  Got:      ${actualHash}\n` +
+      `Mirror REJECTED — keeping existing local mirror.`
+    );
+  }
+
+  // Atomic replace: write to temp, then rename
+  if (!existsSync(MIRROR_DIR)) mkdirSync(MIRROR_DIR, { recursive: true });
+  const tmpPath = MIRROR_DB_PATH + '.tmp';
+  writeFileSync(tmpPath, dbData);
+
+  // Backup existing mirror
+  if (existsSync(MIRROR_DB_PATH)) {
+    const backupPath = MIRROR_DB_PATH + '.bak';
+    try { renameSync(MIRROR_DB_PATH, backupPath); } catch {}
+  }
+
+  renameSync(tmpPath, MIRROR_DB_PATH);
+
+  // Update state
+  state.lastSync = new Date().toISOString();
+  state.lastHash = meta.hash;
+  state.lastSize = dbData.length;
+  saveState(state);
+
+  process.stderr.write(
+    `[mirror-sync] updated: ${(dbData.length / 1024 / 1024).toFixed(1)}MB, ` +
+    `hash=${meta.hash.slice(0, 12)}..., pushed=${meta.pushed_at}\n`
+  );
+
+  // Confirm receipt — Worker deletes all mirror blobs
+  for (const blob of listData.blobs) {
+    try {
+      await fetch(`${RELAY_URL}/confirm/mirror/${blob.id}`, {
+        method: 'DELETE',
+        headers: { 'Authorization': `Bearer ${RELAY_TOKEN}` },
+      });
+    } catch {} // Best effort cleanup
+  }
+
+  return true;
+}
+
+// ── CLI ──
+
+const args = process.argv.slice(2);
+
+if (args.includes('--status')) {
+  const state = loadState();
+  const hasDb = existsSync(MIRROR_DB_PATH);
+  console.log('Mirror sync status:');
+  console.log(`  Relay URL:    ${RELAY_URL || '(not set)'}`);
+  console.log(`  Local mirror: ${hasDb ? MIRROR_DB_PATH : '(none)'}`);
+  console.log(`  Last sync:    ${state.lastSync || 'never'}`);
+  console.log(`  Last hash:    ${state.lastHash ? state.lastHash.slice(0, 16) + '...' : '(none)'}`);
+  console.log(`  Last size:    ${state.lastSize ? (state.lastSize / 1024 / 1024).toFixed(1) + 'MB' : '(none)'}`);
+  process.exit(0);
+}
+
+const force = args.includes('--force');
+
+pullMirror(force)
+  .then(updated => {
+    if (updated) {
+      process.stderr.write('[mirror-sync] done\n');
+    }
+    process.exit(0);
+  })
+  .catch(err => {
+    process.stderr.write(`[mirror-sync] error: ${err.message}\n`);
+    process.exit(1);
+  });

package/src/openclaw.ts

@@ -0,0 +1,250 @@
+// memory-crystal/openclaw.ts — OpenClaw plugin wrapper.
+// Thin layer calling core.ts via api.registerTool() and api.on().
+// Replaces context-embeddings plugin.
+
+import { Crystal, resolveConfig, type Chunk } from './core.js';
+import { runDevUpdate } from './dev-update.js';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const CONFIG_DIR = join(process.env.HOME || '', '.openclaw');
+const PRIVATE_MODE_PATH = join(CONFIG_DIR, 'memory', 'memory-capture-state.json');
+
+function isPrivateMode(): boolean {
+  try {
+    if (existsSync(PRIVATE_MODE_PATH)) {
+      const state = JSON.parse(readFileSync(PRIVATE_MODE_PATH, 'utf-8'));
+      return state.enabled === false;
+    }
+  } catch {
+    // corrupted file = default to enabled (capture on)
+  }
+  return false;
+}
+
+// getPrivateState and setPrivateMode moved to lesa-private-mode plugin.
+// Only isPrivateMode() is needed here for agent_end and crystal_remember checks.
+
+export default {
+  register(api: any) {
+    const crystal = new Crystal(resolveConfig());
+    let initialized = false;
+
+    async function ensureInit() {
+      if (!initialized) {
+        await crystal.init();
+        initialized = true;
+      }
+    }
+
+    // ── Hook: agent_end (continuous conversation ingestion) ──
+
+    api.on('agent_end', async (event: any, ctx: any) => {
+      // Private mode check
+      if (isPrivateMode()) return;
+
+      await ensureInit();
+
+      const messages = event.messages;
+      if (!messages || messages.length === 0) return;
+
+      const agentId = ctx.agentId || 'main';
+      const sessionKey = ctx.sessionKey || 'unknown';
+
+      // Check capture state
+      const state = crystal.getCaptureState(agentId, sessionKey);
+      const storedCount = state.lastMessageCount;
+
+      // Detect compaction: messages array shrank below stored counter
+      let startIndex = storedCount;
+      if (messages.length < storedCount) {
+        api.logger.info(`memory-crystal: compaction detected (${storedCount} → ${messages.length} messages), resetting capture position`);
+        startIndex = 0;
+      }
+
+      if (messages.length <= startIndex) return; // Nothing new
+
+      // Extract new conversation turns
+      const newTurns: Chunk[] = [];
+      for (let i = startIndex; i < messages.length; i++) {
+        const msg = messages[i];
+        if (!msg.content) continue;
+
+        const role = msg.role;
+        if (role !== 'user' && role !== 'assistant') continue;
+
+        // Extract text from content (string or array)
+        let text = '';
+        if (typeof msg.content === 'string') {
+          text = msg.content;
+        } else if (Array.isArray(msg.content)) {
+          text = msg.content
+            .filter((b: any) => b.type === 'text')
+            .map((b: any) => b.text)
+            .join('\n');
+        }
+
+        if (!text || text.length < 50) continue; // Skip tiny messages
+
+        // Turn-boundary chunking: one message = one chunk.
+        // Only fall back to chunkText() for very long messages (>2000 tokens).
+        const maxSingleChunkChars = 2000 * 4;
+        if (text.length <= maxSingleChunkChars) {
+          newTurns.push({
+            text,
+            role: role as 'user' | 'assistant',
+            source_type: 'conversation',
+            source_id: sessionKey,
+            agent_id: agentId,
+            token_count: Math.ceil(text.length / 4),
+            created_at: new Date().toISOString(),
+          });
+        } else {
+          // Very long message: chunk it, but preserve turn context
+          const chunks = crystal.chunkText(text);
+          for (const chunkText of chunks) {
+            newTurns.push({
+              text: chunkText,
+              role: role as 'user' | 'assistant',
+              source_type: 'conversation',
+              source_id: sessionKey,
+              agent_id: agentId,
+              token_count: Math.ceil(chunkText.length / 4),
+              created_at: new Date().toISOString(),
+            });
+          }
+        }
+      }
+
+      // Skip if not enough new content
+      const totalTokens = newTurns.reduce((sum, c) => sum + c.token_count, 0);
+      if (totalTokens < 500) return;
+
+      // Ingest
+      try {
+        const count = await crystal.ingest(newTurns);
+        crystal.setCaptureState(agentId, sessionKey, messages.length, state.captureCount + 1);
+        api.logger.info(`memory-crystal: ingested ${count} chunks from ${sessionKey} (cycle ${state.captureCount + 1})`);
+      } catch (err: any) {
+        api.logger.error(`memory-crystal: ingest error: ${err.message}`);
+      }
+    });
+
+    // ── Tools ──
+    // OpenClaw expects { content: [{ type: "text", text }] } return format
+
+    function toolResult(text: string, isError = false) {
+      return {
+        content: [{ type: 'text' as const, text }],
+        ...(isError ? { isError: true } : {}),
+      };
+    }
+
+    api.registerTool(
+      {
+        name: 'crystal_search',
+        label: 'Search Memory Crystal',
+        description: 'Search memory crystal — semantic search across all conversations and stored memories.',
+        parameters: {
+          type: 'object',
+          properties: {
+            query: { type: 'string', description: 'What to search for' },
+            limit: { type: 'number', description: 'Max results (default: 5)' },
+            agent_id: { type: 'string', description: 'Filter by agent' },
+          },
+          required: ['query'],
+        },
+        async execute(_id: string, params: any) {
+          try {
+            await ensureInit();
+            const results = await crystal.search(
+              params.query,
+              params.limit || 5,
+              params.agent_id ? { agent_id: params.agent_id } : undefined
+            );
+            if (results.length === 0) return toolResult('No results found.');
+            const formatted = results.map((r, i) => {
+              const score = (r.score * 100).toFixed(1);
+              const date = r.created_at?.slice(0, 10) || 'unknown';
+              return `[${i + 1}] (${score}%, ${r.agent_id}, ${date}, ${r.role})\n${r.text}`;
+            }).join('\n\n---\n\n');
+            return toolResult(formatted);
+          } catch (err: any) {
+            return toolResult(`crystal_search error: ${err.message}`, true);
+          }
+        },
+      },
+      { optional: true }
+    );
+
+    api.registerTool(
+      {
+        name: 'crystal_remember',
+        label: 'Remember in Crystal',
+        description: 'Store a fact, preference, or observation in memory crystal.',
+        parameters: {
+          type: 'object',
+          properties: {
+            text: { type: 'string', description: 'The fact to remember' },
+            category: { type: 'string', enum: ['fact', 'preference', 'event', 'opinion', 'skill'] },
+          },
+          required: ['text'],
+        },
+        async execute(_id: string, params: any) {
+          // Private mode blocks explicit memory writes too
+          if (isPrivateMode()) {
+            return toolResult('Private mode is on. No memories are being stored. Use /private-mode off to resume.');
+          }
+          try {
+            await ensureInit();
+            const id = await crystal.remember(params.text, params.category || 'fact');
+            return toolResult(`Remembered (id: ${id}): ${params.text}`);
+          } catch (err: any) {
+            return toolResult(`crystal_remember error: ${err.message}`, true);
+          }
+        },
+      },
+      { optional: true }
+    );
+
+    api.registerTool(
+      {
+        name: 'crystal_forget',
+        label: 'Forget Memory',
+        description: 'Deprecate a memory by ID.',
+        parameters: {
+          type: 'object',
+          properties: {
+            id: { type: 'number', description: 'Memory ID to deprecate' },
+          },
+          required: ['id'],
+        },
+        async execute(_id: string, params: any) {
+          try {
+            await ensureInit();
+            const ok = crystal.forget(params.id);
+            return toolResult(ok ? `Forgot memory ${params.id}` : `Memory ${params.id} not found`);
+          } catch (err: any) {
+            return toolResult(`crystal_forget error: ${err.message}`, true);
+          }
+        },
+      },
+      { optional: true }
+    );
+
+    // ── Hook: before_compaction (auto dev updates) ──
+
+    api.on('before_compaction', (_event: any, _ctx: any) => {
+      try {
+        const result = runDevUpdate('lesa');
+        if (result.reposUpdated > 0) {
+          api.logger.info(`memory-crystal: auto-dev-update wrote ${result.reposUpdated} updates before compaction`);
+        }
+      } catch (err: any) {
+        api.logger.warn(`memory-crystal: auto-dev-update failed: ${err.message}`);
+      }
+    });
+
+    api.logger.info('memory-crystal plugin registered');
+  }
+};