memory-crystal 0.2.0

package/dist/worker.js

@@ -0,0 +1,137 @@
+// src/worker.ts
+function authenticate(request, env) {
+  const auth = request.headers.get("Authorization");
+  if (!auth?.startsWith("Bearer ")) {
+    return json({ error: "Missing Authorization header" }, 401);
+  }
+  const token = auth.slice(7);
+  const tokenMap = {};
+  if (env.AUTH_TOKEN_CC_AIR) tokenMap[env.AUTH_TOKEN_CC_AIR] = "cc-air";
+  if (env.AUTH_TOKEN_CC_MINI) tokenMap[env.AUTH_TOKEN_CC_MINI] = "cc-mini";
+  if (env.AUTH_TOKEN_LESA) tokenMap[env.AUTH_TOKEN_LESA] = "lesa-mini";
+  const agentId = tokenMap[token];
+  if (!agentId) {
+    return json({ error: "Invalid token" }, 403);
+  }
+  return { agentId };
+}
+var VALID_CHANNELS = ["conversations", "mirror"];
+function isValidChannel(channel) {
+  return VALID_CHANNELS.includes(channel);
+}
+async function handleDrop(request, env, agentId, channel) {
+  if (!isValidChannel(channel)) {
+    return json({ error: `Invalid channel: ${channel}. Valid: ${VALID_CHANNELS.join(", ")}` }, 400);
+  }
+  const body = await request.arrayBuffer();
+  if (body.byteLength === 0) {
+    return json({ error: "Empty payload" }, 400);
+  }
+  if (body.byteLength > 100 * 1024 * 1024) {
+    return json({ error: "Payload too large (max 100MB)" }, 413);
+  }
+  const id = crypto.randomUUID();
+  const key = `${channel}/${id}`;
+  const now = (/* @__PURE__ */ new Date()).toISOString();
+  await env.RELAY.put(key, body, {
+    customMetadata: {
+      agent_id: agentId,
+      dropped_at: now,
+      size: String(body.byteLength)
+    }
+  });
+  return json({ ok: true, id, channel, size: body.byteLength, dropped_at: now });
+}
+async function handlePickupList(env, channel) {
+  if (!isValidChannel(channel)) {
+    return json({ error: `Invalid channel: ${channel}` }, 400);
+  }
+  const listed = await env.RELAY.list({ prefix: `${channel}/` });
+  const blobs = listed.objects.map((obj) => ({
+    id: obj.key.split("/")[1],
+    size: obj.size,
+    dropped_at: obj.customMetadata?.dropped_at || obj.uploaded.toISOString(),
+    agent_id: obj.customMetadata?.agent_id || "unknown"
+  }));
+  return json({ channel, count: blobs.length, blobs });
+}
+async function handlePickup(env, channel, id) {
+  if (!isValidChannel(channel)) {
+    return json({ error: `Invalid channel: ${channel}` }, 400);
+  }
+  const key = `${channel}/${id}`;
+  const obj = await env.RELAY.get(key);
+  if (!obj) {
+    return json({ error: "Blob not found (already picked up or expired)" }, 404);
+  }
+  return new Response(obj.body, {
+    headers: {
+      "Content-Type": "application/octet-stream",
+      "X-Agent-Id": obj.customMetadata?.agent_id || "unknown",
+      "X-Dropped-At": obj.customMetadata?.dropped_at || ""
+    }
+  });
+}
+async function handleConfirm(env, channel, id) {
+  if (!isValidChannel(channel)) {
+    return json({ error: `Invalid channel: ${channel}` }, 400);
+  }
+  const key = `${channel}/${id}`;
+  const obj = await env.RELAY.head(key);
+  if (!obj) {
+    return json({ error: "Blob not found (already confirmed or expired)" }, 404);
+  }
+  await env.RELAY.delete(key);
+  return json({ ok: true, deleted: key });
+}
+function json(data, status = 200) {
+  return new Response(JSON.stringify(data), {
+    status,
+    headers: { "Content-Type": "application/json" }
+  });
+}
+var worker_default = {
+  async fetch(request, env) {
+    const url = new URL(request.url);
+    const parts = url.pathname.split("/").filter(Boolean);
+    if (parts[0] === "health" && request.method === "GET") {
+      return json({ ok: true, service: "memory-crystal-relay", mode: "ephemeral" });
+    }
+    const authResult = authenticate(request, env);
+    if (authResult instanceof Response) return authResult;
+    const { agentId } = authResult;
+    try {
+      if (parts[0] === "drop" && parts[1] && request.method === "POST") {
+        return handleDrop(request, env, agentId, parts[1]);
+      }
+      if (parts[0] === "pickup" && parts[1] && request.method === "GET") {
+        if (parts[2]) {
+          return handlePickup(env, parts[1], parts[2]);
+        }
+        return handlePickupList(env, parts[1]);
+      }
+      if (parts[0] === "confirm" && parts[1] && parts[2] && request.method === "DELETE") {
+        return handleConfirm(env, parts[1], parts[2]);
+      }
+      return json({ error: "Not found" }, 404);
+    } catch (err) {
+      return json({ error: err.message || "Internal error" }, 500);
+    }
+  },
+  // Scheduled cleanup: delete blobs older than 24h (TTL safety net)
+  async scheduled(event, env) {
+    const cutoff = Date.now() - 24 * 60 * 60 * 1e3;
+    for (const channel of VALID_CHANNELS) {
+      const listed = await env.RELAY.list({ prefix: `${channel}/` });
+      for (const obj of listed.objects) {
+        const droppedAt = obj.customMetadata?.dropped_at;
+        if (droppedAt && new Date(droppedAt).getTime() < cutoff) {
+          await env.RELAY.delete(obj.key);
+        }
+      }
+    }
+  }
+};
+export {
+  worker_default as default
+};

package/openclaw.plugin.json

@@ -0,0 +1,11 @@
+{
+  "id": "memory-crystal",
+  "name": "Memory Crystal",
+  "description": "Sovereign memory system — search, remember, forget across all agent conversations and files.",
+  "skills": ["./skills"],
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "memory-crystal",
+  "version": "0.2.0",
+  "description": "Sovereign memory system — local-first with ephemeral encrypted relay. Your memory, your machine, your rules.",
+  "type": "module",
+  "main": "dist/core.js",
+  "openclaw": {
+    "extensions": [
+      "./dist/openclaw.js"
+    ]
+  },
+  "bin": {
+    "crystal": "./dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsup src/core.ts src/cli.ts src/mcp-server.ts src/openclaw.ts src/migrate.ts src/cc-hook.ts src/crypto.ts src/poller.ts src/mirror-sync.ts src/ldm.ts src/summarize.ts --format esm --dts --outDir dist && tsup src/worker.ts --format esm --outDir dist --no-dts",
+    "build:local": "tsup src/core.ts src/cli.ts src/mcp-server.ts src/openclaw.ts src/migrate.ts src/cc-hook.ts src/crypto.ts src/poller.ts src/mirror-sync.ts src/ldm.ts src/summarize.ts --format esm --dts --outDir dist",
+    "build:worker": "tsup src/worker.ts --format esm --outDir dist --no-dts",
+    "dev": "tsup src/core.ts src/cli.ts src/mcp-server.ts src/openclaw.ts src/migrate.ts src/cc-hook.ts src/crypto.ts --format esm --watch --outDir dist",
+    "check": "node dist/cli.js status",
+    "search": "node dist/cli.js search",
+    "migrate": "node dist/migrate.js"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.15.0",
+    "better-sqlite3": "^11.8.1",
+    "@modelcontextprotocol/sdk": "^1.12.1",
+    "apache-arrow": "^18.1.0",
+    "sqlite-vec": "^0.1.7-alpha.2"
+  },
+  "optionalDependencies": {
+    "sqlite-vec-darwin-arm64": "^0.1.7-alpha.2"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^22.0.0",
+    "tsup": "^8.0.0",
+    "typescript": "^5.7.0"
+  }
+}

package/scripts/migrate-lance-to-sqlite.mjs

@@ -0,0 +1,217 @@
+#!/usr/bin/env node
+// migrate-lance-to-sqlite.mjs — Copy all chunks + vectors from LanceDB to sqlite-vec.
+// Reads vectors directly from LanceDB (no re-embedding needed).
+// Deduplicates by SHA-256 hash of text content.
+//
+// Usage:
+//   node scripts/migrate-lance-to-sqlite.mjs [--dry-run] [--batch-size N]
+//
+// Data dir: ~/.openclaw/memory-crystal/
+
+import * as lancedb from '@lancedb/lancedb';
+import Database from 'better-sqlite3';
+import * as sqliteVec from 'sqlite-vec';
+import { createHash } from 'node:crypto';
+import { existsSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+const BATCH_SIZE = 500;
+
+async function main() {
+  const args = process.argv.slice(2);
+  const dryRun = args.includes('--dry-run');
+  const batchSizeArg = args.find((_, i) => args[i - 1] === '--batch-size');
+  const batchSize = batchSizeArg ? parseInt(batchSizeArg) : BATCH_SIZE;
+
+  const openclawHome = process.env.OPENCLAW_HOME || join(process.env.HOME || '/Users/lesa', '.openclaw');
+  const dataDir = join(openclawHome, 'memory-crystal');
+  const lanceDir = join(dataDir, 'lance');
+  const sqlitePath = join(dataDir, 'crystal.db');
+
+  if (!existsSync(lanceDir)) {
+    console.error(`LanceDB directory not found: ${lanceDir}`);
+    process.exit(1);
+  }
+
+  // Open LanceDB
+  const lanceDb = await lancedb.connect(lanceDir);
+  const tableNames = await lanceDb.tableNames();
+  if (!tableNames.includes('chunks')) {
+    console.error('No "chunks" table in LanceDB');
+    process.exit(1);
+  }
+  const lanceTable = await lanceDb.openTable('chunks');
+  const totalLance = await lanceTable.countRows();
+  console.log(`LanceDB chunks: ${totalLance.toLocaleString()}`);
+
+  // Open SQLite + load sqlite-vec
+  const db = new Database(sqlitePath);
+  db.pragma('journal_mode = WAL');
+  sqliteVec.load(db);
+
+  // Ensure tables exist
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS chunks (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      text TEXT NOT NULL,
+      text_hash TEXT NOT NULL,
+      role TEXT,
+      source_type TEXT,
+      source_id TEXT,
+      agent_id TEXT,
+      token_count INTEGER,
+      created_at TEXT NOT NULL
+    );
+    CREATE INDEX IF NOT EXISTS idx_chunks_agent ON chunks(agent_id);
+    CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source_type);
+    CREATE INDEX IF NOT EXISTS idx_chunks_hash ON chunks(text_hash);
+    CREATE INDEX IF NOT EXISTS idx_chunks_created ON chunks(created_at);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+      text,
+      tokenize='porter unicode61'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS chunks_fts_insert AFTER INSERT ON chunks
+    BEGIN
+      INSERT INTO chunks_fts(rowid, text) VALUES (NEW.id, NEW.text);
+    END;
+  `);
+
+  const existingSqlite = (db.prepare('SELECT COUNT(*) as count FROM chunks').get()).count;
+  console.log(`SQLite chunks (before): ${existingSqlite.toLocaleString()}`);
+
+  if (dryRun) {
+    // Sample some rows
+    const sample = await lanceTable.query().limit(3).toArray();
+    console.log('\nSample (3 rows):');
+    for (const row of sample) {
+      console.log(`  [${row.source_type}] [${row.agent_id}] ${row.text?.slice(0, 80)}...`);
+      console.log(`    vector: ${row.vector?.length} dims, created: ${row.created_at}`);
+    }
+    console.log(`\nWould migrate ${totalLance.toLocaleString()} chunks.`);
+    console.log(`Estimated crystal.db growth: ~${Math.round(totalLance * 1536 * 4 / 1024 / 1024)}MB vectors + text`);
+    db.close();
+    return;
+  }
+
+  // Detect dimensions from first row
+  const [firstRow] = await lanceTable.query().limit(1).toArray();
+  const dimensions = firstRow.vector?.length;
+  if (!dimensions) {
+    console.error('Could not determine vector dimensions from LanceDB');
+    process.exit(1);
+  }
+  console.log(`Vector dimensions: ${dimensions}`);
+
+  // Create vec table if needed
+  const vecExists = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name='chunks_vec'`).get();
+  if (!vecExists) {
+    db.exec(`CREATE VIRTUAL TABLE chunks_vec USING vec0(
+      chunk_id INTEGER PRIMARY KEY,
+      embedding float[${dimensions}] distance_metric=cosine
+    )`);
+    console.log(`Created chunks_vec table (${dimensions} dims)`);
+  }
+
+  // Build hash set of existing chunks for dedup
+  console.log('Building dedup hash set...');
+  const existingHashes = new Set();
+  const hashRows = db.prepare('SELECT text_hash FROM chunks').all();
+  for (const row of hashRows) {
+    existingHashes.add(row.text_hash);
+  }
+  console.log(`Existing unique hashes: ${existingHashes.size.toLocaleString()}`);
+
+  // Prepare insert statements
+  const insertChunk = db.prepare(`
+    INSERT INTO chunks (text, text_hash, role, source_type, source_id, agent_id, token_count, created_at)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `);
+  const insertVec = db.prepare(`
+    INSERT INTO chunks_vec (chunk_id, embedding) VALUES (?, ?)
+  `);
+
+  // Read all rows from LanceDB in batches using offset/limit
+  let migrated = 0;
+  let skippedDedup = 0;
+  let offset = 0;
+  const startTime = Date.now();
+
+  while (offset < totalLance) {
+    const rows = await lanceTable.query().limit(batchSize).offset(offset).toArray();
+    if (rows.length === 0) break;
+
+    const transaction = db.transaction(() => {
+      for (const row of rows) {
+        const text = row.text || '';
+        const hash = createHash('sha256').update(text).digest('hex');
+
+        if (existingHashes.has(hash)) {
+          skippedDedup++;
+          continue;
+        }
+        existingHashes.add(hash);
+
+        const result = insertChunk.run(
+          text,
+          hash,
+          row.role || null,
+          row.source_type || null,
+          row.source_id || null,
+          row.agent_id || null,
+          row.token_count || Math.ceil(text.length / 4),
+          row.created_at || new Date().toISOString()
+        );
+
+        // sqlite-vec needs BigInt for integer primary keys
+        const chunkId = typeof result.lastInsertRowid === 'bigint'
+          ? result.lastInsertRowid
+          : BigInt(result.lastInsertRowid);
+
+        // Convert vector to Float32Array
+        const vector = row.vector;
+        const f32 = vector instanceof Float32Array ? vector : new Float32Array(Array.from(vector));
+        insertVec.run(chunkId, f32);
+
+        migrated++;
+      }
+    });
+    transaction();
+
+    offset += rows.length;
+    const elapsed = (Date.now() - startTime) / 1000;
+    const rate = Math.round(offset / elapsed);
+    const eta = Math.round((totalLance - offset) / rate);
+    process.stdout.write(
+      `\r  ${offset.toLocaleString()}/${totalLance.toLocaleString()} (${Math.round(offset / totalLance * 100)}%) ` +
+      `| migrated: ${migrated.toLocaleString()} | dedup: ${skippedDedup.toLocaleString()} ` +
+      `| ${rate}/s | ETA: ${eta}s   `
+    );
+  }
+
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  console.log(`\n\nMigration complete in ${elapsed}s:`);
+  console.log(`  Migrated:    ${migrated.toLocaleString()}`);
+  console.log(`  Dedup skip:  ${skippedDedup.toLocaleString()}`);
+
+  // Verify
+  const finalCount = (db.prepare('SELECT COUNT(*) as count FROM chunks').get()).count;
+  const ftsCount = (db.prepare('SELECT COUNT(*) as count FROM chunks_fts').get()).count;
+  console.log(`  SQLite chunks: ${finalCount.toLocaleString()}`);
+  console.log(`  FTS entries:   ${ftsCount.toLocaleString()}`);
+  console.log(`  LanceDB:       ${totalLance.toLocaleString()}`);
+
+  if (finalCount === ftsCount) {
+    console.log('  FTS sync: OK');
+  } else {
+    console.warn(`  WARNING: FTS count mismatch (${ftsCount} vs ${finalCount})`);
+  }
+
+  db.close();
+}
+
+main().catch(err => {
+  console.error(`Migration failed: ${err.message}`);
+  process.exit(1);
+});

package/skills/memory/SKILL.md

@@ -0,0 +1,61 @@
+---
+name: memory
+description: Search and manage the shared memory crystal. Use when user says "do you remember", "search memory", "remember this", "forget that", "memory status", "what do you know about", or needs to recall past discussions, store facts, or check what's in memory.
+---
+
+# Memory Crystal Skill
+
+## When to use
+- Searching for past conversations, decisions, or context
+- Storing important facts, preferences, or observations that should persist
+- Checking what's in memory and how much is stored
+
+## Tools
+
+### crystal_search
+Search across all stored memory — conversations, files, manually stored facts.
+
+```
+crystal_search query="OpenAI API key configuration" limit=5
+crystal_search query="Parker's preferences" agent_id="main"
+```
+
+### crystal_remember
+Store a fact or observation that should persist across sessions.
+
+```
+crystal_remember text="Parker prefers Opus for complex tasks, Sonnet for quick ones" category="preference"
+crystal_remember text="Gateway auth token is required since v2026.2.2" category="fact"
+```
+
+Categories: fact, preference, event, opinion, skill
+
+### crystal_forget
+Deprecate a stored memory by ID (doesn't delete — marks as deprecated).
+
+```
+crystal_forget id=42
+```
+
+### crystal_status
+Check memory crystal health — chunk count, agents, provider.
+
+```
+crystal_status
+```
+
+## CLI (via bash)
+```bash
+crystal search "query"
+crystal search "query" -n 10 --agent main
+crystal remember "fact to store" --category preference
+crystal forget 42
+crystal status
+crystal status --provider ollama  # check with different provider
+```
+
+## Tips
+- Search is semantic — "how do plugins work" will find conversations about plugin architecture even if those exact words weren't used
+- Store preferences and decisions as memories — they survive compaction
+- Use agent_id filter when you only want results from a specific agent
+- Available providers: openai (default), ollama (local, free), google