@wipcomputer/memory-crystal 0.7.34-alpha.1 → 0.7.34-alpha.3

package/dist/file-sync.js

@@ -1,13 +1,411 @@
+// src/crypto.ts
+import { readFileSync as readFileSync2, existsSync as existsSync2 } from "fs";
+import { createCipheriv, createDecipheriv, createHmac, randomBytes, hkdfSync } from "crypto";
+
+// src/ldm.ts
+import { existsSync, mkdirSync, readFileSync, writeFileSync, copyFileSync, chmodSync, readdirSync } from "fs";
+import { join, dirname } from "path";
+import { execSync } from "child_process";
+import { fileURLToPath } from "url";
+var HOME = process.env.HOME || "";
+var LDM_ROOT = join(HOME, ".ldm");
+function loadAgentConfig(id) {
+  const cfgPath = join(LDM_ROOT, "agents", id, "config.json");
+  try {
+    if (existsSync(cfgPath)) return JSON.parse(readFileSync(cfgPath, "utf-8"));
+  } catch {
+  }
+  return null;
+}
+function getAgentId(harnessHint) {
+  if (process.env.CRYSTAL_AGENT_ID) return process.env.CRYSTAL_AGENT_ID;
+  const agentsDir = join(LDM_ROOT, "agents");
+  if (existsSync(agentsDir)) {
+    try {
+      for (const d of readdirSync(agentsDir)) {
+        const cfg = loadAgentConfig(d);
+        if (!cfg || !cfg.agentId) continue;
+        if (!harnessHint) return cfg.agentId;
+        if (harnessHint === "claude-code" && cfg.harness === "claude-code-cli") return cfg.agentId;
+        if (harnessHint === "openclaw" && cfg.harness === "openclaw") return cfg.agentId;
+      }
+    } catch {
+    }
+  }
+  return harnessHint === "openclaw" ? "oc-lesa-mini" : "cc-mini";
+}
+function ldmPaths(agentId) {
+  const id = agentId || getAgentId();
+  const agentRoot = join(LDM_ROOT, "agents", id);
+  return {
+    root: LDM_ROOT,
+    bin: join(LDM_ROOT, "bin"),
+    secrets: join(LDM_ROOT, "secrets"),
+    state: join(LDM_ROOT, "state"),
+    config: join(LDM_ROOT, "config.json"),
+    crystalDb: join(LDM_ROOT, "memory", "crystal.db"),
+    crystalLance: join(LDM_ROOT, "memory", "lance"),
+    agentRoot,
+    transcripts: join(agentRoot, "memory", "transcripts"),
+    sessions: join(agentRoot, "memory", "sessions"),
+    daily: join(agentRoot, "memory", "daily"),
+    journals: join(agentRoot, "memory", "journals"),
+    workspace: join(agentRoot, "memory", "workspace")
+  };
+}
+var LEGACY_OC_DIR = join(HOME, ".openclaw");
+function resolveStatePath(filename) {
+  const paths = ldmPaths();
+  const ldmPath = join(paths.state, filename);
+  if (existsSync(ldmPath)) return ldmPath;
+  const legacyPath = join(LEGACY_OC_DIR, "memory", filename);
+  if (existsSync(legacyPath)) return legacyPath;
+  return ldmPath;
+}
+function stateWritePath(filename) {
+  const paths = ldmPaths();
+  const dir = paths.state;
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  return join(dir, filename);
+}
+function resolveSecretPath(filename) {
+  const paths = ldmPaths();
+  const ldmPath = join(paths.secrets, filename);
+  if (existsSync(ldmPath)) return ldmPath;
+  const legacyPath = join(LEGACY_OC_DIR, "secrets", filename);
+  if (existsSync(legacyPath)) return legacyPath;
+  return ldmPath;
+}
+
+// src/crypto.ts
+import { createHash } from "crypto";
+var KEY_PATH = process.env.CRYSTAL_RELAY_KEY_PATH || resolveSecretPath("crystal-relay-key");
+function loadRelayKey() {
+  if (!existsSync2(KEY_PATH)) {
+    throw new Error(
+      `Relay key not found at ${KEY_PATH}
+Generate one: mkdir -p ~/.ldm/secrets && openssl rand -base64 32 > ~/.ldm/secrets/crystal-relay-key && chmod 600 ~/.ldm/secrets/crystal-relay-key
+Or run: crystal pair`
+    );
+  }
+  const raw = readFileSync2(KEY_PATH, "utf-8").trim();
+  const key = Buffer.from(raw, "base64");
+  if (key.length !== 32) {
+    throw new Error(`Relay key must be 32 bytes (256 bits). Got ${key.length} bytes. Regenerate with: openssl rand -base64 32`);
+  }
+  return key;
+}
+function deriveSigningKey(masterKey) {
+  return Buffer.from(hkdfSync("sha256", masterKey, "", "crystal-relay-sign", 32));
+}
+function encrypt(plaintext, masterKey) {
+  const nonce = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", masterKey, nonce);
+  const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  const signingKey = deriveSigningKey(masterKey);
+  const hmacData = Buffer.concat([nonce, ciphertext, tag]);
+  const hmac = createHmac("sha256", signingKey).update(hmacData).digest("hex");
+  return {
+    v: 1,
+    nonce: nonce.toString("base64"),
+    ciphertext: ciphertext.toString("base64"),
+    tag: tag.toString("base64"),
+    hmac
+  };
+}
+function decrypt(payload, masterKey) {
+  if (payload.v !== 1) {
+    throw new Error(`Unknown payload version: ${payload.v}`);
+  }
+  const nonce = Buffer.from(payload.nonce, "base64");
+  const ciphertext = Buffer.from(payload.ciphertext, "base64");
+  const tag = Buffer.from(payload.tag, "base64");
+  const signingKey = deriveSigningKey(masterKey);
+  const hmacData = Buffer.concat([nonce, ciphertext, tag]);
+  const expectedHmac = createHmac("sha256", signingKey).update(hmacData).digest("hex");
+  if (payload.hmac !== expectedHmac) {
+    throw new Error("HMAC verification failed \u2014 blob rejected (tampered or wrong key)");
+  }
+  const decipher = createDecipheriv("aes-256-gcm", masterKey, nonce);
+  decipher.setAuthTag(tag);
+  return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+}
+function encryptJSON(data, masterKey) {
+  const plaintext = Buffer.from(JSON.stringify(data), "utf-8");
+  return encrypt(plaintext, masterKey);
+}
+function decryptJSON(payload, masterKey) {
+  const plaintext = decrypt(payload, masterKey);
+  return JSON.parse(plaintext.toString("utf-8"));
+}
+
+// src/file-sync.ts
+import { createHash as createHash2 } from "crypto";
 import {
-  compareManifest,
-  generateManifest,
-  loadFileSyncState,
-  pullFileSync,
-  pushFileSync,
-  saveFileSyncState
-} from "./chunk-CGIDSAJB.js";
-import "./chunk-D3MACYZ4.js";
-import "./chunk-DFQ72B7M.js";
+  existsSync as existsSync3,
+  mkdirSync as mkdirSync2,
+  readFileSync as readFileSync3,
+  writeFileSync as writeFileSync2,
+  unlinkSync,
+  readdirSync as readdirSync2,
+  statSync
+} from "fs";
+import { join as join2, relative, dirname as dirname2 } from "path";
+var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
+var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
+var EXCLUDE_PATTERNS = [
+  "memory/crystal.db",
+  // DB syncs via delta chunks, not file copy
+  "memory/crystal.db-wal",
+  "memory/crystal.db-shm",
+  "memory/crystal.db.bak",
+  "memory/crystal.db.tmp",
+  "memory/lance/",
+  // LanceDB (deprecated, not synced)
+  "state/",
+  // Local state files (watermarks, etc.)
+  "secrets/",
+  // Encryption keys, tokens
+  "staging/",
+  // Staging pipeline (Core-only)
+  "bin/",
+  // Local scripts
+  ".DS_Store"
+];
+function shouldExclude(relativePath) {
+  for (const pattern of EXCLUDE_PATTERNS) {
+    if (relativePath === pattern || relativePath.startsWith(pattern)) return true;
+  }
+  const parts = relativePath.split("/");
+  for (const part of parts) {
+    if (part.startsWith(".") && part !== ".ldm") return true;
+  }
+  return false;
+}
+function scanDir(baseDir, currentDir, entries) {
+  if (!existsSync3(currentDir)) return;
+  const items = readdirSync2(currentDir);
+  for (const item of items) {
+    const fullPath = join2(currentDir, item);
+    const relPath = relative(baseDir, fullPath);
+    if (shouldExclude(relPath)) continue;
+    let stat;
+    try {
+      stat = statSync(fullPath);
+    } catch {
+      continue;
+    }
+    if (stat.isDirectory()) {
+      scanDir(baseDir, fullPath, entries);
+    } else if (stat.isFile()) {
+      if (stat.size > 50 * 1024 * 1024) continue;
+      const content = readFileSync3(fullPath);
+      const sha256 = createHash2("sha256").update(content).digest("hex");
+      entries.push({ path: relPath, sha256, size: stat.size });
+    }
+  }
+}
+function generateManifest() {
+  const paths = ldmPaths();
+  const entries = [];
+  scanDir(paths.root, paths.root, entries);
+  return {
+    version: 1,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    fileCount: entries.length,
+    entries
+  };
+}
+function compareManifest(coreManifest) {
+  const paths = ldmPaths();
+  const upsert = [];
+  const toDelete = [];
+  const localManifest = generateManifest();
+  const localMap = new Map(localManifest.entries.map((e) => [e.path, e]));
+  const coreMap = new Map(coreManifest.entries.map((e) => [e.path, e]));
+  for (const entry of coreManifest.entries) {
+    const local = localMap.get(entry.path);
+    if (!local || local.sha256 !== entry.sha256) {
+      upsert.push({ path: entry.path, sha256: entry.sha256 });
+    }
+  }
+  for (const entry of localManifest.entries) {
+    if (!coreMap.has(entry.path)) {
+      toDelete.push(entry.path);
+    }
+  }
+  return { upsert, delete: toDelete };
+}
+function loadFileSyncState() {
+  const statePath = resolveStatePath("file-sync-state.json");
+  try {
+    if (existsSync3(statePath)) {
+      return JSON.parse(readFileSync3(statePath, "utf-8"));
+    }
+  } catch {
+  }
+  return { lastSync: null, lastManifestHash: null, filesTransferred: 0, filesDeleted: 0 };
+}
+function saveFileSyncState(state) {
+  const writePath = stateWritePath("file-sync-state.json");
+  writeFileSync2(writePath, JSON.stringify(state, null, 2));
+}
+async function pushFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const manifest = generateManifest();
+  const manifestJson = JSON.stringify(manifest.entries.map((e) => `${e.path}:${e.sha256}`));
+  const manifestHash = createHash2("sha256").update(manifestJson).digest("hex");
+  const state = loadFileSyncState();
+  if (state.lastManifestHash === manifestHash) {
+    return { manifest: 0, files: 0 };
+  }
+  const encryptedManifest = encryptJSON(manifest, relayKey);
+  const manifestResp = await fetch(`${RELAY_URL}/drop/files`, {
+    method: "POST",
+    headers: {
+      "Authorization": `Bearer ${RELAY_TOKEN}`,
+      "Content-Type": "application/octet-stream",
+      "X-File-Type": "manifest"
+    },
+    body: JSON.stringify(encryptedManifest)
+  });
+  if (!manifestResp.ok) {
+    throw new Error(`Manifest push failed: ${manifestResp.status} ${await manifestResp.text()}`);
+  }
+  let filesPushed = 0;
+  for (const entry of manifest.entries) {
+    const fullPath = join2(paths.root, entry.path);
+    if (!existsSync3(fullPath)) continue;
+    const content = readFileSync3(fullPath);
+    const encrypted = encrypt(content, relayKey);
+    const filePayload = JSON.stringify({
+      path: entry.path,
+      sha256: entry.sha256,
+      size: entry.size,
+      data: encrypted
+    });
+    const fileResp = await fetch(`${RELAY_URL}/drop/files`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${RELAY_TOKEN}`,
+        "Content-Type": "application/octet-stream",
+        "X-File-Type": "file"
+      },
+      body: filePayload
+    });
+    if (fileResp.ok) filesPushed++;
+  }
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.lastManifestHash = manifestHash;
+  state.filesTransferred += filesPushed;
+  saveFileSyncState(state);
+  return { manifest: manifest.fileCount, files: filesPushed };
+}
+async function pullFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const listResp = await fetch(`${RELAY_URL}/pickup/files`, {
+    headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+  });
+  if (!listResp.ok) {
+    throw new Error(`File sync list failed: ${listResp.status} ${await listResp.text()}`);
+  }
+  const listData = await listResp.json();
+  if (listData.count === 0) {
+    return { imported: 0, deleted: 0 };
+  }
+  let coreManifest = null;
+  const fileBlobs = [];
+  for (const blob of listData.blobs) {
+    try {
+      const blobResp = await fetch(`${RELAY_URL}/pickup/files/${blob.id}`, {
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+      if (!blobResp.ok) continue;
+      const text = await blobResp.text();
+      const parsed = JSON.parse(text);
+      if (parsed.v !== void 0 && parsed.nonce !== void 0) {
+        try {
+          const manifest = decryptJSON(parsed, relayKey);
+          if (manifest.version && manifest.entries) {
+            coreManifest = manifest;
+          }
+        } catch {
+        }
+      } else if (parsed.path && parsed.data) {
+        fileBlobs.push({
+          id: blob.id,
+          path: parsed.path,
+          sha256: parsed.sha256,
+          data: parsed.data
+        });
+      }
+      await fetch(`${RELAY_URL}/confirm/files/${blob.id}`, {
+        method: "DELETE",
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+    } catch (err) {
+      process.stderr.write(`[file-sync] error processing blob ${blob.id}: ${err.message}
+`);
+    }
+  }
+  if (!coreManifest) {
+    process.stderr.write("[file-sync] no manifest found in relay\n");
+    return { imported: 0, deleted: 0 };
+  }
+  const delta = compareManifest(coreManifest);
+  const fileBlobMap = new Map(fileBlobs.map((f) => [f.path, f]));
+  let imported = 0;
+  for (const entry of delta.upsert) {
+    const fileBlob = fileBlobMap.get(entry.path);
+    if (!fileBlob) {
+      continue;
+    }
+    try {
+      const content = decrypt(fileBlob.data, relayKey);
+      const actualHash = createHash2("sha256").update(content).digest("hex");
+      if (actualHash !== entry.sha256) {
+        process.stderr.write(`[file-sync] hash mismatch for ${entry.path}, skipping
+`);
+        continue;
+      }
+      const destPath = join2(paths.root, entry.path);
+      mkdirSync2(dirname2(destPath), { recursive: true });
+      writeFileSync2(destPath, content);
+      imported++;
+    } catch (err) {
+      process.stderr.write(`[file-sync] failed to write ${entry.path}: ${err.message}
+`);
+    }
+  }
+  let deleted = 0;
+  for (const path of delta.delete) {
+    const destPath = join2(paths.root, path);
+    if (existsSync3(destPath)) {
+      try {
+        unlinkSync(destPath);
+        deleted++;
+      } catch (err) {
+        process.stderr.write(`[file-sync] failed to delete ${path}: ${err.message}
+`);
+      }
+    }
+  }
+  const state = loadFileSyncState();
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.filesTransferred += imported;
+  state.filesDeleted += deleted;
+  saveFileSyncState(state);
+  return { imported, deleted };
+}
 export {
   compareManifest,
   generateManifest,