@wipcomputer/memory-crystal 0.7.30 → 0.7.33

package/dist/chunk-437F27T6.js

@@ -0,0 +1,97 @@
+import {
+  getAgentId,
+  ldmPaths,
+  resolveSecretPath,
+  resolveStatePath,
+  stateWritePath
+} from "./chunk-DFQ72B7M.js";
+
+// src/role.ts
+import { existsSync, readFileSync, writeFileSync } from "fs";
+var STATE_FILE = "crystal-role.json";
+function loadRoleState() {
+  try {
+    const path = resolveStatePath(STATE_FILE);
+    if (existsSync(path)) {
+      return JSON.parse(readFileSync(path, "utf-8"));
+    }
+  } catch {
+  }
+  return null;
+}
+function saveRoleState(state) {
+  const writePath = stateWritePath(STATE_FILE);
+  writeFileSync(writePath, JSON.stringify(state, null, 2) + "\n");
+}
+function hasLocalEmbeddingProvider() {
+  if (process.env.OPENAI_API_KEY) return true;
+  if (process.env.GOOGLE_API_KEY && process.env.CRYSTAL_EMBEDDING_PROVIDER === "google") return true;
+  if (process.env.CRYSTAL_EMBEDDING_PROVIDER === "ollama") return true;
+  return false;
+}
+function hasRelayKey() {
+  const keyPath = resolveSecretPath("crystal-relay-key");
+  return existsSync(keyPath);
+}
+function detectRole() {
+  const agentId = getAgentId();
+  const paths = ldmPaths(agentId);
+  const relayUrl = process.env.CRYSTAL_RELAY_URL || null;
+  const relayToken = !!process.env.CRYSTAL_RELAY_TOKEN;
+  const relayKeyExists = hasRelayKey();
+  const localEmbeddings = hasLocalEmbeddingProvider();
+  const localDb = existsSync(paths.crystalDb);
+  const state = loadRoleState();
+  if (state && state.override) {
+    return {
+      role: state.role,
+      source: "state-file",
+      relayUrl,
+      relayToken,
+      relayKeyExists,
+      agentId,
+      hasLocalEmbeddings: localEmbeddings,
+      hasLocalDb: localDb
+    };
+  }
+  let role = "core";
+  if ((relayUrl || relayKeyExists) && !localEmbeddings) {
+    role = "node";
+  } else if ((relayUrl || relayKeyExists) && localEmbeddings) {
+    role = "core";
+  }
+  return {
+    role,
+    source: "auto-detected",
+    relayUrl,
+    relayToken,
+    relayKeyExists,
+    agentId,
+    hasLocalEmbeddings: localEmbeddings,
+    hasLocalDb: localDb
+  };
+}
+function promoteToCore() {
+  saveRoleState({
+    role: "core",
+    override: true,
+    agentId: getAgentId(),
+    setAt: (/* @__PURE__ */ new Date()).toISOString()
+  });
+}
+function demoteToNode(relayUrl) {
+  saveRoleState({
+    role: "node",
+    override: true,
+    relayUrl,
+    agentId: getAgentId(),
+    setAt: (/* @__PURE__ */ new Date()).toISOString()
+  });
+}
+
+export {
+  loadRoleState,
+  detectRole,
+  promoteToCore,
+  demoteToNode
+};

package/dist/chunk-5I7GMRDN.js

@@ -0,0 +1,146 @@
+import {
+  ensureLdm,
+  ldmPaths
+} from "./chunk-DFQ72B7M.js";
+
+// src/staging.ts
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync
+} from "fs";
+import { join } from "path";
+import { execSync } from "child_process";
+var HOME = process.env.HOME || "";
+var STAGING_ROOT = join(HOME, ".ldm", "staging");
+function stagingPaths(agentId) {
+  const root = join(STAGING_ROOT, agentId);
+  return {
+    root,
+    transcripts: join(root, "transcripts"),
+    readyFile: join(root, "READY")
+  };
+}
+function ensureStaging(agentId) {
+  const paths = stagingPaths(agentId);
+  mkdirSync(paths.transcripts, { recursive: true });
+  return paths;
+}
+function markReady(agentId) {
+  const paths = stagingPaths(agentId);
+  writeFileSync(paths.readyFile, JSON.stringify({
+    markedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    agentId
+  }));
+}
+function isNewAgent(agentId) {
+  const paths = ldmPaths(agentId);
+  return !existsSync(paths.agentRoot);
+}
+function hasStagedData(agentId) {
+  const paths = stagingPaths(agentId);
+  return existsSync(paths.readyFile);
+}
+function listStagedAgents() {
+  if (!existsSync(STAGING_ROOT)) return [];
+  const agents = [];
+  for (const entry of readdirSync(STAGING_ROOT, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (existsSync(join(STAGING_ROOT, entry.name, "READY"))) {
+      agents.push(entry.name);
+    }
+  }
+  return agents;
+}
+async function processStagedAgent(agentId) {
+  const startTime = Date.now();
+  const staging = stagingPaths(agentId);
+  if (!existsSync(staging.readyFile)) {
+    throw new Error(`No READY file for agent ${agentId}`);
+  }
+  const agentPaths = ensureLdm(agentId);
+  const stagedFiles = existsSync(staging.transcripts) ? readdirSync(staging.transcripts).filter((f) => f.endsWith(".jsonl")) : [];
+  if (stagedFiles.length === 0) {
+    unlinkSync(staging.readyFile);
+    return {
+      agentId,
+      transcriptsProcessed: 0,
+      backfillChunks: 0,
+      dreamWeaverRan: false,
+      durationMs: Date.now() - startTime
+    };
+  }
+  for (const file of stagedFiles) {
+    const src = join(staging.transcripts, file);
+    const dest = join(agentPaths.transcripts, file);
+    try {
+      renameSync(src, dest);
+    } catch {
+      const { copyFileSync } = await import("fs");
+      copyFileSync(src, dest);
+      unlinkSync(src);
+    }
+  }
+  let backfillChunks = 0;
+  let dreamWeaverRan = false;
+  try {
+    const output = execSync(
+      `crystal backfill --agent ${agentId}`,
+      { encoding: "utf-8", timeout: 6e5 }
+    );
+    const match = output.match(/(\d+) chunks embedded/);
+    if (match) backfillChunks = parseInt(match[1], 10);
+  } catch (err) {
+    process.stderr.write(`[staging] backfill failed for ${agentId}: ${err.message}
+`);
+  }
+  try {
+    execSync(
+      `crystal dream-weave --agent ${agentId} --mode full`,
+      { encoding: "utf-8", timeout: 6e5 }
+    );
+    dreamWeaverRan = true;
+  } catch (err) {
+    process.stderr.write(`[staging] dream-weave failed for ${agentId}: ${err.message}
+`);
+  }
+  try {
+    unlinkSync(staging.readyFile);
+  } catch {
+  }
+  return {
+    agentId,
+    transcriptsProcessed: stagedFiles.length,
+    backfillChunks,
+    dreamWeaverRan,
+    durationMs: Date.now() - startTime
+  };
+}
+async function processAllStaged() {
+  const agents = listStagedAgents();
+  const results = [];
+  for (const agentId of agents) {
+    try {
+      const result = await processStagedAgent(agentId);
+      results.push(result);
+    } catch (err) {
+      process.stderr.write(`[staging] failed to process ${agentId}: ${err.message}
+`);
+    }
+  }
+  return results;
+}
+
+export {
+  stagingPaths,
+  ensureStaging,
+  markReady,
+  isNewAgent,
+  hasStagedData,
+  listStagedAgents,
+  processStagedAgent,
+  processAllStaged
+};

package/dist/chunk-CGIDSAJB.js

@@ -0,0 +1,288 @@
+import {
+  decrypt,
+  decryptJSON,
+  encrypt,
+  encryptJSON,
+  loadRelayKey
+} from "./chunk-D3MACYZ4.js";
+import {
+  ldmPaths,
+  resolveStatePath,
+  stateWritePath
+} from "./chunk-DFQ72B7M.js";
+
+// src/file-sync.ts
+import { createHash } from "crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  unlinkSync,
+  readdirSync,
+  statSync
+} from "fs";
+import { join, relative, dirname } from "path";
+var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
+var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
+var EXCLUDE_PATTERNS = [
+  "memory/crystal.db",
+  // DB syncs via delta chunks, not file copy
+  "memory/crystal.db-wal",
+  "memory/crystal.db-shm",
+  "memory/crystal.db.bak",
+  "memory/crystal.db.tmp",
+  "memory/lance/",
+  // LanceDB (deprecated, not synced)
+  "state/",
+  // Local state files (watermarks, etc.)
+  "secrets/",
+  // Encryption keys, tokens
+  "staging/",
+  // Staging pipeline (Core-only)
+  "bin/",
+  // Local scripts
+  ".DS_Store"
+];
+function shouldExclude(relativePath) {
+  for (const pattern of EXCLUDE_PATTERNS) {
+    if (relativePath === pattern || relativePath.startsWith(pattern)) return true;
+  }
+  const parts = relativePath.split("/");
+  for (const part of parts) {
+    if (part.startsWith(".") && part !== ".ldm") return true;
+  }
+  return false;
+}
+function scanDir(baseDir, currentDir, entries) {
+  if (!existsSync(currentDir)) return;
+  const items = readdirSync(currentDir);
+  for (const item of items) {
+    const fullPath = join(currentDir, item);
+    const relPath = relative(baseDir, fullPath);
+    if (shouldExclude(relPath)) continue;
+    let stat;
+    try {
+      stat = statSync(fullPath);
+    } catch {
+      continue;
+    }
+    if (stat.isDirectory()) {
+      scanDir(baseDir, fullPath, entries);
+    } else if (stat.isFile()) {
+      if (stat.size > 50 * 1024 * 1024) continue;
+      const content = readFileSync(fullPath);
+      const sha256 = createHash("sha256").update(content).digest("hex");
+      entries.push({ path: relPath, sha256, size: stat.size });
+    }
+  }
+}
+function generateManifest() {
+  const paths = ldmPaths();
+  const entries = [];
+  scanDir(paths.root, paths.root, entries);
+  return {
+    version: 1,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    fileCount: entries.length,
+    entries
+  };
+}
+function compareManifest(coreManifest) {
+  const paths = ldmPaths();
+  const upsert = [];
+  const toDelete = [];
+  const localManifest = generateManifest();
+  const localMap = new Map(localManifest.entries.map((e) => [e.path, e]));
+  const coreMap = new Map(coreManifest.entries.map((e) => [e.path, e]));
+  for (const entry of coreManifest.entries) {
+    const local = localMap.get(entry.path);
+    if (!local || local.sha256 !== entry.sha256) {
+      upsert.push({ path: entry.path, sha256: entry.sha256 });
+    }
+  }
+  for (const entry of localManifest.entries) {
+    if (!coreMap.has(entry.path)) {
+      toDelete.push(entry.path);
+    }
+  }
+  return { upsert, delete: toDelete };
+}
+function loadFileSyncState() {
+  const statePath = resolveStatePath("file-sync-state.json");
+  try {
+    if (existsSync(statePath)) {
+      return JSON.parse(readFileSync(statePath, "utf-8"));
+    }
+  } catch {
+  }
+  return { lastSync: null, lastManifestHash: null, filesTransferred: 0, filesDeleted: 0 };
+}
+function saveFileSyncState(state) {
+  const writePath = stateWritePath("file-sync-state.json");
+  writeFileSync(writePath, JSON.stringify(state, null, 2));
+}
+async function pushFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const manifest = generateManifest();
+  const manifestJson = JSON.stringify(manifest.entries.map((e) => `${e.path}:${e.sha256}`));
+  const manifestHash = createHash("sha256").update(manifestJson).digest("hex");
+  const state = loadFileSyncState();
+  if (state.lastManifestHash === manifestHash) {
+    return { manifest: 0, files: 0 };
+  }
+  const encryptedManifest = encryptJSON(manifest, relayKey);
+  const manifestResp = await fetch(`${RELAY_URL}/drop/files`, {
+    method: "POST",
+    headers: {
+      "Authorization": `Bearer ${RELAY_TOKEN}`,
+      "Content-Type": "application/octet-stream",
+      "X-File-Type": "manifest"
+    },
+    body: JSON.stringify(encryptedManifest)
+  });
+  if (!manifestResp.ok) {
+    throw new Error(`Manifest push failed: ${manifestResp.status} ${await manifestResp.text()}`);
+  }
+  let filesPushed = 0;
+  for (const entry of manifest.entries) {
+    const fullPath = join(paths.root, entry.path);
+    if (!existsSync(fullPath)) continue;
+    const content = readFileSync(fullPath);
+    const encrypted = encrypt(content, relayKey);
+    const filePayload = JSON.stringify({
+      path: entry.path,
+      sha256: entry.sha256,
+      size: entry.size,
+      data: encrypted
+    });
+    const fileResp = await fetch(`${RELAY_URL}/drop/files`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${RELAY_TOKEN}`,
+        "Content-Type": "application/octet-stream",
+        "X-File-Type": "file"
+      },
+      body: filePayload
+    });
+    if (fileResp.ok) filesPushed++;
+  }
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.lastManifestHash = manifestHash;
+  state.filesTransferred += filesPushed;
+  saveFileSyncState(state);
+  return { manifest: manifest.fileCount, files: filesPushed };
+}
+async function pullFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const listResp = await fetch(`${RELAY_URL}/pickup/files`, {
+    headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+  });
+  if (!listResp.ok) {
+    throw new Error(`File sync list failed: ${listResp.status} ${await listResp.text()}`);
+  }
+  const listData = await listResp.json();
+  if (listData.count === 0) {
+    return { imported: 0, deleted: 0 };
+  }
+  let coreManifest = null;
+  const fileBlobs = [];
+  for (const blob of listData.blobs) {
+    try {
+      const blobResp = await fetch(`${RELAY_URL}/pickup/files/${blob.id}`, {
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+      if (!blobResp.ok) continue;
+      const text = await blobResp.text();
+      const parsed = JSON.parse(text);
+      if (parsed.v !== void 0 && parsed.nonce !== void 0) {
+        try {
+          const manifest = decryptJSON(parsed, relayKey);
+          if (manifest.version && manifest.entries) {
+            coreManifest = manifest;
+          }
+        } catch {
+        }
+      } else if (parsed.path && parsed.data) {
+        fileBlobs.push({
+          id: blob.id,
+          path: parsed.path,
+          sha256: parsed.sha256,
+          data: parsed.data
+        });
+      }
+      await fetch(`${RELAY_URL}/confirm/files/${blob.id}`, {
+        method: "DELETE",
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+    } catch (err) {
+      process.stderr.write(`[file-sync] error processing blob ${blob.id}: ${err.message}
+`);
+    }
+  }
+  if (!coreManifest) {
+    process.stderr.write("[file-sync] no manifest found in relay\n");
+    return { imported: 0, deleted: 0 };
+  }
+  const delta = compareManifest(coreManifest);
+  const fileBlobMap = new Map(fileBlobs.map((f) => [f.path, f]));
+  let imported = 0;
+  for (const entry of delta.upsert) {
+    const fileBlob = fileBlobMap.get(entry.path);
+    if (!fileBlob) {
+      continue;
+    }
+    try {
+      const content = decrypt(fileBlob.data, relayKey);
+      const actualHash = createHash("sha256").update(content).digest("hex");
+      if (actualHash !== entry.sha256) {
+        process.stderr.write(`[file-sync] hash mismatch for ${entry.path}, skipping
+`);
+        continue;
+      }
+      const destPath = join(paths.root, entry.path);
+      mkdirSync(dirname(destPath), { recursive: true });
+      writeFileSync(destPath, content);
+      imported++;
+    } catch (err) {
+      process.stderr.write(`[file-sync] failed to write ${entry.path}: ${err.message}
+`);
+    }
+  }
+  let deleted = 0;
+  for (const path of delta.delete) {
+    const destPath = join(paths.root, path);
+    if (existsSync(destPath)) {
+      try {
+        unlinkSync(destPath);
+        deleted++;
+      } catch (err) {
+        process.stderr.write(`[file-sync] failed to delete ${path}: ${err.message}
+`);
+      }
+    }
+  }
+  const state = loadFileSyncState();
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.filesTransferred += imported;
+  state.filesDeleted += deleted;
+  saveFileSyncState(state);
+  return { imported, deleted };
+}
+
+export {
+  generateManifest,
+  compareManifest,
+  loadFileSyncState,
+  saveFileSyncState,
+  pushFileSync,
+  pullFileSync
+};

package/dist/chunk-D3MACYZ4.js

@@ -0,0 +1,108 @@
+import {
+  resolveSecretPath
+} from "./chunk-DFQ72B7M.js";
+
+// src/crypto.ts
+import { readFileSync, existsSync } from "fs";
+import { createCipheriv, createDecipheriv, createHmac, randomBytes, hkdfSync } from "crypto";
+import { createHash } from "crypto";
+var KEY_PATH = process.env.CRYSTAL_RELAY_KEY_PATH || resolveSecretPath("crystal-relay-key");
+function loadRelayKey() {
+  if (!existsSync(KEY_PATH)) {
+    throw new Error(
+      `Relay key not found at ${KEY_PATH}
+Generate one: mkdir -p ~/.ldm/secrets && openssl rand -base64 32 > ~/.ldm/secrets/crystal-relay-key && chmod 600 ~/.ldm/secrets/crystal-relay-key
+Or run: crystal pair`
+    );
+  }
+  const raw = readFileSync(KEY_PATH, "utf-8").trim();
+  const key = Buffer.from(raw, "base64");
+  if (key.length !== 32) {
+    throw new Error(`Relay key must be 32 bytes (256 bits). Got ${key.length} bytes. Regenerate with: openssl rand -base64 32`);
+  }
+  return key;
+}
+function deriveSigningKey(masterKey) {
+  return Buffer.from(hkdfSync("sha256", masterKey, "", "crystal-relay-sign", 32));
+}
+function encrypt(plaintext, masterKey) {
+  const nonce = randomBytes(12);
+  const cipher = createCipheriv("aes-256-gcm", masterKey, nonce);
+  const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]);
+  const tag = cipher.getAuthTag();
+  const signingKey = deriveSigningKey(masterKey);
+  const hmacData = Buffer.concat([nonce, ciphertext, tag]);
+  const hmac = createHmac("sha256", signingKey).update(hmacData).digest("hex");
+  return {
+    v: 1,
+    nonce: nonce.toString("base64"),
+    ciphertext: ciphertext.toString("base64"),
+    tag: tag.toString("base64"),
+    hmac
+  };
+}
+function decrypt(payload, masterKey) {
+  if (payload.v !== 1) {
+    throw new Error(`Unknown payload version: ${payload.v}`);
+  }
+  const nonce = Buffer.from(payload.nonce, "base64");
+  const ciphertext = Buffer.from(payload.ciphertext, "base64");
+  const tag = Buffer.from(payload.tag, "base64");
+  const signingKey = deriveSigningKey(masterKey);
+  const hmacData = Buffer.concat([nonce, ciphertext, tag]);
+  const expectedHmac = createHmac("sha256", signingKey).update(hmacData).digest("hex");
+  if (payload.hmac !== expectedHmac) {
+    throw new Error("HMAC verification failed \u2014 blob rejected (tampered or wrong key)");
+  }
+  const decipher = createDecipheriv("aes-256-gcm", masterKey, nonce);
+  decipher.setAuthTag(tag);
+  return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+}
+function encryptJSON(data, masterKey) {
+  const plaintext = Buffer.from(JSON.stringify(data), "utf-8");
+  return encrypt(plaintext, masterKey);
+}
+function decryptJSON(payload, masterKey) {
+  const plaintext = decrypt(payload, masterKey);
+  return JSON.parse(plaintext.toString("utf-8"));
+}
+function encryptFile(filePath, masterKey) {
+  const plaintext = readFileSync(filePath);
+  return encrypt(plaintext, masterKey);
+}
+var RELAY_KEY_PATH = KEY_PATH;
+function generateRelayKey() {
+  return randomBytes(32);
+}
+function encodePairingString(key) {
+  if (key.length !== 32) throw new Error("Key must be 32 bytes");
+  return `mc1:${key.toString("base64")}`;
+}
+function decodePairingString(str) {
+  const trimmed = str.trim();
+  if (!trimmed.startsWith("mc1:")) {
+    throw new Error("Invalid pairing string (expected mc1: prefix)");
+  }
+  const key = Buffer.from(trimmed.slice(4), "base64");
+  if (key.length !== 32) {
+    throw new Error(`Invalid key length: expected 32 bytes, got ${key.length}`);
+  }
+  return key;
+}
+function hashBuffer(data) {
+  return createHash("sha256").update(data).digest("hex");
+}
+
+export {
+  loadRelayKey,
+  encrypt,
+  decrypt,
+  encryptJSON,
+  decryptJSON,
+  encryptFile,
+  RELAY_KEY_PATH,
+  generateRelayKey,
+  encodePairingString,
+  decodePairingString,
+  hashBuffer
+};