@wipcomputer/memory-crystal 0.7.32 → 0.7.34-alpha.1

package/dist/chunk-CCYI5O3D.js

@@ -0,0 +1,148 @@
+import {
+  ensureLdm,
+  init_ldm,
+  ldmPaths
+} from "./chunk-PSQZURHO.js";
+
+// src/staging.ts
+init_ldm();
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync
+} from "fs";
+import { join } from "path";
+import { execSync } from "child_process";
+var HOME = process.env.HOME || "";
+var STAGING_ROOT = join(HOME, ".ldm", "staging");
+function stagingPaths(agentId) {
+  const root = join(STAGING_ROOT, agentId);
+  return {
+    root,
+    transcripts: join(root, "transcripts"),
+    readyFile: join(root, "READY")
+  };
+}
+function ensureStaging(agentId) {
+  const paths = stagingPaths(agentId);
+  mkdirSync(paths.transcripts, { recursive: true });
+  return paths;
+}
+function markReady(agentId) {
+  const paths = stagingPaths(agentId);
+  writeFileSync(paths.readyFile, JSON.stringify({
+    markedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    agentId
+  }));
+}
+function isNewAgent(agentId) {
+  const paths = ldmPaths(agentId);
+  return !existsSync(paths.agentRoot);
+}
+function hasStagedData(agentId) {
+  const paths = stagingPaths(agentId);
+  return existsSync(paths.readyFile);
+}
+function listStagedAgents() {
+  if (!existsSync(STAGING_ROOT)) return [];
+  const agents = [];
+  for (const entry of readdirSync(STAGING_ROOT, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    if (existsSync(join(STAGING_ROOT, entry.name, "READY"))) {
+      agents.push(entry.name);
+    }
+  }
+  return agents;
+}
+async function processStagedAgent(agentId) {
+  const startTime = Date.now();
+  const staging = stagingPaths(agentId);
+  if (!existsSync(staging.readyFile)) {
+    throw new Error(`No READY file for agent ${agentId}`);
+  }
+  const agentPaths = ensureLdm(agentId);
+  const stagedFiles = existsSync(staging.transcripts) ? readdirSync(staging.transcripts).filter((f) => f.endsWith(".jsonl")) : [];
+  if (stagedFiles.length === 0) {
+    unlinkSync(staging.readyFile);
+    return {
+      agentId,
+      transcriptsProcessed: 0,
+      backfillChunks: 0,
+      dreamWeaverRan: false,
+      durationMs: Date.now() - startTime
+    };
+  }
+  for (const file of stagedFiles) {
+    const src = join(staging.transcripts, file);
+    const dest = join(agentPaths.transcripts, file);
+    try {
+      renameSync(src, dest);
+    } catch {
+      const { copyFileSync } = await import("fs");
+      copyFileSync(src, dest);
+      unlinkSync(src);
+    }
+  }
+  let backfillChunks = 0;
+  let dreamWeaverRan = false;
+  try {
+    const output = execSync(
+      `crystal backfill --agent ${agentId}`,
+      { encoding: "utf-8", timeout: 6e5 }
+    );
+    const match = output.match(/(\d+) chunks embedded/);
+    if (match) backfillChunks = parseInt(match[1], 10);
+  } catch (err) {
+    process.stderr.write(`[staging] backfill failed for ${agentId}: ${err.message}
+`);
+  }
+  try {
+    execSync(
+      `crystal dream-weave --agent ${agentId} --mode full`,
+      { encoding: "utf-8", timeout: 6e5 }
+    );
+    dreamWeaverRan = true;
+  } catch (err) {
+    process.stderr.write(`[staging] dream-weave failed for ${agentId}: ${err.message}
+`);
+  }
+  try {
+    unlinkSync(staging.readyFile);
+  } catch {
+  }
+  return {
+    agentId,
+    transcriptsProcessed: stagedFiles.length,
+    backfillChunks,
+    dreamWeaverRan,
+    durationMs: Date.now() - startTime
+  };
+}
+async function processAllStaged() {
+  const agents = listStagedAgents();
+  const results = [];
+  for (const agentId of agents) {
+    try {
+      const result = await processStagedAgent(agentId);
+      results.push(result);
+    } catch (err) {
+      process.stderr.write(`[staging] failed to process ${agentId}: ${err.message}
+`);
+    }
+  }
+  return results;
+}
+
+export {
+  stagingPaths,
+  ensureStaging,
+  markReady,
+  isNewAgent,
+  hasStagedData,
+  listStagedAgents,
+  processStagedAgent,
+  processAllStaged
+};

package/dist/chunk-CGIDSAJB.js

@@ -0,0 +1,288 @@
+import {
+  decrypt,
+  decryptJSON,
+  encrypt,
+  encryptJSON,
+  loadRelayKey
+} from "./chunk-D3MACYZ4.js";
+import {
+  ldmPaths,
+  resolveStatePath,
+  stateWritePath
+} from "./chunk-DFQ72B7M.js";
+
+// src/file-sync.ts
+import { createHash } from "crypto";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  writeFileSync,
+  unlinkSync,
+  readdirSync,
+  statSync
+} from "fs";
+import { join, relative, dirname } from "path";
+var RELAY_URL = process.env.CRYSTAL_RELAY_URL || "";
+var RELAY_TOKEN = process.env.CRYSTAL_RELAY_TOKEN || "";
+var EXCLUDE_PATTERNS = [
+  "memory/crystal.db",
+  // DB syncs via delta chunks, not file copy
+  "memory/crystal.db-wal",
+  "memory/crystal.db-shm",
+  "memory/crystal.db.bak",
+  "memory/crystal.db.tmp",
+  "memory/lance/",
+  // LanceDB (deprecated, not synced)
+  "state/",
+  // Local state files (watermarks, etc.)
+  "secrets/",
+  // Encryption keys, tokens
+  "staging/",
+  // Staging pipeline (Core-only)
+  "bin/",
+  // Local scripts
+  ".DS_Store"
+];
+function shouldExclude(relativePath) {
+  for (const pattern of EXCLUDE_PATTERNS) {
+    if (relativePath === pattern || relativePath.startsWith(pattern)) return true;
+  }
+  const parts = relativePath.split("/");
+  for (const part of parts) {
+    if (part.startsWith(".") && part !== ".ldm") return true;
+  }
+  return false;
+}
+function scanDir(baseDir, currentDir, entries) {
+  if (!existsSync(currentDir)) return;
+  const items = readdirSync(currentDir);
+  for (const item of items) {
+    const fullPath = join(currentDir, item);
+    const relPath = relative(baseDir, fullPath);
+    if (shouldExclude(relPath)) continue;
+    let stat;
+    try {
+      stat = statSync(fullPath);
+    } catch {
+      continue;
+    }
+    if (stat.isDirectory()) {
+      scanDir(baseDir, fullPath, entries);
+    } else if (stat.isFile()) {
+      if (stat.size > 50 * 1024 * 1024) continue;
+      const content = readFileSync(fullPath);
+      const sha256 = createHash("sha256").update(content).digest("hex");
+      entries.push({ path: relPath, sha256, size: stat.size });
+    }
+  }
+}
+function generateManifest() {
+  const paths = ldmPaths();
+  const entries = [];
+  scanDir(paths.root, paths.root, entries);
+  return {
+    version: 1,
+    generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
+    fileCount: entries.length,
+    entries
+  };
+}
+function compareManifest(coreManifest) {
+  const paths = ldmPaths();
+  const upsert = [];
+  const toDelete = [];
+  const localManifest = generateManifest();
+  const localMap = new Map(localManifest.entries.map((e) => [e.path, e]));
+  const coreMap = new Map(coreManifest.entries.map((e) => [e.path, e]));
+  for (const entry of coreManifest.entries) {
+    const local = localMap.get(entry.path);
+    if (!local || local.sha256 !== entry.sha256) {
+      upsert.push({ path: entry.path, sha256: entry.sha256 });
+    }
+  }
+  for (const entry of localManifest.entries) {
+    if (!coreMap.has(entry.path)) {
+      toDelete.push(entry.path);
+    }
+  }
+  return { upsert, delete: toDelete };
+}
+function loadFileSyncState() {
+  const statePath = resolveStatePath("file-sync-state.json");
+  try {
+    if (existsSync(statePath)) {
+      return JSON.parse(readFileSync(statePath, "utf-8"));
+    }
+  } catch {
+  }
+  return { lastSync: null, lastManifestHash: null, filesTransferred: 0, filesDeleted: 0 };
+}
+function saveFileSyncState(state) {
+  const writePath = stateWritePath("file-sync-state.json");
+  writeFileSync(writePath, JSON.stringify(state, null, 2));
+}
+async function pushFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const manifest = generateManifest();
+  const manifestJson = JSON.stringify(manifest.entries.map((e) => `${e.path}:${e.sha256}`));
+  const manifestHash = createHash("sha256").update(manifestJson).digest("hex");
+  const state = loadFileSyncState();
+  if (state.lastManifestHash === manifestHash) {
+    return { manifest: 0, files: 0 };
+  }
+  const encryptedManifest = encryptJSON(manifest, relayKey);
+  const manifestResp = await fetch(`${RELAY_URL}/drop/files`, {
+    method: "POST",
+    headers: {
+      "Authorization": `Bearer ${RELAY_TOKEN}`,
+      "Content-Type": "application/octet-stream",
+      "X-File-Type": "manifest"
+    },
+    body: JSON.stringify(encryptedManifest)
+  });
+  if (!manifestResp.ok) {
+    throw new Error(`Manifest push failed: ${manifestResp.status} ${await manifestResp.text()}`);
+  }
+  let filesPushed = 0;
+  for (const entry of manifest.entries) {
+    const fullPath = join(paths.root, entry.path);
+    if (!existsSync(fullPath)) continue;
+    const content = readFileSync(fullPath);
+    const encrypted = encrypt(content, relayKey);
+    const filePayload = JSON.stringify({
+      path: entry.path,
+      sha256: entry.sha256,
+      size: entry.size,
+      data: encrypted
+    });
+    const fileResp = await fetch(`${RELAY_URL}/drop/files`, {
+      method: "POST",
+      headers: {
+        "Authorization": `Bearer ${RELAY_TOKEN}`,
+        "Content-Type": "application/octet-stream",
+        "X-File-Type": "file"
+      },
+      body: filePayload
+    });
+    if (fileResp.ok) filesPushed++;
+  }
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.lastManifestHash = manifestHash;
+  state.filesTransferred += filesPushed;
+  saveFileSyncState(state);
+  return { manifest: manifest.fileCount, files: filesPushed };
+}
+async function pullFileSync() {
+  if (!RELAY_URL || !RELAY_TOKEN) {
+    throw new Error("CRYSTAL_RELAY_URL and CRYSTAL_RELAY_TOKEN must be set");
+  }
+  const relayKey = loadRelayKey();
+  const paths = ldmPaths();
+  const listResp = await fetch(`${RELAY_URL}/pickup/files`, {
+    headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+  });
+  if (!listResp.ok) {
+    throw new Error(`File sync list failed: ${listResp.status} ${await listResp.text()}`);
+  }
+  const listData = await listResp.json();
+  if (listData.count === 0) {
+    return { imported: 0, deleted: 0 };
+  }
+  let coreManifest = null;
+  const fileBlobs = [];
+  for (const blob of listData.blobs) {
+    try {
+      const blobResp = await fetch(`${RELAY_URL}/pickup/files/${blob.id}`, {
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+      if (!blobResp.ok) continue;
+      const text = await blobResp.text();
+      const parsed = JSON.parse(text);
+      if (parsed.v !== void 0 && parsed.nonce !== void 0) {
+        try {
+          const manifest = decryptJSON(parsed, relayKey);
+          if (manifest.version && manifest.entries) {
+            coreManifest = manifest;
+          }
+        } catch {
+        }
+      } else if (parsed.path && parsed.data) {
+        fileBlobs.push({
+          id: blob.id,
+          path: parsed.path,
+          sha256: parsed.sha256,
+          data: parsed.data
+        });
+      }
+      await fetch(`${RELAY_URL}/confirm/files/${blob.id}`, {
+        method: "DELETE",
+        headers: { "Authorization": `Bearer ${RELAY_TOKEN}` }
+      });
+    } catch (err) {
+      process.stderr.write(`[file-sync] error processing blob ${blob.id}: ${err.message}
+`);
+    }
+  }
+  if (!coreManifest) {
+    process.stderr.write("[file-sync] no manifest found in relay\n");
+    return { imported: 0, deleted: 0 };
+  }
+  const delta = compareManifest(coreManifest);
+  const fileBlobMap = new Map(fileBlobs.map((f) => [f.path, f]));
+  let imported = 0;
+  for (const entry of delta.upsert) {
+    const fileBlob = fileBlobMap.get(entry.path);
+    if (!fileBlob) {
+      continue;
+    }
+    try {
+      const content = decrypt(fileBlob.data, relayKey);
+      const actualHash = createHash("sha256").update(content).digest("hex");
+      if (actualHash !== entry.sha256) {
+        process.stderr.write(`[file-sync] hash mismatch for ${entry.path}, skipping
+`);
+        continue;
+      }
+      const destPath = join(paths.root, entry.path);
+      mkdirSync(dirname(destPath), { recursive: true });
+      writeFileSync(destPath, content);
+      imported++;
+    } catch (err) {
+      process.stderr.write(`[file-sync] failed to write ${entry.path}: ${err.message}
+`);
+    }
+  }
+  let deleted = 0;
+  for (const path of delta.delete) {
+    const destPath = join(paths.root, path);
+    if (existsSync(destPath)) {
+      try {
+        unlinkSync(destPath);
+        deleted++;
+      } catch (err) {
+        process.stderr.write(`[file-sync] failed to delete ${path}: ${err.message}
+`);
+      }
+    }
+  }
+  const state = loadFileSyncState();
+  state.lastSync = (/* @__PURE__ */ new Date()).toISOString();
+  state.filesTransferred += imported;
+  state.filesDeleted += deleted;
+  saveFileSyncState(state);
+  return { imported, deleted };
+}
+
+export {
+  generateManifest,
+  compareManifest,
+  loadFileSyncState,
+  saveFileSyncState,
+  pushFileSync,
+  pullFileSync
+};