mcard-studio 0.0.0-stage → 0.0.2

package/src/services/databaseSyncService.ts

@@ -0,0 +1,477 @@
+/**
+ * Sovereign SQLite CAS Database Synchronization Service
+ * Canonical Path: src/services/databaseSyncService.ts
+ * Deliverables: DEL-287-01 (Export), DEL-288-02 (Ingestion & Mount)
+ * Invariants: INV-287-01, INV-287-02, INV-288-01, INV-288-02
+ *
+ * Provides bidirectional compilation between active MCard CAS stores and
+ * standard SQLite 3 binary database files.
+ */
+
+import { $mcardTree, type MCardFileNode } from '../stores/mcardStore';
+import { studioMCardFs, triggerIndexedDbSync } from './mcardVfs';
+import { Handle, ContentHash, AgentDid, MCard } from 'clm-kernel';
+import { sniffMimeType, detectCardType } from './cardTypeDetector';
+import { PATHS, getEndpoint } from '../config/pathsConfig.js';
+
+export interface ExportResult {
+  bytes: Uint8Array;
+  filename: string;
+}
+
+export interface ImportOptions {
+  mode: 'merge' | 'mount';
+  rootPrefix?: string;
+}
+
+export interface ImportResult {
+  imported: number;
+  skipped: number;
+  rejected: Array<{ handle: string; reason: string }>;
+  mode: 'merge' | 'mount';
+}
+
+/**
+ * Checks if binary data has the canonical SQLite 3 magic header.
+ */
+export function isSqlite3Binary(bytes: Uint8Array): boolean {
+  if (!bytes || bytes.length < 16) return false;
+  const magic = new TextDecoder('utf-8').decode(bytes.subarray(0, 16));
+  return magic.startsWith('SQLite format 3\0') || magic.startsWith('SQLite format 3');
+}
+
+/**
+ * Helper to build SQLite 3 binary via headless kernel clm-kernel (INV-REF-04).
+ */
+async function compileSqliteInNode(
+  cards: any[],
+  options?: { history?: Array<{ handle: string; previous_hash: string; changed_at: string }> }
+): Promise<Uint8Array> {
+  const { compilePortableSqlite } = await import('clm-kernel');
+  // EXP-03 / INV-EXP-03: THE KERNEL'S EXPORT IS A PURE FUNCTION; THE UI SUPPLIES THE TIME.
+  //
+  // `exported_at` is now CALLER-SUPPLIED, because a timestamp inside the payload gives the
+  // export a different address every time it runs — and an export that cannot be addressed is
+  // not a card. The KERNEL therefore writes no timestamp by default, and the determinism is
+  // the kernel's to keep.
+  //
+  // A USER-INITIATED export is a different thing from a card: a human downloading their
+  // workspace wants to know WHEN. So the UI passes the time, and the layering is honest —
+  // determinism where it is required, a timestamp where it is wanted.
+  return compilePortableSqlite(cards, { exportedAt: new Date().toISOString(), ...options });
+}
+
+/**
+ * Helper to parse SQLite 3 binary via headless kernel clm-kernel (INV-REF-04).
+ */
+async function parseSqliteInNode(bytes: Uint8Array): Promise<{
+  tables: string[];
+  cards: any[];
+  handles: any[];
+  history: any[];
+  meta: any[];
+}> {
+  const { parsePortableSqlite } = await import('clm-kernel');
+  return parsePortableSqlite(bytes);
+}
+
+/**
+ * Compiles the entire active MCard CAS directory and handle graph into a single,
+ * self-contained SQLite 3 binary file.
+ * Deliverable: DEL-287-01 (INV-287-01, INV-287-02)
+ */
+export async function exportMCardDatabase(): Promise<ExportResult> {
+  const tree = $mcardTree.get();
+  const uniqueCardsMap = new Map<string, { hash: string; content: string; handle?: string; path?: string; created_at?: string }>();
+
+  // Collect cards from $mcardTree
+  for (const node of Object.values(tree)) {
+    if (node && node.hash && !uniqueCardsMap.has(node.hash)) {
+      uniqueCardsMap.set(node.hash, {
+        hash: node.hash,
+        content: typeof node.content === 'string'
+          ? node.content
+          : node.content instanceof Uint8Array
+            ? new TextDecoder('utf-8').decode(node.content)
+            : '',
+        handle: node.handle,
+        path: node.path,
+        created_at: new Date().toISOString(),
+      });
+    }
+  }
+
+  // Also include any raw cards in storageBackend
+  try {
+    const backendCards = studioMCardFs.storageBackend.list();
+    for (const card of backendCards) {
+      const hHex = card.hash.asHex();
+      if (!uniqueCardsMap.has(hHex)) {
+        uniqueCardsMap.set(hHex, {
+          hash: hHex,
+          content: JSON.stringify(card.toJSON()),
+          created_at: new Date().toISOString(),
+        });
+      }
+    }
+  } catch {
+    // Non-fatal if list() is unsupported
+  }
+
+  const cardList = Array.from(uniqueCardsMap.values());
+
+  const d = new Date();
+  const pad = (n: number) => n.toString().padStart(2, '0');
+  const ts = `${d.getFullYear()}${pad(d.getMonth() + 1)}${pad(d.getDate())}_${pad(d.getHours())}${pad(d.getMinutes())}${pad(d.getSeconds())}`;
+  const filename = `clm_mcard_workspace_${ts}.db`;
+
+  let bytes: Uint8Array;
+
+  const rawHistory = typeof (studioMCardFs as any).exportHistory === "function"
+    ? (studioMCardFs as any).exportHistory()
+    : [];
+  const historyList = rawHistory.map((h: any) => ({
+    handle: h.handle,
+    previous_hash: h.previousHash,
+    changed_at: h.changedAt,
+  }));
+
+  // ── EXP-04 / INV-EXP-01: THE LOCAL EXPORT COMES FIRST ──────────────────────────────
+  //
+  // This read: "if in browser, use the configured DB export endpoint" — and it took the
+  // browser branch whenever `window` existed. **But `window` existing is not the same as
+  // `node:sqlite` being absent**, and a test runner (jsdom) has BOTH. So every unit test
+  // fetched a server that was not running, for a compression the kernel could perform locally.
+  //
+  // The order is now the one the architecture implies: **compressions run where the cards
+  // are.** The local engine is tried FIRST; the server route is the DECLARED FALLBACK for the
+  // case the local engine genuinely cannot run — a real browser, which has no `node:sqlite`.
+  //
+  // `INV-EXP-05`: the tests assert the CONTRACT, not the transport, so they must not need a
+  // server to reach a function the runtime can run.
+  let localError: unknown = null;
+  try {
+    bytes = await compileSqliteInNode(cardList, { history: historyList });
+  } catch (err) {
+    localError = err;
+  }
+
+  let serverError: unknown = null;
+  if (bytes! === undefined && typeof window !== 'undefined' && typeof fetch === 'function') {
+    // THE DECLARED FALLBACK: a browser with no local engine. The condition is stated rather
+    // than inferred from `window`, which is what made the local path unreachable in tests.
+    console.warn('[databaseSyncService] local export unavailable, using the server route:', localError);
+    try {
+      const res = await fetch(getEndpoint('vfs_db_export'), {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ cards: cardList, history: historyList }),
+      });
+
+      if (res.ok) {
+        bytes = new Uint8Array(await res.arrayBuffer());
+      } else {
+        throw new Error(`Export API returned HTTP ${res.status}`);
+      }
+    } catch (fetchErr) {
+      serverError = fetchErr;
+      console.warn('[databaseSyncService] Export API fetch failed, attempting cached snapshot fallback:', fetchErr);
+      const { getSqliteBytesFromIndexedDb } = await import('./vfs/vfsPersistence');
+      const cachedBytes = await getSqliteBytesFromIndexedDb();
+      if (cachedBytes && isSqlite3Binary(cachedBytes)) {
+        bytes = cachedBytes;
+      } else {
+        try {
+          bytes = await compileSqliteInNode(cardList, { history: historyList });
+        } catch {
+          // Both paths failed, cached snapshot missing
+        }
+      }
+    }
+  }
+
+  if (bytes! === undefined) {
+    const errorDetails = [
+      localError ? `local export failed: ${localError instanceof Error ? localError.message : String(localError)}` : null,
+      serverError ? `server fallback failed: ${serverError instanceof Error ? serverError.message : String(serverError)}` : null,
+    ].filter(Boolean).join('; ');
+
+    throw new Error(
+      `Offline Export Failed: ${errorDetails || 'neither local engine nor server route could export'}`
+    );
+  }
+
+  // Enforce INV-287-01: SQLite 3 Binary Magic Byte Conformance
+  if (!isSqlite3Binary(bytes)) {
+    throw new Error('INV-287-01 Violation: Exported database bytes do not conform to SQLite 3 binary header.');
+  }
+
+  return { bytes, filename };
+}
+
+/**
+ * Validates, decrypts, and ingests an external SQLite 3 database into the active MCard CAS store.
+ * Supports Mode A (Direct G-Set Join Merge) and Mode B (Scoped Root Mount).
+ * Deliverable: DEL-288-02 (INV-288-01, INV-288-02)
+ */
+export async function importMCardDatabase(
+  bytes: Uint8Array,
+  options: ImportOptions = { mode: 'mount', rootPrefix: 'imported/backup_1/' }
+): Promise<ImportResult> {
+  // 1. Assert magic bytes (INV-287-01 / INV-288-01)
+  if (!isSqlite3Binary(bytes)) {
+    throw new Error('INV-288-01: Corrupted or non-SQLite database file (magic bytes mismatch)');
+  }
+
+  // 2. Parse SQLite database structure
+  let dbData: {
+    tables: string[];
+    cards: any[];
+    handles: any[];
+    history: any[];
+    meta: any[];
+  };
+
+  if (typeof window !== 'undefined' && typeof fetch === 'function' && !process.env.VITEST) {
+    const res = await fetch(getEndpoint('vfs_db_import'), {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/x-sqlite3' },
+      body: bytes as BodyInit,
+    });
+
+    if (!res.ok) {
+      const errJson = await res.json().catch(() => ({}));
+      throw new Error(errJson.error || `Import endpoint failed with HTTP ${res.status}`);
+    }
+
+    dbData = await res.json();
+  } else {
+    dbData = await parseSqliteInNode(bytes);
+  }
+
+  // 3. Zero-Trust Cryptographic Hash Verification (INV-288-01)
+  const { cards, handles } = dbData;
+  if (!cards || !Array.isArray(cards)) {
+    throw new Error('INV-288-01: Invalid database schema — missing card table');
+  }
+
+  for (const card of cards) {
+    if (!card.hash || typeof card.hash !== 'string') {
+      throw new Error('INV-288-01: Malformed card record missing hash');
+    }
+    // Verify hash is valid hex representation (64 chars for Blake3/sha256)
+    if (!/^[0-9a-fA-F]{64}$/.test(card.hash)) {
+      throw new Error(`INV-288-01: Invalid cryptographic hash format: '${card.hash}'`);
+    }
+  }
+
+  // 4. Handle Ingestion by Mode
+  const mode = options.mode || 'mount';
+  let importedCount = 0;
+  let skippedCount = 0;
+  const rejected: Array<{ handle: string; reason: string }> = [];
+
+  const existingTree = $mcardTree.get();
+  const existingHandles = new Set(Object.keys(existingTree));
+
+  const author = AgentDid.create('did:key:z6MkhaXgBZDvotDkL5257faiz48Z8x28qqwjp64guC54Po39');
+  const decodePortableCard = (card: any): { content: string; original?: MCard } => {
+    const content = typeof card.content === 'string'
+      ? card.content
+      : new TextDecoder('utf-8').decode(card.content);
+    let serialized: any;
+    try {
+      serialized = JSON.parse(content);
+    } catch {
+      return { content };
+    }
+    if (!serialized || typeof serialized !== 'object' || !('payload' in serialized || 'uri' in serialized || 'hash' in serialized)) {
+      return { content };
+    }
+    if (typeof serialized.uri !== 'string' || typeof serialized.hash !== 'string' || !('payload' in serialized)) {
+      throw new Error('Malformed serialized MCard');
+    }
+    const original = MCard.fromJSON(serialized);
+    const rowHash = ContentHash.parse(card.hash);
+    if (!original.hash.equals(rowHash)) throw new Error('Serialized MCard hash does not match the database row');
+    const recomputed = MCard.create(original.uri, original.payload, original.author, original.sequence);
+    if (!recomputed.hash.equals(rowHash)) throw new Error('Serialized MCard payload does not match its content hash');
+    return { content: original.payload.kind === 'text' ? original.payload.value : original.content, original };
+  };
+  const storePortableCard = (path: string, decoded: { content: string; original?: MCard }): MCard => {
+    if (!decoded.original) return studioMCardFs.writeFile(path, decoded.content, author);
+    studioMCardFs.storageBackend.put(decoded.original.hash, decoded.original);
+    return decoded.original;
+  };
+
+  const rawPrefix = options.rootPrefix || 'imported/backup_1/';
+  const cleanPrefix = rawPrefix.replace(/^\/+/, '').replace(/\/+$/, '');
+  const slashPrefix = `${cleanPrefix}/`;
+  const colonPrefix = `${cleanPrefix.replace(/\//g, ':')}:`;
+
+  if (mode === 'mount') {
+    // Mode B: Scoped Root Mount (INV-288-02)
+
+    for (const card of cards) {
+      try {
+        const decodedCard = decodePortableCard(card);
+        const contentStr = decodedCard.content;
+
+        // Find associated handles for this card
+        const cardHandles = (handles || []).filter((h: any) => h.hash === card.hash);
+
+        if (cardHandles.length === 0) {
+          // Fallback path if no explicit handle entry
+          const fallbackPath = `${slashPrefix}card_${card.hash.slice(0, 8)}.json`;
+          const coreCard = storePortableCard(fallbackPath, decodedCard);
+          if (decodedCard.original) studioMCardFs.storageBackend.registerHandle(Handle.fromPath(fallbackPath), coreCard.hash);
+          importedCount++;
+        } else {
+          for (const h of cardHandles) {
+            const originalHandle = String(h.handle);
+            // Prefix handle and path under scoped namespace
+            const scopedHandle = `${colonPrefix}${originalHandle.replace(/\//g, ':')}`;
+            const scopedPath = `${slashPrefix}${originalHandle.replace(/:/g, '/')}`;
+
+            if (existingHandles.has(scopedHandle) || existingHandles.has(scopedPath)) {
+              // INV-288-02 guarantee: Scoped mounts should be collision-free
+              rejected.push({ handle: scopedHandle, reason: 'Unexpected collision in scoped root mount' });
+              continue;
+            }
+
+            const coreCard = storePortableCard(scopedPath, decodedCard);
+            const coreHash = coreCard.hash;
+            studioMCardFs.storageBackend.registerHandle(Handle.parse(scopedHandle), coreHash);
+            studioMCardFs.storageBackend.registerHandle(Handle.parse(scopedPath), coreHash);
+
+            const mime = sniffMimeType(scopedPath);
+            const typeInfo = detectCardType({ path: scopedPath, mimeType: mime, content: contentStr });
+
+            const newNode: MCardFileNode = {
+              path: scopedPath,
+              name: scopedPath.split('/').pop() || 'file',
+              handle: scopedHandle,
+              hash: coreHash.asHex(),
+              content: contentStr,
+              mimeType: mime,
+              detectedType: typeInfo.detectedType,
+              epistemicStatus: 'definition',
+              readOnly: false,
+              subclass: typeInfo.subclass,
+              coreCard,
+              sizeBytes: contentStr.length,
+            };
+
+            $mcardTree.setKey(scopedPath, newNode);
+            $mcardTree.setKey(scopedHandle, newNode);
+            importedCount++;
+          }
+        }
+      } catch (err: any) {
+        rejected.push({ handle: card.hash, reason: err.message });
+      }
+    }
+  } else {
+    // Mode A: Direct Merge into Current Workspace (G-Set join)
+    for (const card of cards) {
+      try {
+        const decodedCard = decodePortableCard(card);
+        const contentStr = decodedCard.content;
+
+        const cardHandles = (handles || []).filter((h: any) => h.hash === card.hash);
+
+        if (cardHandles.length === 0) {
+          const fallbackPath = PATHS.paths.import_fallback_template.replace('{hash8}', card.hash.slice(0, 8));
+          const coreCard = storePortableCard(fallbackPath, decodedCard);
+          if (decodedCard.original) studioMCardFs.storageBackend.registerHandle(Handle.fromPath(fallbackPath), coreCard.hash);
+          importedCount++;
+        } else {
+          for (const h of cardHandles) {
+            const hStr = String(h.handle);
+            const pathStr = hStr.includes('/') ? hStr : hStr.replace(/:/g, '/');
+
+            // G-Set semantics: If handle already points to this exact hash, skip
+            const existingNode = existingTree[pathStr] || existingTree[hStr];
+            if (existingNode && existingNode.hash === card.hash) {
+              skippedCount++;
+              continue;
+            }
+
+            const coreCard = storePortableCard(pathStr, decodedCard);
+            const coreHash = coreCard.hash;
+            studioMCardFs.storageBackend.registerHandle(Handle.parse(hStr), coreHash);
+            if (pathStr !== hStr) {
+              studioMCardFs.storageBackend.registerHandle(Handle.parse(pathStr), coreHash);
+            }
+
+            const mime = sniffMimeType(pathStr);
+            const typeInfo = detectCardType({ path: pathStr, mimeType: mime, content: contentStr });
+
+            const newNode: MCardFileNode = {
+              path: pathStr,
+              name: pathStr.split('/').pop() || 'file',
+              handle: hStr,
+              hash: coreHash.asHex(),
+              content: contentStr,
+              mimeType: mime,
+              detectedType: typeInfo.detectedType,
+              epistemicStatus: 'definition',
+              readOnly: false,
+              subclass: typeInfo.subclass,
+              coreCard,
+              sizeBytes: contentStr.length,
+            };
+
+            $mcardTree.setKey(pathStr, newNode);
+            $mcardTree.setKey(hStr, newNode);
+            importedCount++;
+          }
+        }
+      } catch (err: any) {
+        rejected.push({ handle: card.hash, reason: err.message });
+      }
+    }
+  }
+
+  // 5. Replay persisted handle_history into the in-memory provenance trail (INV-467).
+  //    parsePortableSqlite returns `previous_hash`/`changed_at` (with `hash`/`created_at` aliases);
+  //    records are dual-registered under slash and colon spellings to match
+  //    getMCardVersionList's dual history() queries.
+  const historyRows: any[] = (dbData as any).history || [];
+  if (historyRows.length > 0) {
+    const remap: Array<{ handle: string; previousHash: string; changedAt: string }> = [];
+    for (const h of historyRows) {
+      const prev = h.previous_hash ?? h.hash;
+      const at = h.changed_at ?? h.created_at;
+      const handleStr = String(h.handle ?? '');
+      if (!handleStr || !prev || !at) continue;
+      if (mode === 'mount') {
+        remap.push(
+          { handle: `${slashPrefix}${handleStr.replace(/:/g, '/')}`, previousHash: prev, changedAt: at },
+          { handle: `${colonPrefix}${handleStr.replace(/\//g, ':')}`, previousHash: prev, changedAt: at },
+        );
+      } else {
+        const pathForm = handleStr.includes('/') ? handleStr : handleStr.replace(/:/g, '/');
+        remap.push({ handle: handleStr, previousHash: prev, changedAt: at });
+        if (pathForm !== handleStr) {
+          remap.push({ handle: pathForm, previousHash: prev, changedAt: at });
+        }
+      }
+    }
+    try {
+      (studioMCardFs as any).importHistory?.(remap);
+    } catch (histErr) {
+      console.warn('[databaseSyncService] Failed to replay handle_history into VFS:', histErr);
+    }
+  }
+
+  // Persist snapshot to IndexedDB
+  triggerIndexedDbSync();
+
+  return {
+    imported: importedCount,
+    skipped: skippedCount,
+    rejected,
+    mode,
+  };
+}