@doubling/compound-sync 1.2.0 → 1.3.0

package/dual-write.js

@@ -0,0 +1,226 @@
+// Dual-write helpers for the sync daemon and integration tests.
+//
+// Each function takes its dependencies (filesRef, storage, orgId, etc.)
+// as parameters so the logic is importable into a test harness that
+// provides emulator-backed Firebase handles. The sync daemon (sync.js)
+// is the production caller and supplies real-Firebase handles.
+//
+// Side effect ordering on writes:
+//   1. Write the Firestore metadata document (Storage rules require
+//      the metadata doc to exist before the upload is authorized).
+//   2. Upload the blob to Cloud Storage at the deterministic path
+//      derived from orgId + fileId.
+//   3. Storage failures are returned in the result, not thrown, so the
+//      caller can decide whether to surface as a soft warning while the
+//      Firestore content field still holds the up-to-date value.
+
+import crypto from 'crypto';
+import {
+  doc, getDocs, setDoc, updateDoc, deleteDoc, query, where,
+} from 'firebase/firestore';
+import {
+  ref as storageRef, uploadString, deleteObject, getBytes,
+} from 'firebase/storage';
+
+// ---- Pure helpers ----
+
+export function computeContentHash(content) {
+  return crypto.createHash('sha256').update(content || '', 'utf-8').digest('hex');
+}
+
+export function storagePathFor(orgId, fileId) {
+  return `orgs/${orgId}/files/${fileId}/current.md`;
+}
+
+// ---- Storage primitives ----
+
+export async function uploadBlob({ storage, orgId, fileId, content }) {
+  const blobPath = storagePathFor(orgId, fileId);
+  await uploadString(storageRef(storage, blobPath), content || '', 'raw', {
+    contentType: 'text/markdown',
+  });
+}
+
+export async function deleteBlob({ storage, orgId, fileId }) {
+  const blobPath = storagePathFor(orgId, fileId);
+  try {
+    await deleteObject(storageRef(storage, blobPath));
+  } catch (err) {
+    if (err && err.code === 'storage/object-not-found') {
+      return; // tolerate missing blob (older files written before PR 2)
+    }
+    throw err;
+  }
+}
+
+/**
+ * Read file content from Cloud Storage. Throws on any failure
+ * (missing blob, permission error, network). The Firestore content
+ * field is no longer written or read as of PR 6, so the read path
+ * has no fallback. Backfill (PR 4) is a prerequisite.
+ */
+export async function readFileContent({ storage, orgId, fileId }) {
+  const blobPath = storagePathFor(orgId, fileId);
+  const bytes = await getBytes(storageRef(storage, blobPath));
+  return new TextDecoder('utf-8').decode(new Uint8Array(bytes));
+}
+
+// ---- Core dual-write ----
+
+/**
+ * Write a file's content to Cloud Storage at
+ * orgs/{orgId}/files/{fileId}/current.md and update the Firestore
+ * metadata doc with storagePath/contentHash/size. As of PR 6 the
+ * Firestore `content` field is no longer written; the blob is the
+ * sole source of file content.
+ *
+ * Returns one of:
+ *   { action: 'skipped', reason: 'shared-scope' }
+ *   { action: 'noop',    fileId }
+ *   { action: 'created', fileId, fileData, storageError? }
+ *   { action: 'updated', fileId, storageError? }
+ *
+ * `storageError` is populated when the Firestore write succeeded but
+ * the Storage upload failed. Callers should log and retry; the next
+ * save attempts the upload again.
+ */
+export async function pushFileToCloud({
+  filesRef,
+  storage,
+  orgId,
+  userId,
+  scope,
+  teamId,
+  docPath,
+  content,
+  now = new Date().toISOString(),
+}) {
+  if (scope === 'shared-with-me' || scope === 'shared-by-me') {
+    return { action: 'skipped', reason: 'shared-scope' };
+  }
+
+  let snapshot;
+  if (scope === 'private') {
+    snapshot = await getDocs(query(filesRef,
+      where('scope', '==', 'private'),
+      where('ownerId', '==', userId),
+      where('path', '==', docPath)
+    ));
+  } else if (scope === 'team' && teamId) {
+    snapshot = await getDocs(query(filesRef,
+      where('scope', '==', 'team'),
+      where('teamId', '==', teamId),
+      where('path', '==', docPath)
+    ));
+  } else {
+    snapshot = await getDocs(query(filesRef, where('path', '==', docPath)));
+  }
+
+  const contentHash = computeContentHash(content);
+  const size = Buffer.byteLength(content || '', 'utf-8');
+
+  if (!snapshot || snapshot.empty) {
+    const newDocRef = doc(filesRef);
+    const fileId = newDocRef.id;
+    const fileData = {
+      path: docPath,
+      createdAt: now,
+      updatedAt: now,
+      scope,
+      teamId: scope === 'team' ? (teamId || null) : null,
+      ownerId: scope === 'private' ? userId : null,
+      sharedWith: [],
+      storagePath: storagePathFor(orgId, fileId),
+      contentHash,
+      size,
+    };
+    await setDoc(newDocRef, fileData);
+    let storageError = null;
+    try {
+      await uploadBlob({ storage, orgId, fileId, content });
+    } catch (err) {
+      storageError = err;
+    }
+    return { action: 'created', fileId, fileData, storageError };
+  }
+
+  const docSnap = snapshot.docs[0];
+  const existing = docSnap.data();
+  if (existing.contentHash === contentHash) {
+    return { action: 'noop', fileId: docSnap.id };
+  }
+  await updateDoc(docSnap.ref, {
+    updatedAt: now,
+    storagePath: storagePathFor(orgId, docSnap.id),
+    contentHash,
+    size,
+  });
+  let storageError = null;
+  try {
+    await uploadBlob({ storage, orgId, fileId: docSnap.id, content });
+  } catch (err) {
+    storageError = err;
+  }
+  return { action: 'updated', fileId: docSnap.id, storageError };
+}
+
+/**
+ * Delete a file from Firestore and the corresponding Storage blob.
+ *
+ * Returns one of:
+ *   { action: 'skipped',        reason: 'shared-scope' }
+ *   { action: 'not-found' }
+ *   { action: 'scope-mismatch' }
+ *   { action: 'deleted',        fileId }
+ *
+ * Blob deletion tolerates `storage/object-not-found`.
+ */
+export async function deleteFileFromCloud({
+  filesRef,
+  storage,
+  orgId,
+  userId,
+  scope,
+  teamId,
+  docPath,
+}) {
+  if (scope === 'shared-with-me' || scope === 'shared-by-me') {
+    return { action: 'skipped', reason: 'shared-scope' };
+  }
+
+  // The list query must mirror the Firestore rule's per-branch
+  // constraints (scope + ownerId for private, scope + teamId for team)
+  // so the rules engine can statically authorize it.
+  let snapshot;
+  if (scope === 'private') {
+    snapshot = await getDocs(query(
+      filesRef,
+      where('scope', '==', 'private'),
+      where('ownerId', '==', userId),
+      where('path', '==', docPath)
+    ));
+  } else if (scope === 'team' && teamId) {
+    snapshot = await getDocs(query(
+      filesRef,
+      where('scope', '==', 'team'),
+      where('teamId', '==', teamId),
+      where('path', '==', docPath)
+    ));
+  } else {
+    return { action: 'not-found' };
+  }
+
+  if (snapshot.empty) {
+    return { action: 'not-found' };
+  }
+  const docSnap = snapshot.docs[0];
+  const fileId = docSnap.id;
+  // Delete the Storage blob FIRST while the metadata doc still exists,
+  // because the Storage rule's authorization check looks up the doc.
+  // After Firestore deletion the rule would deny (firestore.get fails).
+  // deleteBlob tolerates storage/object-not-found for files that were
+  // written before PR 2 and so have no blob.
+  await deleteBlob({ storage, orgId, fileId });
+  await deleteDoc(docSnap.ref);
+  return { action: 'deleted', fileId };
+}

package/package.json

@@ -1,16 +1,28 @@
 {
   "name": "@doubling/compound-sync",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Bidirectional sync between Compound and local markdown files",
   "type": "module",
-  "bin": "./sync.js",
+  "bin": {
+    "compound-sync": "./sync.js"
+  },
+  "files": [
+    "sync.js",
+    "dual-write.js",
+    "README.md"
+  ],
   "scripts": {
-    "sync": "node sync.js"
+    "sync": "node sync.js",
+    "test": "firebase emulators:exec --only firestore,storage 'node --test dual-write.test.js'"
   },
   "dependencies": {
     "firebase": "^11.7.0",
     "chokidar": "^4.0.0"
   },
+  "devDependencies": {
+    "@firebase/rules-unit-testing": "^4.0.0",
+    "firebase-tools": "^15.0.0"
+  },
   "publishConfig": {
     "access": "public"
   }

package/sync.js

@@ -20,10 +20,9 @@ import {
   getFirestore, collection, doc, getDocs, getDoc, setDoc, addDoc, updateDoc, deleteDoc,
   query, where, onSnapshot
 } from 'firebase/firestore';
-import { getStorage, ref as storageRef, uploadString, deleteObject } from 'firebase/storage';
+import { getStorage } from 'firebase/storage';
 import { initializeAppCheck, CustomProvider } from 'firebase/app-check';
 import chokidar from 'chokidar';
-import crypto from 'crypto';
 import fs from 'fs';
 import path from 'path';
 import readline from 'readline';
@@ -31,6 +30,7 @@ import os from 'os';
 import http from 'http';
 import { exec } from 'child_process';
 import { fileURLToPath } from 'url';
+import { pushFileToCloud, deleteFileFromCloud, readFileContent } from './dual-write.js';
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -498,43 +498,15 @@ const LOCAL_PATH = expandHome(config.localPath || path.join(os.homedir(), 'Sync'
 
 const filesRef = collection(db, `orgs/${ORG_ID}/files`);
 
-// --- Cloud Storage for file blobs ---
+// --- Cloud Storage handle ---
 //
-// PR 2 of the Storage migration: dual-write file content to Cloud
-// Storage in addition to the Firestore `content` field. Reads still
-// come from Firestore until PR 5 cuts over. Storage rules require the
-// file metadata document to exist, so the ordering on create is
-// (1) write Firestore metadata, (2) upload to Storage, (3) update
-// Firestore with the hash/path/size fields.
+// Dual-write file content to Cloud Storage in addition to the Firestore
+// `content` field. Implementation lives in ./dual-write.js so it can be
+// imported by tests and other callers without triggering the daemon's
+// browser-auth side effects.
 
 const storage = getStorage(app);
 
-function computeContentHash(content) {
-  return crypto.createHash('sha256').update(content || '', 'utf-8').digest('hex');
-}
-
-function storagePathFor(fileId) {
-  return `orgs/${ORG_ID}/files/${fileId}/current.md`;
-}
-
-async function uploadBlob(fileId, content) {
-  const blobPath = storagePathFor(fileId);
-  await uploadString(storageRef(storage, blobPath), content || '', 'raw', {
-    contentType: 'text/markdown',
-  });
-}
-
-async function deleteBlob(fileId) {
-  const blobPath = storagePathFor(fileId);
-  try {
-    await deleteObject(storageRef(storage, blobPath));
-  } catch (err) {
-    if (err && err.code !== 'storage/object-not-found') {
-      console.error(`  [storage] delete failed for ${blobPath}: ${err.message}`);
-    }
-  }
-}
-
 // --- Load teams ---
 
 const teams = new Map();
@@ -643,7 +615,7 @@ ensureFolderStructure();
 
 // --- Firestore → Local (multiple listeners) ---
 
-function handleFirestoreChange(change, getLocalPath) {
+async function handleFirestoreChange(change, getLocalPath) {
   const data = change.doc.data();
   const docPath = data.path;
   if (!docPath) return;
@@ -676,7 +648,16 @@ function handleFirestoreChange(change, getLocalPath) {
   console.log(`  [firestore → local] ${action} ${docPath} (${data.scope || 'team'})`);
   ensureDir(localPath);
   suppressLocal.add(localPath);
-  fs.writeFileSync(localPath, data.content || '', 'utf-8');
+
+  // Read content from Cloud Storage. As of PR 6 there is no
+  // fallback; any read failure propagates to the caller's catch
+  // handler (handleFirestoreChange's promise rejection logger).
+  const content = await readFileContent({
+    storage,
+    orgId: ORG_ID,
+    fileId: change.doc.id,
+  });
+  fs.writeFileSync(localPath, content || '', 'utf-8');
   lastKnownUpdate.set(trackingKey, remoteUpdated);
   setTimeout(() => suppressLocal.delete(localPath), 1000);
 }
@@ -691,7 +672,7 @@ function startFirestoreListeners() {
       snapshot.docChanges().forEach(change => {
         handleFirestoreChange(change, (data) => {
           return path.join(LOCAL_PATH, teamFolder, data.path);
-        });
+        }).catch(err => console.error('Firestore change handler error:', err && err.message));
       });
     }, err => console.error(`Team "${teamFolder}" listener error:`, err));
     console.log(`  Listening: ${teamFolder}`);
@@ -704,7 +685,8 @@ function startFirestoreListeners() {
         const data = change.doc.data();
         const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data.path);
 
-        handleFirestoreChange(change, () => realPath);
+        handleFirestoreChange(change, () => realPath)
+          .catch(err => console.error('Firestore change handler error:', err && err.message));
 
         if (change.type !== 'removed') {
           updateSharedByMeSymlink(data, realPath);
@@ -723,7 +705,7 @@ function startFirestoreListeners() {
         if (data.ownerId === USER_ID) return;
         handleFirestoreChange(change, () => {
           return path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data.path);
-        });
+        }).catch(err => console.error('Firestore change handler error:', err && err.message));
       });
     }, err => console.error('Shared with me listener error:', err));
     console.log('  Listening: Shared with Me');
@@ -738,72 +720,25 @@ async function pushToFirestore(filePath, content) {
   const now = new Date().toISOString();
   lastKnownUpdate.set(trackingKey, now);
 
-  if (scope === 'shared-with-me' || scope === 'shared-by-me') return;
-
-  let snapshot;
-  if (scope === 'private') {
-    snapshot = await getDocs(query(filesRef,
-      where('scope', '==', 'private'),
-      where('ownerId', '==', USER_ID),
-      where('path', '==', docPath)
-    ));
-  } else if (scope === 'team' && teamId) {
-    snapshot = await getDocs(query(filesRef,
-      where('scope', '==', 'team'),
-      where('teamId', '==', teamId),
-      where('path', '==', docPath)
-    ));
-  } else {
-    snapshot = await getDocs(query(filesRef, where('path', '==', docPath)));
-  }
-
-  const contentHash = computeContentHash(content);
-  const size = Buffer.byteLength(content || '', 'utf-8');
+  const result = await pushFileToCloud({
+    filesRef,
+    storage,
+    orgId: ORG_ID,
+    userId: USER_ID,
+    scope,
+    teamId,
+    docPath,
+    content,
+    now,
+  });
 
-  if (!snapshot || snapshot.empty) {
+  if (result.action === 'created') {
     console.log(`  [local → firestore] CREATE ${docPath} (${scope})`);
-    // Pre-generate a doc ref so we can compute the deterministic
-    // storagePath before the first Firestore write.
-    const newDocRef = doc(filesRef);
-    const fileId = newDocRef.id;
-    const fileData = {
-      path: docPath,
-      content,
-      createdAt: now,
-      updatedAt: now,
-      scope,
-      teamId: scope === 'team' ? (teamId || null) : null,
-      ownerId: scope === 'private' ? USER_ID : null,
-      sharedWith: [],
-      storagePath: storagePathFor(fileId),
-      contentHash,
-      size,
-    };
-    await setDoc(newDocRef, fileData);
-    // Upload to Storage after Firestore so the Storage rule (which
-    // reads the file metadata to authorize) sees an existing doc.
-    try {
-      await uploadBlob(fileId, content);
-    } catch (err) {
-      console.error(`  [storage] upload failed for ${docPath}: ${err && err.message}`);
-    }
-  } else {
-    const docSnap = snapshot.docs[0];
-    const existing = docSnap.data();
-    if (existing.content === content && existing.contentHash === contentHash) return;
+  } else if (result.action === 'updated') {
     console.log(`  [local → firestore] UPDATE ${docPath} (${scope})`);
-    await updateDoc(docSnap.ref, {
-      content,
-      updatedAt: now,
-      storagePath: storagePathFor(docSnap.id),
-      contentHash,
-      size,
-    });
-    try {
-      await uploadBlob(docSnap.id, content);
-    } catch (err) {
-      console.error(`  [storage] upload failed for ${docPath}: ${err && err.message}`);
-    }
+  }
+  if (result.storageError) {
+    console.error(`  [storage] upload failed for ${docPath}: ${result.storageError.message}`);
   }
 }
 
@@ -812,20 +747,18 @@ async function deleteFromFirestore(filePath) {
   const trackingKey = `${scope}:${docPath}`;
   lastKnownUpdate.delete(trackingKey);
 
-  if (scope === 'shared-with-me' || scope === 'shared-by-me') return;
-
-  const snapshot = await getDocs(query(filesRef, where('path', '==', docPath)));
-  if (!snapshot.empty) {
-    const data = snapshot.docs[0].data();
-    if ((data.scope || 'team') === scope) {
-      console.log(`  [local → firestore] DELETE ${docPath} (${scope})`);
-      const fileId = snapshot.docs[0].id;
-      await deleteDoc(snapshot.docs[0].ref);
-      // Best-effort blob cleanup after the metadata is gone. Tolerates
-      // missing blobs (older files written before PR 2). Any leak is
-      // recoverable via a lifecycle rule or a cleanup script later.
-      await deleteBlob(fileId);
-    }
+  const result = await deleteFileFromCloud({
+    filesRef,
+    storage,
+    orgId: ORG_ID,
+    userId: USER_ID,
+    scope,
+    teamId,
+    docPath,
+  });
+
+  if (result.action === 'deleted') {
+    console.log(`  [local → firestore] DELETE ${docPath} (${scope})`);
   }
 }