@doubling/compound-sync 1.12.1 → 1.12.2

package/files.js

@@ -37,6 +37,8 @@ import {
   extFromPath,
   contentByteSize,
   computeContentHash,
+  folderDocId,
+  fileDocId,
   toBytes,
 } from './storage-helpers.js';
 import { planFolderChain, splitPath } from './folder-chain.js';
@@ -127,15 +129,28 @@ async function loadScopedFolders({ filesRef, scope, teamId, userId }) {
 async function applyFolderChainPlan({ filesRef, plan, now }) {
   const tempToReal = new Map();
   for (const create of plan.creates) {
-    const newRef = doc(filesRef);
+    // Deterministic id by {scope, teamId, ownerId, path}; concurrent
+    // sibling creates for the same path collapse onto the same doc
+    // instead of producing duplicates (DOU-240).
+    const realId = folderDocId({
+      scope: create.doc.scope,
+      teamId: create.doc.teamId,
+      ownerId: create.doc.ownerId,
+      path: create.doc.path,
+    });
+    const newRef = doc(filesRef, realId);
     const docData = {
       ...create.doc,
       parentId: tempToReal.get(create.doc.parentId) || create.doc.parentId,
       createdAt: now,
       updatedAt: now,
     };
+    // setDoc with merge:false is idempotent on identical input; the
+    // last writer wins on differing fields. For folder docs the only
+    // fields that vary between concurrent callers are timestamps, so
+    // any tie-break is acceptable.
     await setDoc(newRef, docData);
-    tempToReal.set(create.tempId, newRef.id);
+    tempToReal.set(create.tempId, realId);
   }
   for (const bf of plan.backfills) {
     await updateDoc(doc(filesRef, bf.id), { path: bf.path, updatedAt: now });
@@ -280,11 +295,25 @@ export async function deleteFolderFromCloud({
   for (const { id, ref, data } of descendants) {
     if (data.type !== 'folder') {
       const ext = extFromPath(data.path);
-      try {
-        await deleteBlob({ storage, orgId, fileId: id, ext });
-      } catch (err) {
-        // Orphan blob is a soft failure; the doc delete is the
-        // load-bearing state transition for the user's view.
+      // Re-check the doc still exists right before calling the
+      // Storage SDK. If a concurrent caller (chokidar's per-file
+      // unlink fan-out, or another cascade) already deleted the
+      // metadata doc, the Storage rule's cross-service
+      // firestore.get(...).data.scope reads null and throws "Null
+      // value error", and the Storage SDK retries for several
+      // seconds before surfacing storage/unauthorized. The retry
+      // latency is what blew the test budget on DOU-242 + DOU-251.
+      // Skipping the SDK call avoids both the rule throw and the
+      // retry; the orphan blob (if any) is cleaned up by the
+      // data-integrity audit (DOU-244).
+      const recheck = await getDoc(ref);
+      if (recheck.exists()) {
+        try {
+          await deleteBlob({ storage, orgId, fileId: id, ext });
+        } catch (err) {
+          // Orphan blob is a soft failure; the doc delete is the
+          // load-bearing state transition for the user's view.
+        }
       }
     }
     await deleteDoc(ref);
@@ -387,8 +416,16 @@ export async function pushFileToCloud({
     : null;
 
   if (!snapshot || snapshot.empty) {
-    const newDocRef = doc(filesRef);
-    const fileId = newDocRef.id;
+    // Deterministic id by {scope, teamId, ownerId, path}; concurrent
+    // sibling creates for the same path collapse onto the same doc
+    // instead of producing duplicates. Mirrors the DOU-240 folder fix.
+    const fileId = fileDocId({
+      scope,
+      teamId: scope === 'team' ? (teamId || null) : null,
+      ownerId: scope === 'private' ? userId : null,
+      path: docPath,
+    });
+    const newDocRef = doc(filesRef, fileId);
     const resolvedMime = mimeType || 'text/markdown';
     const fileData = {
       type: 'file',
@@ -502,21 +539,45 @@ export async function deleteFileFromCloud({
   // Delete the Storage blob FIRST while the metadata doc still exists,
   // because the Storage rule's authorization check looks up the doc.
   //
-  // Race tolerance (DOU-231 follow-up): a concurrent delete can remove
+  // Race tolerance (DOU-242 / DOU-252): a concurrent delete can remove
   // the metadata doc between our snapshot read and our blob-delete
   // (most commonly the cascade in deleteFolderFromCloud firing first
   // for a path chokidar's per-file `unlink` event then re-tries via
-  // this function). After that delete, the Storage rule's
-  // `firestore.get(...).data.scope` reads null, the rule throws
-  // "Null value error", and the Storage SDK surfaces it as
-  // `storage/unauthorized`. In that specific case the desired end
-  // state (no doc, no blob) is already achieved, so treat as
-  // already-deleted instead of propagating a noisy auth error.
+  // this function). Pre-check the doc still exists right before the
+  // Storage SDK call: if it's gone, skip the blob delete entirely.
+  // This avoids the Storage rule's cross-service firestore.get(...)
+  // reading null and throwing "Null value error", and avoids the
+  // Storage SDK's multi-second retry chain that breaks test budgets.
+  const preRecheck = await getDoc(docSnap.ref);
+  if (!preRecheck.exists()) {
+    return { action: 'not-found', fileId };
+  }
   try {
     await deleteBlob({ storage, orgId, fileId, ext });
   } catch (err) {
     const code = err && err.code;
     if (code === 'storage/unauthorized') {
+      // Last-resort safety net for the narrow window between
+      // preRecheck above and the SDK actually firing. Treat as
+      // already-deleted if the doc is now gone.
+      const recheck = await getDoc(docSnap.ref);
+      if (!recheck.exists()) {
+        return { action: 'not-found', fileId };
+      }
+    }
+    throw err;
+  }
+  // Concurrent-delete tolerance (DOU-242): the Firestore delete rule
+  // evaluates canAccessFile() which reads resource.data; if another
+  // caller already deleted this doc, resource is null and the rule
+  // throws "Null value error" → SDK surfaces permission-denied. The
+  // desired end state (no doc) is already achieved, so treat as
+  // already-deleted instead of propagating.
+  try {
+    await deleteDoc(docSnap.ref);
+  } catch (err) {
+    const code = err && err.code;
+    if (code === 'permission-denied') {
       const recheck = await getDoc(docSnap.ref);
       if (!recheck.exists()) {
         return { action: 'not-found', fileId };
@@ -524,6 +585,5 @@ export async function deleteFileFromCloud({
     }
     throw err;
   }
-  await deleteDoc(docSnap.ref);
   return { action: 'deleted', fileId };
 }

package/org-sync.js

@@ -135,13 +135,49 @@ export async function startOrgSync({
   // Track which fileIds already have a binding (or one being set
   // up) so we never double-create. `stoppedRef` is read by the
   // background retry loops so they can abandon work after sync.stop.
-  // `pendingPromises` lets stop() await any in-flight setup before
-  // returning so the next test's clearFirestore does not race
-  // dangling onSnapshot streams.
+  // `pendingPromises` lets stop() await any in-flight setup OR any
+  // fire-and-forget snapshot/chokidar handler work before returning.
+  // Without this, the next test's clearFirestore + terminate can
+  // race a handler that's still flushing storage from this test,
+  // surfacing as "Firebase app was deleted" errors in the daemon log
+  // and (suspected) accumulating gRPC stream state that the next
+  // listener attach receives as a single oversize Listen message.
   const pendingBindings = new Set();
   const pendingPromises = new Set();
   const stoppedRef = { value: false };
 
+  // Helper: register a promise with pendingPromises so stop() awaits
+  // it, and auto-remove on settle. Returns the same promise so call
+  // sites can keep their chaining shape.
+  function track(p) {
+    pendingPromises.add(p);
+    p.finally(() => pendingPromises.delete(p));
+    return p;
+  }
+
+  // DOU-254: per-local-path operation queue. chokidar can fire `add`
+  // then `unlink` for the same path within a few hundred ms (a test
+  // writes a file, waits only for the Firestore doc to appear, then
+  // deletes locally). Without serialization, the push (setDoc +
+  // uploadBlob) and the delete (preRecheck + deleteBlob + deleteDoc)
+  // race against each other on the same fileId, and uploadBlob can
+  // land AFTER deleteBlob, leaving an orphaned blob after the doc is
+  // gone. Chain ops keyed by local path so the next op starts only
+  // after the previous one settles.
+  const localOpQueues = new Map();
+  function enqueueLocalOp(filePath, op) {
+    const prev = localOpQueues.get(filePath) || Promise.resolve();
+    // Swallow the previous op's rejection so a failed push doesn't
+    // skip the subsequent delete (or vice versa); each op handles
+    // its own errors via the .catch() at the call site.
+    const next = prev.catch(() => {}).then(op);
+    localOpQueues.set(filePath, next);
+    next.finally(() => {
+      if (localOpQueues.get(filePath) === next) localOpQueues.delete(filePath);
+    });
+    return next;
+  }
+
   // Fire-and-forget binding setup. Returns nothing because the
   // common shape "ensureBinding then read its ytext" runs straight
   // into a race: the daemon's onSnapshot fires on Alice's own
@@ -512,9 +548,9 @@ export async function startOrgSync({
     const q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId));
     const unsubscribe = onSnapshot(q, snapshot => {
       snapshot.docChanges().forEach(change => {
-        handleFirestoreChange(change, data =>
+        track(handleFirestoreChange(change, data =>
           path.join(LOCAL_PATH, teamFolder, data.path),
-        ).catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message));
+        ).catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message)));
       });
     }, err => console.error(`  [${orgId}] Team "${teamFolder}" listener error:`, err));
 
@@ -563,8 +599,8 @@ export async function startOrgSync({
           const data = change.doc.data();
           const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data.path);
 
-          handleFirestoreChange(change, () => realPath)
-            .catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message));
+          track(handleFirestoreChange(change, () => realPath)
+            .catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message)));
 
           if (change.type !== 'removed') {
             updateSharedByMeSymlink(data, realPath);
@@ -590,9 +626,9 @@ export async function startOrgSync({
         snapshot.docChanges().forEach(change => {
           const data = change.doc.data();
           if (data.ownerId === userId) return;
-          handleFirestoreChange(change, () =>
+          track(handleFirestoreChange(change, () =>
             path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data.path),
-          ).catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message));
+          ).catch(err => console.error(`  [${orgId}] Firestore change handler error:`, err && err.message)));
         });
       }, err => console.error(`  [${orgId}] Shared with me listener error:`, err));
       unsubscribers.push(sharedUnsub);
@@ -827,27 +863,29 @@ export async function startOrgSync({
       if (binding) binding.applyDiskText(data);
     }
 
-    pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
-      console.error(`  [${orgId}] Push error for ${rel}: ${err.message}`)
-    );
+    track(enqueueLocalOp(filePath, () =>
+      pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
+        console.error(`  [${orgId}] Push error for ${rel}: ${err.message}`)
+      )
+    ));
   }
 
   function handleLocalDirAdded(dirPath) {
     if (suppressLocal.has(dirPath)) return;
     const rel = path.relative(LOCAL_PATH, dirPath);
     if (!isSyncableLocalDir(rel)) return;
-    pushDirToFirestore(dirPath).catch(err =>
+    track(pushDirToFirestore(dirPath).catch(err =>
       console.error(`  [${orgId}] Folder push error for ${rel}: ${err.message}`)
-    );
+    ));
   }
 
   function handleLocalDirRemoved(dirPath) {
     if (suppressLocal.has(dirPath)) return;
     const rel = path.relative(LOCAL_PATH, dirPath);
     if (!isSyncableLocalDir(rel)) return;
-    deleteDirFromFirestore(dirPath).catch(err =>
+    track(deleteDirFromFirestore(dirPath).catch(err =>
       console.error(`  [${orgId}] Folder delete error for ${rel}: ${err.message}`)
-    );
+    ));
   }
 
   function startLocalWatcher() {
@@ -871,15 +909,44 @@ export async function startOrgSync({
       },
     });
 
-    watcher.on('add', filePath => handleLocalChange(filePath));
-    watcher.on('change', filePath => handleLocalChange(filePath));
+    // Reusable per-event trace logging, gated by COMPOUND_CHOKIDAR_DEBUG.
+    // Writes to both stdout AND a file (default /tmp/chokidar-debug.log)
+    // so traces survive a process kill or buffered stdout. Used by the
+    // Test-on-demand workflow's `Dump diagnostic logs` step for
+    // chokidar-related diagnostic runs. Zero cost when the env var is
+    // unset.
+    const chokidarDebug = process.env.COMPOUND_CHOKIDAR_DEBUG === '1';
+    const chokidarLogPath = process.env.COMPOUND_CHOKIDAR_LOG || '/tmp/chokidar-debug.log';
+    function logEvent(event, p) {
+      if (chokidarDebug) {
+        const rel = path.relative(LOCAL_PATH, p);
+        const line = `  [${orgId}] [chokidar ${event}] ${rel} @ ${new Date().toISOString()}\n`;
+        try { fs.appendFileSync(chokidarLogPath, line); } catch (_) {}
+        console.log(line.trimEnd());
+      }
+    }
+
+    watcher.on('add', filePath => { logEvent('add', filePath); handleLocalChange(filePath); });
+    watcher.on('change', filePath => { logEvent('change', filePath); handleLocalChange(filePath); });
     watcher.on('unlink', filePath => {
+      logEvent('unlink', filePath);
       if (suppressLocal.has(filePath)) return;
       const rel = path.relative(LOCAL_PATH, filePath);
-      deleteFromFirestore(filePath).catch(err => console.error(`  [${orgId}] Delete error for ${rel}: ${err.message}`));
+      track(enqueueLocalOp(filePath, () =>
+        deleteFromFirestore(filePath).catch(err =>
+          console.error(`  [${orgId}] Delete error for ${rel}: ${err.message}`)
+        )
+      ));
     });
-    watcher.on('addDir', dirPath => handleLocalDirAdded(dirPath));
-    watcher.on('unlinkDir', dirPath => handleLocalDirRemoved(dirPath));
+    watcher.on('addDir', dirPath => { logEvent('addDir', dirPath); handleLocalDirAdded(dirPath); });
+    watcher.on('unlinkDir', dirPath => { logEvent('unlinkDir', dirPath); handleLocalDirRemoved(dirPath); });
+    if (chokidarDebug) {
+      watcher.on('raw', (event, p, details) => {
+        const line = `  [${orgId}] [chokidar raw ${event}] ${p} @ ${new Date().toISOString()} ${JSON.stringify(details)}\n`;
+        try { fs.appendFileSync(chokidarLogPath, line); } catch (_) {}
+        console.log(line.trimEnd());
+      });
+    }
 
     // Resolve only once chokidar has finished its initial scan and
     // registered inotify (or FSEvents) watches on every existing
@@ -899,9 +966,10 @@ export async function startOrgSync({
   await startLocalWatcher();
 
   return {
-    // Tear down all listeners and the watcher. Returns the watcher's
-    // close() promise so callers/tests can await full shutdown.
-    stop() {
+    // Tear down all listeners and the watcher. Awaits every tracked
+    // in-flight handler so callers/tests get a strong "no daemon work
+    // is still running" guarantee on the resolved promise.
+    async stop() {
       stoppedRef.value = true;
       for (const unsub of unsubscribers) {
         try { unsub(); } catch (e) { /* ignore */ }
@@ -910,15 +978,32 @@ export async function startOrgSync({
         try { unsub(); } catch (e) { /* ignore */ }
       }
       teamUnsubscribers.clear();
-      const setupDrain = Promise.all([...pendingPromises]);
-      const bindingDrain = setupDrain.then(() => Promise.all(
+
+      // Drain tracked promises in a loop, bounded by a 5s ceiling.
+      // A handler currently in flight (e.g., a snapshot callback
+      // already dispatched before unsub()) can chain follow-up work
+      // that gets added back to pendingPromises. Loop until the set
+      // quiesces so no daemon task outlives stop() in the common
+      // case. The cap protects against tests where a handler is
+      // genuinely stuck (e.g., emulator pause), so stop() never
+      // hangs indefinitely; anything still pending past 5s is
+      // abandoned and the next test's clearFirestore will sweep it.
+      const drainDeadline = Date.now() + 5000;
+      while (pendingPromises.size > 0 && Date.now() < drainDeadline) {
+        const remaining = Math.max(0, drainDeadline - Date.now());
+        await Promise.race([
+          Promise.all([...pendingPromises]),
+          new Promise((r) => setTimeout(r, remaining)),
+        ]);
+      }
+
+      await Promise.all(
         [...bindings.values()].map((b) => b.stop().catch(() => {})),
-      )).then(() => {
-        bindings.clear();
-        bindingsByPath.clear();
-      });
-      const watcherClose = watcher ? watcher.close() : Promise.resolve();
-      return Promise.all([bindingDrain, watcherClose]).then(() => undefined);
+      );
+      bindings.clear();
+      bindingsByPath.clear();
+
+      if (watcher) await watcher.close();
     },
   };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doubling/compound-sync",
-  "version": "1.12.1",
+  "version": "1.12.2",
   "description": "Bidirectional sync between Compound and local markdown files",
   "type": "module",
   "bin": {
@@ -32,7 +32,7 @@
     "typecheck": "tsc --noEmit",
     "prepack": "npm run build",
     "test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js yjs-file-binding.test.js yjs-binding-state.test.js",
-    "test:integration": "firebase emulators:exec --only firestore,storage 'node --import tsx --test --test-concurrency=1 files-helpers.test.js files-push-create-private.test.js files-push-create-team.test.js files-push-update-private.test.js files-push-binary.test.js files-push-staleness.test.js files-push-storage-fail.test.js files-push-no-content.test.js files-push-stamps.test.js files-push-folder.test.js files-delete.test.js files-delete-folder.test.js files-read-blob.test.js org-sync-local-edit.test.js org-sync-file-ops.test.js org-sync-mtime.test.js org-sync-rename.test.js org-sync-yjs-pull.test.js org-sync-yjs-push.test.js org-sync-cold-start.test.js two-user-sync.test.js'",
+    "test:integration": "firebase emulators:exec --only firestore,storage 'node --import tsx --test --test-concurrency=1 --test-timeout=120000 files-helpers.test.js files-push-create-private.test.js files-push-create-team.test.js files-push-update-private.test.js files-push-binary.test.js files-push-staleness.test.js files-push-storage-fail.test.js files-push-no-content.test.js files-push-stamps.test.js files-push-folder.test.js files-delete.test.js files-delete-folder.test.js files-read-blob.test.js org-sync-local-edit.test.js org-sync-file-ops.test.js org-sync-mtime.test.js org-sync-rename.test.js org-sync-yjs-pull.test.js org-sync-yjs-push.test.js org-sync-cold-start.test.js org-sync-special-chars.test.js org-sync-copy.test.js two-user-happy-paths.test.js two-user-yjs-convergence.test.js files-concurrency-folder-create.test.js files-concurrency-file-create.test.js files-concurrency-file-delete.test.js files-concurrency-folder-delete.test.js'",
     "test": "npm run test:unit && npm run test:integration"
   },
   "dependencies": {

package/storage-helpers.js

@@ -36,6 +36,30 @@ export function computeContentHash(data) {
   return crypto.createHash('sha256').update(buf).digest('hex');
 }
 
+// Deterministic Firestore document id for a folder doc identified by
+// scope + ownership + path. Two concurrent creates collapse onto the
+// same id, so a setDoc by all of them is idempotent (no duplicates).
+// See DOU-240 for the race this prevents.
+export function folderDocId({ scope, teamId, ownerId, path }) {
+  const key = `folder|${scope}|${teamId ?? ''}|${ownerId ?? ''}|${path}`;
+  return 'f-' + crypto.createHash('sha256').update(key).digest('hex').slice(0, 20);
+}
+
+// Deterministic Firestore document id for a file doc identified by
+// scope + ownership + path. Same purpose as folderDocId: concurrent
+// creates at the same path collapse onto one doc.
+//
+// Caveat: rename happens as unlink(old) + add(new) (DOU-205 / chokidar
+// event sequence), so each rename produces a fresh doc with a new
+// path-derived id. Two distinct files at the same path AT THE SAME
+// TIME are the impossible state we want to prevent; sequential
+// reuse of a path (e.g., delete then recreate) is fine because the
+// old doc was already removed by unlink before the new add fires.
+export function fileDocId({ scope, teamId, ownerId, path }) {
+  const key = `file|${scope}|${teamId ?? ''}|${ownerId ?? ''}|${path}`;
+  return 'x-' + crypto.createHash('sha256').update(key).digest('hex').slice(0, 20);
+}
+
 export function toBytes(data) {
   if (data == null) return Buffer.alloc(0);
   if (Buffer.isBuffer(data)) return data;