@doubling/compound-sync 1.9.5 → 1.10.2

package/files.js

@@ -291,6 +291,13 @@ export async function pushFileToCloud({
   mimeType,
   now = new Date().toISOString(),
   localModifiedAt = null,
+  // The contentHash this caller saw as "current" when it last pulled.
+  // If the existing remote doc has a different contentHash, another
+  // writer landed an edit in the meantime; we refuse to overwrite and
+  // return action='conflict' so the caller can write a conflict copy
+  // and pull the new remote state. Undefined disables the check (used
+  // for create-from-nothing where no baseline exists).
+  expectedRemoteHash = undefined,
 }) {
   if (scope === 'shared-with-me' || scope === 'shared-by-me') {
     return { action: 'skipped', reason: 'shared-scope' };
@@ -372,6 +379,23 @@ export async function pushFileToCloud({
       && (existing.parentId == null ? null : existing.parentId) === parentId) {
     return { action: 'noop', fileId: docSnap.id };
   }
+  // Compare-and-swap (DOU-167): if the caller passed a baseline of
+  // what they last saw as current, refuse the push when the remote
+  // has moved on. The daemon will write the proposed local content
+  // to a sibling conflict file and pull the new remote state into
+  // the live path. Without this guard, a stale push silently
+  // overwrites another user's freshly-saved edits (data loss).
+  if (
+    expectedRemoteHash !== undefined &&
+    existing.contentHash !== expectedRemoteHash
+  ) {
+    return {
+      action: 'conflict',
+      fileId: docSnap.id,
+      remoteHash: existing.contentHash,
+      remoteData: { id: docSnap.id, ...existing },
+    };
+  }
   // Last-write-wins: never let an older local file overwrite a newer
   // server edit. The caller (sync daemon) passes the local file's mtime;
   // if the server doc was updated more recently, leave it for the

package/org-sync.js

@@ -40,6 +40,36 @@ import { loadSyncState, saveSyncState } from './sync-state.js';
 
 const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
 
+// Retry a blob download until either the bytes appear at all (covers
+// the create race: doc written before blob upload) AND the content
+// hash matches the metadata's `contentHash` (covers the update race:
+// the blob still has the old bytes when the new doc is already
+// visible). Backs off 100/200/400/800 ms (1.5s total) across both
+// failure modes. Other errors throw immediately.
+async function readBlobWithRetry(read, { expectedHash = null, maxAttempts = 5 } = {}) {
+  let delay = 100;
+  let lastErr;
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      const content = await read();
+      if (!expectedHash || computeContentHash(content) === expectedHash) {
+        return content;
+      }
+      lastErr = new Error(
+        `blob hash does not match metadata contentHash (attempt ${attempt}/${maxAttempts})`,
+      );
+    } catch (err) {
+      if (err?.code !== 'storage/object-not-found') throw err;
+      lastErr = err;
+    }
+    if (attempt < maxAttempts) {
+      await new Promise((resolve) => setTimeout(resolve, delay));
+      delay *= 2;
+    }
+  }
+  throw lastErr;
+}
+
 // Start the per-org sync loop. Returns { stop } to unsubscribe all
 // listeners and close the watcher (used by tests and by graceful
 // shutdown). `localPath` must already be absolute/expanded.
@@ -243,21 +273,29 @@ export async function startOrgSync({
     suppressLocal.add(localFilePath);
 
     const ext = extFromPath(docPath);
-    if (isTextMimeType(data.mimeType)) {
-      const content = await readBlobAsText({
-        storage,
-        orgId,
-        fileId: change.doc.id,
-        ext,
-      });
+    // Extension is used as a fallback authority when the stored
+    // mimeType claims non-text but the extension maps to a text mime
+    // in EXT_TO_MIME. This heals files uploaded before their extension
+    // was registered as text (e.g. .base files uploaded prior to the
+    // text/yaml mapping landing): on the next pull the daemon writes
+    // them as UTF-8 so text-aware features (line diffing, conflict
+    // markers) can apply.
+    const extDerivedMime = mimeTypeFromExt(ext);
+    const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
+    // Storage rules require the Firestore doc to exist before they
+    // authorize a blob upload, so writers must create the doc first
+    // and upload second. That leaves a window where the metadata is
+    // visible (this listener fires) but the blob is not yet uploaded.
+    // Retry on object-not-found with exponential backoff covers that
+    // race; on a persistent miss we fall through to the original
+    // catch upstream.
+    const blobReadArgs = { storage, orgId, fileId: change.doc.id, ext };
+    const retryOpts = { expectedHash: data.contentHash };
+    if (isText) {
+      const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
       fs.writeFileSync(localFilePath, content || '', 'utf-8');
     } else {
-      const bytes = await readBlob({
-        storage,
-        orgId,
-        fileId: change.doc.id,
-        ext,
-      });
+      const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
       fs.writeFileSync(localFilePath, bytes);
     }
     lastKnownUpdate.set(trackingKey, remoteUpdated);
@@ -387,6 +425,12 @@ export async function startOrgSync({
     // duplicate doc). Pre-setting it suppresses the echo.
     lastKnownUpdate.set(trackingKey, now);
 
+    // Baseline = what we last saw as the remote contentHash for this
+    // path. pushFileToCloud uses it as a compare-and-swap precondition
+    // (DOU-167): if the remote has moved past this baseline, refuse
+    // the push and surface the conflict to the caller.
+    const baseline = lastSyncedHash.get(trackingKey);
+
     const result = await pushFileToCloud({
       filesRef,
       storage,
@@ -399,6 +443,7 @@ export async function startOrgSync({
       mimeType,
       now,
       localModifiedAt,
+      expectedRemoteHash: baseline,
     });
 
     // Record the synced content baseline whenever local matches the
@@ -411,6 +456,21 @@ export async function startOrgSync({
       // its firestore -> local snapshot is allowed through to update the
       // local file.
       lastKnownUpdate.delete(trackingKey);
+    } else if (result.action === 'conflict') {
+      // Another writer landed an edit in the gap between our last sync
+      // and this push. Preserve the user's would-be push as a sibling
+      // conflict copy, pull the new remote state into the live file,
+      // and advance the baseline so the next push (or further edits)
+      // can proceed against the new ground truth.
+      const conflictPath = writeConflictCopy(filePath, data);
+      await pullRemoteIntoLocal(result.remoteData, filePath);
+      setBaseline(trackingKey, result.remoteHash);
+      lastKnownUpdate.delete(trackingKey);
+      console.log(
+        `  [${orgId}] [local → firestore] CONFLICT ${docPath}: ` +
+        `remote changed; local saved to ${path.basename(conflictPath)}`,
+      );
+      return;
     }
 
     if (result.action === 'created') {
@@ -425,6 +485,38 @@ export async function startOrgSync({
     }
   }
 
+  /** Write the proposed (rejected) push to a sibling conflict file.
+   * Returns the new path. Suppresses chokidar so the conflict file's
+   * creation doesn't trigger its own push. */
+  function writeConflictCopy(filePath, data) {
+    const parsed = path.parse(filePath);
+    const ts = new Date().toISOString().replace(/[:.]/g, '-');
+    const conflictPath = path.join(parsed.dir, `${parsed.name}.conflict-${ts}${parsed.ext}`);
+    suppressLocal.add(conflictPath);
+    fs.writeFileSync(conflictPath, data);
+    setTimeout(() => suppressLocal.delete(conflictPath), 1000);
+    return conflictPath;
+  }
+
+  /** Pull the remote blob into the live local path after a conflict.
+   * Uses suppressLocal so chokidar doesn't re-emit the write. */
+  async function pullRemoteIntoLocal(remoteData, localFilePath) {
+    const ext = extFromPath(remoteData.path);
+    const extDerivedMime = mimeTypeFromExt(ext);
+    const isText = isTextMimeType(remoteData.mimeType) || isTextMimeType(extDerivedMime);
+    suppressLocal.add(localFilePath);
+    const blobReadArgs = { storage, orgId, fileId: remoteData.id, ext };
+    const retryOpts = { expectedHash: remoteData.contentHash };
+    if (isText) {
+      const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
+      fs.writeFileSync(localFilePath, content || '', 'utf-8');
+    } else {
+      const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
+      fs.writeFileSync(localFilePath, bytes);
+    }
+    setTimeout(() => suppressLocal.delete(localFilePath), 1000);
+  }
+
   async function deleteFromFirestore(filePath) {
     const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
     const trackingKey = `${scope}:${docPath}`;
@@ -584,11 +676,22 @@ export async function startOrgSync({
     watcher.on('addDir', dirPath => handleLocalDirAdded(dirPath));
     watcher.on('unlinkDir', dirPath => handleLocalDirRemoved(dirPath));
 
-    watcher.on('ready', () => console.log(`  [${orgId}] Local watcher ready.`));
+    // Resolve only once chokidar has finished its initial scan and
+    // registered inotify (or FSEvents) watches on every existing
+    // subdirectory. Until 'ready' fires, files written to the watched
+    // tree can be silently missed because the OS-level watch hasn't
+    // attached yet. Callers awaiting startOrgSync get a guarantee
+    // that any local writes after the await are seen by the daemon.
+    return new Promise((resolve) => {
+      watcher.on('ready', () => {
+        console.log(`  [${orgId}] Local watcher ready.`);
+        resolve();
+      });
+    });
   }
 
   startFirestoreListeners();
-  startLocalWatcher();
+  await startLocalWatcher();
 
   return {
     // Tear down all listeners and the watcher. Returns the watcher's

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doubling/compound-sync",
-  "version": "1.9.5",
+  "version": "1.10.2",
   "description": "Bidirectional sync between Compound and local markdown files",
   "type": "module",
   "bin": {
@@ -24,7 +24,7 @@
   "scripts": {
     "sync": "node sync.js",
     "test:unit": "node --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js",
-    "test:integration": "firebase emulators:exec --only firestore,storage 'node --test --test-concurrency=1 files.test.js org-sync.test.js'",
+    "test:integration": "firebase emulators:exec --only firestore,storage 'node --test --test-concurrency=1 files.test.js org-sync.test.js two-user-sync.test.js'",
     "test": "npm run test:unit && npm run test:integration"
   },
   "dependencies": {
@@ -32,7 +32,7 @@
     "chokidar": "^5.0.0"
   },
   "devDependencies": {
-    "@firebase/rules-unit-testing": "^4.0.0",
+    "@firebase/rules-unit-testing": "^5.0.1",
     "firebase-tools": "^15.0.0"
   },
   "publishConfig": {

package/storage-helpers.js

@@ -65,6 +65,13 @@ const EXT_TO_MIME = {
   htm: 'text/html',
   css: 'text/css',
   js: 'text/javascript',
+  // Obsidian Bases (.base) files are YAML. Mapping to text/yaml here
+  // (in addition to the web copy) ensures the daemon uploads .base as
+  // a UTF-8 text blob with the right Content-Type so the web client's
+  // eager-load path picks up entity.content for the Bases renderer.
+  base: 'text/yaml',
+  yaml: 'text/yaml',
+  yml: 'text/yaml',
   json: 'application/json',
   pdf: 'application/pdf',
   png: 'image/png',