@doubling/compound-sync 1.4.1 → 1.5.0

package/{dual-write.js → files.js}

@@ -1,88 +1,116 @@
-// Dual-write helpers for the sync daemon and integration tests.
+// Cloud-side file persistence for the sync daemon and integration tests.
 //
 // Each function takes its dependencies (filesRef, storage, orgId, etc.)
 // as parameters so the logic is importable into a test harness that
 // provides emulator-backed Firebase handles. The sync daemon (sync.js)
-// is the production caller and supplies real-Firebase handles.
+// is the production caller and supplies real Firebase handles.
 //
-// Side effect ordering on writes:
-//   1. Write the Firestore metadata document (Storage rules require
-//      the metadata doc to exist before the upload is authorized).
-//   2. Upload the blob to Cloud Storage at the deterministic path
-//      derived from orgId + fileId.
-//   3. Storage failures are returned in the result, not thrown, so the
-//      caller can decide whether to surface as a soft warning while the
-//      Firestore content field still holds the up-to-date value.
-
-import crypto from 'crypto';
+// A "file" is a Firestore metadata document at
+//   orgs/{orgId}/files/{fileId}
+// plus a Storage blob at
+//   orgs/{orgId}/files/{fileId}/current.{ext}
+// where {ext} is derived from the file's path so Storage can serve the
+// correct Content-Type header automatically.
+//
+// Ordering on writes:
+//   1. Write/update the Firestore metadata document.
+//      (Storage rules require the metadata doc to exist before they
+//      authorize the blob upload.)
+//   2. Upload the blob to Cloud Storage.
+//   Storage failures are returned in the result, not thrown, so the
+//   caller can decide whether to surface as a soft warning and retry.
+//
+// Ordering on deletes:
+//   1. Delete the Storage blob (while metadata still exists so the
+//      Storage rule can authorize).
+//   2. Delete the Firestore metadata document.
+
 import {
   doc, getDocs, setDoc, updateDoc, deleteDoc, query, where,
 } from 'firebase/firestore';
 import {
-  ref as storageRef, uploadString, deleteObject, getBytes,
+  ref as storageRef, uploadBytes, deleteObject, getBytes,
 } from 'firebase/storage';
 
-// ---- Pure helpers ----
-
-export function computeContentHash(content) {
-  return crypto.createHash('sha256').update(content || '', 'utf-8').digest('hex');
-}
-
-export function storagePathFor(orgId, fileId) {
-  return `orgs/${orgId}/files/${fileId}/current.md`;
-}
+import {
+  storagePathFor,
+  extFromPath,
+  contentByteSize,
+  computeContentHash,
+  toBytes,
+} from './storage-helpers.js';
+
+// Re-export pure helpers so callers (and existing tests) can import
+// either from here or directly from storage-helpers.js.
+export {
+  storagePathFor,
+  extFromPath,
+  contentByteSize,
+  computeContentHash,
+  isTextMimeType,
+  mimeTypeFromExt,
+} from './storage-helpers.js';
 
 // ---- Storage primitives ----
 
-export async function uploadBlob({ storage, orgId, fileId, content }) {
-  const blobPath = storagePathFor(orgId, fileId);
-  await uploadString(storageRef(storage, blobPath), content || '', 'raw', {
-    contentType: 'text/markdown',
+export async function uploadBlob({ storage, orgId, fileId, ext, data, mimeType }) {
+  const blobPath = storagePathFor(orgId, fileId, ext);
+  const payload = toBytes(data);
+  await uploadBytes(storageRef(storage, blobPath), payload, {
+    contentType: mimeType || 'application/octet-stream',
   });
 }
 
-export async function deleteBlob({ storage, orgId, fileId }) {
-  const blobPath = storagePathFor(orgId, fileId);
+export async function deleteBlob({ storage, orgId, fileId, ext }) {
+  const blobPath = storagePathFor(orgId, fileId, ext);
   try {
     await deleteObject(storageRef(storage, blobPath));
   } catch (err) {
     if (err && err.code === 'storage/object-not-found') {
-      return; // tolerate missing blob (older files written before PR 2)
+      return; // tolerate missing blob
     }
     throw err;
   }
 }
 
 /**
- * Read file content from Cloud Storage. Throws on any failure
- * (missing blob, permission error, network). The Firestore content
- * field is no longer written or read as of PR 6, so the read path
- * has no fallback. Backfill (PR 4) is a prerequisite.
+ * Read a blob from Cloud Storage as raw bytes. Throws on any failure
+ * (missing blob, permission error, network).
  */
-export async function readFileContent({ storage, orgId, fileId }) {
-  const blobPath = storagePathFor(orgId, fileId);
-  const bytes = await getBytes(storageRef(storage, blobPath));
-  return new TextDecoder('utf-8').decode(new Uint8Array(bytes));
+export async function readBlob({ storage, orgId, fileId, ext }) {
+  const blobPath = storagePathFor(orgId, fileId, ext);
+  const buf = await getBytes(storageRef(storage, blobPath));
+  return Buffer.from(buf);
 }
 
-// ---- Core dual-write ----
+/**
+ * Read a blob and decode as UTF-8 text. Convenience for text/markdown
+ * callers.
+ */
+export async function readBlobAsText({ storage, orgId, fileId, ext }) {
+  const buf = await readBlob({ storage, orgId, fileId, ext });
+  return buf.toString('utf-8');
+}
+
+// ---- High-level coordinator ----
 
 /**
- * Write a file's content to Cloud Storage at
- * orgs/{orgId}/files/{fileId}/current.md and update the Firestore
- * metadata doc with storagePath/contentHash/size. As of PR 6 the
- * Firestore `content` field is no longer written; the blob is the
- * sole source of file content.
+ * Push a file's content to the cloud: upsert the Firestore metadata
+ * doc and upload the blob.
+ *
+ * Args:
+ *   - data: the file's content. String, Buffer, or Uint8Array.
+ *   - mimeType: the file's mime type. Stored on the metadata doc and
+ *               used as the blob's Content-Type. Optional; if absent
+ *               for an existing file the existing mime is preserved,
+ *               and for new files defaults to 'text/markdown' to keep
+ *               legacy callers working.
  *
  * Returns one of:
  *   { action: 'skipped', reason: 'shared-scope' }
  *   { action: 'noop',    fileId }
  *   { action: 'created', fileId, fileData, storageError? }
  *   { action: 'updated', fileId, storageError? }
- *
- * `storageError` is populated when the Firestore write succeeded but
- * the Storage upload failed. Callers should log and retry; the next
- * save attempts the upload again.
  */
 export async function pushFileToCloud({
   filesRef,
@@ -92,7 +120,8 @@ export async function pushFileToCloud({
   scope,
   teamId,
   docPath,
-  content,
+  data,
+  mimeType,
   now = new Date().toISOString(),
 }) {
   if (scope === 'shared-with-me' || scope === 'shared-by-me') {
@@ -116,12 +145,14 @@ export async function pushFileToCloud({
     snapshot = await getDocs(query(filesRef, where('path', '==', docPath)));
   }
 
-  const contentHash = computeContentHash(content);
-  const size = Buffer.byteLength(content || '', 'utf-8');
+  const ext = extFromPath(docPath);
+  const contentHash = computeContentHash(data);
+  const size = contentByteSize(data);
 
   if (!snapshot || snapshot.empty) {
     const newDocRef = doc(filesRef);
     const fileId = newDocRef.id;
+    const resolvedMime = mimeType || 'text/markdown';
     const fileData = {
       path: docPath,
       createdAt: now,
@@ -130,14 +161,15 @@ export async function pushFileToCloud({
       teamId: scope === 'team' ? (teamId || null) : null,
       ownerId: scope === 'private' ? userId : null,
       sharedWith: [],
-      storagePath: storagePathFor(orgId, fileId),
+      storagePath: storagePathFor(orgId, fileId, ext),
       contentHash,
       size,
+      mimeType: resolvedMime,
     };
     await setDoc(newDocRef, fileData);
     let storageError = null;
     try {
-      await uploadBlob({ storage, orgId, fileId, content });
+      await uploadBlob({ storage, orgId, fileId, ext, data, mimeType: resolvedMime });
     } catch (err) {
       storageError = err;
     }
@@ -149,15 +181,17 @@ export async function pushFileToCloud({
   if (existing.contentHash === contentHash) {
     return { action: 'noop', fileId: docSnap.id };
   }
+  const resolvedMime = mimeType || existing.mimeType || 'text/markdown';
   await updateDoc(docSnap.ref, {
     updatedAt: now,
-    storagePath: storagePathFor(orgId, docSnap.id),
+    storagePath: storagePathFor(orgId, docSnap.id, ext),
     contentHash,
     size,
+    mimeType: resolvedMime,
   });
   let storageError = null;
   try {
-    await uploadBlob({ storage, orgId, fileId: docSnap.id, content });
+    await uploadBlob({ storage, orgId, fileId: docSnap.id, ext, data, mimeType: resolvedMime });
   } catch (err) {
     storageError = err;
   }
@@ -170,7 +204,6 @@ export async function pushFileToCloud({
  * Returns one of:
  *   { action: 'skipped',        reason: 'shared-scope' }
  *   { action: 'not-found' }
- *   { action: 'scope-mismatch' }
  *   { action: 'deleted',        fileId }
  *
  * Blob deletion tolerates `storage/object-not-found`.
@@ -188,9 +221,6 @@ export async function deleteFileFromCloud({
     return { action: 'skipped', reason: 'shared-scope' };
   }
 
-  // The list query must mirror the Firestore rule's per-branch
-  // constraints (scope + ownerId for private, scope + teamId for team)
-  // so the rules engine can statically authorize it.
   let snapshot;
   if (scope === 'private') {
     snapshot = await getDocs(query(
@@ -215,12 +245,10 @@ export async function deleteFileFromCloud({
   }
   const docSnap = snapshot.docs[0];
   const fileId = docSnap.id;
+  const ext = extFromPath(docPath);
   // Delete the Storage blob FIRST while the metadata doc still exists,
   // because the Storage rule's authorization check looks up the doc.
-  // After Firestore deletion the rule would deny (firestore.get fails).
-  // deleteBlob tolerates storage/object-not-found for files that were
-  // written before PR 2 and so have no blob.
-  await deleteBlob({ storage, orgId, fileId });
+  await deleteBlob({ storage, orgId, fileId, ext });
   await deleteDoc(docSnap.ref);
   return { action: 'deleted', fileId };
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@doubling/compound-sync",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "Bidirectional sync between Compound and local markdown files",
   "type": "module",
   "bin": {
@@ -10,13 +10,14 @@
     "sync.js",
     "config.js",
     "paths.js",
-    "dual-write.js",
+    "files.js",
+    "storage-helpers.js",
     "README.md"
   ],
   "scripts": {
     "sync": "node sync.js",
-    "test:unit": "node --test config.test.js paths.test.js",
-    "test:integration": "firebase emulators:exec --only firestore,storage 'node --test dual-write.test.js'",
+    "test:unit": "node --test config.test.js paths.test.js storage-helpers.test.js",
+    "test:integration": "firebase emulators:exec --only firestore,storage 'node --test files.test.js'",
     "test": "npm run test:unit && npm run test:integration"
   },
   "dependencies": {

package/storage-helpers.js

@@ -0,0 +1,88 @@
+// Pure Node-side storage helpers. No Firebase imports.
+//
+// Mirrors public/js/storage-helpers.js but uses Node-native APIs
+// (Buffer, crypto.createHash) instead of the browser equivalents
+// (Uint8Array, crypto.subtle). Function names and return types
+// match so that callers on either side can be reasoned about
+// uniformly.
+//
+// Storage layout convention:
+//   orgs/{orgId}/files/{fileId}/current.{ext}
+
+import crypto from 'crypto';
+
+export function extFromPath(path) {
+  const name = (((path == null ? '' : path) + '').split('/').pop()) || '';
+  const idx = name.lastIndexOf('.');
+  if (idx <= 0 || idx === name.length - 1) return 'bin';
+  return name.slice(idx + 1).toLowerCase();
+}
+
+export function storagePathFor(orgId, fileId, ext) {
+  return `orgs/${orgId}/files/${fileId}/current.${ext || 'bin'}`;
+}
+
+export function contentByteSize(data) {
+  if (data == null) return 0;
+  if (typeof data === 'string') return Buffer.byteLength(data, 'utf-8');
+  if (Buffer.isBuffer(data)) return data.length;
+  if (data instanceof Uint8Array) return data.byteLength;
+  if (data instanceof ArrayBuffer) return data.byteLength;
+  return Buffer.byteLength(String(data), 'utf-8');
+}
+
+export function computeContentHash(data) {
+  const buf = toBytes(data);
+  return crypto.createHash('sha256').update(buf).digest('hex');
+}
+
+export function toBytes(data) {
+  if (data == null) return Buffer.alloc(0);
+  if (Buffer.isBuffer(data)) return data;
+  if (data instanceof Uint8Array) return Buffer.from(data);
+  if (data instanceof ArrayBuffer) return Buffer.from(data);
+  return Buffer.from(String(data), 'utf-8');
+}
+
+// Files with these mime types are treated as inline-readable text by
+// the sync daemon: read with utf-8 encoding from disk, written with
+// utf-8 encoding to disk. Everything else is treated as binary and
+// transferred as raw bytes.
+//
+// Missing mime type is treated as text for backward compatibility
+// with markdown files written before mimeType was tracked.
+export function isTextMimeType(mimeType) {
+  if (mimeType == null || mimeType === '') return true;
+  return typeof mimeType === 'string' && mimeType.startsWith('text/');
+}
+
+const EXT_TO_MIME = {
+  md: 'text/markdown',
+  markdown: 'text/markdown',
+  txt: 'text/plain',
+  csv: 'text/csv',
+  html: 'text/html',
+  htm: 'text/html',
+  css: 'text/css',
+  js: 'text/javascript',
+  json: 'application/json',
+  pdf: 'application/pdf',
+  png: 'image/png',
+  jpg: 'image/jpeg',
+  jpeg: 'image/jpeg',
+  gif: 'image/gif',
+  webp: 'image/webp',
+  svg: 'image/svg+xml',
+  bmp: 'image/bmp',
+  mp3: 'audio/mpeg',
+  wav: 'audio/wav',
+  mp4: 'video/mp4',
+  webm: 'video/webm',
+  mov: 'video/quicktime',
+  zip: 'application/zip',
+};
+
+export function mimeTypeFromExt(ext) {
+  if (ext == null) return 'application/octet-stream';
+  return EXT_TO_MIME[String(ext).toLowerCase()] || 'application/octet-stream';
+}

package/sync.js

@@ -32,7 +32,18 @@ import os from 'os';
 import http from 'http';
 import { exec } from 'child_process';
 import { fileURLToPath } from 'url';
-import { pushFileToCloud, deleteFileFromCloud, readFileContent } from './dual-write.js';
+import {
+  pushFileToCloud,
+  deleteFileFromCloud,
+  readBlob,
+  readBlobAsText,
+  extFromPath,
+  isTextMimeType,
+  mimeTypeFromExt,
+} from './files.js';
+
+// Max file size enforced before upload. Mirrored in storage.rules.
+const MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
 import { resolveConfigPath, loadConfig, saveConfig } from './config.js';
 import {
   scopeFromLocalPath,
@@ -659,12 +670,24 @@ async function setupOrgSync({ orgId, localPath: rawLocalPath }) {
     ensureDir(localFilePath);
     suppressLocal.add(localFilePath);
 
-    const content = await readFileContent({
-      storage,
-      orgId,
-      fileId: change.doc.id,
-    });
-    fs.writeFileSync(localFilePath, content || '', 'utf-8');
+    const ext = extFromPath(docPath);
+    if (isTextMimeType(data.mimeType)) {
+      const content = await readBlobAsText({
+        storage,
+        orgId,
+        fileId: change.doc.id,
+        ext,
+      });
+      fs.writeFileSync(localFilePath, content || '', 'utf-8');
+    } else {
+      const bytes = await readBlob({
+        storage,
+        orgId,
+        fileId: change.doc.id,
+        ext,
+      });
+      fs.writeFileSync(localFilePath, bytes);
+    }
     lastKnownUpdate.set(trackingKey, remoteUpdated);
     setTimeout(() => suppressLocal.delete(localFilePath), 1000);
   }
@@ -719,7 +742,7 @@ async function setupOrgSync({ orgId, localPath: rawLocalPath }) {
     }
   }
 
-  async function pushToFirestore(filePath, content) {
+  async function pushToFirestore(filePath, data, mimeType) {
     const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
     const trackingKey = `${scope}:${docPath}`;
     const now = new Date().toISOString();
@@ -733,12 +756,13 @@ async function setupOrgSync({ orgId, localPath: rawLocalPath }) {
       scope,
       teamId,
       docPath,
-      content,
+      data,
+      mimeType,
       now,
     });
 
     if (result.action === 'created') {
-      console.log(`  [${orgId}] [local → firestore] CREATE ${docPath} (${scope})`);
+      console.log(`  [${orgId}] [local → firestore] CREATE ${docPath} (${scope}, ${mimeType})`);
     } else if (result.action === 'updated') {
       console.log(`  [${orgId}] [local → firestore] UPDATE ${docPath} (${scope})`);
     }
@@ -771,13 +795,32 @@ async function setupOrgSync({ orgId, localPath: rawLocalPath }) {
     if (suppressLocal.has(filePath)) return;
 
     const rel = path.relative(LOCAL_PATH, filePath);
-    if (!rel.endsWith('.md') && !rel.endsWith('.txt')) return;
-
     if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return;
     if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return;
 
-    const content = fs.readFileSync(filePath, 'utf-8');
-    pushToFirestore(filePath, content).catch(err => console.error(`  [${orgId}] Push error: ${err.message}`));
+    let stat;
+    try {
+      stat = fs.statSync(filePath);
+    } catch (err) {
+      return; // disappeared between event and read
+    }
+    if (!stat.isFile()) return;
+    if (stat.size > MAX_UPLOAD_BYTES) {
+      console.error(`  [${orgId}] [skip] ${rel} exceeds ${MAX_UPLOAD_BYTES} byte cap (${stat.size} bytes)`);
+      return;
+    }
+
+    const ext = extFromPath(rel);
+    const mimeType = mimeTypeFromExt(ext);
+    // Text files: read as a UTF-8 string so future text-aware features
+    // (line-level diffing, conflict markers) can work. Binary files:
+    // read as a Buffer so bytes round-trip without corruption.
+    const data = isTextMimeType(mimeType)
+      ? fs.readFileSync(filePath, 'utf-8')
+      : fs.readFileSync(filePath);
+    pushToFirestore(filePath, data, mimeType).catch(err =>
+      console.error(`  [${orgId}] Push error: ${err.message}`)
+    );
   }
 
   function startLocalWatcher() {