@doubling/compound-sync 1.10.4 → 1.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -14,6 +14,16 @@ That's it. A browser window will open for sign-in (Google or email/password), th
14
14
 
15
15
  - [Node.js](https://nodejs.org/) (v18 or later)
16
16
 
17
+ ### TLS certificates
18
+
19
+ On Node 22+ the daemon auto-detects the OS system certificate store (`/etc/ssl/cert.pem` on macOS, `/etc/ssl/certs/ca-certificates.crt` or `/etc/pki/tls/certs/ca-bundle.crt` on Linux) and re-execs itself with `NODE_EXTRA_CA_CERTS` set; otherwise `fetch()` to Google Identity / Firebase Auth endpoints fails with `auth/network-request-failed` because Node's bundled CA store doesn't match what those services use.
20
+
21
+ If you're behind a corporate proxy with custom roots, set `NODE_EXTRA_CA_CERTS` explicitly to your bundle and the daemon will use that instead:
22
+
23
+ ```bash
24
+ NODE_EXTRA_CA_CERTS=/path/to/corporate-bundle.pem npx @doubling/compound-sync
25
+ ```
26
+
17
27
  ## What it does
18
28
 
19
29
  Compound Sync watches a local folder and your Compound workspace simultaneously. Changes in either direction are synced automatically:
@@ -95,6 +105,19 @@ npm run test:integration
95
105
  npm test
96
106
  ```
97
107
 
108
+ ### TypeScript conventions (DOU-181)
109
+
110
+ `sync/` is being migrated to TypeScript file-by-file. The conventions:
111
+
112
+ - **Source layout: in-place.** Each `.ts` file emits `.js`, `.d.ts`, and source maps as siblings via `tsc` with `outDir: "."`. The emitted `.js` is a gitignored build artifact, never edited by hand; the `.ts` is the source.
113
+ - **Imports always use the `.js` extension** (NodeNext convention). At runtime: in dev/test the `tsx` loader maps `./paths.js` to `./paths.ts` so source runs without a build step; in production the compiled `paths.js` exists alongside and resolves directly.
114
+ - **Dev (no build needed):** `npm run sync` runs through `node --import tsx`, so `.ts` files load on demand.
115
+ - **CI / publish:** `npm run build` compiles every `.ts` source to its `.js` sibling. `prepack` runs build automatically before `npm publish` so the npm tarball ships compiled JS. The desktop staging script also runs `npm run -w sync build` before copying.
116
+ - **Typecheck:** `npm run typecheck` (alias for `tsc --noEmit`) at the workspace level, or at repo root `npm run typecheck` which runs web + sync together.
117
+ - **Strictness:** `strict: true`, `noUncheckedIndexedAccess: true`, `noImplicitOverride: true`. No `any`. No `// @ts-ignore`. If a third-party module lacks types, declare its shape in a `.d.ts` or PR types upstream.
118
+
119
+ Phase 1 migrated `paths.js` to `paths.ts` as proof of pattern. Phase 3 ([[DOU-183]]) migrates the remaining source files; everything in `sync/` will be `.ts` by the end.
120
+
98
121
  ### Publishing to npm
99
122
 
100
123
  After making changes to the sync daemon, publish a new version:
package/files.js CHANGED
@@ -291,13 +291,6 @@ export async function pushFileToCloud({
291
291
  mimeType,
292
292
  now = new Date().toISOString(),
293
293
  localModifiedAt = null,
294
- // The contentHash this caller saw as "current" when it last pulled.
295
- // If the existing remote doc has a different contentHash, another
296
- // writer landed an edit in the meantime; we refuse to overwrite and
297
- // return action='conflict' so the caller can write a conflict copy
298
- // and pull the new remote state. Undefined disables the check (used
299
- // for create-from-nothing where no baseline exists).
300
- expectedRemoteHash = undefined,
301
294
  }) {
302
295
  if (scope === 'shared-with-me' || scope === 'shared-by-me') {
303
296
  return { action: 'skipped', reason: 'shared-scope' };
@@ -379,23 +372,6 @@ export async function pushFileToCloud({
379
372
  && (existing.parentId == null ? null : existing.parentId) === parentId) {
380
373
  return { action: 'noop', fileId: docSnap.id };
381
374
  }
382
- // Compare-and-swap (DOU-167): if the caller passed a baseline of
383
- // what they last saw as current, refuse the push when the remote
384
- // has moved on. The daemon will write the proposed local content
385
- // to a sibling conflict file and pull the new remote state into
386
- // the live path. Without this guard, a stale push silently
387
- // overwrites another user's freshly-saved edits (data loss).
388
- if (
389
- expectedRemoteHash !== undefined &&
390
- existing.contentHash !== expectedRemoteHash
391
- ) {
392
- return {
393
- action: 'conflict',
394
- fileId: docSnap.id,
395
- remoteHash: existing.contentHash,
396
- remoteData: { id: docSnap.id, ...existing },
397
- };
398
- }
399
375
  // Last-write-wins: never let an older local file overwrite a newer
400
376
  // server edit. The caller (sync daemon) passes the local file's mtime;
401
377
  // if the server doc was updated more recently, leave it for the
package/org-sync.js CHANGED
@@ -37,6 +37,11 @@ import {
37
37
  } from './paths.js';
38
38
  import { TeamRegistry } from './team-registry.js';
39
39
  import { loadSyncState, saveSyncState } from './sync-state.js';
40
+ import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
41
+ import { YjsFileBinding } from './yjs-file-binding.js';
42
+ import {
43
+ loadBindingState, saveBindingState, deleteBindingState,
44
+ } from './yjs-binding-state.js';
40
45
 
41
46
  const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
42
47
 
@@ -106,6 +111,122 @@ export async function startOrgSync({
106
111
  function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
107
112
  function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
108
113
  const suppressLocal = new Set();
114
+
115
+ // Per-file Yjs bindings, created on first sight of a text file
116
+ // (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
117
+ // orgs/.../yupdates and the debounced disk flush. We keep two
118
+ // indexes: by fileId (so handleFirestoreChange can dispose on
119
+ // 'removed' and re-use across events) and by local path (so the
120
+ // chokidar handler can route disk edits into the binding without
121
+ // re-resolving the fileId).
122
+ const bindings = new Map();
123
+ const bindingsByPath = new Map();
124
+
125
+ // DOU-205: per-fileId record of the most recently observed Firestore
126
+ // metadata (local path + tracking key). Used to detect renames /
127
+ // moves on subsequent 'modified' snapshot events: when the file
128
+ // doc's path changes server-side (via the web rename UI), the local
129
+ // path computed from the new data also changes, and this map lets
130
+ // the handler relocate the local file in place (fs.renameSync,
131
+ // which preserves Obsidian's inode-level tracking) rather than
132
+ // leaving the old file orphaned at its previous location.
133
+ const knownPaths = new Map();
134
+
135
+ // Track which fileIds already have a binding (or one being set
136
+ // up) so we never double-create. `stoppedRef` is read by the
137
+ // background retry loops so they can abandon work after sync.stop.
138
+ // `pendingPromises` lets stop() await any in-flight setup before
139
+ // returning so the next test's clearFirestore does not race
140
+ // dangling onSnapshot streams.
141
+ const pendingBindings = new Set();
142
+ const pendingPromises = new Set();
143
+ const stoppedRef = { value: false };
144
+
145
+ // Fire-and-forget binding setup. Returns nothing because the
146
+ // common shape "ensureBinding then read its ytext" runs straight
147
+ // into a race: the daemon's onSnapshot fires on Alice's own
148
+ // optimistic write before the file doc is server-committed, and
149
+ // the Firestore rules engine evaluates the yupdates/yjs nested
150
+ // rule via a server-side get() against the parent files/{id}
151
+ // doc. That get() sees no doc and returns null, so binding.start
152
+ // fails with permission-denied; the right behavior is to retry
153
+ // until the commit propagates, not to block handleFirestoreChange
154
+ // for seconds. Callers fall back to the blob path while the
155
+ // binding is still being set up; once it lands in
156
+ // bindings/bindingsByPath, subsequent events take the Yjs path.
157
+ function ensureBinding(fileId, localFilePath) {
158
+ if (bindings.has(fileId) || pendingBindings.has(fileId)) return;
159
+ pendingBindings.add(fileId);
160
+ const onDiskWrite = (text) => {
161
+ suppressLocal.add(localFilePath);
162
+ ensureDir(localFilePath);
163
+ fs.writeFileSync(localFilePath, text, 'utf-8');
164
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
165
+ };
166
+ // DOU-209: persist the Y.Doc binary state after every stable
167
+ // change so the next daemon restart has the right baseline to
168
+ // merge offline disk edits without losing offline web edits.
169
+ const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
170
+ // DOU-209: feed the binding's startup-merge with the daemon's
171
+ // last view (initialDocState) and the current disk content. If
172
+ // they diverge, the binding applies the diff as Yjs ops so disk-
173
+ // side offline edits survive the cloud-side replay.
174
+ const initialDocState = loadBindingState(LOCAL_PATH, fileId);
175
+ let currentDiskText = null;
176
+ try {
177
+ if (fs.existsSync(localFilePath)) {
178
+ currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
179
+ }
180
+ } catch (err) {
181
+ console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err && err.message}`);
182
+ }
183
+ const setupPromise = (async () => {
184
+ // Retry binding.start on permission-denied. See note above
185
+ // about the optimistic-write race with the nested-rule get().
186
+ // YjsProvider rejects double-start, so each retry needs a
187
+ // fresh binding (and a fresh provider underneath it).
188
+ for (let attempt = 0; attempt < 8; attempt++) {
189
+ if (stoppedRef.value) return;
190
+ const store = new FirestoreUpdateStore(db, orgId, fileId);
191
+ const binding = new YjsFileBinding(store, {
192
+ onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
193
+ });
194
+ try {
195
+ await binding.start();
196
+ if (stoppedRef.value) {
197
+ await binding.stop().catch(() => {});
198
+ return;
199
+ }
200
+ bindings.set(fileId, binding);
201
+ bindingsByPath.set(localFilePath, binding);
202
+ return;
203
+ } catch (err) {
204
+ if (stoppedRef.value) return;
205
+ if (err?.code !== 'permission-denied' || attempt === 7) {
206
+ console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err && err.message}`);
207
+ return;
208
+ }
209
+ await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
210
+ }
211
+ }
212
+ })().finally(() => {
213
+ pendingBindings.delete(fileId);
214
+ pendingPromises.delete(setupPromise);
215
+ });
216
+ pendingPromises.add(setupPromise);
217
+ }
218
+
219
+ async function disposeBinding(fileId, localFilePath) {
220
+ const binding = bindings.get(fileId);
221
+ if (!binding) return;
222
+ bindings.delete(fileId);
223
+ if (localFilePath) bindingsByPath.delete(localFilePath);
224
+ await binding.stop();
225
+ // DOU-209: drop the per-file Y.Doc state so deleted files don't
226
+ // accumulate stale binding-state blobs under .compound-yjs-binding/.
227
+ deleteBindingState(LOCAL_PATH, fileId);
228
+ }
229
+
109
230
  // Per-team Firestore-listener unsubscribe handles, keyed by teamId.
110
231
  const teamUnsubscribers = new Map();
111
232
  // Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
@@ -188,6 +309,48 @@ export async function startOrgSync({
188
309
  const trackingKey = `${data.scope || 'team'}:${docPath}`;
189
310
  const isFolder = data.type === 'folder';
190
311
 
312
+ // DOU-205: rename / move detection. On a 'modified' snapshot for
313
+ // a fileId we've seen before, if the computed local path differs
314
+ // from what we last recorded, the server's path field has
315
+ // changed. Move the local file in place (fs.renameSync preserves
316
+ // Obsidian's inode tracking so notes don't lose their metadata,
317
+ // backlinks, or recent-files entries) and migrate the tracking
318
+ // maps from the old key to the new. We do this before the
319
+ // freshness check below because the rename happens irrespective
320
+ // of whether the content changed in the same update.
321
+ const fileId = change.doc.id;
322
+ const prevKnown = knownPaths.get(fileId);
323
+ if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
324
+ suppressLocal.add(prevKnown.localPath);
325
+ suppressLocal.add(localFilePath);
326
+ try {
327
+ if (fs.existsSync(prevKnown.localPath)) {
328
+ ensureDir(localFilePath);
329
+ fs.renameSync(prevKnown.localPath, localFilePath);
330
+ console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
331
+ }
332
+ } catch (err) {
333
+ console.error(` [${orgId}] [firestore → local] RENAME failed: ${err && err.message}`);
334
+ }
335
+ if (lastKnownUpdate.has(prevKnown.trackingKey)) {
336
+ lastKnownUpdate.set(trackingKey, lastKnownUpdate.get(prevKnown.trackingKey));
337
+ lastKnownUpdate.delete(prevKnown.trackingKey);
338
+ }
339
+ if (lastSyncedHash.has(prevKnown.trackingKey)) {
340
+ setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
341
+ deleteBaseline(prevKnown.trackingKey);
342
+ }
343
+ const binding = bindings.get(fileId);
344
+ if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
345
+ bindingsByPath.delete(prevKnown.localPath);
346
+ bindingsByPath.set(localFilePath, binding);
347
+ }
348
+ setTimeout(() => {
349
+ suppressLocal.delete(prevKnown.localPath);
350
+ suppressLocal.delete(localFilePath);
351
+ }, 1000);
352
+ }
353
+
191
354
  if (change.type === 'removed') {
192
355
  if (isFolder) {
193
356
  // Best-effort directory removal. The daemon's own file-delete
@@ -213,9 +376,27 @@ export async function startOrgSync({
213
376
  }
214
377
  lastKnownUpdate.delete(trackingKey);
215
378
  deleteBaseline(trackingKey);
379
+ knownPaths.delete(fileId);
380
+ await disposeBinding(change.doc.id, localFilePath);
216
381
  return;
217
382
  }
218
383
 
384
+ // Record (or refresh) the per-fileId path so the next 'modified'
385
+ // event can detect a rename / move by comparing against this.
386
+ knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
387
+
388
+ // Kick off Yjs binding setup for text files. Fire-and-forget so
389
+ // a slow start (retrying past the rules-engine race on a fresh
390
+ // doc) does not stall the legacy blob pull. Once the binding is
391
+ // registered, future events take the Yjs path; until then the
392
+ // blob path still keeps disk in sync.
393
+ if (!isFolder) {
394
+ const ext = extFromPath(docPath);
395
+ const extDerivedMime = mimeTypeFromExt(ext);
396
+ const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
397
+ if (isText) ensureBinding(change.doc.id, localFilePath);
398
+ }
399
+
219
400
  const remoteUpdated = data.updatedAt || data.createdAt || '';
220
401
  const lastKnown = lastKnownUpdate.get(trackingKey);
221
402
  if (lastKnown && lastKnown >= remoteUpdated) return;
@@ -282,6 +463,23 @@ export async function startOrgSync({
282
463
  // markers) can apply.
283
464
  const extDerivedMime = mimeTypeFromExt(ext);
284
465
  const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
466
+
467
+ // Yjs path (DOU-178 PR 3): if a binding for this file is already
468
+ // up and carries non-empty ytext, the debounced flush owns the
469
+ // disk write and the legacy blob pull is skipped. Bindings that
470
+ // are still being set up, or that loaded empty ytext (pre-Yjs
471
+ // files), fall through to the blob path; PR 4 retires the blob
472
+ // path entirely.
473
+ if (isText) {
474
+ const binding = bindings.get(change.doc.id);
475
+ if (binding && binding.ytext.length > 0) {
476
+ lastKnownUpdate.set(trackingKey, remoteUpdated);
477
+ setBaseline(trackingKey, data.contentHash);
478
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
479
+ return;
480
+ }
481
+ }
482
+
285
483
  // Storage rules require the Firestore doc to exist before they
286
484
  // authorize a blob upload, so writers must create the doc first
287
485
  // and upload second. That leaves a window where the metadata is
@@ -425,12 +623,6 @@ export async function startOrgSync({
425
623
  // duplicate doc). Pre-setting it suppresses the echo.
426
624
  lastKnownUpdate.set(trackingKey, now);
427
625
 
428
- // Baseline = what we last saw as the remote contentHash for this
429
- // path. pushFileToCloud uses it as a compare-and-swap precondition
430
- // (DOU-167): if the remote has moved past this baseline, refuse
431
- // the push and surface the conflict to the caller.
432
- const baseline = lastSyncedHash.get(trackingKey);
433
-
434
626
  const result = await pushFileToCloud({
435
627
  filesRef,
436
628
  storage,
@@ -443,7 +635,6 @@ export async function startOrgSync({
443
635
  mimeType,
444
636
  now,
445
637
  localModifiedAt,
446
- expectedRemoteHash: baseline,
447
638
  });
448
639
 
449
640
  // Record the synced content baseline whenever local matches the
@@ -456,21 +647,6 @@ export async function startOrgSync({
456
647
  // its firestore -> local snapshot is allowed through to update the
457
648
  // local file.
458
649
  lastKnownUpdate.delete(trackingKey);
459
- } else if (result.action === 'conflict') {
460
- // Another writer landed an edit in the gap between our last sync
461
- // and this push. Preserve the user's would-be push as a sibling
462
- // conflict copy, pull the new remote state into the live file,
463
- // and advance the baseline so the next push (or further edits)
464
- // can proceed against the new ground truth.
465
- const conflictPath = writeConflictCopy(filePath, data);
466
- await pullRemoteIntoLocal(result.remoteData, filePath);
467
- setBaseline(trackingKey, result.remoteHash);
468
- lastKnownUpdate.delete(trackingKey);
469
- console.log(
470
- ` [${orgId}] [local → firestore] CONFLICT ${docPath}: ` +
471
- `remote changed; local saved to ${path.basename(conflictPath)}`,
472
- );
473
- return;
474
650
  }
475
651
 
476
652
  if (result.action === 'created') {
@@ -485,38 +661,6 @@ export async function startOrgSync({
485
661
  }
486
662
  }
487
663
 
488
- /** Write the proposed (rejected) push to a sibling conflict file.
489
- * Returns the new path. Suppresses chokidar so the conflict file's
490
- * creation doesn't trigger its own push. */
491
- function writeConflictCopy(filePath, data) {
492
- const parsed = path.parse(filePath);
493
- const ts = new Date().toISOString().replace(/[:.]/g, '-');
494
- const conflictPath = path.join(parsed.dir, `${parsed.name}.conflict-${ts}${parsed.ext}`);
495
- suppressLocal.add(conflictPath);
496
- fs.writeFileSync(conflictPath, data);
497
- setTimeout(() => suppressLocal.delete(conflictPath), 1000);
498
- return conflictPath;
499
- }
500
-
501
- /** Pull the remote blob into the live local path after a conflict.
502
- * Uses suppressLocal so chokidar doesn't re-emit the write. */
503
- async function pullRemoteIntoLocal(remoteData, localFilePath) {
504
- const ext = extFromPath(remoteData.path);
505
- const extDerivedMime = mimeTypeFromExt(ext);
506
- const isText = isTextMimeType(remoteData.mimeType) || isTextMimeType(extDerivedMime);
507
- suppressLocal.add(localFilePath);
508
- const blobReadArgs = { storage, orgId, fileId: remoteData.id, ext };
509
- const retryOpts = { expectedHash: remoteData.contentHash };
510
- if (isText) {
511
- const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
512
- fs.writeFileSync(localFilePath, content || '', 'utf-8');
513
- } else {
514
- const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
515
- fs.writeFileSync(localFilePath, bytes);
516
- }
517
- setTimeout(() => suppressLocal.delete(localFilePath), 1000);
518
- }
519
-
520
664
  async function deleteFromFirestore(filePath) {
521
665
  const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
522
666
  const trackingKey = `${scope}:${docPath}`;
@@ -623,8 +767,21 @@ export async function startOrgSync({
623
767
  ? fs.readFileSync(filePath, 'utf-8')
624
768
  : fs.readFileSync(filePath);
625
769
  const localModifiedAt = stat.mtime.toISOString();
770
+
771
+ // Yjs path (DOU-178 PR 3): route text edits into the binding so
772
+ // the disk delta is published as a yupdate (preserving concurrent
773
+ // web edits via fast-diff). The binding only exists once the
774
+ // Firestore listener has seen the file; for the very first
775
+ // local-only save the binding hasn't been created yet, and the
776
+ // blob path is still the only writer. Subsequent saves go through
777
+ // both paths until PR 4 retires the blob upload.
778
+ if (isTextMimeType(mimeType)) {
779
+ const binding = bindingsByPath.get(filePath);
780
+ if (binding) binding.applyDiskText(data);
781
+ }
782
+
626
783
  pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
627
- console.error(` [${orgId}] Push error: ${err.message}`)
784
+ console.error(` [${orgId}] Push error for ${rel}: ${err.message}`)
628
785
  );
629
786
  }
630
787
 
@@ -671,7 +828,8 @@ export async function startOrgSync({
671
828
  watcher.on('change', filePath => handleLocalChange(filePath));
672
829
  watcher.on('unlink', filePath => {
673
830
  if (suppressLocal.has(filePath)) return;
674
- deleteFromFirestore(filePath).catch(err => console.error(` [${orgId}] Delete error: ${err.message}`));
831
+ const rel = path.relative(LOCAL_PATH, filePath);
832
+ deleteFromFirestore(filePath).catch(err => console.error(` [${orgId}] Delete error for ${rel}: ${err.message}`));
675
833
  });
676
834
  watcher.on('addDir', dirPath => handleLocalDirAdded(dirPath));
677
835
  watcher.on('unlinkDir', dirPath => handleLocalDirRemoved(dirPath));
@@ -697,6 +855,7 @@ export async function startOrgSync({
697
855
  // Tear down all listeners and the watcher. Returns the watcher's
698
856
  // close() promise so callers/tests can await full shutdown.
699
857
  stop() {
858
+ stoppedRef.value = true;
700
859
  for (const unsub of unsubscribers) {
701
860
  try { unsub(); } catch (e) { /* ignore */ }
702
861
  }
@@ -704,7 +863,15 @@ export async function startOrgSync({
704
863
  try { unsub(); } catch (e) { /* ignore */ }
705
864
  }
706
865
  teamUnsubscribers.clear();
707
- return watcher ? watcher.close() : Promise.resolve();
866
+ const setupDrain = Promise.all([...pendingPromises]);
867
+ const bindingDrain = setupDrain.then(() => Promise.all(
868
+ [...bindings.values()].map((b) => b.stop().catch(() => {})),
869
+ )).then(() => {
870
+ bindings.clear();
871
+ bindingsByPath.clear();
872
+ });
873
+ const watcherClose = watcher ? watcher.close() : Promise.resolve();
874
+ return Promise.all([bindingDrain, watcherClose]).then(() => undefined);
708
875
  },
709
876
  };
710
877
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@doubling/compound-sync",
3
- "version": "1.10.4",
3
+ "version": "1.11.1",
4
4
  "description": "Bidirectional sync between Compound and local markdown files",
5
5
  "type": "module",
6
6
  "bin": {
@@ -12,6 +12,7 @@
12
12
  "sync-state.js",
13
13
  "config.js",
14
14
  "paths.js",
15
+ "paths.d.ts",
15
16
  "files.js",
16
17
  "folder-chain.js",
17
18
  "storage-helpers.js",
@@ -19,21 +20,33 @@
19
20
  "auth-persistence.js",
20
21
  "pre-mint-app-check.js",
21
22
  "setup-auth-with-pre-mint.js",
23
+ "yjs-firestore-update-store.js",
24
+ "yjs-provider.js",
25
+ "yjs-file-binding.js",
26
+ "yjs-binding-state.js",
22
27
  "README.md"
23
28
  ],
24
29
  "scripts": {
25
- "sync": "node sync.js",
26
- "test:unit": "node --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js",
27
- "test:integration": "firebase emulators:exec --only firestore,storage 'node --test --test-concurrency=1 files.test.js org-sync.test.js two-user-sync.test.js'",
30
+ "sync": "node --import tsx ./sync.js",
31
+ "build": "tsc",
32
+ "typecheck": "tsc --noEmit",
33
+ "prepack": "npm run build",
34
+ "test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js yjs-file-binding.test.js yjs-binding-state.test.js",
35
+ "test:integration": "firebase emulators:exec --only firestore,storage 'node --import tsx --test --test-concurrency=1 files.test.js org-sync.test.js two-user-sync.test.js'",
28
36
  "test": "npm run test:unit && npm run test:integration"
29
37
  },
30
38
  "dependencies": {
39
+ "chokidar": "^5.0.0",
40
+ "fast-diff": "^1.3.0",
31
41
  "firebase": "^12.14.0",
32
- "chokidar": "^5.0.0"
42
+ "yjs": "^13.6.31"
33
43
  },
34
44
  "devDependencies": {
35
45
  "@firebase/rules-unit-testing": "^5.0.1",
36
- "firebase-tools": "^15.0.0"
46
+ "@types/node": "^26.0.0",
47
+ "firebase-tools": "^15.20.0",
48
+ "tsx": "^4.22.4",
49
+ "typescript": "^5.9.3"
37
50
  },
38
51
  "publishConfig": {
39
52
  "access": "public"
package/paths.d.ts ADDED
@@ -0,0 +1,12 @@
1
+ export declare const PRIVATE_FOLDER = "Private";
2
+ export declare const SHARED_WITH_ME_FOLDER = "Shared with Me";
3
+ export declare const SHARED_BY_ME_FOLDER = "Shared by Me";
4
+ export type Scope = 'team' | 'private' | 'shared-with-me' | 'shared-by-me';
5
+ export interface ScopedDocPath {
6
+ scope: Scope;
7
+ teamId?: string;
8
+ docPath: string;
9
+ }
10
+ export declare function teamFolderName(teamName: string): string;
11
+ export declare function scopeFromLocalPath(filePath: string, localPath: string, teamIdsByName: Map<string, string>): ScopedDocPath;
12
+ //# sourceMappingURL=paths.d.ts.map
package/paths.js CHANGED
@@ -5,48 +5,43 @@
5
5
  // expected to be swapped if/when authorization moves to a Zanzibar-
6
6
  // style relationship model. Keep this module's interface narrow so the
7
7
  // swap stays localized.
8
-
9
8
  import path from 'node:path';
10
-
11
9
  export const PRIVATE_FOLDER = 'Private';
12
10
  export const SHARED_WITH_ME_FOLDER = 'Shared with Me';
13
11
  export const SHARED_BY_ME_FOLDER = 'Shared by Me';
14
12
  const TEAMSPACE_SUFFIX = ' Teamspace';
15
-
16
13
  export function teamFolderName(teamName) {
17
- return teamName + TEAMSPACE_SUFFIX;
14
+ return teamName + TEAMSPACE_SUFFIX;
18
15
  }
19
-
20
16
  // Given an absolute file path within one of the daemon's local sync
21
17
  // roots, return { scope, teamId?, docPath }. The caller passes the
22
- // matching localPath root and the team-name team-id map.
18
+ // matching localPath root and the team-name -> team-id map.
23
19
  export function scopeFromLocalPath(filePath, localPath, teamIdsByName) {
24
- const rel = path.relative(localPath, filePath);
25
-
26
- if (rel.startsWith(PRIVATE_FOLDER + path.sep)) {
27
- return {
28
- scope: 'private',
29
- docPath: rel.slice(PRIVATE_FOLDER.length + 1),
30
- };
31
- }
32
- if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) {
33
- return {
34
- scope: 'shared-with-me',
35
- docPath: rel.slice(SHARED_WITH_ME_FOLDER.length + 1),
36
- };
37
- }
38
- if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) {
39
- return {
40
- scope: 'shared-by-me',
41
- docPath: rel.slice(SHARED_BY_ME_FOLDER.length + 1),
42
- };
43
- }
44
-
45
- const parts = rel.split(path.sep);
46
- const folderName = parts[0];
47
- const teamName = folderName.endsWith(TEAMSPACE_SUFFIX)
48
- ? folderName.slice(0, -TEAMSPACE_SUFFIX.length)
49
- : folderName;
50
- const teamId = teamIdsByName.get(teamName);
51
- return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
20
+ const rel = path.relative(localPath, filePath);
21
+ if (rel.startsWith(PRIVATE_FOLDER + path.sep)) {
22
+ return {
23
+ scope: 'private',
24
+ docPath: rel.slice(PRIVATE_FOLDER.length + 1),
25
+ };
26
+ }
27
+ if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) {
28
+ return {
29
+ scope: 'shared-with-me',
30
+ docPath: rel.slice(SHARED_WITH_ME_FOLDER.length + 1),
31
+ };
32
+ }
33
+ if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) {
34
+ return {
35
+ scope: 'shared-by-me',
36
+ docPath: rel.slice(SHARED_BY_ME_FOLDER.length + 1),
37
+ };
38
+ }
39
+ const parts = rel.split(path.sep);
40
+ const folderName = parts[0] ?? '';
41
+ const teamName = folderName.endsWith(TEAMSPACE_SUFFIX)
42
+ ? folderName.slice(0, -TEAMSPACE_SUFFIX.length)
43
+ : folderName;
44
+ const teamId = teamIdsByName.get(teamName);
45
+ return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
52
46
  }
47
+ //# sourceMappingURL=paths.js.map
@@ -81,6 +81,8 @@ const EXT_TO_MIME = {
81
81
  webp: 'image/webp',
82
82
  svg: 'image/svg+xml',
83
83
  bmp: 'image/bmp',
84
+ heic: 'image/heic',
85
+ heif: 'image/heif',
84
86
  mp3: 'audio/mpeg',
85
87
  wav: 'audio/wav',
86
88
  mp4: 'video/mp4',
package/sync.js CHANGED
@@ -16,6 +16,43 @@
16
16
  * node sync.js --help
17
17
  */
18
18
 
19
+ // DOU-208: Node 22+ ships its own CA bundle and does not consult the
20
+ // macOS system trust store. Without NODE_EXTRA_CA_CERTS pointing at
21
+ // /etc/ssl/cert.pem, fetch() to securetoken.googleapis.com fails
22
+ // with a bare `fetch failed`, which Firebase Auth then surfaces as
23
+ // auth/network-request-failed. The web e2e harness's run-emulator.js
24
+ // already does this for child processes; here we need it for our own
25
+ // process. NODE_EXTRA_CA_CERTS is read at startup, so setting it on
26
+ // process.env after init is a no-op — instead, re-exec ourselves
27
+ // with the env var set when missing. The child inherits stdio +
28
+ // argv; we exit with its status. No-op when already set (the user
29
+ // or a wrapper explicitly chose a different bundle).
30
+ import { spawnSync as __dou208_spawnSync } from 'node:child_process';
31
+ import { existsSync as __dou208_existsSync } from 'node:fs';
32
+ if (!process.env.NODE_EXTRA_CA_CERTS) {
33
+ const __dou208_candidates = process.platform === 'darwin'
34
+ ? ['/etc/ssl/cert.pem']
35
+ : process.platform === 'linux'
36
+ ? ['/etc/ssl/certs/ca-certificates.crt', '/etc/pki/tls/certs/ca-bundle.crt']
37
+ : [];
38
+ for (const __dou208_candidate of __dou208_candidates) {
39
+ if (__dou208_existsSync(__dou208_candidate)) {
40
+ const __dou208_result = __dou208_spawnSync(
41
+ process.execPath,
42
+ process.argv.slice(1),
43
+ {
44
+ stdio: 'inherit',
45
+ env: { ...process.env, NODE_EXTRA_CA_CERTS: __dou208_candidate },
46
+ },
47
+ );
48
+ process.exit(__dou208_result.status ?? 1);
49
+ }
50
+ }
51
+ // No candidate found (Windows, unusual Linux distro): fall through
52
+ // and let Node's bundled CA bundle handle TLS. The user can still
53
+ // set NODE_EXTRA_CA_CERTS manually for corporate cert chains.
54
+ }
55
+
19
56
  import { initializeApp } from 'firebase/app';
20
57
  import { getAuth, initializeAuth, signInWithEmailAndPassword, signInWithCredential, GoogleAuthProvider } from 'firebase/auth';
21
58
  import {
@@ -0,0 +1,61 @@
1
+ // Per-file Y.Doc binary state persistence (DOU-209).
2
+ //
3
+ // On daemon restart we need to merge offline edits made on disk with
4
+ // offline edits made via the web. Yjs converges deterministically when
5
+ // both sides apply their ops to a common baseline. The daemon's last
6
+ // view of the file (== ytext at last write, == disk content at last
7
+ // write) is that baseline. Persisting the Y.Doc binary state per file
8
+ // lets the daemon restore that baseline on startup so:
9
+ // 1. ytext == lastDisk after restore
10
+ // 2. fast-diff(ytext, currentDisk) gives positions valid in ytext
11
+ // 3. cloud yupdates apply on top via CRDT merge (idempotent for ops
12
+ // already in the loaded state, additive for new cloud ops).
13
+ //
14
+ // Storage layout: one file per Y.Doc, keyed by fileId, under
15
+ // `<localPath>/.compound-yjs-binding/<fileId>.yjs`. The binary blob is
16
+ // the output of Y.encodeStateAsUpdate(doc); the daemon applies it via
17
+ // Y.applyUpdate(doc, blob) on restore.
18
+
19
+ import fs from 'node:fs';
20
+ import path from 'node:path';
21
+
22
+ const STATE_DIR = '.compound-yjs-binding';
23
+
24
+ function statePathFor(localPath, fileId) {
25
+ return path.join(localPath, STATE_DIR, `${fileId}.yjs`);
26
+ }
27
+
28
+ // Load the encoded Y.Doc state for a file. Returns null if no state
29
+ // exists yet (first run, or this fileId hasn't been synced before) or
30
+ // if the file is unreadable for any reason (corruption, permissions).
31
+ // Treating unreadable as "no state" is safe: the binding falls back to
32
+ // its first-run behavior, which writes ytext to disk on initial flush.
33
+ export function loadBindingState(localPath, fileId) {
34
+ try {
35
+ return fs.readFileSync(statePathFor(localPath, fileId));
36
+ } catch {
37
+ return null;
38
+ }
39
+ }
40
+
41
+ // Persist the encoded Y.Doc state for a file. Atomic via write-to-tmp +
42
+ // rename so a crash mid-write can't leave a half-written .yjs file that
43
+ // would corrupt the next startup. Best-effort: a failed save costs us a
44
+ // cold-start reconciliation next launch, never data.
45
+ export function saveBindingState(localPath, fileId, encoded) {
46
+ const finalPath = statePathFor(localPath, fileId);
47
+ const tmpPath = `${finalPath}.${process.pid}.${Date.now()}.tmp`;
48
+ try {
49
+ fs.mkdirSync(path.dirname(finalPath), { recursive: true });
50
+ fs.writeFileSync(tmpPath, encoded);
51
+ fs.renameSync(tmpPath, finalPath);
52
+ } catch {
53
+ try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
54
+ }
55
+ }
56
+
57
+ // Best-effort cleanup when a file is deleted from the workspace, so
58
+ // stale .yjs blobs don't accumulate. Failing to delete is not fatal.
59
+ export function deleteBindingState(localPath, fileId) {
60
+ try { fs.unlinkSync(statePathFor(localPath, fileId)); } catch { /* ignore */ }
61
+ }
@@ -0,0 +1,236 @@
1
+ // YjsFileBinding: per-file glue that keeps a local Markdown file
2
+ // converged with the per-file Y.Doc that lives in Firestore.
3
+ //
4
+ // Two directions, both flowing through the Y.Doc:
5
+ // cloud -> disk: provider applies remote yupdates to ytext; an
6
+ // observer debounces the resulting text into a single disk write
7
+ // so a burst of web keystrokes coalesces into one fs.writeFile.
8
+ // disk -> cloud: when the local file changes, caller invokes
9
+ // applyDiskText(newText); we diff against ytext (fast-diff for a
10
+ // char-level edit script) and apply inserts/deletes in one
11
+ // transaction so the provider's local-update listener emits a
12
+ // single yupdate carrying just the changed regions, preserving
13
+ // concurrent web edits to unrelated regions.
14
+ //
15
+ // Disk I/O is injected via onDiskWrite so tests can record without
16
+ // touching the filesystem and so the daemon can plumb its existing
17
+ // suppressLocal set (avoids the disk write feeding back through
18
+ // chokidar as a fake user edit).
19
+
20
+ import * as Y from 'yjs';
21
+ import fastDiff from 'fast-diff';
22
+ import { YjsProvider } from './yjs-provider.js';
23
+
24
+ export class YjsFileBinding {
25
+ constructor(store, {
26
+ debounceMs = 300, onDiskWrite, onDocStateChange,
27
+ initialDocState = null, currentDiskText = null,
28
+ }) {
29
+ if (!onDiskWrite) throw new Error('YjsFileBinding: onDiskWrite is required');
30
+ this.doc = new Y.Doc();
31
+ this.ytext = this.doc.getText('content');
32
+ this.store = store;
33
+ this.provider = new YjsProvider(this.doc, store);
34
+ this.debounceMs = debounceMs;
35
+ this.onDiskWrite = onDiskWrite;
36
+ // Called after every Y.Doc mutation reaches a stable state, with
37
+ // the encoded Y.Doc binary state. The daemon persists this to
38
+ // disk so the next startup can restore the baseline and merge
39
+ // offline disk edits without losing offline cloud edits (DOU-209).
40
+ this.onDocStateChange = onDocStateChange ?? null;
41
+ // Restore options for offline-merge (DOU-209).
42
+ this.initialDocState = initialDocState;
43
+ this.currentDiskText = currentDiskText;
44
+ this.flushTimer = null;
45
+ this.lastDiskText = null;
46
+ this.observer = null;
47
+ this.stopped = false;
48
+ }
49
+
50
+ async start() {
51
+ // DOU-209: offline-merge startup. Three-step process that turns the
52
+ // daemon's restart into a deterministic CRDT merge:
53
+ //
54
+ // 1. Restore Y.Doc from the daemon's last-saved binary state (if
55
+ // any). After this, ytext equals what disk was at the daemon's
56
+ // last write. This is the common ancestor for any divergence
57
+ // that happened while the daemon was offline.
58
+ // 2. If the current disk content differs from the restored ytext,
59
+ // the difference is disk-only edits made during downtime.
60
+ // Apply them as Y.Doc operations (positions are valid in the
61
+ // restored ytext since restored-ytext == lastDisk == fast-diff
62
+ // input).
63
+ // 3. provider.loadInitial() pulls cloud's yupdates on top. Y.applyUpdate
64
+ // is idempotent (Yjs's state-vector dedup), so already-known
65
+ // ops are no-ops. Only NEW cloud ops (offline web edits) take
66
+ // effect. They CRDT-merge with the disk ops from step 2.
67
+ //
68
+ // Without step 1, fast-diff(post-loadInitial-ytext, currentDisk)
69
+ // treats web's offline insertions as "things present in ytext but
70
+ // missing from disk" and DELETES them. That's the bug this fixes.
71
+ // Seed lastDiskText from the disk content we observed at construction
72
+ // time so the initial flush is a no-op when ytext already matches
73
+ // what's on disk. Without this, the binding's initial flush always
74
+ // wrote ytext to disk on startup, which clobbered any content the
75
+ // daemon's legacy contentHash + Storage blob pull (the fallthrough
76
+ // path in handleFirestoreChange) may have written first for non-Yjs
77
+ // cloud edits. Regression case: "after restart, pulls a web edit
78
+ // made while stopped despite a fresh local mtime" in
79
+ // sync/org-sync.test.js.
80
+ this.lastDiskText = this.currentDiskText;
81
+
82
+ if (this.initialDocState) {
83
+ Y.applyUpdate(this.doc, this.initialDocState);
84
+ }
85
+
86
+ // Disk-merge ops emitted below need to reach the store so other
87
+ // clients see the offline disk edits. The provider's local-update
88
+ // handler isn't attached yet (it's wired in startSubscription,
89
+ // which we can't call until after loadInitial), so we capture the
90
+ // updates synchronously here and push them after the subscription
91
+ // is set up. Capture by Y.Doc 'update' event with a 'disk' origin
92
+ // filter so we only grab the ops from our own _applyTextDiff and
93
+ // not anything spurious.
94
+ //
95
+ // The disk-merge is only safe when we have an `initialDocState`:
96
+ // that gives ytext a known baseline (the daemon's last view of the
97
+ // file) before we diff against currentDiskText, so fast-diff
98
+ // positions are valid AND the resulting ops are scoped to JUST the
99
+ // disk-only changes. Without a baseline, applying currentDiskText
100
+ // as fresh inserts on top of an empty ytext would then collide
101
+ // with whatever loadInitial pulls from the cloud: Yjs would see
102
+ // both the daemon's "fresh insert" and the cloud's "original
103
+ // insert" as concurrent ops, producing duplicated content. So on
104
+ // first contact (no saved state) we skip the disk merge and let
105
+ // loadInitial + initial flush behave the same as before DOU-209.
106
+ const capturedDiskUpdates = [];
107
+ if (this.initialDocState && this.currentDiskText != null) {
108
+ const restored = this.ytext.toString();
109
+ if (this.currentDiskText !== restored) {
110
+ const captureHandler = (update, origin) => {
111
+ if (origin === 'disk') capturedDiskUpdates.push(update);
112
+ };
113
+ this.doc.on('update', captureHandler);
114
+ // Same diff-and-apply shape as applyDiskText, but we call the
115
+ // internal helper directly: applyDiskText is the live-chokidar
116
+ // path and also touches lastDiskText / _emitDocState in ways
117
+ // that aren't appropriate during the startup-merge phase.
118
+ this._applyTextDiff(restored, this.currentDiskText);
119
+ this.doc.off('update', captureHandler);
120
+ }
121
+ }
122
+
123
+ // DOU-204: two-phase provider start. Phase 1 loads snapshot +
124
+ // initial updates without subscribing. We then attach the
125
+ // Y.Text observer that drives the debounced disk write. Phase 2
126
+ // subscribes to remote updates; the first listener fire is the
127
+ // initial query result (re-applies the already-loaded docs
128
+ // idempotently), and any update that lands after loadInitial
129
+ // routes through the now-attached observer instead of mutating
130
+ // Y.Text silently between provider.start and observer attach.
131
+ await this.provider.loadInitial();
132
+ this.observer = () => {
133
+ if (this.stopped) return;
134
+ if (this.flushTimer) clearTimeout(this.flushTimer);
135
+ this.flushTimer = setTimeout(() => this._flushNow(), this.debounceMs);
136
+ };
137
+ this.ytext.observe(this.observer);
138
+ this.provider.startSubscription();
139
+
140
+ // Push the disk-side updates captured above so the store records
141
+ // the offline disk edits and other clients receive them. Y.applyUpdate
142
+ // is idempotent so the daemon's own subscription echo (when the
143
+ // store fires its own append back through subscribeUpdates) is a
144
+ // no-op on our local Y.Doc.
145
+ for (const update of capturedDiskUpdates) {
146
+ try { await this.store.appendUpdate(update); }
147
+ catch (err) { console.error('YjsFileBinding: disk-merge appendUpdate failed:', err && err.message); }
148
+ }
149
+
150
+ // Initial flush: ytext may be non-empty after replay (file has
151
+ // existing Yjs state). Writes the loaded content to disk so the
152
+ // local file is up-to-date the moment the binding is wired in.
153
+ if (this.ytext.length > 0) this._flushNow();
154
+ // Persist the merged baseline so the next restart has the right
155
+ // anchor (the binding's state right now == disk after the flush
156
+ // above; future disk-side edits should diff against this state).
157
+ this._emitDocState();
158
+ }
159
+
160
+ _applyTextDiff(fromText, toText) {
161
+ if (fromText === toText) return;
162
+ const diffs = fastDiff(fromText, toText);
163
+ this.doc.transact(() => {
164
+ let idx = 0;
165
+ for (const [op, str] of diffs) {
166
+ if (op === 0) {
167
+ idx += str.length;
168
+ } else if (op === 1) {
169
+ this.ytext.insert(idx, str);
170
+ idx += str.length;
171
+ } else {
172
+ this.ytext.delete(idx, str.length);
173
+ }
174
+ }
175
+ }, 'disk');
176
+ }
177
+
178
+ _emitDocState() {
179
+ if (this.stopped || !this.onDocStateChange) return;
180
+ try {
181
+ this.onDocStateChange(Y.encodeStateAsUpdate(this.doc));
182
+ } catch (err) {
183
+ console.error('YjsFileBinding: onDocStateChange failed:', err && err.message);
184
+ }
185
+ }
186
+
187
+ // Sync flush used by both the debounced observer and the final
188
+ // drain in stop().
189
+ _flushNow() {
190
+ if (this.stopped) return;
191
+ if (this.flushTimer) {
192
+ clearTimeout(this.flushTimer);
193
+ this.flushTimer = null;
194
+ }
195
+ const text = this.ytext.toString();
196
+ if (text === this.lastDiskText) return;
197
+ this.lastDiskText = text;
198
+ this.onDiskWrite(text);
199
+ // The disk now matches ytext: snapshot the Y.Doc state so the next
200
+ // restart has the right baseline for offline-merge (DOU-209).
201
+ this._emitDocState();
202
+ }
203
+
204
+ // The local disk file changed (chokidar). Diff against the current
205
+ // ytext and apply as a sequence of inserts/deletes inside one
206
+ // transaction. Using a distinct origin ('disk') keeps the provider's
207
+ // local-update listener pushing the resulting yupdate to Firestore.
208
+ applyDiskText(newText) {
209
+ if (this.stopped) return;
210
+ const oldText = this.ytext.toString();
211
+ if (oldText === newText) return;
212
+ this._applyTextDiff(oldText, newText);
213
+ // Suppress the debounced echo: lastDiskText now matches the disk
214
+ // content the caller just observed, so the observer's pending
215
+ // flush is a no-op.
216
+ this.lastDiskText = newText;
217
+ // Snapshot the Y.Doc state so the next restart's baseline includes
218
+ // this disk-originated edit (DOU-209).
219
+ this._emitDocState();
220
+ }
221
+
222
+ async stop() {
223
+ if (this.stopped) return;
224
+ this.stopped = true;
225
+ if (this.observer) {
226
+ this.ytext.unobserve(this.observer);
227
+ this.observer = null;
228
+ }
229
+ if (this.flushTimer) {
230
+ clearTimeout(this.flushTimer);
231
+ this.flushTimer = null;
232
+ }
233
+ this.provider.stop();
234
+ this.doc.destroy();
235
+ }
236
+ }
@@ -0,0 +1,117 @@
1
+ // FirestoreUpdateStore: Firestore-backed store for per-file Yjs
2
+ // CRDT updates. Ported from web/src/yjs/firestore-update-store.ts so
3
+ // the published @doubling/compound-sync package does not reach into
4
+ // the web source tree. Keep the API identical to the web copy; a
5
+ // follow-up will unify the two into a shared workspace package once
6
+ // PR 4 finishes retiring the legacy blob path.
7
+
8
+ import {
9
+ collection,
10
+ doc,
11
+ addDoc,
12
+ getDoc,
13
+ getDocs,
14
+ setDoc,
15
+ query,
16
+ where,
17
+ orderBy,
18
+ onSnapshot,
19
+ serverTimestamp,
20
+ writeBatch,
21
+ Bytes,
22
+ Timestamp,
23
+ } from 'firebase/firestore';
24
+
25
+ function formatId(createdAt, docId) {
26
+ return `${createdAt.toMillis().toString().padStart(16, '0')}_${docId}`;
27
+ }
28
+
29
+ function parseTimestampFromId(id) {
30
+ return Timestamp.fromMillis(Number(id.split('_')[0]));
31
+ }
32
+
33
+ export class FirestoreUpdateStore {
34
+ constructor(db, orgId, fileId) {
35
+ this.updatesCol = collection(db, 'orgs', orgId, 'files', fileId, 'yupdates');
36
+ this.snapshotDoc = doc(db, 'orgs', orgId, 'files', fileId, 'yjs', 'state');
37
+ }
38
+
39
+ async loadSnapshot() {
40
+ const snap = await getDoc(this.snapshotDoc);
41
+ if (!snap.exists()) return null;
42
+ const data = snap.data();
43
+ return {
44
+ state: data.state.toUint8Array(),
45
+ baselineId: data.baselineId ?? null,
46
+ };
47
+ }
48
+
49
+ async loadUpdatesAfter(baselineId) {
50
+ const q =
51
+ baselineId === null
52
+ ? query(this.updatesCol, orderBy('createdAt', 'asc'))
53
+ : query(
54
+ this.updatesCol,
55
+ where('createdAt', '>', parseTimestampFromId(baselineId)),
56
+ orderBy('createdAt', 'asc'),
57
+ );
58
+ const snap = await getDocs(q);
59
+ return snap.docs
60
+ .map((d) => {
61
+ const data = d.data();
62
+ if (!data.createdAt) return null;
63
+ return { id: formatId(data.createdAt, d.id), update: data.update.toUint8Array() };
64
+ })
65
+ .filter((r) => r !== null);
66
+ }
67
+
68
+ async appendUpdate(update) {
69
+ const docRef = await addDoc(this.updatesCol, {
70
+ update: Bytes.fromUint8Array(update),
71
+ createdAt: serverTimestamp(),
72
+ });
73
+ const fresh = await getDoc(docRef);
74
+ const data = fresh.data();
75
+ if (!data || !data.createdAt) {
76
+ throw new Error('FirestoreUpdateStore.appendUpdate: createdAt unresolved');
77
+ }
78
+ return { id: formatId(data.createdAt, docRef.id), update };
79
+ }
80
+
81
+ subscribeUpdates(onUpdate) {
82
+ const q = query(this.updatesCol, orderBy('createdAt', 'asc'));
83
+ // includeMetadataChanges + emitted-id dedup: see the web copy for
84
+ // the full rationale. Skipping pending writes alone drops the
85
+ // server-ack delivery because that fire is metadata-only.
86
+ const emitted = new Set();
87
+ return onSnapshot(q, { includeMetadataChanges: true }, (snap) => {
88
+ for (const change of snap.docChanges({ includeMetadataChanges: true })) {
89
+ if (change.type !== 'added' && change.type !== 'modified') continue;
90
+ const data = change.doc.data();
91
+ if (!data.createdAt) continue;
92
+ if (emitted.has(change.doc.id)) continue;
93
+ emitted.add(change.doc.id);
94
+ onUpdate({
95
+ id: formatId(data.createdAt, change.doc.id),
96
+ update: data.update.toUint8Array(),
97
+ });
98
+ }
99
+ });
100
+ }
101
+
102
+ async saveSnapshotAndCompact(state, newBaselineId) {
103
+ const baselineTs = parseTimestampFromId(newBaselineId);
104
+ await setDoc(this.snapshotDoc, {
105
+ state: Bytes.fromUint8Array(state),
106
+ baselineId: newBaselineId,
107
+ baselineCreatedAt: baselineTs,
108
+ updatedAt: serverTimestamp(),
109
+ });
110
+ const q = query(this.updatesCol, where('createdAt', '<=', baselineTs));
111
+ const stale = await getDocs(q);
112
+ if (stale.empty) return;
113
+ const batch = writeBatch(this.snapshotDoc.firestore);
114
+ for (const d of stale.docs) batch.delete(d.ref);
115
+ await batch.commit();
116
+ }
117
+ }
@@ -0,0 +1,85 @@
1
+ // YjsProvider: binds a Y.Doc to an UpdateStore. Ported from
2
+ // web/src/yjs/provider.ts; see that file for the full design notes
3
+ // (echo handling, snapshot replay, compaction semantics, two-phase
4
+ // start lifecycle from DOU-204). API kept identical so the two copies
5
+ // can be unified into a shared workspace package in a follow-up.
6
+
7
+ import * as Y from 'yjs';
8
+
9
+ export class YjsProvider {
10
+ constructor(doc, store) {
11
+ this.doc = doc;
12
+ this.store = store;
13
+ this.unsubscribeStore = null;
14
+ this.updateHandler = null;
15
+ this.loaded = false;
16
+ this.subscribed = false;
17
+ this.stopped = false;
18
+ }
19
+
20
+ /** Phase 1: snapshot + initial updates → Y.Doc. No remote
21
+ * subscription yet. Callers that intend to attach a Y.Text observer
22
+ * (e.g. a disk-write debouncer here, yCollab on the web) should
23
+ * install that observer between `loadInitial()` and
24
+ * `startSubscription()` so the first remote update routes through
25
+ * to it instead of mutating Y.Text silently in the gap (DOU-204). */
26
+ async loadInitial() {
27
+ if (this.loaded) throw new Error('YjsProvider.loadInitial() called twice');
28
+ this.loaded = true;
29
+
30
+ const snapshot = await this.store.loadSnapshot();
31
+ if (snapshot) Y.applyUpdate(this.doc, snapshot.state, this);
32
+
33
+ const initial = await this.store.loadUpdatesAfter(snapshot?.baselineId ?? null);
34
+ for (const record of initial) Y.applyUpdate(this.doc, record.update, this);
35
+ }
36
+
37
+ /** Phase 2: subscribe to remote updates and push local Y.Doc
38
+ * mutations back to the store. */
39
+ startSubscription() {
40
+ if (!this.loaded) throw new Error('YjsProvider.startSubscription() before loadInitial()');
41
+ if (this.subscribed) throw new Error('YjsProvider.startSubscription() called twice');
42
+ this.subscribed = true;
43
+
44
+ this.unsubscribeStore = this.store.subscribeUpdates((record) => {
45
+ if (this.stopped) return;
46
+ Y.applyUpdate(this.doc, record.update, this);
47
+ });
48
+
49
+ this.updateHandler = (update, origin) => {
50
+ if (this.stopped) return;
51
+ if (origin === this) return;
52
+ this.store.appendUpdate(update).catch((err) => {
53
+ console.error('YjsProvider: appendUpdate failed:', err && err.message);
54
+ });
55
+ };
56
+ this.doc.on('update', this.updateHandler);
57
+ }
58
+
59
+ /** Convenience: load + subscribe in one call. Use only when no
60
+ * view-layer observer needs to attach between the phases. */
61
+ async start() {
62
+ await this.loadInitial();
63
+ this.startSubscription();
64
+ }
65
+
66
+ stop() {
67
+ this.stopped = true;
68
+ if (this.unsubscribeStore) {
69
+ this.unsubscribeStore();
70
+ this.unsubscribeStore = null;
71
+ }
72
+ if (this.updateHandler) {
73
+ this.doc.off('update', this.updateHandler);
74
+ this.updateHandler = null;
75
+ }
76
+ }
77
+
78
+ async compact() {
79
+ const state = Y.encodeStateAsUpdate(this.doc);
80
+ const records = await this.store.loadUpdatesAfter(null);
81
+ const last = records[records.length - 1];
82
+ if (!last) return;
83
+ await this.store.saveSnapshotAndCompact(state, last.id);
84
+ }
85
+ }