@doubling/compound-sync 1.10.5 → 1.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/files.js +95 -45
- package/org-sync.js +274 -60
- package/package.json +19 -6
- package/paths.d.ts +12 -0
- package/paths.js +29 -34
- package/storage-helpers.js +2 -0
- package/sync.js +37 -0
- package/yjs-binding-state.js +61 -0
- package/yjs-file-binding.js +236 -0
- package/yjs-firestore-update-store.js +117 -0
- package/yjs-provider.js +85 -0
package/org-sync.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
import chokidar from 'chokidar';
|
|
16
16
|
import fs from 'fs';
|
|
17
17
|
import path from 'path';
|
|
18
|
-
import { collection, query, where, onSnapshot } from 'firebase/firestore';
|
|
18
|
+
import { collection, query, where, onSnapshot, getDocs } from 'firebase/firestore';
|
|
19
19
|
import {
|
|
20
20
|
pushFileToCloud,
|
|
21
21
|
deleteFileFromCloud,
|
|
@@ -37,6 +37,11 @@ import {
|
|
|
37
37
|
} from './paths.js';
|
|
38
38
|
import { TeamRegistry } from './team-registry.js';
|
|
39
39
|
import { loadSyncState, saveSyncState } from './sync-state.js';
|
|
40
|
+
import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
|
|
41
|
+
import { YjsFileBinding } from './yjs-file-binding.js';
|
|
42
|
+
import {
|
|
43
|
+
loadBindingState, saveBindingState, deleteBindingState,
|
|
44
|
+
} from './yjs-binding-state.js';
|
|
40
45
|
|
|
41
46
|
const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
|
|
42
47
|
|
|
@@ -106,6 +111,122 @@ export async function startOrgSync({
|
|
|
106
111
|
function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
|
|
107
112
|
function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
|
|
108
113
|
const suppressLocal = new Set();
|
|
114
|
+
|
|
115
|
+
// Per-file Yjs bindings, created on first sight of a text file
|
|
116
|
+
// (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
|
|
117
|
+
// orgs/.../yupdates and the debounced disk flush. We keep two
|
|
118
|
+
// indexes: by fileId (so handleFirestoreChange can dispose on
|
|
119
|
+
// 'removed' and re-use across events) and by local path (so the
|
|
120
|
+
// chokidar handler can route disk edits into the binding without
|
|
121
|
+
// re-resolving the fileId).
|
|
122
|
+
const bindings = new Map();
|
|
123
|
+
const bindingsByPath = new Map();
|
|
124
|
+
|
|
125
|
+
// DOU-205: per-fileId record of the most recently observed Firestore
|
|
126
|
+
// metadata (local path + tracking key). Used to detect renames /
|
|
127
|
+
// moves on subsequent 'modified' snapshot events: when the file
|
|
128
|
+
// doc's path changes server-side (via the web rename UI), the local
|
|
129
|
+
// path computed from the new data also changes, and this map lets
|
|
130
|
+
// the handler relocate the local file in place (fs.renameSync,
|
|
131
|
+
// which preserves Obsidian's inode-level tracking) rather than
|
|
132
|
+
// leaving the old file orphaned at its previous location.
|
|
133
|
+
const knownPaths = new Map();
|
|
134
|
+
|
|
135
|
+
// Track which fileIds already have a binding (or one being set
|
|
136
|
+
// up) so we never double-create. `stoppedRef` is read by the
|
|
137
|
+
// background retry loops so they can abandon work after sync.stop.
|
|
138
|
+
// `pendingPromises` lets stop() await any in-flight setup before
|
|
139
|
+
// returning so the next test's clearFirestore does not race
|
|
140
|
+
// dangling onSnapshot streams.
|
|
141
|
+
const pendingBindings = new Set();
|
|
142
|
+
const pendingPromises = new Set();
|
|
143
|
+
const stoppedRef = { value: false };
|
|
144
|
+
|
|
145
|
+
// Fire-and-forget binding setup. Returns nothing because the
|
|
146
|
+
// common shape "ensureBinding then read its ytext" runs straight
|
|
147
|
+
// into a race: the daemon's onSnapshot fires on Alice's own
|
|
148
|
+
// optimistic write before the file doc is server-committed, and
|
|
149
|
+
// the Firestore rules engine evaluates the yupdates/yjs nested
|
|
150
|
+
// rule via a server-side get() against the parent files/{id}
|
|
151
|
+
// doc. That get() sees no doc and returns null, so binding.start
|
|
152
|
+
// fails with permission-denied; the right behavior is to retry
|
|
153
|
+
// until the commit propagates, not to block handleFirestoreChange
|
|
154
|
+
// for seconds. Callers fall back to the blob path while the
|
|
155
|
+
// binding is still being set up; once it lands in
|
|
156
|
+
// bindings/bindingsByPath, subsequent events take the Yjs path.
|
|
157
|
+
function ensureBinding(fileId, localFilePath) {
|
|
158
|
+
if (bindings.has(fileId) || pendingBindings.has(fileId)) return;
|
|
159
|
+
pendingBindings.add(fileId);
|
|
160
|
+
const onDiskWrite = (text) => {
|
|
161
|
+
suppressLocal.add(localFilePath);
|
|
162
|
+
ensureDir(localFilePath);
|
|
163
|
+
fs.writeFileSync(localFilePath, text, 'utf-8');
|
|
164
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
165
|
+
};
|
|
166
|
+
// DOU-209: persist the Y.Doc binary state after every stable
|
|
167
|
+
// change so the next daemon restart has the right baseline to
|
|
168
|
+
// merge offline disk edits without losing offline web edits.
|
|
169
|
+
const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
|
|
170
|
+
// DOU-209: feed the binding's startup-merge with the daemon's
|
|
171
|
+
// last view (initialDocState) and the current disk content. If
|
|
172
|
+
// they diverge, the binding applies the diff as Yjs ops so disk-
|
|
173
|
+
// side offline edits survive the cloud-side replay.
|
|
174
|
+
const initialDocState = loadBindingState(LOCAL_PATH, fileId);
|
|
175
|
+
let currentDiskText = null;
|
|
176
|
+
try {
|
|
177
|
+
if (fs.existsSync(localFilePath)) {
|
|
178
|
+
currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
|
|
179
|
+
}
|
|
180
|
+
} catch (err) {
|
|
181
|
+
console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err && err.message}`);
|
|
182
|
+
}
|
|
183
|
+
const setupPromise = (async () => {
|
|
184
|
+
// Retry binding.start on permission-denied. See note above
|
|
185
|
+
// about the optimistic-write race with the nested-rule get().
|
|
186
|
+
// YjsProvider rejects double-start, so each retry needs a
|
|
187
|
+
// fresh binding (and a fresh provider underneath it).
|
|
188
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
189
|
+
if (stoppedRef.value) return;
|
|
190
|
+
const store = new FirestoreUpdateStore(db, orgId, fileId);
|
|
191
|
+
const binding = new YjsFileBinding(store, {
|
|
192
|
+
onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
|
|
193
|
+
});
|
|
194
|
+
try {
|
|
195
|
+
await binding.start();
|
|
196
|
+
if (stoppedRef.value) {
|
|
197
|
+
await binding.stop().catch(() => {});
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
bindings.set(fileId, binding);
|
|
201
|
+
bindingsByPath.set(localFilePath, binding);
|
|
202
|
+
return;
|
|
203
|
+
} catch (err) {
|
|
204
|
+
if (stoppedRef.value) return;
|
|
205
|
+
if (err?.code !== 'permission-denied' || attempt === 7) {
|
|
206
|
+
console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err && err.message}`);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
})().finally(() => {
|
|
213
|
+
pendingBindings.delete(fileId);
|
|
214
|
+
pendingPromises.delete(setupPromise);
|
|
215
|
+
});
|
|
216
|
+
pendingPromises.add(setupPromise);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async function disposeBinding(fileId, localFilePath) {
|
|
220
|
+
const binding = bindings.get(fileId);
|
|
221
|
+
if (!binding) return;
|
|
222
|
+
bindings.delete(fileId);
|
|
223
|
+
if (localFilePath) bindingsByPath.delete(localFilePath);
|
|
224
|
+
await binding.stop();
|
|
225
|
+
// DOU-209: drop the per-file Y.Doc state so deleted files don't
|
|
226
|
+
// accumulate stale binding-state blobs under .compound-yjs-binding/.
|
|
227
|
+
deleteBindingState(LOCAL_PATH, fileId);
|
|
228
|
+
}
|
|
229
|
+
|
|
109
230
|
// Per-team Firestore-listener unsubscribe handles, keyed by teamId.
|
|
110
231
|
const teamUnsubscribers = new Map();
|
|
111
232
|
// Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
|
|
@@ -188,6 +309,48 @@ export async function startOrgSync({
|
|
|
188
309
|
const trackingKey = `${data.scope || 'team'}:${docPath}`;
|
|
189
310
|
const isFolder = data.type === 'folder';
|
|
190
311
|
|
|
312
|
+
// DOU-205: rename / move detection. On a 'modified' snapshot for
|
|
313
|
+
// a fileId we've seen before, if the computed local path differs
|
|
314
|
+
// from what we last recorded, the server's path field has
|
|
315
|
+
// changed. Move the local file in place (fs.renameSync preserves
|
|
316
|
+
// Obsidian's inode tracking so notes don't lose their metadata,
|
|
317
|
+
// backlinks, or recent-files entries) and migrate the tracking
|
|
318
|
+
// maps from the old key to the new. We do this before the
|
|
319
|
+
// freshness check below because the rename happens irrespective
|
|
320
|
+
// of whether the content changed in the same update.
|
|
321
|
+
const fileId = change.doc.id;
|
|
322
|
+
const prevKnown = knownPaths.get(fileId);
|
|
323
|
+
if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
|
|
324
|
+
suppressLocal.add(prevKnown.localPath);
|
|
325
|
+
suppressLocal.add(localFilePath);
|
|
326
|
+
try {
|
|
327
|
+
if (fs.existsSync(prevKnown.localPath)) {
|
|
328
|
+
ensureDir(localFilePath);
|
|
329
|
+
fs.renameSync(prevKnown.localPath, localFilePath);
|
|
330
|
+
console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
|
|
331
|
+
}
|
|
332
|
+
} catch (err) {
|
|
333
|
+
console.error(` [${orgId}] [firestore → local] RENAME failed: ${err && err.message}`);
|
|
334
|
+
}
|
|
335
|
+
if (lastKnownUpdate.has(prevKnown.trackingKey)) {
|
|
336
|
+
lastKnownUpdate.set(trackingKey, lastKnownUpdate.get(prevKnown.trackingKey));
|
|
337
|
+
lastKnownUpdate.delete(prevKnown.trackingKey);
|
|
338
|
+
}
|
|
339
|
+
if (lastSyncedHash.has(prevKnown.trackingKey)) {
|
|
340
|
+
setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
|
|
341
|
+
deleteBaseline(prevKnown.trackingKey);
|
|
342
|
+
}
|
|
343
|
+
const binding = bindings.get(fileId);
|
|
344
|
+
if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
|
|
345
|
+
bindingsByPath.delete(prevKnown.localPath);
|
|
346
|
+
bindingsByPath.set(localFilePath, binding);
|
|
347
|
+
}
|
|
348
|
+
setTimeout(() => {
|
|
349
|
+
suppressLocal.delete(prevKnown.localPath);
|
|
350
|
+
suppressLocal.delete(localFilePath);
|
|
351
|
+
}, 1000);
|
|
352
|
+
}
|
|
353
|
+
|
|
191
354
|
if (change.type === 'removed') {
|
|
192
355
|
if (isFolder) {
|
|
193
356
|
// Best-effort directory removal. The daemon's own file-delete
|
|
@@ -213,9 +376,27 @@ export async function startOrgSync({
|
|
|
213
376
|
}
|
|
214
377
|
lastKnownUpdate.delete(trackingKey);
|
|
215
378
|
deleteBaseline(trackingKey);
|
|
379
|
+
knownPaths.delete(fileId);
|
|
380
|
+
await disposeBinding(change.doc.id, localFilePath);
|
|
216
381
|
return;
|
|
217
382
|
}
|
|
218
383
|
|
|
384
|
+
// Record (or refresh) the per-fileId path so the next 'modified'
|
|
385
|
+
// event can detect a rename / move by comparing against this.
|
|
386
|
+
knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
|
|
387
|
+
|
|
388
|
+
// Kick off Yjs binding setup for text files. Fire-and-forget so
|
|
389
|
+
// a slow start (retrying past the rules-engine race on a fresh
|
|
390
|
+
// doc) does not stall the legacy blob pull. Once the binding is
|
|
391
|
+
// registered, future events take the Yjs path; until then the
|
|
392
|
+
// blob path still keeps disk in sync.
|
|
393
|
+
if (!isFolder) {
|
|
394
|
+
const ext = extFromPath(docPath);
|
|
395
|
+
const extDerivedMime = mimeTypeFromExt(ext);
|
|
396
|
+
const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
|
|
397
|
+
if (isText) ensureBinding(change.doc.id, localFilePath);
|
|
398
|
+
}
|
|
399
|
+
|
|
219
400
|
const remoteUpdated = data.updatedAt || data.createdAt || '';
|
|
220
401
|
const lastKnown = lastKnownUpdate.get(trackingKey);
|
|
221
402
|
if (lastKnown && lastKnown >= remoteUpdated) return;
|
|
@@ -282,6 +463,23 @@ export async function startOrgSync({
|
|
|
282
463
|
// markers) can apply.
|
|
283
464
|
const extDerivedMime = mimeTypeFromExt(ext);
|
|
284
465
|
const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
|
|
466
|
+
|
|
467
|
+
// Yjs path (DOU-178 PR 3): if a binding for this file is already
|
|
468
|
+
// up and carries non-empty ytext, the debounced flush owns the
|
|
469
|
+
// disk write and the legacy blob pull is skipped. Bindings that
|
|
470
|
+
// are still being set up, or that loaded empty ytext (pre-Yjs
|
|
471
|
+
// files), fall through to the blob path; PR 4 retires the blob
|
|
472
|
+
// path entirely.
|
|
473
|
+
if (isText) {
|
|
474
|
+
const binding = bindings.get(change.doc.id);
|
|
475
|
+
if (binding && binding.ytext.length > 0) {
|
|
476
|
+
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
477
|
+
setBaseline(trackingKey, data.contentHash);
|
|
478
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
285
483
|
// Storage rules require the Firestore doc to exist before they
|
|
286
484
|
// authorize a blob upload, so writers must create the doc first
|
|
287
485
|
// and upload second. That leaves a window where the metadata is
|
|
@@ -402,6 +600,34 @@ export async function startOrgSync({
|
|
|
402
600
|
}
|
|
403
601
|
}
|
|
404
602
|
|
|
603
|
+
// DOU-217 helper: does a doc with this path + matching contentHash
|
|
604
|
+
// exist in Firestore right now? Used to verify a baseline entry
|
|
605
|
+
// before trusting it (so a stale baseline can't silently suppress
|
|
606
|
+
// re-upload of a file whose Firestore doc was deleted out-of-band).
|
|
607
|
+
async function firestoreDocExists({ scope, teamId, docPath, contentHash }) {
|
|
608
|
+
let q;
|
|
609
|
+
if (scope === 'private') {
|
|
610
|
+
q = query(filesRef,
|
|
611
|
+
where('scope', '==', 'private'),
|
|
612
|
+
where('ownerId', '==', userId),
|
|
613
|
+
where('path', '==', docPath));
|
|
614
|
+
} else if (scope === 'team' && teamId) {
|
|
615
|
+
q = query(filesRef,
|
|
616
|
+
where('scope', '==', 'team'),
|
|
617
|
+
where('teamId', '==', teamId),
|
|
618
|
+
where('path', '==', docPath));
|
|
619
|
+
} else {
|
|
620
|
+
return false;
|
|
621
|
+
}
|
|
622
|
+
const snap = await getDocs(q);
|
|
623
|
+
if (snap.empty) return false;
|
|
624
|
+
// Also require the doc's contentHash to match the baseline. A doc
|
|
625
|
+
// with the same path but a different hash means a web edit
|
|
626
|
+
// happened that the daemon didn't observe; treat as "missing" so
|
|
627
|
+
// the push side proceeds and the pull side can reconcile.
|
|
628
|
+
return snap.docs[0].data().contentHash === contentHash;
|
|
629
|
+
}
|
|
630
|
+
|
|
405
631
|
async function pushToFirestore(filePath, data, mimeType, localModifiedAt) {
|
|
406
632
|
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
407
633
|
const trackingKey = `${scope}:${docPath}`;
|
|
@@ -411,8 +637,22 @@ export async function startOrgSync({
|
|
|
411
637
|
// re-saving or touching a file (or the watcher re-emitting it on
|
|
412
638
|
// startup) must not push stale bytes over a newer server edit. Only
|
|
413
639
|
// a genuine content change proceeds.
|
|
640
|
+
//
|
|
641
|
+
// DOU-217: trust the baseline ONLY after verifying the Firestore
|
|
642
|
+
// doc actually exists. The baseline is persisted across daemon
|
|
643
|
+
// restarts, so a doc deleted out-of-band (web delete the daemon
|
|
644
|
+
// missed while offline, manual cleanup, etc.) leaves a permanently
|
|
645
|
+
// stale entry that would silently suppress re-upload forever.
|
|
646
|
+
// Verifying via a single indexed read per touch is cheap; if the
|
|
647
|
+
// doc is gone, clear the baseline and fall through to push.
|
|
414
648
|
const contentHash = computeContentHash(data);
|
|
415
|
-
if (lastSyncedHash.get(trackingKey) === contentHash)
|
|
649
|
+
if (lastSyncedHash.get(trackingKey) === contentHash) {
|
|
650
|
+
if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
653
|
+
console.log(` [${orgId}] [reconcile] dropping stale baseline for ${docPath} (Firestore doc missing); re-pushing`);
|
|
654
|
+
deleteBaseline(trackingKey);
|
|
655
|
+
}
|
|
416
656
|
|
|
417
657
|
const now = new Date().toISOString();
|
|
418
658
|
|
|
@@ -425,12 +665,6 @@ export async function startOrgSync({
|
|
|
425
665
|
// duplicate doc). Pre-setting it suppresses the echo.
|
|
426
666
|
lastKnownUpdate.set(trackingKey, now);
|
|
427
667
|
|
|
428
|
-
// Baseline = what we last saw as the remote contentHash for this
|
|
429
|
-
// path. pushFileToCloud uses it as a compare-and-swap precondition
|
|
430
|
-
// (DOU-167): if the remote has moved past this baseline, refuse
|
|
431
|
-
// the push and surface the conflict to the caller.
|
|
432
|
-
const baseline = lastSyncedHash.get(trackingKey);
|
|
433
|
-
|
|
434
668
|
const result = await pushFileToCloud({
|
|
435
669
|
filesRef,
|
|
436
670
|
storage,
|
|
@@ -443,7 +677,6 @@ export async function startOrgSync({
|
|
|
443
677
|
mimeType,
|
|
444
678
|
now,
|
|
445
679
|
localModifiedAt,
|
|
446
|
-
expectedRemoteHash: baseline,
|
|
447
680
|
});
|
|
448
681
|
|
|
449
682
|
// Record the synced content baseline whenever local matches the
|
|
@@ -456,21 +689,6 @@ export async function startOrgSync({
|
|
|
456
689
|
// its firestore -> local snapshot is allowed through to update the
|
|
457
690
|
// local file.
|
|
458
691
|
lastKnownUpdate.delete(trackingKey);
|
|
459
|
-
} else if (result.action === 'conflict') {
|
|
460
|
-
// Another writer landed an edit in the gap between our last sync
|
|
461
|
-
// and this push. Preserve the user's would-be push as a sibling
|
|
462
|
-
// conflict copy, pull the new remote state into the live file,
|
|
463
|
-
// and advance the baseline so the next push (or further edits)
|
|
464
|
-
// can proceed against the new ground truth.
|
|
465
|
-
const conflictPath = writeConflictCopy(filePath, data);
|
|
466
|
-
await pullRemoteIntoLocal(result.remoteData, filePath);
|
|
467
|
-
setBaseline(trackingKey, result.remoteHash);
|
|
468
|
-
lastKnownUpdate.delete(trackingKey);
|
|
469
|
-
console.log(
|
|
470
|
-
` [${orgId}] [local → firestore] CONFLICT ${docPath}: ` +
|
|
471
|
-
`remote changed; local saved to ${path.basename(conflictPath)}`,
|
|
472
|
-
);
|
|
473
|
-
return;
|
|
474
692
|
}
|
|
475
693
|
|
|
476
694
|
if (result.action === 'created') {
|
|
@@ -485,38 +703,6 @@ export async function startOrgSync({
|
|
|
485
703
|
}
|
|
486
704
|
}
|
|
487
705
|
|
|
488
|
-
/** Write the proposed (rejected) push to a sibling conflict file.
|
|
489
|
-
* Returns the new path. Suppresses chokidar so the conflict file's
|
|
490
|
-
* creation doesn't trigger its own push. */
|
|
491
|
-
function writeConflictCopy(filePath, data) {
|
|
492
|
-
const parsed = path.parse(filePath);
|
|
493
|
-
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
494
|
-
const conflictPath = path.join(parsed.dir, `${parsed.name}.conflict-${ts}${parsed.ext}`);
|
|
495
|
-
suppressLocal.add(conflictPath);
|
|
496
|
-
fs.writeFileSync(conflictPath, data);
|
|
497
|
-
setTimeout(() => suppressLocal.delete(conflictPath), 1000);
|
|
498
|
-
return conflictPath;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
/** Pull the remote blob into the live local path after a conflict.
|
|
502
|
-
* Uses suppressLocal so chokidar doesn't re-emit the write. */
|
|
503
|
-
async function pullRemoteIntoLocal(remoteData, localFilePath) {
|
|
504
|
-
const ext = extFromPath(remoteData.path);
|
|
505
|
-
const extDerivedMime = mimeTypeFromExt(ext);
|
|
506
|
-
const isText = isTextMimeType(remoteData.mimeType) || isTextMimeType(extDerivedMime);
|
|
507
|
-
suppressLocal.add(localFilePath);
|
|
508
|
-
const blobReadArgs = { storage, orgId, fileId: remoteData.id, ext };
|
|
509
|
-
const retryOpts = { expectedHash: remoteData.contentHash };
|
|
510
|
-
if (isText) {
|
|
511
|
-
const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
|
|
512
|
-
fs.writeFileSync(localFilePath, content || '', 'utf-8');
|
|
513
|
-
} else {
|
|
514
|
-
const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
|
|
515
|
-
fs.writeFileSync(localFilePath, bytes);
|
|
516
|
-
}
|
|
517
|
-
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
518
|
-
}
|
|
519
|
-
|
|
520
706
|
async function deleteFromFirestore(filePath) {
|
|
521
707
|
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
522
708
|
const trackingKey = `${scope}:${docPath}`;
|
|
@@ -584,6 +770,8 @@ export async function startOrgSync({
|
|
|
584
770
|
|
|
585
771
|
const result = await deleteFolderFromCloud({
|
|
586
772
|
filesRef,
|
|
773
|
+
storage,
|
|
774
|
+
orgId,
|
|
587
775
|
scope,
|
|
588
776
|
teamId,
|
|
589
777
|
userId,
|
|
@@ -591,7 +779,10 @@ export async function startOrgSync({
|
|
|
591
779
|
});
|
|
592
780
|
|
|
593
781
|
if (result.action === 'deleted') {
|
|
594
|
-
|
|
782
|
+
// childCount > 0 means cascade did real work; useful in logs
|
|
783
|
+
// when investigating future folder-delete reports.
|
|
784
|
+
const childSuffix = result.childCount > 0 ? ` (+${result.childCount} children)` : '';
|
|
785
|
+
console.log(` [${orgId}] [local → firestore] RMFOLDER ${docPath} (${scope})${childSuffix}`);
|
|
595
786
|
}
|
|
596
787
|
}
|
|
597
788
|
|
|
@@ -623,8 +814,21 @@ export async function startOrgSync({
|
|
|
623
814
|
? fs.readFileSync(filePath, 'utf-8')
|
|
624
815
|
: fs.readFileSync(filePath);
|
|
625
816
|
const localModifiedAt = stat.mtime.toISOString();
|
|
817
|
+
|
|
818
|
+
// Yjs path (DOU-178 PR 3): route text edits into the binding so
|
|
819
|
+
// the disk delta is published as a yupdate (preserving concurrent
|
|
820
|
+
// web edits via fast-diff). The binding only exists once the
|
|
821
|
+
// Firestore listener has seen the file; for the very first
|
|
822
|
+
// local-only save the binding hasn't been created yet, and the
|
|
823
|
+
// blob path is still the only writer. Subsequent saves go through
|
|
824
|
+
// both paths until PR 4 retires the blob upload.
|
|
825
|
+
if (isTextMimeType(mimeType)) {
|
|
826
|
+
const binding = bindingsByPath.get(filePath);
|
|
827
|
+
if (binding) binding.applyDiskText(data);
|
|
828
|
+
}
|
|
829
|
+
|
|
626
830
|
pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
|
|
627
|
-
console.error(` [${orgId}] Push error: ${err.message}`)
|
|
831
|
+
console.error(` [${orgId}] Push error for ${rel}: ${err.message}`)
|
|
628
832
|
);
|
|
629
833
|
}
|
|
630
834
|
|
|
@@ -671,7 +875,8 @@ export async function startOrgSync({
|
|
|
671
875
|
watcher.on('change', filePath => handleLocalChange(filePath));
|
|
672
876
|
watcher.on('unlink', filePath => {
|
|
673
877
|
if (suppressLocal.has(filePath)) return;
|
|
674
|
-
|
|
878
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
879
|
+
deleteFromFirestore(filePath).catch(err => console.error(` [${orgId}] Delete error for ${rel}: ${err.message}`));
|
|
675
880
|
});
|
|
676
881
|
watcher.on('addDir', dirPath => handleLocalDirAdded(dirPath));
|
|
677
882
|
watcher.on('unlinkDir', dirPath => handleLocalDirRemoved(dirPath));
|
|
@@ -697,6 +902,7 @@ export async function startOrgSync({
|
|
|
697
902
|
// Tear down all listeners and the watcher. Returns the watcher's
|
|
698
903
|
// close() promise so callers/tests can await full shutdown.
|
|
699
904
|
stop() {
|
|
905
|
+
stoppedRef.value = true;
|
|
700
906
|
for (const unsub of unsubscribers) {
|
|
701
907
|
try { unsub(); } catch (e) { /* ignore */ }
|
|
702
908
|
}
|
|
@@ -704,7 +910,15 @@ export async function startOrgSync({
|
|
|
704
910
|
try { unsub(); } catch (e) { /* ignore */ }
|
|
705
911
|
}
|
|
706
912
|
teamUnsubscribers.clear();
|
|
707
|
-
|
|
913
|
+
const setupDrain = Promise.all([...pendingPromises]);
|
|
914
|
+
const bindingDrain = setupDrain.then(() => Promise.all(
|
|
915
|
+
[...bindings.values()].map((b) => b.stop().catch(() => {})),
|
|
916
|
+
)).then(() => {
|
|
917
|
+
bindings.clear();
|
|
918
|
+
bindingsByPath.clear();
|
|
919
|
+
});
|
|
920
|
+
const watcherClose = watcher ? watcher.close() : Promise.resolve();
|
|
921
|
+
return Promise.all([bindingDrain, watcherClose]).then(() => undefined);
|
|
708
922
|
},
|
|
709
923
|
};
|
|
710
924
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@doubling/compound-sync",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.12.0",
|
|
4
4
|
"description": "Bidirectional sync between Compound and local markdown files",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
"sync-state.js",
|
|
13
13
|
"config.js",
|
|
14
14
|
"paths.js",
|
|
15
|
+
"paths.d.ts",
|
|
15
16
|
"files.js",
|
|
16
17
|
"folder-chain.js",
|
|
17
18
|
"storage-helpers.js",
|
|
@@ -19,21 +20,33 @@
|
|
|
19
20
|
"auth-persistence.js",
|
|
20
21
|
"pre-mint-app-check.js",
|
|
21
22
|
"setup-auth-with-pre-mint.js",
|
|
23
|
+
"yjs-firestore-update-store.js",
|
|
24
|
+
"yjs-provider.js",
|
|
25
|
+
"yjs-file-binding.js",
|
|
26
|
+
"yjs-binding-state.js",
|
|
22
27
|
"README.md"
|
|
23
28
|
],
|
|
24
29
|
"scripts": {
|
|
25
|
-
"sync": "node sync.js",
|
|
26
|
-
"
|
|
27
|
-
"
|
|
30
|
+
"sync": "node --import tsx ./sync.js",
|
|
31
|
+
"build": "tsc",
|
|
32
|
+
"typecheck": "tsc --noEmit",
|
|
33
|
+
"prepack": "npm run build",
|
|
34
|
+
"test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js yjs-file-binding.test.js yjs-binding-state.test.js",
|
|
35
|
+
"test:integration": "firebase emulators:exec --only firestore,storage 'node --import tsx --test --test-concurrency=1 files.test.js org-sync.test.js two-user-sync.test.js'",
|
|
28
36
|
"test": "npm run test:unit && npm run test:integration"
|
|
29
37
|
},
|
|
30
38
|
"dependencies": {
|
|
39
|
+
"chokidar": "^5.0.0",
|
|
40
|
+
"fast-diff": "^1.3.0",
|
|
31
41
|
"firebase": "^12.14.0",
|
|
32
|
-
"
|
|
42
|
+
"yjs": "^13.6.31"
|
|
33
43
|
},
|
|
34
44
|
"devDependencies": {
|
|
35
45
|
"@firebase/rules-unit-testing": "^5.0.1",
|
|
36
|
-
"
|
|
46
|
+
"@types/node": "^26.0.0",
|
|
47
|
+
"firebase-tools": "^15.20.0",
|
|
48
|
+
"tsx": "^4.22.4",
|
|
49
|
+
"typescript": "^5.9.3"
|
|
37
50
|
},
|
|
38
51
|
"publishConfig": {
|
|
39
52
|
"access": "public"
|
package/paths.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare const PRIVATE_FOLDER = "Private";
|
|
2
|
+
export declare const SHARED_WITH_ME_FOLDER = "Shared with Me";
|
|
3
|
+
export declare const SHARED_BY_ME_FOLDER = "Shared by Me";
|
|
4
|
+
export type Scope = 'team' | 'private' | 'shared-with-me' | 'shared-by-me';
|
|
5
|
+
export interface ScopedDocPath {
|
|
6
|
+
scope: Scope;
|
|
7
|
+
teamId?: string;
|
|
8
|
+
docPath: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function teamFolderName(teamName: string): string;
|
|
11
|
+
export declare function scopeFromLocalPath(filePath: string, localPath: string, teamIdsByName: Map<string, string>): ScopedDocPath;
|
|
12
|
+
//# sourceMappingURL=paths.d.ts.map
|
package/paths.js
CHANGED
|
@@ -5,48 +5,43 @@
|
|
|
5
5
|
// expected to be swapped if/when authorization moves to a Zanzibar-
|
|
6
6
|
// style relationship model. Keep this module's interface narrow so the
|
|
7
7
|
// swap stays localized.
|
|
8
|
-
|
|
9
8
|
import path from 'node:path';
|
|
10
|
-
|
|
11
9
|
export const PRIVATE_FOLDER = 'Private';
|
|
12
10
|
export const SHARED_WITH_ME_FOLDER = 'Shared with Me';
|
|
13
11
|
export const SHARED_BY_ME_FOLDER = 'Shared by Me';
|
|
14
12
|
const TEAMSPACE_SUFFIX = ' Teamspace';
|
|
15
|
-
|
|
16
13
|
export function teamFolderName(teamName) {
|
|
17
|
-
|
|
14
|
+
return teamName + TEAMSPACE_SUFFIX;
|
|
18
15
|
}
|
|
19
|
-
|
|
20
16
|
// Given an absolute file path within one of the daemon's local sync
|
|
21
17
|
// roots, return { scope, teamId?, docPath }. The caller passes the
|
|
22
|
-
// matching localPath root and the team-name
|
|
18
|
+
// matching localPath root and the team-name -> team-id map.
|
|
23
19
|
export function scopeFromLocalPath(filePath, localPath, teamIdsByName) {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
:
|
|
50
|
-
const teamId = teamIdsByName.get(teamName);
|
|
51
|
-
return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
|
|
20
|
+
const rel = path.relative(localPath, filePath);
|
|
21
|
+
if (rel.startsWith(PRIVATE_FOLDER + path.sep)) {
|
|
22
|
+
return {
|
|
23
|
+
scope: 'private',
|
|
24
|
+
docPath: rel.slice(PRIVATE_FOLDER.length + 1),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) {
|
|
28
|
+
return {
|
|
29
|
+
scope: 'shared-with-me',
|
|
30
|
+
docPath: rel.slice(SHARED_WITH_ME_FOLDER.length + 1),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) {
|
|
34
|
+
return {
|
|
35
|
+
scope: 'shared-by-me',
|
|
36
|
+
docPath: rel.slice(SHARED_BY_ME_FOLDER.length + 1),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const parts = rel.split(path.sep);
|
|
40
|
+
const folderName = parts[0] ?? '';
|
|
41
|
+
const teamName = folderName.endsWith(TEAMSPACE_SUFFIX)
|
|
42
|
+
? folderName.slice(0, -TEAMSPACE_SUFFIX.length)
|
|
43
|
+
: folderName;
|
|
44
|
+
const teamId = teamIdsByName.get(teamName);
|
|
45
|
+
return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
|
|
52
46
|
}
|
|
47
|
+
//# sourceMappingURL=paths.js.map
|
package/storage-helpers.js
CHANGED
package/sync.js
CHANGED
|
@@ -16,6 +16,43 @@
|
|
|
16
16
|
* node sync.js --help
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
// DOU-208: Node 22+ ships its own CA bundle and does not consult the
|
|
20
|
+
// macOS system trust store. Without NODE_EXTRA_CA_CERTS pointing at
|
|
21
|
+
// /etc/ssl/cert.pem, fetch() to securetoken.googleapis.com fails
|
|
22
|
+
// with a bare `fetch failed`, which Firebase Auth then surfaces as
|
|
23
|
+
// auth/network-request-failed. The web e2e harness's run-emulator.js
|
|
24
|
+
// already does this for child processes; here we need it for our own
|
|
25
|
+
// process. NODE_EXTRA_CA_CERTS is read at startup, so setting it on
|
|
26
|
+
// process.env after init is a no-op — instead, re-exec ourselves
|
|
27
|
+
// with the env var set when missing. The child inherits stdio +
|
|
28
|
+
// argv; we exit with its status. No-op when already set (the user
|
|
29
|
+
// or a wrapper explicitly chose a different bundle).
|
|
30
|
+
import { spawnSync as __dou208_spawnSync } from 'node:child_process';
|
|
31
|
+
import { existsSync as __dou208_existsSync } from 'node:fs';
|
|
32
|
+
if (!process.env.NODE_EXTRA_CA_CERTS) {
|
|
33
|
+
const __dou208_candidates = process.platform === 'darwin'
|
|
34
|
+
? ['/etc/ssl/cert.pem']
|
|
35
|
+
: process.platform === 'linux'
|
|
36
|
+
? ['/etc/ssl/certs/ca-certificates.crt', '/etc/pki/tls/certs/ca-bundle.crt']
|
|
37
|
+
: [];
|
|
38
|
+
for (const __dou208_candidate of __dou208_candidates) {
|
|
39
|
+
if (__dou208_existsSync(__dou208_candidate)) {
|
|
40
|
+
const __dou208_result = __dou208_spawnSync(
|
|
41
|
+
process.execPath,
|
|
42
|
+
process.argv.slice(1),
|
|
43
|
+
{
|
|
44
|
+
stdio: 'inherit',
|
|
45
|
+
env: { ...process.env, NODE_EXTRA_CA_CERTS: __dou208_candidate },
|
|
46
|
+
},
|
|
47
|
+
);
|
|
48
|
+
process.exit(__dou208_result.status ?? 1);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// No candidate found (Windows, unusual Linux distro): fall through
|
|
52
|
+
// and let Node's bundled CA bundle handle TLS. The user can still
|
|
53
|
+
// set NODE_EXTRA_CA_CERTS manually for corporate cert chains.
|
|
54
|
+
}
|
|
55
|
+
|
|
19
56
|
import { initializeApp } from 'firebase/app';
|
|
20
57
|
import { getAuth, initializeAuth, signInWithEmailAndPassword, signInWithCredential, GoogleAuthProvider } from 'firebase/auth';
|
|
21
58
|
import {
|