@doubling/compound-sync 1.10.4 → 1.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/files.js +0 -24
- package/org-sync.js +224 -57
- package/package.json +19 -6
- package/paths.d.ts +12 -0
- package/paths.js +29 -34
- package/storage-helpers.js +2 -0
- package/sync.js +37 -0
- package/yjs-binding-state.js +61 -0
- package/yjs-file-binding.js +236 -0
- package/yjs-firestore-update-store.js +117 -0
- package/yjs-provider.js +85 -0
package/README.md
CHANGED
|
@@ -14,6 +14,16 @@ That's it. A browser window will open for sign-in (Google or email/password), th
|
|
|
14
14
|
|
|
15
15
|
- [Node.js](https://nodejs.org/) (v18 or later)
|
|
16
16
|
|
|
17
|
+
### TLS certificates
|
|
18
|
+
|
|
19
|
+
On Node 22+ the daemon auto-detects the OS system certificate store (`/etc/ssl/cert.pem` on macOS, `/etc/ssl/certs/ca-certificates.crt` or `/etc/pki/tls/certs/ca-bundle.crt` on Linux) and re-execs itself with `NODE_EXTRA_CA_CERTS` set; otherwise `fetch()` to Google Identity / Firebase Auth endpoints fails with `auth/network-request-failed` because Node's bundled CA store doesn't match what those services use.
|
|
20
|
+
|
|
21
|
+
If you're behind a corporate proxy with custom roots, set `NODE_EXTRA_CA_CERTS` explicitly to your bundle and the daemon will use that instead:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
NODE_EXTRA_CA_CERTS=/path/to/corporate-bundle.pem npx @doubling/compound-sync
|
|
25
|
+
```
|
|
26
|
+
|
|
17
27
|
## What it does
|
|
18
28
|
|
|
19
29
|
Compound Sync watches a local folder and your Compound workspace simultaneously. Changes in either direction are synced automatically:
|
|
@@ -95,6 +105,19 @@ npm run test:integration
|
|
|
95
105
|
npm test
|
|
96
106
|
```
|
|
97
107
|
|
|
108
|
+
### TypeScript conventions (DOU-181)
|
|
109
|
+
|
|
110
|
+
`sync/` is being migrated to TypeScript file-by-file. The conventions:
|
|
111
|
+
|
|
112
|
+
- **Source layout: in-place.** Each `.ts` file emits `.js`, `.d.ts`, and source maps as siblings via `tsc` with `outDir: "."`. The emitted `.js` is a gitignored build artifact, never edited by hand; the `.ts` is the source.
|
|
113
|
+
- **Imports always use the `.js` extension** (NodeNext convention). At runtime: in dev/test the `tsx` loader maps `./paths.js` to `./paths.ts` so source runs without a build step; in production the compiled `paths.js` exists alongside and resolves directly.
|
|
114
|
+
- **Dev (no build needed):** `npm run sync` runs through `node --import tsx`, so `.ts` files load on demand.
|
|
115
|
+
- **CI / publish:** `npm run build` compiles every `.ts` source to its `.js` sibling. `prepack` runs build automatically before `npm publish` so the npm tarball ships compiled JS. The desktop staging script also runs `npm run -w sync build` before copying.
|
|
116
|
+
- **Typecheck:** `npm run typecheck` (alias for `tsc --noEmit`) at the workspace level, or at repo root `npm run typecheck` which runs web + sync together.
|
|
117
|
+
- **Strictness:** `strict: true`, `noUncheckedIndexedAccess: true`, `noImplicitOverride: true`. No `any`. No `// @ts-ignore`. If a third-party module lacks types, declare its shape in a `.d.ts` or PR types upstream.
|
|
118
|
+
|
|
119
|
+
Phase 1 migrated `paths.js` to `paths.ts` as proof of pattern. Phase 3 ([[DOU-183]]) migrates the remaining source files; everything in `sync/` will be `.ts` by the end.
|
|
120
|
+
|
|
98
121
|
### Publishing to npm
|
|
99
122
|
|
|
100
123
|
After making changes to the sync daemon, publish a new version:
|
package/files.js
CHANGED
|
@@ -291,13 +291,6 @@ export async function pushFileToCloud({
|
|
|
291
291
|
mimeType,
|
|
292
292
|
now = new Date().toISOString(),
|
|
293
293
|
localModifiedAt = null,
|
|
294
|
-
// The contentHash this caller saw as "current" when it last pulled.
|
|
295
|
-
// If the existing remote doc has a different contentHash, another
|
|
296
|
-
// writer landed an edit in the meantime; we refuse to overwrite and
|
|
297
|
-
// return action='conflict' so the caller can write a conflict copy
|
|
298
|
-
// and pull the new remote state. Undefined disables the check (used
|
|
299
|
-
// for create-from-nothing where no baseline exists).
|
|
300
|
-
expectedRemoteHash = undefined,
|
|
301
294
|
}) {
|
|
302
295
|
if (scope === 'shared-with-me' || scope === 'shared-by-me') {
|
|
303
296
|
return { action: 'skipped', reason: 'shared-scope' };
|
|
@@ -379,23 +372,6 @@ export async function pushFileToCloud({
|
|
|
379
372
|
&& (existing.parentId == null ? null : existing.parentId) === parentId) {
|
|
380
373
|
return { action: 'noop', fileId: docSnap.id };
|
|
381
374
|
}
|
|
382
|
-
// Compare-and-swap (DOU-167): if the caller passed a baseline of
|
|
383
|
-
// what they last saw as current, refuse the push when the remote
|
|
384
|
-
// has moved on. The daemon will write the proposed local content
|
|
385
|
-
// to a sibling conflict file and pull the new remote state into
|
|
386
|
-
// the live path. Without this guard, a stale push silently
|
|
387
|
-
// overwrites another user's freshly-saved edits (data loss).
|
|
388
|
-
if (
|
|
389
|
-
expectedRemoteHash !== undefined &&
|
|
390
|
-
existing.contentHash !== expectedRemoteHash
|
|
391
|
-
) {
|
|
392
|
-
return {
|
|
393
|
-
action: 'conflict',
|
|
394
|
-
fileId: docSnap.id,
|
|
395
|
-
remoteHash: existing.contentHash,
|
|
396
|
-
remoteData: { id: docSnap.id, ...existing },
|
|
397
|
-
};
|
|
398
|
-
}
|
|
399
375
|
// Last-write-wins: never let an older local file overwrite a newer
|
|
400
376
|
// server edit. The caller (sync daemon) passes the local file's mtime;
|
|
401
377
|
// if the server doc was updated more recently, leave it for the
|
package/org-sync.js
CHANGED
|
@@ -37,6 +37,11 @@ import {
|
|
|
37
37
|
} from './paths.js';
|
|
38
38
|
import { TeamRegistry } from './team-registry.js';
|
|
39
39
|
import { loadSyncState, saveSyncState } from './sync-state.js';
|
|
40
|
+
import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
|
|
41
|
+
import { YjsFileBinding } from './yjs-file-binding.js';
|
|
42
|
+
import {
|
|
43
|
+
loadBindingState, saveBindingState, deleteBindingState,
|
|
44
|
+
} from './yjs-binding-state.js';
|
|
40
45
|
|
|
41
46
|
const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
|
|
42
47
|
|
|
@@ -106,6 +111,122 @@ export async function startOrgSync({
|
|
|
106
111
|
function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
|
|
107
112
|
function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
|
|
108
113
|
const suppressLocal = new Set();
|
|
114
|
+
|
|
115
|
+
// Per-file Yjs bindings, created on first sight of a text file
|
|
116
|
+
// (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
|
|
117
|
+
// orgs/.../yupdates and the debounced disk flush. We keep two
|
|
118
|
+
// indexes: by fileId (so handleFirestoreChange can dispose on
|
|
119
|
+
// 'removed' and re-use across events) and by local path (so the
|
|
120
|
+
// chokidar handler can route disk edits into the binding without
|
|
121
|
+
// re-resolving the fileId).
|
|
122
|
+
const bindings = new Map();
|
|
123
|
+
const bindingsByPath = new Map();
|
|
124
|
+
|
|
125
|
+
// DOU-205: per-fileId record of the most recently observed Firestore
|
|
126
|
+
// metadata (local path + tracking key). Used to detect renames /
|
|
127
|
+
// moves on subsequent 'modified' snapshot events: when the file
|
|
128
|
+
// doc's path changes server-side (via the web rename UI), the local
|
|
129
|
+
// path computed from the new data also changes, and this map lets
|
|
130
|
+
// the handler relocate the local file in place (fs.renameSync,
|
|
131
|
+
// which preserves Obsidian's inode-level tracking) rather than
|
|
132
|
+
// leaving the old file orphaned at its previous location.
|
|
133
|
+
const knownPaths = new Map();
|
|
134
|
+
|
|
135
|
+
// Track which fileIds already have a binding (or one being set
|
|
136
|
+
// up) so we never double-create. `stoppedRef` is read by the
|
|
137
|
+
// background retry loops so they can abandon work after sync.stop.
|
|
138
|
+
// `pendingPromises` lets stop() await any in-flight setup before
|
|
139
|
+
// returning so the next test's clearFirestore does not race
|
|
140
|
+
// dangling onSnapshot streams.
|
|
141
|
+
const pendingBindings = new Set();
|
|
142
|
+
const pendingPromises = new Set();
|
|
143
|
+
const stoppedRef = { value: false };
|
|
144
|
+
|
|
145
|
+
// Fire-and-forget binding setup. Returns nothing because the
|
|
146
|
+
// common shape "ensureBinding then read its ytext" runs straight
|
|
147
|
+
// into a race: the daemon's onSnapshot fires on Alice's own
|
|
148
|
+
// optimistic write before the file doc is server-committed, and
|
|
149
|
+
// the Firestore rules engine evaluates the yupdates/yjs nested
|
|
150
|
+
// rule via a server-side get() against the parent files/{id}
|
|
151
|
+
// doc. That get() sees no doc and returns null, so binding.start
|
|
152
|
+
// fails with permission-denied; the right behavior is to retry
|
|
153
|
+
// until the commit propagates, not to block handleFirestoreChange
|
|
154
|
+
// for seconds. Callers fall back to the blob path while the
|
|
155
|
+
// binding is still being set up; once it lands in
|
|
156
|
+
// bindings/bindingsByPath, subsequent events take the Yjs path.
|
|
157
|
+
function ensureBinding(fileId, localFilePath) {
|
|
158
|
+
if (bindings.has(fileId) || pendingBindings.has(fileId)) return;
|
|
159
|
+
pendingBindings.add(fileId);
|
|
160
|
+
const onDiskWrite = (text) => {
|
|
161
|
+
suppressLocal.add(localFilePath);
|
|
162
|
+
ensureDir(localFilePath);
|
|
163
|
+
fs.writeFileSync(localFilePath, text, 'utf-8');
|
|
164
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
165
|
+
};
|
|
166
|
+
// DOU-209: persist the Y.Doc binary state after every stable
|
|
167
|
+
// change so the next daemon restart has the right baseline to
|
|
168
|
+
// merge offline disk edits without losing offline web edits.
|
|
169
|
+
const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
|
|
170
|
+
// DOU-209: feed the binding's startup-merge with the daemon's
|
|
171
|
+
// last view (initialDocState) and the current disk content. If
|
|
172
|
+
// they diverge, the binding applies the diff as Yjs ops so disk-
|
|
173
|
+
// side offline edits survive the cloud-side replay.
|
|
174
|
+
const initialDocState = loadBindingState(LOCAL_PATH, fileId);
|
|
175
|
+
let currentDiskText = null;
|
|
176
|
+
try {
|
|
177
|
+
if (fs.existsSync(localFilePath)) {
|
|
178
|
+
currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
|
|
179
|
+
}
|
|
180
|
+
} catch (err) {
|
|
181
|
+
console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err && err.message}`);
|
|
182
|
+
}
|
|
183
|
+
const setupPromise = (async () => {
|
|
184
|
+
// Retry binding.start on permission-denied. See note above
|
|
185
|
+
// about the optimistic-write race with the nested-rule get().
|
|
186
|
+
// YjsProvider rejects double-start, so each retry needs a
|
|
187
|
+
// fresh binding (and a fresh provider underneath it).
|
|
188
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
189
|
+
if (stoppedRef.value) return;
|
|
190
|
+
const store = new FirestoreUpdateStore(db, orgId, fileId);
|
|
191
|
+
const binding = new YjsFileBinding(store, {
|
|
192
|
+
onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
|
|
193
|
+
});
|
|
194
|
+
try {
|
|
195
|
+
await binding.start();
|
|
196
|
+
if (stoppedRef.value) {
|
|
197
|
+
await binding.stop().catch(() => {});
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
bindings.set(fileId, binding);
|
|
201
|
+
bindingsByPath.set(localFilePath, binding);
|
|
202
|
+
return;
|
|
203
|
+
} catch (err) {
|
|
204
|
+
if (stoppedRef.value) return;
|
|
205
|
+
if (err?.code !== 'permission-denied' || attempt === 7) {
|
|
206
|
+
console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err && err.message}`);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
})().finally(() => {
|
|
213
|
+
pendingBindings.delete(fileId);
|
|
214
|
+
pendingPromises.delete(setupPromise);
|
|
215
|
+
});
|
|
216
|
+
pendingPromises.add(setupPromise);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async function disposeBinding(fileId, localFilePath) {
|
|
220
|
+
const binding = bindings.get(fileId);
|
|
221
|
+
if (!binding) return;
|
|
222
|
+
bindings.delete(fileId);
|
|
223
|
+
if (localFilePath) bindingsByPath.delete(localFilePath);
|
|
224
|
+
await binding.stop();
|
|
225
|
+
// DOU-209: drop the per-file Y.Doc state so deleted files don't
|
|
226
|
+
// accumulate stale binding-state blobs under .compound-yjs-binding/.
|
|
227
|
+
deleteBindingState(LOCAL_PATH, fileId);
|
|
228
|
+
}
|
|
229
|
+
|
|
109
230
|
// Per-team Firestore-listener unsubscribe handles, keyed by teamId.
|
|
110
231
|
const teamUnsubscribers = new Map();
|
|
111
232
|
// Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
|
|
@@ -188,6 +309,48 @@ export async function startOrgSync({
|
|
|
188
309
|
const trackingKey = `${data.scope || 'team'}:${docPath}`;
|
|
189
310
|
const isFolder = data.type === 'folder';
|
|
190
311
|
|
|
312
|
+
// DOU-205: rename / move detection. On a 'modified' snapshot for
|
|
313
|
+
// a fileId we've seen before, if the computed local path differs
|
|
314
|
+
// from what we last recorded, the server's path field has
|
|
315
|
+
// changed. Move the local file in place (fs.renameSync preserves
|
|
316
|
+
// Obsidian's inode tracking so notes don't lose their metadata,
|
|
317
|
+
// backlinks, or recent-files entries) and migrate the tracking
|
|
318
|
+
// maps from the old key to the new. We do this before the
|
|
319
|
+
// freshness check below because the rename happens irrespective
|
|
320
|
+
// of whether the content changed in the same update.
|
|
321
|
+
const fileId = change.doc.id;
|
|
322
|
+
const prevKnown = knownPaths.get(fileId);
|
|
323
|
+
if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
|
|
324
|
+
suppressLocal.add(prevKnown.localPath);
|
|
325
|
+
suppressLocal.add(localFilePath);
|
|
326
|
+
try {
|
|
327
|
+
if (fs.existsSync(prevKnown.localPath)) {
|
|
328
|
+
ensureDir(localFilePath);
|
|
329
|
+
fs.renameSync(prevKnown.localPath, localFilePath);
|
|
330
|
+
console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
|
|
331
|
+
}
|
|
332
|
+
} catch (err) {
|
|
333
|
+
console.error(` [${orgId}] [firestore → local] RENAME failed: ${err && err.message}`);
|
|
334
|
+
}
|
|
335
|
+
if (lastKnownUpdate.has(prevKnown.trackingKey)) {
|
|
336
|
+
lastKnownUpdate.set(trackingKey, lastKnownUpdate.get(prevKnown.trackingKey));
|
|
337
|
+
lastKnownUpdate.delete(prevKnown.trackingKey);
|
|
338
|
+
}
|
|
339
|
+
if (lastSyncedHash.has(prevKnown.trackingKey)) {
|
|
340
|
+
setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
|
|
341
|
+
deleteBaseline(prevKnown.trackingKey);
|
|
342
|
+
}
|
|
343
|
+
const binding = bindings.get(fileId);
|
|
344
|
+
if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
|
|
345
|
+
bindingsByPath.delete(prevKnown.localPath);
|
|
346
|
+
bindingsByPath.set(localFilePath, binding);
|
|
347
|
+
}
|
|
348
|
+
setTimeout(() => {
|
|
349
|
+
suppressLocal.delete(prevKnown.localPath);
|
|
350
|
+
suppressLocal.delete(localFilePath);
|
|
351
|
+
}, 1000);
|
|
352
|
+
}
|
|
353
|
+
|
|
191
354
|
if (change.type === 'removed') {
|
|
192
355
|
if (isFolder) {
|
|
193
356
|
// Best-effort directory removal. The daemon's own file-delete
|
|
@@ -213,9 +376,27 @@ export async function startOrgSync({
|
|
|
213
376
|
}
|
|
214
377
|
lastKnownUpdate.delete(trackingKey);
|
|
215
378
|
deleteBaseline(trackingKey);
|
|
379
|
+
knownPaths.delete(fileId);
|
|
380
|
+
await disposeBinding(change.doc.id, localFilePath);
|
|
216
381
|
return;
|
|
217
382
|
}
|
|
218
383
|
|
|
384
|
+
// Record (or refresh) the per-fileId path so the next 'modified'
|
|
385
|
+
// event can detect a rename / move by comparing against this.
|
|
386
|
+
knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
|
|
387
|
+
|
|
388
|
+
// Kick off Yjs binding setup for text files. Fire-and-forget so
|
|
389
|
+
// a slow start (retrying past the rules-engine race on a fresh
|
|
390
|
+
// doc) does not stall the legacy blob pull. Once the binding is
|
|
391
|
+
// registered, future events take the Yjs path; until then the
|
|
392
|
+
// blob path still keeps disk in sync.
|
|
393
|
+
if (!isFolder) {
|
|
394
|
+
const ext = extFromPath(docPath);
|
|
395
|
+
const extDerivedMime = mimeTypeFromExt(ext);
|
|
396
|
+
const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
|
|
397
|
+
if (isText) ensureBinding(change.doc.id, localFilePath);
|
|
398
|
+
}
|
|
399
|
+
|
|
219
400
|
const remoteUpdated = data.updatedAt || data.createdAt || '';
|
|
220
401
|
const lastKnown = lastKnownUpdate.get(trackingKey);
|
|
221
402
|
if (lastKnown && lastKnown >= remoteUpdated) return;
|
|
@@ -282,6 +463,23 @@ export async function startOrgSync({
|
|
|
282
463
|
// markers) can apply.
|
|
283
464
|
const extDerivedMime = mimeTypeFromExt(ext);
|
|
284
465
|
const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
|
|
466
|
+
|
|
467
|
+
// Yjs path (DOU-178 PR 3): if a binding for this file is already
|
|
468
|
+
// up and carries non-empty ytext, the debounced flush owns the
|
|
469
|
+
// disk write and the legacy blob pull is skipped. Bindings that
|
|
470
|
+
// are still being set up, or that loaded empty ytext (pre-Yjs
|
|
471
|
+
// files), fall through to the blob path; PR 4 retires the blob
|
|
472
|
+
// path entirely.
|
|
473
|
+
if (isText) {
|
|
474
|
+
const binding = bindings.get(change.doc.id);
|
|
475
|
+
if (binding && binding.ytext.length > 0) {
|
|
476
|
+
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
477
|
+
setBaseline(trackingKey, data.contentHash);
|
|
478
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
|
|
285
483
|
// Storage rules require the Firestore doc to exist before they
|
|
286
484
|
// authorize a blob upload, so writers must create the doc first
|
|
287
485
|
// and upload second. That leaves a window where the metadata is
|
|
@@ -425,12 +623,6 @@ export async function startOrgSync({
|
|
|
425
623
|
// duplicate doc). Pre-setting it suppresses the echo.
|
|
426
624
|
lastKnownUpdate.set(trackingKey, now);
|
|
427
625
|
|
|
428
|
-
// Baseline = what we last saw as the remote contentHash for this
|
|
429
|
-
// path. pushFileToCloud uses it as a compare-and-swap precondition
|
|
430
|
-
// (DOU-167): if the remote has moved past this baseline, refuse
|
|
431
|
-
// the push and surface the conflict to the caller.
|
|
432
|
-
const baseline = lastSyncedHash.get(trackingKey);
|
|
433
|
-
|
|
434
626
|
const result = await pushFileToCloud({
|
|
435
627
|
filesRef,
|
|
436
628
|
storage,
|
|
@@ -443,7 +635,6 @@ export async function startOrgSync({
|
|
|
443
635
|
mimeType,
|
|
444
636
|
now,
|
|
445
637
|
localModifiedAt,
|
|
446
|
-
expectedRemoteHash: baseline,
|
|
447
638
|
});
|
|
448
639
|
|
|
449
640
|
// Record the synced content baseline whenever local matches the
|
|
@@ -456,21 +647,6 @@ export async function startOrgSync({
|
|
|
456
647
|
// its firestore -> local snapshot is allowed through to update the
|
|
457
648
|
// local file.
|
|
458
649
|
lastKnownUpdate.delete(trackingKey);
|
|
459
|
-
} else if (result.action === 'conflict') {
|
|
460
|
-
// Another writer landed an edit in the gap between our last sync
|
|
461
|
-
// and this push. Preserve the user's would-be push as a sibling
|
|
462
|
-
// conflict copy, pull the new remote state into the live file,
|
|
463
|
-
// and advance the baseline so the next push (or further edits)
|
|
464
|
-
// can proceed against the new ground truth.
|
|
465
|
-
const conflictPath = writeConflictCopy(filePath, data);
|
|
466
|
-
await pullRemoteIntoLocal(result.remoteData, filePath);
|
|
467
|
-
setBaseline(trackingKey, result.remoteHash);
|
|
468
|
-
lastKnownUpdate.delete(trackingKey);
|
|
469
|
-
console.log(
|
|
470
|
-
` [${orgId}] [local → firestore] CONFLICT ${docPath}: ` +
|
|
471
|
-
`remote changed; local saved to ${path.basename(conflictPath)}`,
|
|
472
|
-
);
|
|
473
|
-
return;
|
|
474
650
|
}
|
|
475
651
|
|
|
476
652
|
if (result.action === 'created') {
|
|
@@ -485,38 +661,6 @@ export async function startOrgSync({
|
|
|
485
661
|
}
|
|
486
662
|
}
|
|
487
663
|
|
|
488
|
-
/** Write the proposed (rejected) push to a sibling conflict file.
|
|
489
|
-
* Returns the new path. Suppresses chokidar so the conflict file's
|
|
490
|
-
* creation doesn't trigger its own push. */
|
|
491
|
-
function writeConflictCopy(filePath, data) {
|
|
492
|
-
const parsed = path.parse(filePath);
|
|
493
|
-
const ts = new Date().toISOString().replace(/[:.]/g, '-');
|
|
494
|
-
const conflictPath = path.join(parsed.dir, `${parsed.name}.conflict-${ts}${parsed.ext}`);
|
|
495
|
-
suppressLocal.add(conflictPath);
|
|
496
|
-
fs.writeFileSync(conflictPath, data);
|
|
497
|
-
setTimeout(() => suppressLocal.delete(conflictPath), 1000);
|
|
498
|
-
return conflictPath;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
/** Pull the remote blob into the live local path after a conflict.
|
|
502
|
-
* Uses suppressLocal so chokidar doesn't re-emit the write. */
|
|
503
|
-
async function pullRemoteIntoLocal(remoteData, localFilePath) {
|
|
504
|
-
const ext = extFromPath(remoteData.path);
|
|
505
|
-
const extDerivedMime = mimeTypeFromExt(ext);
|
|
506
|
-
const isText = isTextMimeType(remoteData.mimeType) || isTextMimeType(extDerivedMime);
|
|
507
|
-
suppressLocal.add(localFilePath);
|
|
508
|
-
const blobReadArgs = { storage, orgId, fileId: remoteData.id, ext };
|
|
509
|
-
const retryOpts = { expectedHash: remoteData.contentHash };
|
|
510
|
-
if (isText) {
|
|
511
|
-
const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
|
|
512
|
-
fs.writeFileSync(localFilePath, content || '', 'utf-8');
|
|
513
|
-
} else {
|
|
514
|
-
const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
|
|
515
|
-
fs.writeFileSync(localFilePath, bytes);
|
|
516
|
-
}
|
|
517
|
-
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
518
|
-
}
|
|
519
|
-
|
|
520
664
|
async function deleteFromFirestore(filePath) {
|
|
521
665
|
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
522
666
|
const trackingKey = `${scope}:${docPath}`;
|
|
@@ -623,8 +767,21 @@ export async function startOrgSync({
|
|
|
623
767
|
? fs.readFileSync(filePath, 'utf-8')
|
|
624
768
|
: fs.readFileSync(filePath);
|
|
625
769
|
const localModifiedAt = stat.mtime.toISOString();
|
|
770
|
+
|
|
771
|
+
// Yjs path (DOU-178 PR 3): route text edits into the binding so
|
|
772
|
+
// the disk delta is published as a yupdate (preserving concurrent
|
|
773
|
+
// web edits via fast-diff). The binding only exists once the
|
|
774
|
+
// Firestore listener has seen the file; for the very first
|
|
775
|
+
// local-only save the binding hasn't been created yet, and the
|
|
776
|
+
// blob path is still the only writer. Subsequent saves go through
|
|
777
|
+
// both paths until PR 4 retires the blob upload.
|
|
778
|
+
if (isTextMimeType(mimeType)) {
|
|
779
|
+
const binding = bindingsByPath.get(filePath);
|
|
780
|
+
if (binding) binding.applyDiskText(data);
|
|
781
|
+
}
|
|
782
|
+
|
|
626
783
|
pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
|
|
627
|
-
console.error(` [${orgId}] Push error: ${err.message}`)
|
|
784
|
+
console.error(` [${orgId}] Push error for ${rel}: ${err.message}`)
|
|
628
785
|
);
|
|
629
786
|
}
|
|
630
787
|
|
|
@@ -671,7 +828,8 @@ export async function startOrgSync({
|
|
|
671
828
|
watcher.on('change', filePath => handleLocalChange(filePath));
|
|
672
829
|
watcher.on('unlink', filePath => {
|
|
673
830
|
if (suppressLocal.has(filePath)) return;
|
|
674
|
-
|
|
831
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
832
|
+
deleteFromFirestore(filePath).catch(err => console.error(` [${orgId}] Delete error for ${rel}: ${err.message}`));
|
|
675
833
|
});
|
|
676
834
|
watcher.on('addDir', dirPath => handleLocalDirAdded(dirPath));
|
|
677
835
|
watcher.on('unlinkDir', dirPath => handleLocalDirRemoved(dirPath));
|
|
@@ -697,6 +855,7 @@ export async function startOrgSync({
|
|
|
697
855
|
// Tear down all listeners and the watcher. Returns the watcher's
|
|
698
856
|
// close() promise so callers/tests can await full shutdown.
|
|
699
857
|
stop() {
|
|
858
|
+
stoppedRef.value = true;
|
|
700
859
|
for (const unsub of unsubscribers) {
|
|
701
860
|
try { unsub(); } catch (e) { /* ignore */ }
|
|
702
861
|
}
|
|
@@ -704,7 +863,15 @@ export async function startOrgSync({
|
|
|
704
863
|
try { unsub(); } catch (e) { /* ignore */ }
|
|
705
864
|
}
|
|
706
865
|
teamUnsubscribers.clear();
|
|
707
|
-
|
|
866
|
+
const setupDrain = Promise.all([...pendingPromises]);
|
|
867
|
+
const bindingDrain = setupDrain.then(() => Promise.all(
|
|
868
|
+
[...bindings.values()].map((b) => b.stop().catch(() => {})),
|
|
869
|
+
)).then(() => {
|
|
870
|
+
bindings.clear();
|
|
871
|
+
bindingsByPath.clear();
|
|
872
|
+
});
|
|
873
|
+
const watcherClose = watcher ? watcher.close() : Promise.resolve();
|
|
874
|
+
return Promise.all([bindingDrain, watcherClose]).then(() => undefined);
|
|
708
875
|
},
|
|
709
876
|
};
|
|
710
877
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@doubling/compound-sync",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.11.1",
|
|
4
4
|
"description": "Bidirectional sync between Compound and local markdown files",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
"sync-state.js",
|
|
13
13
|
"config.js",
|
|
14
14
|
"paths.js",
|
|
15
|
+
"paths.d.ts",
|
|
15
16
|
"files.js",
|
|
16
17
|
"folder-chain.js",
|
|
17
18
|
"storage-helpers.js",
|
|
@@ -19,21 +20,33 @@
|
|
|
19
20
|
"auth-persistence.js",
|
|
20
21
|
"pre-mint-app-check.js",
|
|
21
22
|
"setup-auth-with-pre-mint.js",
|
|
23
|
+
"yjs-firestore-update-store.js",
|
|
24
|
+
"yjs-provider.js",
|
|
25
|
+
"yjs-file-binding.js",
|
|
26
|
+
"yjs-binding-state.js",
|
|
22
27
|
"README.md"
|
|
23
28
|
],
|
|
24
29
|
"scripts": {
|
|
25
|
-
"sync": "node sync.js",
|
|
26
|
-
"
|
|
27
|
-
"
|
|
30
|
+
"sync": "node --import tsx ./sync.js",
|
|
31
|
+
"build": "tsc",
|
|
32
|
+
"typecheck": "tsc --noEmit",
|
|
33
|
+
"prepack": "npm run build",
|
|
34
|
+
"test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js yjs-file-binding.test.js yjs-binding-state.test.js",
|
|
35
|
+
"test:integration": "firebase emulators:exec --only firestore,storage 'node --import tsx --test --test-concurrency=1 files.test.js org-sync.test.js two-user-sync.test.js'",
|
|
28
36
|
"test": "npm run test:unit && npm run test:integration"
|
|
29
37
|
},
|
|
30
38
|
"dependencies": {
|
|
39
|
+
"chokidar": "^5.0.0",
|
|
40
|
+
"fast-diff": "^1.3.0",
|
|
31
41
|
"firebase": "^12.14.0",
|
|
32
|
-
"
|
|
42
|
+
"yjs": "^13.6.31"
|
|
33
43
|
},
|
|
34
44
|
"devDependencies": {
|
|
35
45
|
"@firebase/rules-unit-testing": "^5.0.1",
|
|
36
|
-
"
|
|
46
|
+
"@types/node": "^26.0.0",
|
|
47
|
+
"firebase-tools": "^15.20.0",
|
|
48
|
+
"tsx": "^4.22.4",
|
|
49
|
+
"typescript": "^5.9.3"
|
|
37
50
|
},
|
|
38
51
|
"publishConfig": {
|
|
39
52
|
"access": "public"
|
package/paths.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare const PRIVATE_FOLDER = "Private";
|
|
2
|
+
export declare const SHARED_WITH_ME_FOLDER = "Shared with Me";
|
|
3
|
+
export declare const SHARED_BY_ME_FOLDER = "Shared by Me";
|
|
4
|
+
export type Scope = 'team' | 'private' | 'shared-with-me' | 'shared-by-me';
|
|
5
|
+
export interface ScopedDocPath {
|
|
6
|
+
scope: Scope;
|
|
7
|
+
teamId?: string;
|
|
8
|
+
docPath: string;
|
|
9
|
+
}
|
|
10
|
+
export declare function teamFolderName(teamName: string): string;
|
|
11
|
+
export declare function scopeFromLocalPath(filePath: string, localPath: string, teamIdsByName: Map<string, string>): ScopedDocPath;
|
|
12
|
+
//# sourceMappingURL=paths.d.ts.map
|
package/paths.js
CHANGED
|
@@ -5,48 +5,43 @@
|
|
|
5
5
|
// expected to be swapped if/when authorization moves to a Zanzibar-
|
|
6
6
|
// style relationship model. Keep this module's interface narrow so the
|
|
7
7
|
// swap stays localized.
|
|
8
|
-
|
|
9
8
|
import path from 'node:path';
|
|
10
|
-
|
|
11
9
|
export const PRIVATE_FOLDER = 'Private';
|
|
12
10
|
export const SHARED_WITH_ME_FOLDER = 'Shared with Me';
|
|
13
11
|
export const SHARED_BY_ME_FOLDER = 'Shared by Me';
|
|
14
12
|
const TEAMSPACE_SUFFIX = ' Teamspace';
|
|
15
|
-
|
|
16
13
|
export function teamFolderName(teamName) {
|
|
17
|
-
|
|
14
|
+
return teamName + TEAMSPACE_SUFFIX;
|
|
18
15
|
}
|
|
19
|
-
|
|
20
16
|
// Given an absolute file path within one of the daemon's local sync
|
|
21
17
|
// roots, return { scope, teamId?, docPath }. The caller passes the
|
|
22
|
-
// matching localPath root and the team-name
|
|
18
|
+
// matching localPath root and the team-name -> team-id map.
|
|
23
19
|
export function scopeFromLocalPath(filePath, localPath, teamIdsByName) {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
:
|
|
50
|
-
const teamId = teamIdsByName.get(teamName);
|
|
51
|
-
return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
|
|
20
|
+
const rel = path.relative(localPath, filePath);
|
|
21
|
+
if (rel.startsWith(PRIVATE_FOLDER + path.sep)) {
|
|
22
|
+
return {
|
|
23
|
+
scope: 'private',
|
|
24
|
+
docPath: rel.slice(PRIVATE_FOLDER.length + 1),
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) {
|
|
28
|
+
return {
|
|
29
|
+
scope: 'shared-with-me',
|
|
30
|
+
docPath: rel.slice(SHARED_WITH_ME_FOLDER.length + 1),
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) {
|
|
34
|
+
return {
|
|
35
|
+
scope: 'shared-by-me',
|
|
36
|
+
docPath: rel.slice(SHARED_BY_ME_FOLDER.length + 1),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const parts = rel.split(path.sep);
|
|
40
|
+
const folderName = parts[0] ?? '';
|
|
41
|
+
const teamName = folderName.endsWith(TEAMSPACE_SUFFIX)
|
|
42
|
+
? folderName.slice(0, -TEAMSPACE_SUFFIX.length)
|
|
43
|
+
: folderName;
|
|
44
|
+
const teamId = teamIdsByName.get(teamName);
|
|
45
|
+
return { scope: 'team', teamId, docPath: parts.slice(1).join('/') };
|
|
52
46
|
}
|
|
47
|
+
//# sourceMappingURL=paths.js.map
|
package/storage-helpers.js
CHANGED
package/sync.js
CHANGED
|
@@ -16,6 +16,43 @@
|
|
|
16
16
|
* node sync.js --help
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
// DOU-208: Node 22+ ships its own CA bundle and does not consult the
|
|
20
|
+
// macOS system trust store. Without NODE_EXTRA_CA_CERTS pointing at
|
|
21
|
+
// /etc/ssl/cert.pem, fetch() to securetoken.googleapis.com fails
|
|
22
|
+
// with a bare `fetch failed`, which Firebase Auth then surfaces as
|
|
23
|
+
// auth/network-request-failed. The web e2e harness's run-emulator.js
|
|
24
|
+
// already does this for child processes; here we need it for our own
|
|
25
|
+
// process. NODE_EXTRA_CA_CERTS is read at startup, so setting it on
|
|
26
|
+
// process.env after init is a no-op — instead, re-exec ourselves
|
|
27
|
+
// with the env var set when missing. The child inherits stdio +
|
|
28
|
+
// argv; we exit with its status. No-op when already set (the user
|
|
29
|
+
// or a wrapper explicitly chose a different bundle).
|
|
30
|
+
import { spawnSync as __dou208_spawnSync } from 'node:child_process';
|
|
31
|
+
import { existsSync as __dou208_existsSync } from 'node:fs';
|
|
32
|
+
if (!process.env.NODE_EXTRA_CA_CERTS) {
|
|
33
|
+
const __dou208_candidates = process.platform === 'darwin'
|
|
34
|
+
? ['/etc/ssl/cert.pem']
|
|
35
|
+
: process.platform === 'linux'
|
|
36
|
+
? ['/etc/ssl/certs/ca-certificates.crt', '/etc/pki/tls/certs/ca-bundle.crt']
|
|
37
|
+
: [];
|
|
38
|
+
for (const __dou208_candidate of __dou208_candidates) {
|
|
39
|
+
if (__dou208_existsSync(__dou208_candidate)) {
|
|
40
|
+
const __dou208_result = __dou208_spawnSync(
|
|
41
|
+
process.execPath,
|
|
42
|
+
process.argv.slice(1),
|
|
43
|
+
{
|
|
44
|
+
stdio: 'inherit',
|
|
45
|
+
env: { ...process.env, NODE_EXTRA_CA_CERTS: __dou208_candidate },
|
|
46
|
+
},
|
|
47
|
+
);
|
|
48
|
+
process.exit(__dou208_result.status ?? 1);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// No candidate found (Windows, unusual Linux distro): fall through
|
|
52
|
+
// and let Node's bundled CA bundle handle TLS. The user can still
|
|
53
|
+
// set NODE_EXTRA_CA_CERTS manually for corporate cert chains.
|
|
54
|
+
}
|
|
55
|
+
|
|
19
56
|
import { initializeApp } from 'firebase/app';
|
|
20
57
|
import { getAuth, initializeAuth, signInWithEmailAndPassword, signInWithCredential, GoogleAuthProvider } from 'firebase/auth';
|
|
21
58
|
import {
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Per-file Y.Doc binary state persistence (DOU-209).
|
|
2
|
+
//
|
|
3
|
+
// On daemon restart we need to merge offline edits made on disk with
|
|
4
|
+
// offline edits made via the web. Yjs converges deterministically when
|
|
5
|
+
// both sides apply their ops to a common baseline. The daemon's last
|
|
6
|
+
// view of the file (== ytext at last write, == disk content at last
|
|
7
|
+
// write) is that baseline. Persisting the Y.Doc binary state per file
|
|
8
|
+
// lets the daemon restore that baseline on startup so:
|
|
9
|
+
// 1. ytext == lastDisk after restore
|
|
10
|
+
// 2. fast-diff(ytext, currentDisk) gives positions valid in ytext
|
|
11
|
+
// 3. cloud yupdates apply on top via CRDT merge (idempotent for ops
|
|
12
|
+
// already in the loaded state, additive for new cloud ops).
|
|
13
|
+
//
|
|
14
|
+
// Storage layout: one file per Y.Doc, keyed by fileId, under
|
|
15
|
+
// `<localPath>/.compound-yjs-binding/<fileId>.yjs`. The binary blob is
|
|
16
|
+
// the output of Y.encodeStateAsUpdate(doc); the daemon applies it via
|
|
17
|
+
// Y.applyUpdate(doc, blob) on restore.
|
|
18
|
+
|
|
19
|
+
import fs from 'node:fs';
|
|
20
|
+
import path from 'node:path';
|
|
21
|
+
|
|
22
|
+
const STATE_DIR = '.compound-yjs-binding';
|
|
23
|
+
|
|
24
|
+
function statePathFor(localPath, fileId) {
|
|
25
|
+
return path.join(localPath, STATE_DIR, `${fileId}.yjs`);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// Load the encoded Y.Doc state for a file. Returns null if no state
|
|
29
|
+
// exists yet (first run, or this fileId hasn't been synced before) or
|
|
30
|
+
// if the file is unreadable for any reason (corruption, permissions).
|
|
31
|
+
// Treating unreadable as "no state" is safe: the binding falls back to
|
|
32
|
+
// its first-run behavior, which writes ytext to disk on initial flush.
|
|
33
|
+
export function loadBindingState(localPath, fileId) {
|
|
34
|
+
try {
|
|
35
|
+
return fs.readFileSync(statePathFor(localPath, fileId));
|
|
36
|
+
} catch {
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// Persist the encoded Y.Doc state for a file. Atomic via write-to-tmp +
|
|
42
|
+
// rename so a crash mid-write can't leave a half-written .yjs file that
|
|
43
|
+
// would corrupt the next startup. Best-effort: a failed save costs us a
|
|
44
|
+
// cold-start reconciliation next launch, never data.
|
|
45
|
+
export function saveBindingState(localPath, fileId, encoded) {
|
|
46
|
+
const finalPath = statePathFor(localPath, fileId);
|
|
47
|
+
const tmpPath = `${finalPath}.${process.pid}.${Date.now()}.tmp`;
|
|
48
|
+
try {
|
|
49
|
+
fs.mkdirSync(path.dirname(finalPath), { recursive: true });
|
|
50
|
+
fs.writeFileSync(tmpPath, encoded);
|
|
51
|
+
fs.renameSync(tmpPath, finalPath);
|
|
52
|
+
} catch {
|
|
53
|
+
try { fs.unlinkSync(tmpPath); } catch { /* ignore */ }
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// Best-effort cleanup when a file is deleted from the workspace, so
|
|
58
|
+
// stale .yjs blobs don't accumulate. Failing to delete is not fatal.
|
|
59
|
+
export function deleteBindingState(localPath, fileId) {
|
|
60
|
+
try { fs.unlinkSync(statePathFor(localPath, fileId)); } catch { /* ignore */ }
|
|
61
|
+
}
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// YjsFileBinding: per-file glue that keeps a local Markdown file
|
|
2
|
+
// converged with the per-file Y.Doc that lives in Firestore.
|
|
3
|
+
//
|
|
4
|
+
// Two directions, both flowing through the Y.Doc:
|
|
5
|
+
// cloud -> disk: provider applies remote yupdates to ytext; an
|
|
6
|
+
// observer debounces the resulting text into a single disk write
|
|
7
|
+
// so a burst of web keystrokes coalesces into one fs.writeFile.
|
|
8
|
+
// disk -> cloud: when the local file changes, caller invokes
|
|
9
|
+
// applyDiskText(newText); we diff against ytext (fast-diff for a
|
|
10
|
+
// char-level edit script) and apply inserts/deletes in one
|
|
11
|
+
// transaction so the provider's local-update listener emits a
|
|
12
|
+
// single yupdate carrying just the changed regions, preserving
|
|
13
|
+
// concurrent web edits to unrelated regions.
|
|
14
|
+
//
|
|
15
|
+
// Disk I/O is injected via onDiskWrite so tests can record without
|
|
16
|
+
// touching the filesystem and so the daemon can plumb its existing
|
|
17
|
+
// suppressLocal set (avoids the disk write feeding back through
|
|
18
|
+
// chokidar as a fake user edit).
|
|
19
|
+
|
|
20
|
+
import * as Y from 'yjs';
|
|
21
|
+
import fastDiff from 'fast-diff';
|
|
22
|
+
import { YjsProvider } from './yjs-provider.js';
|
|
23
|
+
|
|
24
|
+
export class YjsFileBinding {
|
|
25
|
+
constructor(store, {
|
|
26
|
+
debounceMs = 300, onDiskWrite, onDocStateChange,
|
|
27
|
+
initialDocState = null, currentDiskText = null,
|
|
28
|
+
}) {
|
|
29
|
+
if (!onDiskWrite) throw new Error('YjsFileBinding: onDiskWrite is required');
|
|
30
|
+
this.doc = new Y.Doc();
|
|
31
|
+
this.ytext = this.doc.getText('content');
|
|
32
|
+
this.store = store;
|
|
33
|
+
this.provider = new YjsProvider(this.doc, store);
|
|
34
|
+
this.debounceMs = debounceMs;
|
|
35
|
+
this.onDiskWrite = onDiskWrite;
|
|
36
|
+
// Called after every Y.Doc mutation reaches a stable state, with
|
|
37
|
+
// the encoded Y.Doc binary state. The daemon persists this to
|
|
38
|
+
// disk so the next startup can restore the baseline and merge
|
|
39
|
+
// offline disk edits without losing offline cloud edits (DOU-209).
|
|
40
|
+
this.onDocStateChange = onDocStateChange ?? null;
|
|
41
|
+
// Restore options for offline-merge (DOU-209).
|
|
42
|
+
this.initialDocState = initialDocState;
|
|
43
|
+
this.currentDiskText = currentDiskText;
|
|
44
|
+
this.flushTimer = null;
|
|
45
|
+
this.lastDiskText = null;
|
|
46
|
+
this.observer = null;
|
|
47
|
+
this.stopped = false;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async start() {
|
|
51
|
+
// DOU-209: offline-merge startup. Three-step process that turns the
|
|
52
|
+
// daemon's restart into a deterministic CRDT merge:
|
|
53
|
+
//
|
|
54
|
+
// 1. Restore Y.Doc from the daemon's last-saved binary state (if
|
|
55
|
+
// any). After this, ytext equals what disk was at the daemon's
|
|
56
|
+
// last write. This is the common ancestor for any divergence
|
|
57
|
+
// that happened while the daemon was offline.
|
|
58
|
+
// 2. If the current disk content differs from the restored ytext,
|
|
59
|
+
// the difference is disk-only edits made during downtime.
|
|
60
|
+
// Apply them as Y.Doc operations (positions are valid in the
|
|
61
|
+
// restored ytext since restored-ytext == lastDisk == fast-diff
|
|
62
|
+
// input).
|
|
63
|
+
// 3. provider.loadInitial() pulls cloud's yupdates on top. Y.applyUpdate
|
|
64
|
+
// is idempotent (Yjs's state-vector dedup), so already-known
|
|
65
|
+
// ops are no-ops. Only NEW cloud ops (offline web edits) take
|
|
66
|
+
// effect. They CRDT-merge with the disk ops from step 2.
|
|
67
|
+
//
|
|
68
|
+
// Without step 1, fast-diff(post-loadInitial-ytext, currentDisk)
|
|
69
|
+
// treats web's offline insertions as "things present in ytext but
|
|
70
|
+
// missing from disk" and DELETES them. That's the bug this fixes.
|
|
71
|
+
// Seed lastDiskText from the disk content we observed at construction
|
|
72
|
+
// time so the initial flush is a no-op when ytext already matches
|
|
73
|
+
// what's on disk. Without this, the binding's initial flush always
|
|
74
|
+
// wrote ytext to disk on startup, which clobbered any content the
|
|
75
|
+
// daemon's legacy contentHash + Storage blob pull (the fallthrough
|
|
76
|
+
// path in handleFirestoreChange) may have written first for non-Yjs
|
|
77
|
+
// cloud edits. Regression case: "after restart, pulls a web edit
|
|
78
|
+
// made while stopped despite a fresh local mtime" in
|
|
79
|
+
// sync/org-sync.test.js.
|
|
80
|
+
this.lastDiskText = this.currentDiskText;
|
|
81
|
+
|
|
82
|
+
if (this.initialDocState) {
|
|
83
|
+
Y.applyUpdate(this.doc, this.initialDocState);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Disk-merge ops emitted below need to reach the store so other
|
|
87
|
+
// clients see the offline disk edits. The provider's local-update
|
|
88
|
+
// handler isn't attached yet (it's wired in startSubscription,
|
|
89
|
+
// which we can't call until after loadInitial), so we capture the
|
|
90
|
+
// updates synchronously here and push them after the subscription
|
|
91
|
+
// is set up. Capture by Y.Doc 'update' event with a 'disk' origin
|
|
92
|
+
// filter so we only grab the ops from our own _applyTextDiff and
|
|
93
|
+
// not anything spurious.
|
|
94
|
+
//
|
|
95
|
+
// The disk-merge is only safe when we have an `initialDocState`:
|
|
96
|
+
// that gives ytext a known baseline (the daemon's last view of the
|
|
97
|
+
// file) before we diff against currentDiskText, so fast-diff
|
|
98
|
+
// positions are valid AND the resulting ops are scoped to JUST the
|
|
99
|
+
// disk-only changes. Without a baseline, applying currentDiskText
|
|
100
|
+
// as fresh inserts on top of an empty ytext would then collide
|
|
101
|
+
// with whatever loadInitial pulls from the cloud: Yjs would see
|
|
102
|
+
// both the daemon's "fresh insert" and the cloud's "original
|
|
103
|
+
// insert" as concurrent ops, producing duplicated content. So on
|
|
104
|
+
// first contact (no saved state) we skip the disk merge and let
|
|
105
|
+
// loadInitial + initial flush behave the same as before DOU-209.
|
|
106
|
+
const capturedDiskUpdates = [];
|
|
107
|
+
if (this.initialDocState && this.currentDiskText != null) {
|
|
108
|
+
const restored = this.ytext.toString();
|
|
109
|
+
if (this.currentDiskText !== restored) {
|
|
110
|
+
const captureHandler = (update, origin) => {
|
|
111
|
+
if (origin === 'disk') capturedDiskUpdates.push(update);
|
|
112
|
+
};
|
|
113
|
+
this.doc.on('update', captureHandler);
|
|
114
|
+
// Same diff-and-apply shape as applyDiskText, but we call the
|
|
115
|
+
// internal helper directly: applyDiskText is the live-chokidar
|
|
116
|
+
// path and also touches lastDiskText / _emitDocState in ways
|
|
117
|
+
// that aren't appropriate during the startup-merge phase.
|
|
118
|
+
this._applyTextDiff(restored, this.currentDiskText);
|
|
119
|
+
this.doc.off('update', captureHandler);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
// DOU-204: two-phase provider start. Phase 1 loads snapshot +
|
|
124
|
+
// initial updates without subscribing. We then attach the
|
|
125
|
+
// Y.Text observer that drives the debounced disk write. Phase 2
|
|
126
|
+
// subscribes to remote updates; the first listener fire is the
|
|
127
|
+
// initial query result (re-applies the already-loaded docs
|
|
128
|
+
// idempotently), and any update that lands after loadInitial
|
|
129
|
+
// routes through the now-attached observer instead of mutating
|
|
130
|
+
// Y.Text silently between provider.start and observer attach.
|
|
131
|
+
await this.provider.loadInitial();
|
|
132
|
+
this.observer = () => {
|
|
133
|
+
if (this.stopped) return;
|
|
134
|
+
if (this.flushTimer) clearTimeout(this.flushTimer);
|
|
135
|
+
this.flushTimer = setTimeout(() => this._flushNow(), this.debounceMs);
|
|
136
|
+
};
|
|
137
|
+
this.ytext.observe(this.observer);
|
|
138
|
+
this.provider.startSubscription();
|
|
139
|
+
|
|
140
|
+
// Push the disk-side updates captured above so the store records
|
|
141
|
+
// the offline disk edits and other clients receive them. Y.applyUpdate
|
|
142
|
+
// is idempotent so the daemon's own subscription echo (when the
|
|
143
|
+
// store fires its own append back through subscribeUpdates) is a
|
|
144
|
+
// no-op on our local Y.Doc.
|
|
145
|
+
for (const update of capturedDiskUpdates) {
|
|
146
|
+
try { await this.store.appendUpdate(update); }
|
|
147
|
+
catch (err) { console.error('YjsFileBinding: disk-merge appendUpdate failed:', err && err.message); }
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Initial flush: ytext may be non-empty after replay (file has
|
|
151
|
+
// existing Yjs state). Writes the loaded content to disk so the
|
|
152
|
+
// local file is up-to-date the moment the binding is wired in.
|
|
153
|
+
if (this.ytext.length > 0) this._flushNow();
|
|
154
|
+
// Persist the merged baseline so the next restart has the right
|
|
155
|
+
// anchor (the binding's state right now == disk after the flush
|
|
156
|
+
// above; future disk-side edits should diff against this state).
|
|
157
|
+
this._emitDocState();
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
_applyTextDiff(fromText, toText) {
|
|
161
|
+
if (fromText === toText) return;
|
|
162
|
+
const diffs = fastDiff(fromText, toText);
|
|
163
|
+
this.doc.transact(() => {
|
|
164
|
+
let idx = 0;
|
|
165
|
+
for (const [op, str] of diffs) {
|
|
166
|
+
if (op === 0) {
|
|
167
|
+
idx += str.length;
|
|
168
|
+
} else if (op === 1) {
|
|
169
|
+
this.ytext.insert(idx, str);
|
|
170
|
+
idx += str.length;
|
|
171
|
+
} else {
|
|
172
|
+
this.ytext.delete(idx, str.length);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}, 'disk');
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
_emitDocState() {
|
|
179
|
+
if (this.stopped || !this.onDocStateChange) return;
|
|
180
|
+
try {
|
|
181
|
+
this.onDocStateChange(Y.encodeStateAsUpdate(this.doc));
|
|
182
|
+
} catch (err) {
|
|
183
|
+
console.error('YjsFileBinding: onDocStateChange failed:', err && err.message);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// Sync flush used by both the debounced observer and the final
|
|
188
|
+
// drain in stop().
|
|
189
|
+
_flushNow() {
|
|
190
|
+
if (this.stopped) return;
|
|
191
|
+
if (this.flushTimer) {
|
|
192
|
+
clearTimeout(this.flushTimer);
|
|
193
|
+
this.flushTimer = null;
|
|
194
|
+
}
|
|
195
|
+
const text = this.ytext.toString();
|
|
196
|
+
if (text === this.lastDiskText) return;
|
|
197
|
+
this.lastDiskText = text;
|
|
198
|
+
this.onDiskWrite(text);
|
|
199
|
+
// The disk now matches ytext: snapshot the Y.Doc state so the next
|
|
200
|
+
// restart has the right baseline for offline-merge (DOU-209).
|
|
201
|
+
this._emitDocState();
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// The local disk file changed (chokidar). Diff against the current
|
|
205
|
+
// ytext and apply as a sequence of inserts/deletes inside one
|
|
206
|
+
// transaction. Using a distinct origin ('disk') keeps the provider's
|
|
207
|
+
// local-update listener pushing the resulting yupdate to Firestore.
|
|
208
|
+
applyDiskText(newText) {
|
|
209
|
+
if (this.stopped) return;
|
|
210
|
+
const oldText = this.ytext.toString();
|
|
211
|
+
if (oldText === newText) return;
|
|
212
|
+
this._applyTextDiff(oldText, newText);
|
|
213
|
+
// Suppress the debounced echo: lastDiskText now matches the disk
|
|
214
|
+
// content the caller just observed, so the observer's pending
|
|
215
|
+
// flush is a no-op.
|
|
216
|
+
this.lastDiskText = newText;
|
|
217
|
+
// Snapshot the Y.Doc state so the next restart's baseline includes
|
|
218
|
+
// this disk-originated edit (DOU-209).
|
|
219
|
+
this._emitDocState();
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async stop() {
|
|
223
|
+
if (this.stopped) return;
|
|
224
|
+
this.stopped = true;
|
|
225
|
+
if (this.observer) {
|
|
226
|
+
this.ytext.unobserve(this.observer);
|
|
227
|
+
this.observer = null;
|
|
228
|
+
}
|
|
229
|
+
if (this.flushTimer) {
|
|
230
|
+
clearTimeout(this.flushTimer);
|
|
231
|
+
this.flushTimer = null;
|
|
232
|
+
}
|
|
233
|
+
this.provider.stop();
|
|
234
|
+
this.doc.destroy();
|
|
235
|
+
}
|
|
236
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
// FirestoreUpdateStore: Firestore-backed store for per-file Yjs
|
|
2
|
+
// CRDT updates. Ported from web/src/yjs/firestore-update-store.ts so
|
|
3
|
+
// the published @doubling/compound-sync package does not reach into
|
|
4
|
+
// the web source tree. Keep the API identical to the web copy; a
|
|
5
|
+
// follow-up will unify the two into a shared workspace package once
|
|
6
|
+
// PR 4 finishes retiring the legacy blob path.
|
|
7
|
+
|
|
8
|
+
import {
|
|
9
|
+
collection,
|
|
10
|
+
doc,
|
|
11
|
+
addDoc,
|
|
12
|
+
getDoc,
|
|
13
|
+
getDocs,
|
|
14
|
+
setDoc,
|
|
15
|
+
query,
|
|
16
|
+
where,
|
|
17
|
+
orderBy,
|
|
18
|
+
onSnapshot,
|
|
19
|
+
serverTimestamp,
|
|
20
|
+
writeBatch,
|
|
21
|
+
Bytes,
|
|
22
|
+
Timestamp,
|
|
23
|
+
} from 'firebase/firestore';
|
|
24
|
+
|
|
25
|
+
function formatId(createdAt, docId) {
|
|
26
|
+
return `${createdAt.toMillis().toString().padStart(16, '0')}_${docId}`;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function parseTimestampFromId(id) {
|
|
30
|
+
return Timestamp.fromMillis(Number(id.split('_')[0]));
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export class FirestoreUpdateStore {
|
|
34
|
+
constructor(db, orgId, fileId) {
|
|
35
|
+
this.updatesCol = collection(db, 'orgs', orgId, 'files', fileId, 'yupdates');
|
|
36
|
+
this.snapshotDoc = doc(db, 'orgs', orgId, 'files', fileId, 'yjs', 'state');
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async loadSnapshot() {
|
|
40
|
+
const snap = await getDoc(this.snapshotDoc);
|
|
41
|
+
if (!snap.exists()) return null;
|
|
42
|
+
const data = snap.data();
|
|
43
|
+
return {
|
|
44
|
+
state: data.state.toUint8Array(),
|
|
45
|
+
baselineId: data.baselineId ?? null,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async loadUpdatesAfter(baselineId) {
|
|
50
|
+
const q =
|
|
51
|
+
baselineId === null
|
|
52
|
+
? query(this.updatesCol, orderBy('createdAt', 'asc'))
|
|
53
|
+
: query(
|
|
54
|
+
this.updatesCol,
|
|
55
|
+
where('createdAt', '>', parseTimestampFromId(baselineId)),
|
|
56
|
+
orderBy('createdAt', 'asc'),
|
|
57
|
+
);
|
|
58
|
+
const snap = await getDocs(q);
|
|
59
|
+
return snap.docs
|
|
60
|
+
.map((d) => {
|
|
61
|
+
const data = d.data();
|
|
62
|
+
if (!data.createdAt) return null;
|
|
63
|
+
return { id: formatId(data.createdAt, d.id), update: data.update.toUint8Array() };
|
|
64
|
+
})
|
|
65
|
+
.filter((r) => r !== null);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async appendUpdate(update) {
|
|
69
|
+
const docRef = await addDoc(this.updatesCol, {
|
|
70
|
+
update: Bytes.fromUint8Array(update),
|
|
71
|
+
createdAt: serverTimestamp(),
|
|
72
|
+
});
|
|
73
|
+
const fresh = await getDoc(docRef);
|
|
74
|
+
const data = fresh.data();
|
|
75
|
+
if (!data || !data.createdAt) {
|
|
76
|
+
throw new Error('FirestoreUpdateStore.appendUpdate: createdAt unresolved');
|
|
77
|
+
}
|
|
78
|
+
return { id: formatId(data.createdAt, docRef.id), update };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
subscribeUpdates(onUpdate) {
|
|
82
|
+
const q = query(this.updatesCol, orderBy('createdAt', 'asc'));
|
|
83
|
+
// includeMetadataChanges + emitted-id dedup: see the web copy for
|
|
84
|
+
// the full rationale. Skipping pending writes alone drops the
|
|
85
|
+
// server-ack delivery because that fire is metadata-only.
|
|
86
|
+
const emitted = new Set();
|
|
87
|
+
return onSnapshot(q, { includeMetadataChanges: true }, (snap) => {
|
|
88
|
+
for (const change of snap.docChanges({ includeMetadataChanges: true })) {
|
|
89
|
+
if (change.type !== 'added' && change.type !== 'modified') continue;
|
|
90
|
+
const data = change.doc.data();
|
|
91
|
+
if (!data.createdAt) continue;
|
|
92
|
+
if (emitted.has(change.doc.id)) continue;
|
|
93
|
+
emitted.add(change.doc.id);
|
|
94
|
+
onUpdate({
|
|
95
|
+
id: formatId(data.createdAt, change.doc.id),
|
|
96
|
+
update: data.update.toUint8Array(),
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async saveSnapshotAndCompact(state, newBaselineId) {
|
|
103
|
+
const baselineTs = parseTimestampFromId(newBaselineId);
|
|
104
|
+
await setDoc(this.snapshotDoc, {
|
|
105
|
+
state: Bytes.fromUint8Array(state),
|
|
106
|
+
baselineId: newBaselineId,
|
|
107
|
+
baselineCreatedAt: baselineTs,
|
|
108
|
+
updatedAt: serverTimestamp(),
|
|
109
|
+
});
|
|
110
|
+
const q = query(this.updatesCol, where('createdAt', '<=', baselineTs));
|
|
111
|
+
const stale = await getDocs(q);
|
|
112
|
+
if (stale.empty) return;
|
|
113
|
+
const batch = writeBatch(this.snapshotDoc.firestore);
|
|
114
|
+
for (const d of stale.docs) batch.delete(d.ref);
|
|
115
|
+
await batch.commit();
|
|
116
|
+
}
|
|
117
|
+
}
|
package/yjs-provider.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
// YjsProvider: binds a Y.Doc to an UpdateStore. Ported from
|
|
2
|
+
// web/src/yjs/provider.ts; see that file for the full design notes
|
|
3
|
+
// (echo handling, snapshot replay, compaction semantics, two-phase
|
|
4
|
+
// start lifecycle from DOU-204). API kept identical so the two copies
|
|
5
|
+
// can be unified into a shared workspace package in a follow-up.
|
|
6
|
+
|
|
7
|
+
import * as Y from 'yjs';
|
|
8
|
+
|
|
9
|
+
export class YjsProvider {
|
|
10
|
+
constructor(doc, store) {
|
|
11
|
+
this.doc = doc;
|
|
12
|
+
this.store = store;
|
|
13
|
+
this.unsubscribeStore = null;
|
|
14
|
+
this.updateHandler = null;
|
|
15
|
+
this.loaded = false;
|
|
16
|
+
this.subscribed = false;
|
|
17
|
+
this.stopped = false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** Phase 1: snapshot + initial updates → Y.Doc. No remote
|
|
21
|
+
* subscription yet. Callers that intend to attach a Y.Text observer
|
|
22
|
+
* (e.g. a disk-write debouncer here, yCollab on the web) should
|
|
23
|
+
* install that observer between `loadInitial()` and
|
|
24
|
+
* `startSubscription()` so the first remote update routes through
|
|
25
|
+
* to it instead of mutating Y.Text silently in the gap (DOU-204). */
|
|
26
|
+
async loadInitial() {
|
|
27
|
+
if (this.loaded) throw new Error('YjsProvider.loadInitial() called twice');
|
|
28
|
+
this.loaded = true;
|
|
29
|
+
|
|
30
|
+
const snapshot = await this.store.loadSnapshot();
|
|
31
|
+
if (snapshot) Y.applyUpdate(this.doc, snapshot.state, this);
|
|
32
|
+
|
|
33
|
+
const initial = await this.store.loadUpdatesAfter(snapshot?.baselineId ?? null);
|
|
34
|
+
for (const record of initial) Y.applyUpdate(this.doc, record.update, this);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Phase 2: subscribe to remote updates and push local Y.Doc
|
|
38
|
+
* mutations back to the store. */
|
|
39
|
+
startSubscription() {
|
|
40
|
+
if (!this.loaded) throw new Error('YjsProvider.startSubscription() before loadInitial()');
|
|
41
|
+
if (this.subscribed) throw new Error('YjsProvider.startSubscription() called twice');
|
|
42
|
+
this.subscribed = true;
|
|
43
|
+
|
|
44
|
+
this.unsubscribeStore = this.store.subscribeUpdates((record) => {
|
|
45
|
+
if (this.stopped) return;
|
|
46
|
+
Y.applyUpdate(this.doc, record.update, this);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
this.updateHandler = (update, origin) => {
|
|
50
|
+
if (this.stopped) return;
|
|
51
|
+
if (origin === this) return;
|
|
52
|
+
this.store.appendUpdate(update).catch((err) => {
|
|
53
|
+
console.error('YjsProvider: appendUpdate failed:', err && err.message);
|
|
54
|
+
});
|
|
55
|
+
};
|
|
56
|
+
this.doc.on('update', this.updateHandler);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Convenience: load + subscribe in one call. Use only when no
|
|
60
|
+
* view-layer observer needs to attach between the phases. */
|
|
61
|
+
async start() {
|
|
62
|
+
await this.loadInitial();
|
|
63
|
+
this.startSubscription();
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
stop() {
|
|
67
|
+
this.stopped = true;
|
|
68
|
+
if (this.unsubscribeStore) {
|
|
69
|
+
this.unsubscribeStore();
|
|
70
|
+
this.unsubscribeStore = null;
|
|
71
|
+
}
|
|
72
|
+
if (this.updateHandler) {
|
|
73
|
+
this.doc.off('update', this.updateHandler);
|
|
74
|
+
this.updateHandler = null;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
async compact() {
|
|
79
|
+
const state = Y.encodeStateAsUpdate(this.doc);
|
|
80
|
+
const records = await this.store.loadUpdatesAfter(null);
|
|
81
|
+
const last = records[records.length - 1];
|
|
82
|
+
if (!last) return;
|
|
83
|
+
await this.store.saveSnapshotAndCompact(state, last.id);
|
|
84
|
+
}
|
|
85
|
+
}
|