@doubling/compound-sync 1.12.2 → 1.12.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/manifest.js +127 -0
- package/org-sync.js +311 -40
- package/package.json +6 -3
- package/push-outcome.js +10 -0
- package/team-registry.js +12 -0
package/README.md
CHANGED
|
@@ -42,6 +42,19 @@ Compound Sync watches a local folder and your Compound workspace simultaneously.
|
|
|
42
42
|
Shared with Me/ -- files shared with you (read-only)
|
|
43
43
|
```
|
|
44
44
|
|
|
45
|
+
## Hidden sync metadata
|
|
46
|
+
|
|
47
|
+
The daemon maintains per-machine state under your sync folder (dotfiles and
|
|
48
|
+
directories, not uploaded to Compound):
|
|
49
|
+
|
|
50
|
+
| Path | Purpose |
|
|
51
|
+
| ---- | ------- |
|
|
52
|
+
| `.compound-sync/manifest.json` | Maps synced paths to Firestore file IDs (startup reconcile) |
|
|
53
|
+
| `.compound-sync-state.json` | Content-hash baseline for push/pull decisions |
|
|
54
|
+
| `.compound-yjs-binding/` | Offline Yjs merge state per file |
|
|
55
|
+
|
|
56
|
+
Do not edit these by hand. See [docs/features/sync/local-metadata.md](../docs/features/sync/local-metadata.md).
|
|
57
|
+
|
|
45
58
|
## Running
|
|
46
59
|
|
|
47
60
|
After setup, start syncing with:
|
|
@@ -116,6 +129,10 @@ npm test
|
|
|
116
129
|
- **Typecheck:** `npm run typecheck` (alias for `tsc --noEmit`) at the workspace level, or at repo root `npm run typecheck` which runs web + sync together.
|
|
117
130
|
- **Strictness:** `strict: true`, `noUncheckedIndexedAccess: true`, `noImplicitOverride: true`. No `any`. No `// @ts-ignore`. If a third-party module lacks types, declare its shape in a `.d.ts` or PR types upstream.
|
|
118
131
|
|
|
132
|
+
### Logging convention
|
|
133
|
+
|
|
134
|
+
Org-scoped output: ` [${orgId}] [tag] message`. Use `console.warn` for recoverable issues, `console.error` for failures. Tags like `[local scan]` and `[manifest reconcile]` identify subsystems.
|
|
135
|
+
|
|
119
136
|
Phase 1 migrated `paths.js` to `paths.ts` as proof of pattern. Phase 3 ([[DOU-183]]) migrates the remaining source files; everything in `sync/` will be `.ts` by the end.
|
|
120
137
|
|
|
121
138
|
### Publishing to npm
|
package/manifest.js
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Per-machine sync manifest (DOU-55).
|
|
2
|
+
//
|
|
3
|
+
// Maps `scope:docPath` → Firestore fileId so the daemon can reconcile
|
|
4
|
+
// local disk vs cloud on startup (e.g. web deletes while offline must
|
|
5
|
+
// remove local orphans instead of re-uploading them).
|
|
6
|
+
|
|
7
|
+
import fs from 'node:fs';
|
|
8
|
+
import path from 'node:path';
|
|
9
|
+
|
|
10
|
+
const MANIFEST_DIR = '.compound-sync';
|
|
11
|
+
const MANIFEST_FILE = 'manifest.json';
|
|
12
|
+
const FLUSH_DEBOUNCE_MS = 500;
|
|
13
|
+
|
|
14
|
+
export function manifestPath(localPath) {
|
|
15
|
+
return path.join(localPath, MANIFEST_DIR, MANIFEST_FILE);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function normalizeEntry(raw) {
|
|
19
|
+
if (!raw || typeof raw !== 'object') return null;
|
|
20
|
+
if (typeof raw.fileId !== 'string' || !raw.fileId) return null;
|
|
21
|
+
if (typeof raw.contentHash !== 'string' || !raw.contentHash) return null;
|
|
22
|
+
return {
|
|
23
|
+
fileId: raw.fileId,
|
|
24
|
+
contentHash: raw.contentHash,
|
|
25
|
+
lastSyncedAt: typeof raw.lastSyncedAt === 'string' ? raw.lastSyncedAt : new Date().toISOString(),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function loadManifest(localPath) {
|
|
30
|
+
try {
|
|
31
|
+
const parsed = JSON.parse(fs.readFileSync(manifestPath(localPath), 'utf8'));
|
|
32
|
+
if (parsed && typeof parsed === 'object' && parsed.entries && typeof parsed.entries === 'object') {
|
|
33
|
+
const entries = {};
|
|
34
|
+
for (const [key, raw] of Object.entries(parsed.entries)) {
|
|
35
|
+
const entry = normalizeEntry(raw);
|
|
36
|
+
if (entry) {
|
|
37
|
+
entries[key] = entry;
|
|
38
|
+
} else {
|
|
39
|
+
console.error(`[sync-manifest] dropped invalid entry for ${key}`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return { version: parsed.version ?? 1, entries };
|
|
43
|
+
}
|
|
44
|
+
} catch {
|
|
45
|
+
// Missing or corrupt file reads as empty so first run can't crash sync.
|
|
46
|
+
}
|
|
47
|
+
return { version: 1, entries: {} };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Persists the in-memory manifest state to disk.
|
|
52
|
+
*
|
|
53
|
+
* PERFORMANCE & SAFETY NOTES:
|
|
54
|
+
* 1. Atomic Writes: We write to `.compound-sync/manifest.json.tmp` first,
|
|
55
|
+
* then use `fs.renameSync` to overwrite the actual manifest. This ensures
|
|
56
|
+
* that if the daemon crashes mid-write, we do not corrupt the JSON.
|
|
57
|
+
* 2. Unsorted Keys: We do not sort the object keys before calling
|
|
58
|
+
* `JSON.stringify()`. Sorting requires significant CPU overhead for large
|
|
59
|
+
* workspaces. The in-memory object map is the source of truth; disk
|
|
60
|
+
* order does not matter.
|
|
61
|
+
* 3. Synchronous I/O: All disk operations use `*Sync` APIs. A flush blocks
|
|
62
|
+
* the Node event loop briefly; see docs/features/sync/sync-manifest-design.md.
|
|
63
|
+
*
|
|
64
|
+
* @param {string} localPath - Sync folder root
|
|
65
|
+
* @param {Object} manifestState - The current in-memory manifest dictionary
|
|
66
|
+
*/
|
|
67
|
+
export function saveManifest(localPath, manifestState) {
|
|
68
|
+
const finalPath = manifestPath(localPath);
|
|
69
|
+
const tmpPath = path.join(path.dirname(finalPath), 'manifest.json.tmp');
|
|
70
|
+
try {
|
|
71
|
+
fs.mkdirSync(path.dirname(finalPath), { recursive: true });
|
|
72
|
+
fs.writeFileSync(tmpPath, JSON.stringify(manifestState));
|
|
73
|
+
fs.renameSync(tmpPath, finalPath);
|
|
74
|
+
} catch (err) {
|
|
75
|
+
console.error(
|
|
76
|
+
`[sync-manifest] failed to persist manifest at ${finalPath}: ${err && err.message}`,
|
|
77
|
+
);
|
|
78
|
+
try {
|
|
79
|
+
fs.unlinkSync(tmpPath);
|
|
80
|
+
} catch (cleanupErr) {
|
|
81
|
+
// A leftover .tmp is ignored on load (only the canonical path is read).
|
|
82
|
+
// ENOENT means the tmp never landed or rename already consumed it.
|
|
83
|
+
if (cleanupErr?.code !== 'ENOENT') {
|
|
84
|
+
console.error(
|
|
85
|
+
`[sync-manifest] failed to remove temp manifest ${tmpPath}: ${cleanupErr && cleanupErr.message}`,
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function createManifestStore(localPath, { debounceMs = FLUSH_DEBOUNCE_MS } = {}) {
|
|
93
|
+
const state = loadManifest(localPath);
|
|
94
|
+
let timer = null;
|
|
95
|
+
|
|
96
|
+
function scheduleFlush() {
|
|
97
|
+
if (timer) clearTimeout(timer);
|
|
98
|
+
timer = setTimeout(() => {
|
|
99
|
+
timer = null;
|
|
100
|
+
saveManifest(localPath, state);
|
|
101
|
+
}, debounceMs);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return {
|
|
105
|
+
upsertEntry(trackingKey, entry) {
|
|
106
|
+
state.entries[trackingKey] = entry;
|
|
107
|
+
scheduleFlush();
|
|
108
|
+
},
|
|
109
|
+
removeEntry(trackingKey) {
|
|
110
|
+
delete state.entries[trackingKey];
|
|
111
|
+
scheduleFlush();
|
|
112
|
+
},
|
|
113
|
+
getEntry(trackingKey) {
|
|
114
|
+
return state.entries[trackingKey] ?? null;
|
|
115
|
+
},
|
|
116
|
+
getState() {
|
|
117
|
+
return state;
|
|
118
|
+
},
|
|
119
|
+
flush() {
|
|
120
|
+
if (timer) {
|
|
121
|
+
clearTimeout(timer);
|
|
122
|
+
timer = null;
|
|
123
|
+
}
|
|
124
|
+
saveManifest(localPath, state);
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
package/org-sync.js
CHANGED
|
@@ -37,6 +37,8 @@ import {
|
|
|
37
37
|
} from './paths.js';
|
|
38
38
|
import { TeamRegistry } from './team-registry.js';
|
|
39
39
|
import { loadSyncState, saveSyncState } from './sync-state.js';
|
|
40
|
+
import { createManifestStore } from './manifest.js';
|
|
41
|
+
import { shouldPersistLocalSyncState } from './push-outcome.js';
|
|
40
42
|
import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
|
|
41
43
|
import { YjsFileBinding } from './yjs-file-binding.js';
|
|
42
44
|
import {
|
|
@@ -93,6 +95,7 @@ export async function startOrgSync({
|
|
|
93
95
|
// dotfile in the synced folder (ignored by the watcher), so it travels
|
|
94
96
|
// with the folder and needs no extra wiring from the caller.
|
|
95
97
|
const STATE_PATH = statePath ?? path.join(LOCAL_PATH, '.compound-sync-state.json');
|
|
98
|
+
const manifestStore = createManifestStore(LOCAL_PATH);
|
|
96
99
|
|
|
97
100
|
// Teardown handles: every onSnapshot unsub plus the chokidar watcher.
|
|
98
101
|
const unsubscribers = [];
|
|
@@ -321,8 +324,8 @@ export async function startOrgSync({
|
|
|
321
324
|
}
|
|
322
325
|
}
|
|
323
326
|
|
|
324
|
-
// Static (user-level) folders. Team folders are
|
|
325
|
-
//
|
|
327
|
+
// Static (user-level) folders. Team folders are mkdir'd by setupTeam,
|
|
328
|
+
// which runs after manifest reconcile (see startup ordering below).
|
|
326
329
|
function ensureFolderStructure() {
|
|
327
330
|
const privatePath = path.join(LOCAL_PATH, PRIVATE_FOLDER);
|
|
328
331
|
if (!fs.existsSync(privatePath)) fs.mkdirSync(privatePath, { recursive: true });
|
|
@@ -376,6 +379,11 @@ export async function startOrgSync({
|
|
|
376
379
|
setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
|
|
377
380
|
deleteBaseline(prevKnown.trackingKey);
|
|
378
381
|
}
|
|
382
|
+
const manifestEntry = manifestStore.getEntry(prevKnown.trackingKey);
|
|
383
|
+
if (manifestEntry) {
|
|
384
|
+
manifestStore.removeEntry(prevKnown.trackingKey);
|
|
385
|
+
manifestStore.upsertEntry(trackingKey, manifestEntry);
|
|
386
|
+
}
|
|
379
387
|
const binding = bindings.get(fileId);
|
|
380
388
|
if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
|
|
381
389
|
bindingsByPath.delete(prevKnown.localPath);
|
|
@@ -412,6 +420,7 @@ export async function startOrgSync({
|
|
|
412
420
|
}
|
|
413
421
|
lastKnownUpdate.delete(trackingKey);
|
|
414
422
|
deleteBaseline(trackingKey);
|
|
423
|
+
manifestStore.removeEntry(trackingKey);
|
|
415
424
|
knownPaths.delete(fileId);
|
|
416
425
|
await disposeBinding(change.doc.id, localFilePath);
|
|
417
426
|
return;
|
|
@@ -538,8 +547,9 @@ export async function startOrgSync({
|
|
|
538
547
|
}
|
|
539
548
|
|
|
540
549
|
// Set up per-team resources: mkdir the local folder, register the
|
|
541
|
-
// Firestore listener for that team's files.
|
|
542
|
-
//
|
|
550
|
+
// Firestore listener for that team's files. On startup, called once
|
|
551
|
+
// per bootstrapped team after reconcile; thereafter via TeamRegistry
|
|
552
|
+
// when the live teams-collection listener reports add/modify/remove.
|
|
543
553
|
setupTeam = (teamId, teamData) => {
|
|
544
554
|
const teamFolder = teamFolderName(teamData.name);
|
|
545
555
|
const teamPath = path.join(LOCAL_PATH, teamFolder);
|
|
@@ -558,35 +568,287 @@ export async function startOrgSync({
|
|
|
558
568
|
console.log(` [${orgId}] Listening: ${teamFolder}`);
|
|
559
569
|
};
|
|
560
570
|
|
|
561
|
-
//
|
|
562
|
-
//
|
|
563
|
-
//
|
|
564
|
-
//
|
|
565
|
-
//
|
|
566
|
-
|
|
567
|
-
await
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
571
|
+
// Bootstrap team metadata synchronously so manifest reconcile can
|
|
572
|
+
// query team-scoped cloud files before any file onSnapshot handlers
|
|
573
|
+
// run (DOU-55). Populating teamIdsByName here also lets chokidar
|
|
574
|
+
// route local paths once the watcher starts. Per-team file listeners
|
|
575
|
+
// and the live teams-collection onSnapshot are wired after reconcile
|
|
576
|
+
// completes so their initial snapshots cannot race with startup cleanup.
|
|
577
|
+
const teamsSnap = await getDocs(collection(db, `orgs/${orgId}/teams`));
|
|
578
|
+
for (const teamDoc of teamsSnap.docs) {
|
|
579
|
+
teamRegistry.bootstrap(teamDoc.id, teamDoc.data());
|
|
580
|
+
}
|
|
581
|
+
console.log(` [${orgId}] Teams: ${teamRegistry.names().join(', ') || 'none'}`);
|
|
582
|
+
|
|
583
|
+
function trackingKeyFromDocData(data) {
|
|
584
|
+
return `${data.scope || 'team'}:${data.path}`;
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
// Paths the daemon does not scan or sync. Add checks here as needed
|
|
588
|
+
// (extensions, .compoundignore, etc.).
|
|
589
|
+
function shouldSkipLocalPath(name, rel) {
|
|
590
|
+
if (name.startsWith('.') || name === 'node_modules') return true;
|
|
591
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return true;
|
|
592
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return true;
|
|
593
|
+
return false;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function walkLocalSyncableFiles(absDir, files) {
|
|
597
|
+
let entries;
|
|
598
|
+
try {
|
|
599
|
+
entries = fs.readdirSync(absDir, { withFileTypes: true });
|
|
600
|
+
} catch (err) {
|
|
601
|
+
const rel = path.relative(LOCAL_PATH, absDir) || '.';
|
|
602
|
+
console.warn(` [${orgId}] [local scan] could not read directory ${rel}: ${err.message}`);
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
for (const ent of entries) {
|
|
606
|
+
const abs = path.join(absDir, ent.name);
|
|
607
|
+
const rel = path.relative(LOCAL_PATH, abs);
|
|
608
|
+
if (shouldSkipLocalPath(ent.name, rel)) continue;
|
|
609
|
+
if (ent.isDirectory()) {
|
|
610
|
+
walkLocalSyncableFiles(abs, files);
|
|
611
|
+
} else if (ent.isFile()) {
|
|
612
|
+
files.push(abs);
|
|
581
613
|
}
|
|
582
|
-
}
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
614
|
+
}
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
function collectLocalSyncableFiles() {
|
|
618
|
+
const files = [];
|
|
619
|
+
walkLocalSyncableFiles(LOCAL_PATH, files);
|
|
620
|
+
return files;
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
function readLocalFilePayload(filePath) {
|
|
624
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
625
|
+
const ext = extFromPath(rel);
|
|
626
|
+
const mimeType = mimeTypeFromExt(ext);
|
|
627
|
+
const data = isTextMimeType(mimeType)
|
|
628
|
+
? fs.readFileSync(filePath, 'utf-8')
|
|
629
|
+
: fs.readFileSync(filePath);
|
|
630
|
+
return { data, mimeType, contentHash: computeContentHash(data) };
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
async function pushLocalFileAtPath(filePath) {
|
|
634
|
+
let stat;
|
|
635
|
+
try {
|
|
636
|
+
stat = fs.statSync(filePath);
|
|
637
|
+
} catch {
|
|
638
|
+
return;
|
|
639
|
+
}
|
|
640
|
+
if (!stat.isFile() || stat.size > maxUploadBytes) return;
|
|
641
|
+
const { data, mimeType } = readLocalFilePayload(filePath);
|
|
642
|
+
await pushToFirestore(filePath, data, mimeType, stat.mtime.toISOString());
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
// Sync local cleanup during startup manifest reconcile. Runs before
|
|
646
|
+
// Firestore listeners and Yjs bindings are wired, so no async cloud or
|
|
647
|
+
// binding teardown is needed here.
|
|
648
|
+
function deleteLocalReconcile(localFilePath, fileId, trackingKey) {
|
|
649
|
+
suppressLocal.add(localFilePath);
|
|
650
|
+
if (fs.existsSync(localFilePath)) {
|
|
651
|
+
console.log(` [${orgId}] [manifest reconcile] DELETE local ${path.relative(LOCAL_PATH, localFilePath)}`);
|
|
652
|
+
fs.unlinkSync(localFilePath);
|
|
653
|
+
}
|
|
654
|
+
lastKnownUpdate.delete(trackingKey);
|
|
655
|
+
deleteBaseline(trackingKey);
|
|
656
|
+
manifestStore.removeEntry(trackingKey);
|
|
657
|
+
if (fileId) {
|
|
658
|
+
knownPaths.delete(fileId);
|
|
659
|
+
}
|
|
660
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
async function deleteRemoteReconcile(cloudData, trackingKey) {
|
|
664
|
+
const scope = cloudData.scope || 'team';
|
|
665
|
+
const teamId = cloudData.teamId;
|
|
666
|
+
const docPath = cloudData.path;
|
|
667
|
+
const result = await deleteFileFromCloud({
|
|
668
|
+
filesRef,
|
|
669
|
+
storage,
|
|
670
|
+
orgId,
|
|
671
|
+
userId,
|
|
672
|
+
scope,
|
|
673
|
+
teamId,
|
|
674
|
+
docPath,
|
|
675
|
+
});
|
|
676
|
+
if (result.action === 'deleted') {
|
|
677
|
+
console.log(` [${orgId}] [manifest reconcile] DELETE cloud ${docPath}`);
|
|
678
|
+
}
|
|
679
|
+
manifestStore.removeEntry(trackingKey);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
function localPathForCloudData(data) {
|
|
683
|
+
if (!data?.path) return null;
|
|
684
|
+
if (data.scope === 'private') {
|
|
685
|
+
if (data.ownerId === userId) {
|
|
686
|
+
return path.join(LOCAL_PATH, PRIVATE_FOLDER, data.path);
|
|
587
687
|
}
|
|
688
|
+
return path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data.path);
|
|
689
|
+
}
|
|
690
|
+
if (data.scope === 'team' && data.teamId) {
|
|
691
|
+
const team = teamRegistry.get(data.teamId);
|
|
692
|
+
if (!team?.name) return null;
|
|
693
|
+
return path.join(LOCAL_PATH, teamFolderName(team.name), data.path);
|
|
694
|
+
}
|
|
695
|
+
return null;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
async function fetchCloudFilesByKey() {
|
|
699
|
+
const cloudByKey = new Map();
|
|
700
|
+
|
|
701
|
+
function ingestDoc(d) {
|
|
702
|
+
const data = d.data();
|
|
703
|
+
if (data.type === 'folder' || !data.path) return;
|
|
704
|
+
const key = trackingKeyFromDocData(data);
|
|
705
|
+
cloudByKey.set(key, { fileId: d.id, data });
|
|
706
|
+
}
|
|
707
|
+
|
|
708
|
+
for (const [teamId] of teamRegistry.entries()) {
|
|
709
|
+
const teamSnap = await getDocs(query(
|
|
710
|
+
filesRef,
|
|
711
|
+
where('scope', '==', 'team'),
|
|
712
|
+
where('teamId', '==', teamId),
|
|
713
|
+
));
|
|
714
|
+
teamSnap.docs.forEach(ingestDoc);
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
if (userId) {
|
|
718
|
+
const privateSnap = await getDocs(query(
|
|
719
|
+
filesRef,
|
|
720
|
+
where('scope', '==', 'private'),
|
|
721
|
+
where('ownerId', '==', userId),
|
|
722
|
+
));
|
|
723
|
+
privateSnap.docs.forEach(ingestDoc);
|
|
724
|
+
|
|
725
|
+
const sharedSnap = await getDocs(query(
|
|
726
|
+
filesRef,
|
|
727
|
+
where('scope', '==', 'private'),
|
|
728
|
+
where('sharedWith', 'array-contains', userId),
|
|
729
|
+
));
|
|
730
|
+
sharedSnap.docs.forEach(ingestDoc);
|
|
731
|
+
}
|
|
732
|
+
|
|
733
|
+
return cloudByKey;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
// Runs after team metadata is loaded but before any file listeners or
|
|
737
|
+
// chokidar. See docs/features/sync/sync-manifest-design.md.
|
|
738
|
+
async function reconcileManifestOnStartup() {
|
|
739
|
+
console.log(` [${orgId}] Reconciling local manifest...`);
|
|
740
|
+
const cloudByKey = await fetchCloudFilesByKey();
|
|
741
|
+
|
|
742
|
+
const localByKey = new Map();
|
|
743
|
+
for (const filePath of collectLocalSyncableFiles()) {
|
|
744
|
+
const { scope, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
745
|
+
if (!docPath) continue;
|
|
746
|
+
localByKey.set(`${scope}:${docPath}`, filePath);
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
// Bootstrap manifest entries for files already in sync before DOU-55.
|
|
750
|
+
for (const [key, cloud] of cloudByKey) {
|
|
751
|
+
if (manifestStore.getEntry(key)) continue;
|
|
752
|
+
const localFilePath = localByKey.get(key);
|
|
753
|
+
if (!localFilePath || !fs.existsSync(localFilePath)) continue;
|
|
754
|
+
try {
|
|
755
|
+
const { contentHash } = readLocalFilePayload(localFilePath);
|
|
756
|
+
if (contentHash === cloud.data.contentHash) {
|
|
757
|
+
manifestStore.upsertEntry(key, {
|
|
758
|
+
fileId: cloud.fileId,
|
|
759
|
+
contentHash,
|
|
760
|
+
lastSyncedAt: cloud.data.updatedAt || cloud.data.createdAt || new Date().toISOString(),
|
|
761
|
+
});
|
|
762
|
+
}
|
|
763
|
+
} catch {
|
|
764
|
+
// skip unreadable locals
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
const allKeys = new Set([
|
|
769
|
+
...Object.keys(manifestStore.getState().entries),
|
|
770
|
+
...localByKey.keys(),
|
|
771
|
+
...cloudByKey.keys(),
|
|
772
|
+
]);
|
|
773
|
+
|
|
774
|
+
for (const trackingKey of allKeys) {
|
|
775
|
+
const entry = manifestStore.getEntry(trackingKey);
|
|
776
|
+
let localFilePath = localByKey.get(trackingKey) ?? null;
|
|
777
|
+
const cloud = cloudByKey.get(trackingKey) ?? null;
|
|
778
|
+
if (!localFilePath && cloud) {
|
|
779
|
+
localFilePath = localPathForCloudData(cloud.data);
|
|
780
|
+
}
|
|
781
|
+
const onDisk = !!(localFilePath && fs.existsSync(localFilePath));
|
|
782
|
+
|
|
783
|
+
if (entry && onDisk && !cloud) {
|
|
784
|
+
let localHash;
|
|
785
|
+
try {
|
|
786
|
+
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
787
|
+
} catch {
|
|
788
|
+
localHash = null;
|
|
789
|
+
}
|
|
790
|
+
if (localHash === null) {
|
|
791
|
+
console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
|
|
792
|
+
} else if (entry.contentHash && localHash !== entry.contentHash) {
|
|
793
|
+
await pushLocalFileAtPath(localFilePath);
|
|
794
|
+
} else {
|
|
795
|
+
deleteLocalReconcile(localFilePath, entry.fileId, trackingKey);
|
|
796
|
+
}
|
|
797
|
+
} else if (!entry && onDisk && !cloud) {
|
|
798
|
+
const baselineHash = lastSyncedHash.get(trackingKey);
|
|
799
|
+
let localHash;
|
|
800
|
+
try {
|
|
801
|
+
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
802
|
+
} catch {
|
|
803
|
+
localHash = null;
|
|
804
|
+
}
|
|
805
|
+
if (localHash === null) {
|
|
806
|
+
console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
|
|
807
|
+
} else if (baselineHash && localHash === baselineHash) {
|
|
808
|
+
deleteLocalReconcile(localFilePath, null, trackingKey);
|
|
809
|
+
} else {
|
|
810
|
+
await pushLocalFileAtPath(localFilePath);
|
|
811
|
+
}
|
|
812
|
+
} else if (entry && !onDisk && cloud) {
|
|
813
|
+
await deleteRemoteReconcile(cloud.data, trackingKey);
|
|
814
|
+
} else if ((entry || cloud) && onDisk && cloud) {
|
|
815
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
816
|
+
fileId: cloud.fileId,
|
|
817
|
+
contentHash: cloud.data.contentHash,
|
|
818
|
+
lastSyncedAt: cloud.data.updatedAt || cloud.data.createdAt || new Date().toISOString(),
|
|
819
|
+
});
|
|
820
|
+
} else if (!entry && onDisk && cloud) {
|
|
821
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
822
|
+
fileId: cloud.fileId,
|
|
823
|
+
contentHash: cloud.data.contentHash,
|
|
824
|
+
lastSyncedAt: cloud.data.updatedAt || cloud.data.createdAt || new Date().toISOString(),
|
|
825
|
+
});
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
manifestStore.flush();
|
|
830
|
+
console.log(` [${orgId}] Manifest reconcile complete.`);
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
await reconcileManifestOnStartup();
|
|
834
|
+
|
|
835
|
+
// Wire per-team file listeners and the live teams-collection watcher
|
|
836
|
+
// after reconcile so initial onSnapshot deliveries cannot race with
|
|
837
|
+
// startup cleanup (DOU-55). applyChange on the live listener picks
|
|
838
|
+
// up teamspaces created/renamed/removed after daemon start without
|
|
839
|
+
// restarting sync.
|
|
840
|
+
for (const [teamId, teamData] of teamRegistry.entries()) {
|
|
841
|
+
setupTeam(teamId, teamData);
|
|
842
|
+
}
|
|
843
|
+
const teamsUnsub = onSnapshot(collection(db, `orgs/${orgId}/teams`), (snapshot) => {
|
|
844
|
+
snapshot.docChanges().forEach((change) => {
|
|
845
|
+
teamRegistry.applyChange({
|
|
846
|
+
type: change.type,
|
|
847
|
+
id: change.doc.id,
|
|
848
|
+
data: change.doc.data(),
|
|
849
|
+
});
|
|
588
850
|
});
|
|
589
|
-
});
|
|
851
|
+
}, (err) => console.error(` [${orgId}] Teams listener error:`, err));
|
|
590
852
|
unsubscribers.push(teamsUnsub);
|
|
591
853
|
|
|
592
854
|
function startFirestoreListeners() {
|
|
@@ -637,9 +899,10 @@ export async function startOrgSync({
|
|
|
637
899
|
}
|
|
638
900
|
|
|
639
901
|
// DOU-217 helper: does a doc with this path + matching contentHash
|
|
640
|
-
// exist in Firestore right now?
|
|
641
|
-
//
|
|
642
|
-
// re-
|
|
902
|
+
// exist in Firestore right now? Startup reconcile (DOU-55) handles
|
|
903
|
+
// manifest-present orphans; this check covers the runtime push path
|
|
904
|
+
// when chokidar re-emits a file whose baseline still matches but the
|
|
905
|
+
// Firestore doc is gone (e.g. corrupt/missing manifest).
|
|
643
906
|
async function firestoreDocExists({ scope, teamId, docPath, contentHash }) {
|
|
644
907
|
let q;
|
|
645
908
|
if (scope === 'private') {
|
|
@@ -675,12 +938,10 @@ export async function startOrgSync({
|
|
|
675
938
|
// a genuine content change proceeds.
|
|
676
939
|
//
|
|
677
940
|
// DOU-217: trust the baseline ONLY after verifying the Firestore
|
|
678
|
-
// doc actually exists.
|
|
679
|
-
//
|
|
680
|
-
//
|
|
681
|
-
//
|
|
682
|
-
// Verifying via a single indexed read per touch is cheap; if the
|
|
683
|
-
// doc is gone, clear the baseline and fall through to push.
|
|
941
|
+
// doc actually exists. DOU-55 startup reconcile deletes local orphans
|
|
942
|
+
// when the manifest records a remote delete; this guard applies when
|
|
943
|
+
// pushToFirestore runs at runtime (chokidar add/change) and the
|
|
944
|
+
// baseline would otherwise suppress re-upload forever.
|
|
684
945
|
const contentHash = computeContentHash(data);
|
|
685
946
|
if (lastSyncedHash.get(trackingKey) === contentHash) {
|
|
686
947
|
if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
|
|
@@ -718,8 +979,16 @@ export async function startOrgSync({
|
|
|
718
979
|
// Record the synced content baseline whenever local matches the
|
|
719
980
|
// server (created/updated, or already-equal noop), so the pull side
|
|
720
981
|
// can tell "local unchanged" from "local edited" by content.
|
|
721
|
-
if (result
|
|
982
|
+
if (shouldPersistLocalSyncState(result)) {
|
|
722
983
|
setBaseline(trackingKey, contentHash);
|
|
984
|
+
if (result.fileId) {
|
|
985
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
986
|
+
fileId: result.fileId,
|
|
987
|
+
contentHash,
|
|
988
|
+
lastSyncedAt: now,
|
|
989
|
+
});
|
|
990
|
+
manifestStore.flush();
|
|
991
|
+
}
|
|
723
992
|
} else if (result.action === 'skipped-stale') {
|
|
724
993
|
// The server copy is genuinely newer; clear the pre-set marker so
|
|
725
994
|
// its firestore -> local snapshot is allowed through to update the
|
|
@@ -756,6 +1025,7 @@ export async function startOrgSync({
|
|
|
756
1025
|
|
|
757
1026
|
if (result.action === 'deleted') {
|
|
758
1027
|
console.log(` [${orgId}] [local → firestore] DELETE ${docPath} (${scope})`);
|
|
1028
|
+
manifestStore.removeEntry(trackingKey);
|
|
759
1029
|
}
|
|
760
1030
|
}
|
|
761
1031
|
|
|
@@ -971,6 +1241,7 @@ export async function startOrgSync({
|
|
|
971
1241
|
// is still running" guarantee on the resolved promise.
|
|
972
1242
|
async stop() {
|
|
973
1243
|
stoppedRef.value = true;
|
|
1244
|
+
manifestStore.flush();
|
|
974
1245
|
for (const unsub of unsubscribers) {
|
|
975
1246
|
try { unsub(); } catch (e) { /* ignore */ }
|
|
976
1247
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@doubling/compound-sync",
|
|
3
|
-
"version": "1.12.
|
|
3
|
+
"version": "1.12.4",
|
|
4
4
|
"description": "Bidirectional sync between Compound and local markdown files",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -10,6 +10,8 @@
|
|
|
10
10
|
"sync.js",
|
|
11
11
|
"org-sync.js",
|
|
12
12
|
"sync-state.js",
|
|
13
|
+
"manifest.js",
|
|
14
|
+
"push-outcome.js",
|
|
13
15
|
"config.js",
|
|
14
16
|
"paths.js",
|
|
15
17
|
"paths.d.ts",
|
|
@@ -31,8 +33,9 @@
|
|
|
31
33
|
"build": "tsc",
|
|
32
34
|
"typecheck": "tsc --noEmit",
|
|
33
35
|
"prepack": "npm run build",
|
|
34
|
-
"test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js yjs-file-binding.test.js yjs-binding-state.test.js",
|
|
35
|
-
"test:integration": "
|
|
36
|
+
"test:unit": "node --import tsx --test config.test.js paths.test.js storage-helpers.test.js folder-chain.test.js team-registry.test.js auth-persistence.test.js pre-mint-app-check.test.js setup-auth-with-pre-mint.test.js sync-state.test.js manifest.test.js push-outcome.test.js yjs-file-binding.test.js yjs-binding-state.test.js files-helpers.test.js",
|
|
37
|
+
"test:integration": "node --import tsx --test --test-timeout=180000 files-push-create-private.test.js files-push-create-team.test.js files-push-update-private.test.js files-push-binary.test.js files-push-staleness.test.js files-push-storage-fail.test.js files-push-no-content.test.js files-push-stamps.test.js files-push-folder.test.js files-delete.test.js files-delete-folder.test.js files-read-blob.test.js org-sync-local-edit.test.js org-sync-file-ops.test.js org-sync-mtime.test.js org-sync-rename.test.js org-sync-yjs-pull.test.js org-sync-yjs-push.test.js org-sync-cold-start.test.js org-sync-startup-ordering.test.js org-sync-manifest-reconcile.test.js org-sync-special-chars.test.js org-sync-copy.test.js two-user-happy-paths.test.js two-user-yjs-convergence.test.js files-concurrency-folder-create.test.js files-concurrency-file-create.test.js files-concurrency-file-delete.test.js files-concurrency-folder-delete.test.js",
|
|
38
|
+
"test:integration:emulator": "USE_EMULATOR=true firebase emulators:exec --only firestore,storage,auth 'npm run test:integration'",
|
|
36
39
|
"test": "npm run test:unit && npm run test:integration"
|
|
37
40
|
},
|
|
38
41
|
"dependencies": {
|
package/push-outcome.js
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Whether a pushFileToCloud result should update local sync artifacts
|
|
2
|
+
// (content baseline + manifest). Partial uploads (Firestore OK, blob
|
|
3
|
+
// failed) must not be recorded as successfully synced.
|
|
4
|
+
|
|
5
|
+
export function shouldPersistLocalSyncState(result) {
|
|
6
|
+
if (result.storageError) return false;
|
|
7
|
+
return result.action === 'created'
|
|
8
|
+
|| result.action === 'updated'
|
|
9
|
+
|| result.action === 'noop';
|
|
10
|
+
}
|
package/team-registry.js
CHANGED
|
@@ -26,6 +26,18 @@ export class TeamRegistry {
|
|
|
26
26
|
this.onTeardown = onTeardown;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
// Register a team in-memory without wiring listeners. Used during
|
|
30
|
+
// startup so manifest reconcile can resolve team-scoped paths before
|
|
31
|
+
// any file onSnapshot handlers run (DOU-55 ordering).
|
|
32
|
+
bootstrap(teamId, teamData) {
|
|
33
|
+
if (this.teams.has(teamId)) return false;
|
|
34
|
+
this.teams.set(teamId, teamData);
|
|
35
|
+
if (teamData && teamData.name) {
|
|
36
|
+
this.teamIdsByName.set(teamData.name, teamId);
|
|
37
|
+
}
|
|
38
|
+
return true;
|
|
39
|
+
}
|
|
40
|
+
|
|
29
41
|
// Idempotently register a team. Safe to call with the same team
|
|
30
42
|
// multiple times; only the first call triggers onSetup.
|
|
31
43
|
add(teamId, teamData) {
|