@doubling/compound-sync 1.12.3 → 1.12.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/org-sync.js CHANGED
@@ -11,999 +11,1223 @@
11
11
  // caller. Behavior is otherwise unchanged (including the known
12
12
  // mtime-based decision quirks DOU-154 will rework, those stay so the
13
13
  // scenario tests can pin them red first).
14
-
15
14
  import chokidar from 'chokidar';
16
- import fs from 'fs';
17
- import path from 'path';
18
- import { collection, query, where, onSnapshot, getDocs } from 'firebase/firestore';
19
- import {
20
- pushFileToCloud,
21
- deleteFileFromCloud,
22
- pushFolderToCloud,
23
- deleteFolderFromCloud,
24
- readBlob,
25
- readBlobAsText,
26
- extFromPath,
27
- isTextMimeType,
28
- mimeTypeFromExt,
29
- computeContentHash,
30
- } from './files.js';
31
- import {
32
- scopeFromLocalPath,
33
- teamFolderName,
34
- PRIVATE_FOLDER,
35
- SHARED_WITH_ME_FOLDER,
36
- SHARED_BY_ME_FOLDER,
37
- } from './paths.js';
15
+ import fs from 'node:fs';
16
+ import path from 'node:path';
17
+ import { collection, query, where, onSnapshot, getDocs, } from 'firebase/firestore';
18
+ import { pushFileToCloud, deleteFileFromCloud, pushFolderToCloud, deleteFolderFromCloud, readBlob, readBlobAsText, extFromPath, isTextMimeType, mimeTypeFromExt, computeContentHash, } from './files.js';
19
+ import { scopeFromLocalPath, teamFolderName, PRIVATE_FOLDER, SHARED_WITH_ME_FOLDER, SHARED_BY_ME_FOLDER, } from './paths.js';
38
20
  import { TeamRegistry } from './team-registry.js';
39
21
  import { loadSyncState, saveSyncState } from './sync-state.js';
22
+ import { createManifestStore } from './manifest.js';
23
+ import { shouldPersistLocalSyncState } from './push-outcome.js';
40
24
  import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
41
25
  import { YjsFileBinding } from './yjs-file-binding.js';
42
- import {
43
- loadBindingState, saveBindingState, deleteBindingState,
44
- } from './yjs-binding-state.js';
45
-
26
+ import { loadBindingState, saveBindingState, deleteBindingState, } from './yjs-binding-state.js';
46
27
  const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
47
-
48
- // Retry a blob download until either the bytes appear at all (covers
49
- // the create race: doc written before blob upload) AND the content
50
- // hash matches the metadata's `contentHash` (covers the update race:
51
- // the blob still has the old bytes when the new doc is already
52
- // visible). Backs off 100/200/400/800 ms (1.5s total) across both
53
- // failure modes. Other errors throw immediately.
54
28
  async function readBlobWithRetry(read, { expectedHash = null, maxAttempts = 5 } = {}) {
55
- let delay = 100;
56
- let lastErr;
57
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
58
- try {
59
- const content = await read();
60
- if (!expectedHash || computeContentHash(content) === expectedHash) {
61
- return content;
62
- }
63
- lastErr = new Error(
64
- `blob hash does not match metadata contentHash (attempt ${attempt}/${maxAttempts})`,
65
- );
66
- } catch (err) {
67
- if (err?.code !== 'storage/object-not-found') throw err;
68
- lastErr = err;
69
- }
70
- if (attempt < maxAttempts) {
71
- await new Promise((resolve) => setTimeout(resolve, delay));
72
- delay *= 2;
73
- }
74
- }
75
- throw lastErr;
29
+ let delay = 100;
30
+ let lastErr;
31
+ for (let attempt = 1; attempt <= maxAttempts; attempt++) {
32
+ try {
33
+ const content = await read();
34
+ if (!expectedHash || computeContentHash(content) === expectedHash) {
35
+ return content;
36
+ }
37
+ lastErr = new Error(`blob hash does not match metadata contentHash (attempt ${attempt}/${maxAttempts})`);
38
+ }
39
+ catch (err) {
40
+ if (err?.code !== 'storage/object-not-found')
41
+ throw err;
42
+ lastErr = err;
43
+ }
44
+ if (attempt < maxAttempts) {
45
+ await new Promise((resolve) => setTimeout(resolve, delay));
46
+ delay *= 2;
47
+ }
48
+ }
49
+ throw lastErr;
76
50
  }
77
-
78
51
  // Start the per-org sync loop. Returns { stop } to unsubscribe all
79
52
  // listeners and close the watcher (used by tests and by graceful
80
53
  // shutdown). `localPath` must already be absolute/expanded.
81
- export async function startOrgSync({
82
- db,
83
- storage,
84
- userId,
85
- orgId,
86
- localPath,
87
- maxUploadBytes = DEFAULT_MAX_UPLOAD_BYTES,
88
- statePath,
89
- }) {
90
- const LOCAL_PATH = localPath;
91
- const filesRef = collection(db, `orgs/${orgId}/files`);
92
- // Where the per-folder content baseline is persisted. Default: a
93
- // dotfile in the synced folder (ignored by the watcher), so it travels
94
- // with the folder and needs no extra wiring from the caller.
95
- const STATE_PATH = statePath ?? path.join(LOCAL_PATH, '.compound-sync-state.json');
96
-
97
- // Teardown handles: every onSnapshot unsub plus the chokidar watcher.
98
- const unsubscribers = [];
99
- let watcher = null;
100
-
101
- const lastKnownUpdate = new Map();
102
- // Content hash of each file as of the last successful sync (push or
103
- // pull), keyed by trackingKey. This "baseline" decides pull/push
104
- // direction by CONTENT rather than filesystem mtime (which an editor
105
- // like Obsidian bumps without a content change). Loaded from
106
- // STATE_PATH so it survives restarts; persisted on every change.
107
- const lastSyncedHash = new Map(Object.entries(loadSyncState(STATE_PATH)));
108
- function persistBaseline() {
109
- saveSyncState(STATE_PATH, Object.fromEntries(lastSyncedHash));
110
- }
111
- function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
112
- function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
113
- const suppressLocal = new Set();
114
-
115
- // Per-file Yjs bindings, created on first sight of a text file
116
- // (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
117
- // orgs/.../yupdates and the debounced disk flush. We keep two
118
- // indexes: by fileId (so handleFirestoreChange can dispose on
119
- // 'removed' and re-use across events) and by local path (so the
120
- // chokidar handler can route disk edits into the binding without
121
- // re-resolving the fileId).
122
- const bindings = new Map();
123
- const bindingsByPath = new Map();
124
-
125
- // DOU-205: per-fileId record of the most recently observed Firestore
126
- // metadata (local path + tracking key). Used to detect renames /
127
- // moves on subsequent 'modified' snapshot events: when the file
128
- // doc's path changes server-side (via the web rename UI), the local
129
- // path computed from the new data also changes, and this map lets
130
- // the handler relocate the local file in place (fs.renameSync,
131
- // which preserves Obsidian's inode-level tracking) rather than
132
- // leaving the old file orphaned at its previous location.
133
- const knownPaths = new Map();
134
-
135
- // Track which fileIds already have a binding (or one being set
136
- // up) so we never double-create. `stoppedRef` is read by the
137
- // background retry loops so they can abandon work after sync.stop.
138
- // `pendingPromises` lets stop() await any in-flight setup OR any
139
- // fire-and-forget snapshot/chokidar handler work before returning.
140
- // Without this, the next test's clearFirestore + terminate can
141
- // race a handler that's still flushing storage from this test,
142
- // surfacing as "Firebase app was deleted" errors in the daemon log
143
- // and (suspected) accumulating gRPC stream state that the next
144
- // listener attach receives as a single oversize Listen message.
145
- const pendingBindings = new Set();
146
- const pendingPromises = new Set();
147
- const stoppedRef = { value: false };
148
-
149
- // Helper: register a promise with pendingPromises so stop() awaits
150
- // it, and auto-remove on settle. Returns the same promise so call
151
- // sites can keep their chaining shape.
152
- function track(p) {
153
- pendingPromises.add(p);
154
- p.finally(() => pendingPromises.delete(p));
155
- return p;
156
- }
157
-
158
- // DOU-254: per-local-path operation queue. chokidar can fire `add`
159
- // then `unlink` for the same path within a few hundred ms (a test
160
- // writes a file, waits only for the Firestore doc to appear, then
161
- // deletes locally). Without serialization, the push (setDoc +
162
- // uploadBlob) and the delete (preRecheck + deleteBlob + deleteDoc)
163
- // race against each other on the same fileId, and uploadBlob can
164
- // land AFTER deleteBlob, leaving an orphaned blob after the doc is
165
- // gone. Chain ops keyed by local path so the next op starts only
166
- // after the previous one settles.
167
- const localOpQueues = new Map();
168
- function enqueueLocalOp(filePath, op) {
169
- const prev = localOpQueues.get(filePath) || Promise.resolve();
170
- // Swallow the previous op's rejection so a failed push doesn't
171
- // skip the subsequent delete (or vice versa); each op handles
172
- // its own errors via the .catch() at the call site.
173
- const next = prev.catch(() => {}).then(op);
174
- localOpQueues.set(filePath, next);
175
- next.finally(() => {
176
- if (localOpQueues.get(filePath) === next) localOpQueues.delete(filePath);
177
- });
178
- return next;
179
- }
180
-
181
- // Fire-and-forget binding setup. Returns nothing because the
182
- // common shape "ensureBinding then read its ytext" runs straight
183
- // into a race: the daemon's onSnapshot fires on Alice's own
184
- // optimistic write before the file doc is server-committed, and
185
- // the Firestore rules engine evaluates the yupdates/yjs nested
186
- // rule via a server-side get() against the parent files/{id}
187
- // doc. That get() sees no doc and returns null, so binding.start
188
- // fails with permission-denied; the right behavior is to retry
189
- // until the commit propagates, not to block handleFirestoreChange
190
- // for seconds. Callers fall back to the blob path while the
191
- // binding is still being set up; once it lands in
192
- // bindings/bindingsByPath, subsequent events take the Yjs path.
193
- function ensureBinding(fileId, localFilePath) {
194
- if (bindings.has(fileId) || pendingBindings.has(fileId)) return;
195
- pendingBindings.add(fileId);
196
- const onDiskWrite = (text) => {
197
- suppressLocal.add(localFilePath);
198
- ensureDir(localFilePath);
199
- fs.writeFileSync(localFilePath, text, 'utf-8');
200
- setTimeout(() => suppressLocal.delete(localFilePath), 1000);
201
- };
202
- // DOU-209: persist the Y.Doc binary state after every stable
203
- // change so the next daemon restart has the right baseline to
204
- // merge offline disk edits without losing offline web edits.
205
- const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
206
- // DOU-209: feed the binding's startup-merge with the daemon's
207
- // last view (initialDocState) and the current disk content. If
208
- // they diverge, the binding applies the diff as Yjs ops so disk-
209
- // side offline edits survive the cloud-side replay.
210
- const initialDocState = loadBindingState(LOCAL_PATH, fileId);
211
- let currentDiskText = null;
212
- try {
213
- if (fs.existsSync(localFilePath)) {
214
- currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
215
- }
216
- } catch (err) {
217
- console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err && err.message}`);
218
- }
219
- const setupPromise = (async () => {
220
- // Retry binding.start on permission-denied. See note above
221
- // about the optimistic-write race with the nested-rule get().
222
- // YjsProvider rejects double-start, so each retry needs a
223
- // fresh binding (and a fresh provider underneath it).
224
- for (let attempt = 0; attempt < 8; attempt++) {
225
- if (stoppedRef.value) return;
226
- const store = new FirestoreUpdateStore(db, orgId, fileId);
227
- const binding = new YjsFileBinding(store, {
228
- onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
54
+ export async function startOrgSync({ db, storage, userId, orgId, localPath, maxUploadBytes = DEFAULT_MAX_UPLOAD_BYTES, statePath, }) {
55
+ const LOCAL_PATH = localPath;
56
+ const filesRef = collection(db, `orgs/${orgId}/files`);
57
+ // Where the per-folder content baseline is persisted. Default: a
58
+ // dotfile in the synced folder (ignored by the watcher), so it travels
59
+ // with the folder and needs no extra wiring from the caller.
60
+ const STATE_PATH = statePath ?? path.join(LOCAL_PATH, '.compound-sync-state.json');
61
+ const manifestStore = createManifestStore(LOCAL_PATH);
62
+ // Teardown handles: every onSnapshot unsub plus the chokidar watcher.
63
+ const unsubscribers = [];
64
+ let watcher = null;
65
+ const lastKnownUpdate = new Map();
66
+ // Content hash of each file as of the last successful sync (push or
67
+ // pull), keyed by trackingKey. This "baseline" decides pull/push
68
+ // direction by CONTENT rather than filesystem mtime (which an editor
69
+ // like Obsidian bumps without a content change). Loaded from
70
+ // STATE_PATH so it survives restarts; persisted on every change.
71
+ const lastSyncedHash = new Map(Object.entries(loadSyncState(STATE_PATH)));
72
+ function persistBaseline() {
73
+ saveSyncState(STATE_PATH, Object.fromEntries(lastSyncedHash));
74
+ }
75
+ function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
76
+ function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
77
+ const suppressLocal = new Set();
78
+ // Per-file Yjs bindings, created on first sight of a text file
79
+ // (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
80
+ // orgs/.../yupdates and the debounced disk flush. We keep two
81
+ // indexes: by fileId (so handleFirestoreChange can dispose on
82
+ // 'removed' and re-use across events) and by local path (so the
83
+ // chokidar handler can route disk edits into the binding without
84
+ // re-resolving the fileId).
85
+ const bindings = new Map();
86
+ const bindingsByPath = new Map();
87
+ // DOU-205: per-fileId record of the most recently observed Firestore
88
+ // metadata (local path + tracking key). Used to detect renames /
89
+ // moves on subsequent 'modified' snapshot events: when the file
90
+ // doc's path changes server-side (via the web rename UI), the local
91
+ // path computed from the new data also changes, and this map lets
92
+ // the handler relocate the local file in place (fs.renameSync,
93
+ // which preserves Obsidian's inode-level tracking) rather than
94
+ // leaving the old file orphaned at its previous location.
95
+ const knownPaths = new Map();
96
+ // Track which fileIds already have a binding (or one being set
97
+ // up) so we never double-create. `stoppedRef` is read by the
98
+ // background retry loops so they can abandon work after sync.stop.
99
+ // `pendingPromises` lets stop() await any in-flight setup OR any
100
+ // fire-and-forget snapshot/chokidar handler work before returning.
101
+ // Without this, the next test's clearFirestore + terminate can
102
+ // race a handler that's still flushing storage from this test,
103
+ // surfacing as "Firebase app was deleted" errors in the daemon log
104
+ // and (suspected) accumulating gRPC stream state that the next
105
+ // listener attach receives as a single oversize Listen message.
106
+ const pendingBindings = new Set();
107
+ const pendingPromises = new Set();
108
+ const stoppedRef = { value: false };
109
+ // Helper: register a promise with pendingPromises so stop() awaits
110
+ // it, and auto-remove on settle. Returns the same promise so call
111
+ // sites can keep their chaining shape.
112
+ function track(p) {
113
+ pendingPromises.add(p);
114
+ p.finally(() => pendingPromises.delete(p));
115
+ return p;
116
+ }
117
+ // DOU-254: per-local-path operation queue. chokidar can fire `add`
118
+ // then `unlink` for the same path within a few hundred ms (a test
119
+ // writes a file, waits only for the Firestore doc to appear, then
120
+ // deletes locally). Without serialization, the push (setDoc +
121
+ // uploadBlob) and the delete (preRecheck + deleteBlob + deleteDoc)
122
+ // race against each other on the same fileId, and uploadBlob can
123
+ // land AFTER deleteBlob, leaving an orphaned blob after the doc is
124
+ // gone. Chain ops keyed by local path so the next op starts only
125
+ // after the previous one settles.
126
+ const localOpQueues = new Map();
127
+ function enqueueLocalOp(filePath, op) {
128
+ const prev = localOpQueues.get(filePath) ?? Promise.resolve();
129
+ // Swallow the previous op's rejection so a failed push doesn't
130
+ // skip the subsequent delete (or vice versa); each op handles
131
+ // its own errors via the .catch() at the call site.
132
+ const next = prev.catch(() => undefined).then(op);
133
+ localOpQueues.set(filePath, next);
134
+ next.finally(() => {
135
+ if (localOpQueues.get(filePath) === next)
136
+ localOpQueues.delete(filePath);
229
137
  });
230
- try {
231
- await binding.start();
232
- if (stoppedRef.value) {
233
- await binding.stop().catch(() => {});
138
+ return next;
139
+ }
140
+ // Fire-and-forget binding setup. Returns nothing because the
141
+ // common shape "ensureBinding then read its ytext" runs straight
142
+ // into a race: the daemon's onSnapshot fires on Alice's own
143
+ // optimistic write before the file doc is server-committed, and
144
+ // the Firestore rules engine evaluates the yupdates/yjs nested
145
+ // rule via a server-side get() against the parent files/{id}
146
+ // doc. That get() sees no doc and returns null, so binding.start
147
+ // fails with permission-denied; the right behavior is to retry
148
+ // until the commit propagates, not to block handleFirestoreChange
149
+ // for seconds. Callers fall back to the blob path while the
150
+ // binding is still being set up; once it lands in
151
+ // bindings/bindingsByPath, subsequent events take the Yjs path.
152
+ function ensureBinding(fileId, localFilePath) {
153
+ if (bindings.has(fileId) || pendingBindings.has(fileId))
234
154
  return;
235
- }
236
- bindings.set(fileId, binding);
237
- bindingsByPath.set(localFilePath, binding);
238
- return;
239
- } catch (err) {
240
- if (stoppedRef.value) return;
241
- if (err?.code !== 'permission-denied' || attempt === 7) {
242
- console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err && err.message}`);
155
+ pendingBindings.add(fileId);
156
+ const onDiskWrite = (text) => {
157
+ suppressLocal.add(localFilePath);
158
+ ensureDir(localFilePath);
159
+ fs.writeFileSync(localFilePath, text, 'utf-8');
160
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
161
+ };
162
+ // DOU-209: persist the Y.Doc binary state after every stable
163
+ // change so the next daemon restart has the right baseline to
164
+ // merge offline disk edits without losing offline web edits.
165
+ const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
166
+ // DOU-209: feed the binding's startup-merge with the daemon's
167
+ // last view (initialDocState) and the current disk content. If
168
+ // they diverge, the binding applies the diff as Yjs ops so disk-
169
+ // side offline edits survive the cloud-side replay.
170
+ const initialDocState = loadBindingState(LOCAL_PATH, fileId);
171
+ let currentDiskText = null;
172
+ try {
173
+ if (fs.existsSync(localFilePath)) {
174
+ currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
175
+ }
176
+ }
177
+ catch (err) {
178
+ console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err?.message}`);
179
+ }
180
+ const setupPromise = (async () => {
181
+ // Retry binding.start on permission-denied. See note above
182
+ // about the optimistic-write race with the nested-rule get().
183
+ // YjsProvider rejects double-start, so each retry needs a
184
+ // fresh binding (and a fresh provider underneath it).
185
+ for (let attempt = 0; attempt < 8; attempt++) {
186
+ if (stoppedRef.value)
187
+ return;
188
+ const store = new FirestoreUpdateStore(db, orgId, fileId);
189
+ const binding = new YjsFileBinding(store, {
190
+ onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
191
+ });
192
+ try {
193
+ await binding.start();
194
+ if (stoppedRef.value) {
195
+ await binding.stop().catch(() => undefined);
196
+ return;
197
+ }
198
+ bindings.set(fileId, binding);
199
+ bindingsByPath.set(localFilePath, binding);
200
+ return;
201
+ }
202
+ catch (err) {
203
+ if (stoppedRef.value)
204
+ return;
205
+ const code = err?.code;
206
+ if (code !== 'permission-denied' || attempt === 7) {
207
+ console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err?.message}`);
208
+ return;
209
+ }
210
+ await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
211
+ }
212
+ }
213
+ })().finally(() => {
214
+ pendingBindings.delete(fileId);
215
+ pendingPromises.delete(setupPromise);
216
+ });
217
+ pendingPromises.add(setupPromise);
218
+ }
219
+ async function disposeBinding(fileId, localFilePath) {
220
+ const binding = bindings.get(fileId);
221
+ if (!binding)
243
222
  return;
244
- }
245
- await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
223
+ bindings.delete(fileId);
224
+ if (localFilePath)
225
+ bindingsByPath.delete(localFilePath);
226
+ await binding.stop();
227
+ // DOU-209: drop the per-file Y.Doc state so deleted files don't
228
+ // accumulate stale binding-state blobs under .compound-yjs-binding/.
229
+ deleteBindingState(LOCAL_PATH, fileId);
230
+ }
231
+ // Per-team Firestore-listener unsubscribe handles, keyed by teamId.
232
+ const teamUnsubscribers = new Map();
233
+ // Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
234
+ // which is defined later in this function. Wired below.
235
+ let setupTeam = () => { };
236
+ function teardownTeam(teamId, teamData) {
237
+ const unsub = teamUnsubscribers.get(teamId);
238
+ if (unsub) {
239
+ try {
240
+ unsub();
241
+ }
242
+ catch { /* ignore */ }
243
+ teamUnsubscribers.delete(teamId);
244
+ const folder = teamData && teamData.name ? teamFolderName(teamData.name) : teamId;
245
+ console.log(` [${orgId}] Stopped listening: ${folder}`);
246
246
  }
247
- }
248
- })().finally(() => {
249
- pendingBindings.delete(fileId);
250
- pendingPromises.delete(setupPromise);
247
+ }
248
+ const teamRegistry = new TeamRegistry({
249
+ onSetup: (teamId, teamData) => setupTeam(teamId, teamData),
250
+ onTeardown: teardownTeam,
251
251
  });
252
- pendingPromises.add(setupPromise);
253
- }
254
-
255
- async function disposeBinding(fileId, localFilePath) {
256
- const binding = bindings.get(fileId);
257
- if (!binding) return;
258
- bindings.delete(fileId);
259
- if (localFilePath) bindingsByPath.delete(localFilePath);
260
- await binding.stop();
261
- // DOU-209: drop the per-file Y.Doc state so deleted files don't
262
- // accumulate stale binding-state blobs under .compound-yjs-binding/.
263
- deleteBindingState(LOCAL_PATH, fileId);
264
- }
265
-
266
- // Per-team Firestore-listener unsubscribe handles, keyed by teamId.
267
- const teamUnsubscribers = new Map();
268
- // Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
269
- // which is defined later in this function. Wired below.
270
- let setupTeam = () => {};
271
- function teardownTeam(teamId, teamData) {
272
- const unsub = teamUnsubscribers.get(teamId);
273
- if (unsub) {
274
- try { unsub(); } catch (e) { /* ignore */ }
275
- teamUnsubscribers.delete(teamId);
276
- const folder = teamData && teamData.name ? teamFolderName(teamData.name) : teamId;
277
- console.log(` [${orgId}] Stopped listening: ${folder}`);
278
- }
279
- }
280
- const teamRegistry = new TeamRegistry({
281
- onSetup: (teamId, teamData) => setupTeam(teamId, teamData),
282
- onTeardown: teardownTeam,
283
- });
284
- // scopeFromLocalPath expects a Map<teamName, teamId>; the registry
285
- // exposes one matching that shape.
286
- const teamIdsByName = teamRegistry.teamIdsByName;
287
-
288
- console.log('');
289
- console.log(` [${orgId}] Local: ${LOCAL_PATH}`);
290
-
291
- function ensureDir(filePath) {
292
- const dir = path.dirname(filePath);
293
- if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
294
- }
295
-
296
- function updateSharedByMeSymlink(fileDoc, realPath) {
297
- const sharedWith = fileDoc.sharedWith || [];
298
- const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, fileDoc.path);
299
-
300
- if (sharedWith.length > 0) {
301
- ensureDir(symlinkPath);
302
- const relativeTo = path.relative(path.dirname(symlinkPath), realPath);
303
- try {
304
- if (fs.existsSync(symlinkPath) || fs.lstatSync(symlinkPath).isSymbolicLink()) {
305
- fs.unlinkSync(symlinkPath);
306
- }
307
- } catch (e) { /* doesn't exist */ }
308
- try {
309
- fs.symlinkSync(relativeTo, symlinkPath);
310
- console.log(` [${orgId}] [symlink] ${fileDoc.path} → Shared by Me/`);
311
- } catch (e) {
312
- if (e.code !== 'EEXIST') console.error(` [${orgId}] Symlink error: ${e.message}`);
313
- }
314
- } else {
315
- try {
316
- if (fs.lstatSync(symlinkPath).isSymbolicLink()) {
317
- fs.unlinkSync(symlinkPath);
318
- console.log(` [${orgId}] [symlink] removed Shared by Me/${fileDoc.path}`);
319
- }
320
- } catch (e) { /* doesn't exist */ }
321
- }
322
- }
323
-
324
- // Static (user-level) folders. Team folders are created per-team by
325
- // setupTeam below as soon as the registry sees each teamspace.
326
- function ensureFolderStructure() {
327
- const privatePath = path.join(LOCAL_PATH, PRIVATE_FOLDER);
328
- if (!fs.existsSync(privatePath)) fs.mkdirSync(privatePath, { recursive: true });
329
-
330
- const sharedWithMePath = path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER);
331
- if (!fs.existsSync(sharedWithMePath)) fs.mkdirSync(sharedWithMePath, { recursive: true });
332
-
333
- const sharedByMePath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER);
334
- if (!fs.existsSync(sharedByMePath)) fs.mkdirSync(sharedByMePath, { recursive: true });
335
- }
336
-
337
- ensureFolderStructure();
338
-
339
- async function handleFirestoreChange(change, getLocalPath) {
340
- const data = change.doc.data();
341
- const docPath = data.path;
342
- if (!docPath) return;
343
-
344
- const localFilePath = getLocalPath(data);
345
- const trackingKey = `${data.scope || 'team'}:${docPath}`;
346
- const isFolder = data.type === 'folder';
347
-
348
- // DOU-205: rename / move detection. On a 'modified' snapshot for
349
- // a fileId we've seen before, if the computed local path differs
350
- // from what we last recorded, the server's path field has
351
- // changed. Move the local file in place (fs.renameSync preserves
352
- // Obsidian's inode tracking so notes don't lose their metadata,
353
- // backlinks, or recent-files entries) and migrate the tracking
354
- // maps from the old key to the new. We do this before the
355
- // freshness check below because the rename happens irrespective
356
- // of whether the content changed in the same update.
357
- const fileId = change.doc.id;
358
- const prevKnown = knownPaths.get(fileId);
359
- if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
360
- suppressLocal.add(prevKnown.localPath);
361
- suppressLocal.add(localFilePath);
362
- try {
363
- if (fs.existsSync(prevKnown.localPath)) {
364
- ensureDir(localFilePath);
365
- fs.renameSync(prevKnown.localPath, localFilePath);
366
- console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
367
- }
368
- } catch (err) {
369
- console.error(` [${orgId}] [firestore → local] RENAME failed: ${err && err.message}`);
370
- }
371
- if (lastKnownUpdate.has(prevKnown.trackingKey)) {
372
- lastKnownUpdate.set(trackingKey, lastKnownUpdate.get(prevKnown.trackingKey));
373
- lastKnownUpdate.delete(prevKnown.trackingKey);
374
- }
375
- if (lastSyncedHash.has(prevKnown.trackingKey)) {
376
- setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
377
- deleteBaseline(prevKnown.trackingKey);
378
- }
379
- const binding = bindings.get(fileId);
380
- if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
381
- bindingsByPath.delete(prevKnown.localPath);
382
- bindingsByPath.set(localFilePath, binding);
383
- }
384
- setTimeout(() => {
385
- suppressLocal.delete(prevKnown.localPath);
386
- suppressLocal.delete(localFilePath);
387
- }, 1000);
388
- }
389
-
390
- if (change.type === 'removed') {
391
- if (isFolder) {
392
- // Best-effort directory removal. The daemon's own file-delete
393
- // events should have cleared all children by the time we get
394
- // here; if not, rmdirSync throws ENOTEMPTY and we keep the
395
- // directory rather than risk losing data the user hasn't seen.
396
- try {
397
- suppressLocal.add(localFilePath);
398
- fs.rmdirSync(localFilePath);
399
- console.log(` [${orgId}] [firestore → local] RMDIR ${docPath}`);
400
- setTimeout(() => suppressLocal.delete(localFilePath), 1000);
401
- } catch (err) {
402
- suppressLocal.delete(localFilePath);
403
- if (err.code !== 'ENOENT' && err.code !== 'ENOTEMPTY') {
404
- console.error(` [${orgId}] [firestore → local] RMDIR ${docPath} failed: ${err.message}`);
405
- }
406
- }
407
- } else if (fs.existsSync(localFilePath)) {
408
- console.log(` [${orgId}] [firestore → local] DELETE ${docPath}`);
252
+ // scopeFromLocalPath expects a Map<teamName, teamId>; the registry
253
+ // exposes one matching that shape via teamIdsByNameMap().
254
+ const teamIdsByName = teamRegistry.teamIdsByNameMap();
255
+ console.log('');
256
+ console.log(` [${orgId}] Local: ${LOCAL_PATH}`);
257
+ function ensureDir(filePath) {
258
+ const dir = path.dirname(filePath);
259
+ if (!fs.existsSync(dir))
260
+ fs.mkdirSync(dir, { recursive: true });
261
+ }
262
+ function updateSharedByMeSymlink(fileDoc, realPath) {
263
+ const sharedWith = fileDoc['sharedWith'] ?? [];
264
+ const docPath = fileDoc['path'];
265
+ // Bail out on a missing / non-string path: path.join would resolve
266
+ // to the Shared-by-Me root, and a subsequent symlinkSync /
267
+ // unlinkSync there could clobber the entire folder.
268
+ if (typeof docPath !== 'string' || !docPath)
269
+ return;
270
+ const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, docPath);
271
+ // Check whether `symlinkPath` exists as anything (regular file,
272
+ // directory, or symlink including a dangling one). Uses lstatSync,
273
+ // returns null on ENOENT, rethrows other errors. Beats
274
+ // `existsSync || lstatSync(...).isSymbolicLink()` which leaned on
275
+ // a throw-and-catch for the not-found case and could mask real
276
+ // filesystem errors.
277
+ function lstatOrNull(p) {
278
+ try {
279
+ return fs.lstatSync(p);
280
+ }
281
+ catch (e) {
282
+ if (e?.code === 'ENOENT')
283
+ return null;
284
+ throw e;
285
+ }
286
+ }
287
+ if (sharedWith.length > 0) {
288
+ ensureDir(symlinkPath);
289
+ const relativeTo = path.relative(path.dirname(symlinkPath), realPath);
290
+ const existing = lstatOrNull(symlinkPath);
291
+ if (existing) {
292
+ fs.unlinkSync(symlinkPath);
293
+ }
294
+ try {
295
+ fs.symlinkSync(relativeTo, symlinkPath);
296
+ console.log(` [${orgId}] [symlink] ${docPath} → Shared by Me/`);
297
+ }
298
+ catch (e) {
299
+ const code = e?.code;
300
+ if (code !== 'EEXIST')
301
+ console.error(` [${orgId}] Symlink error: ${e?.message}`);
302
+ }
303
+ }
304
+ else {
305
+ const existing = lstatOrNull(symlinkPath);
306
+ if (existing && existing.isSymbolicLink()) {
307
+ fs.unlinkSync(symlinkPath);
308
+ console.log(` [${orgId}] [symlink] removed Shared by Me/${docPath}`);
309
+ }
310
+ }
311
+ }
312
+ // Static (user-level) folders. Team folders are mkdir'd by setupTeam,
313
+ // which runs after manifest reconcile (see startup ordering below).
314
+ function ensureFolderStructure() {
315
+ const privatePath = path.join(LOCAL_PATH, PRIVATE_FOLDER);
316
+ if (!fs.existsSync(privatePath))
317
+ fs.mkdirSync(privatePath, { recursive: true });
318
+ const sharedWithMePath = path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER);
319
+ if (!fs.existsSync(sharedWithMePath))
320
+ fs.mkdirSync(sharedWithMePath, { recursive: true });
321
+ const sharedByMePath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER);
322
+ if (!fs.existsSync(sharedByMePath))
323
+ fs.mkdirSync(sharedByMePath, { recursive: true });
324
+ }
325
+ ensureFolderStructure();
326
+ async function handleFirestoreChange(change, getLocalPath) {
327
+ const data = change.doc.data();
328
+ const docPath = data['path'];
329
+ if (typeof docPath !== 'string' || !docPath)
330
+ return;
331
+ const localFilePath = getLocalPath(data);
332
+ const trackingKey = `${data['scope'] || 'team'}:${docPath}`;
333
+ const isFolder = data['type'] === 'folder';
334
+ // DOU-205: rename / move detection. On a 'modified' snapshot for
335
+ // a fileId we've seen before, if the computed local path differs
336
+ // from what we last recorded, the server's path field has
337
+ // changed. Move the local file in place (fs.renameSync preserves
338
+ // Obsidian's inode tracking so notes don't lose their metadata,
339
+ // backlinks, or recent-files entries) and migrate the tracking
340
+ // maps from the old key to the new. We do this before the
341
+ // freshness check below because the rename happens irrespective
342
+ // of whether the content changed in the same update.
343
+ const fileId = change.doc.id;
344
+ const prevKnown = knownPaths.get(fileId);
345
+ if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
346
+ suppressLocal.add(prevKnown.localPath);
347
+ suppressLocal.add(localFilePath);
348
+ try {
349
+ if (fs.existsSync(prevKnown.localPath)) {
350
+ ensureDir(localFilePath);
351
+ fs.renameSync(prevKnown.localPath, localFilePath);
352
+ console.log(` [${orgId}] [firestore local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
353
+ }
354
+ }
355
+ catch (err) {
356
+ console.error(` [${orgId}] [firestore local] RENAME failed: ${err?.message}`);
357
+ }
358
+ const prevLastKnown = lastKnownUpdate.get(prevKnown.trackingKey);
359
+ if (prevLastKnown !== undefined) {
360
+ lastKnownUpdate.set(trackingKey, prevLastKnown);
361
+ lastKnownUpdate.delete(prevKnown.trackingKey);
362
+ }
363
+ const prevBaseline = lastSyncedHash.get(prevKnown.trackingKey);
364
+ if (prevBaseline !== undefined) {
365
+ setBaseline(trackingKey, prevBaseline);
366
+ deleteBaseline(prevKnown.trackingKey);
367
+ }
368
+ const manifestEntry = manifestStore.getEntry(prevKnown.trackingKey);
369
+ if (manifestEntry) {
370
+ manifestStore.removeEntry(prevKnown.trackingKey);
371
+ manifestStore.upsertEntry(trackingKey, manifestEntry);
372
+ }
373
+ const binding = bindings.get(fileId);
374
+ if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
375
+ bindingsByPath.delete(prevKnown.localPath);
376
+ bindingsByPath.set(localFilePath, binding);
377
+ }
378
+ setTimeout(() => {
379
+ suppressLocal.delete(prevKnown.localPath);
380
+ suppressLocal.delete(localFilePath);
381
+ }, 1000);
382
+ }
383
+ if (change.type === 'removed') {
384
+ if (isFolder) {
385
+ // Best-effort directory removal. The daemon's own file-delete
386
+ // events should have cleared all children by the time we get
387
+ // here; if not, rmdirSync throws ENOTEMPTY and we keep the
388
+ // directory rather than risk losing data the user hasn't seen.
389
+ try {
390
+ suppressLocal.add(localFilePath);
391
+ fs.rmdirSync(localFilePath);
392
+ console.log(` [${orgId}] [firestore local] RMDIR ${docPath}`);
393
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
394
+ }
395
+ catch (err) {
396
+ suppressLocal.delete(localFilePath);
397
+ const code = err?.code;
398
+ if (code !== 'ENOENT' && code !== 'ENOTEMPTY') {
399
+ console.error(` [${orgId}] [firestore → local] RMDIR ${docPath} failed: ${err?.message}`);
400
+ }
401
+ }
402
+ }
403
+ else if (fs.existsSync(localFilePath)) {
404
+ console.log(` [${orgId}] [firestore → local] DELETE ${docPath}`);
405
+ suppressLocal.add(localFilePath);
406
+ fs.unlinkSync(localFilePath);
407
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
408
+ }
409
+ lastKnownUpdate.delete(trackingKey);
410
+ deleteBaseline(trackingKey);
411
+ manifestStore.removeEntry(trackingKey);
412
+ knownPaths.delete(fileId);
413
+ await disposeBinding(change.doc.id, localFilePath);
414
+ return;
415
+ }
416
+ // Record (or refresh) the per-fileId path so the next 'modified'
417
+ // event can detect a rename / move by comparing against this.
418
+ knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
419
+ // Kick off Yjs binding setup for text files. Fire-and-forget so
420
+ // a slow start (retrying past the rules-engine race on a fresh
421
+ // doc) does not stall the legacy blob pull. Once the binding is
422
+ // registered, future events take the Yjs path; until then the
423
+ // blob path still keeps disk in sync.
424
+ if (!isFolder) {
425
+ const ext = extFromPath(docPath);
426
+ const extDerivedMime = mimeTypeFromExt(ext);
427
+ const isText = isTextMimeType(data['mimeType']) || isTextMimeType(extDerivedMime);
428
+ if (isText)
429
+ ensureBinding(change.doc.id, localFilePath);
430
+ }
431
+ const remoteUpdated = data['updatedAt'] || data['createdAt'] || '';
432
+ const lastKnown = lastKnownUpdate.get(trackingKey);
433
+ if (lastKnown && lastKnown >= remoteUpdated)
434
+ return;
435
+ if (isFolder) {
436
+ // Folder docs mirror to a local directory. No blob, no content
437
+ // comparison; if the directory doesn't exist yet, create it.
438
+ if (!fs.existsSync(localFilePath)) {
439
+ suppressLocal.add(localFilePath);
440
+ fs.mkdirSync(localFilePath, { recursive: true });
441
+ console.log(` [${orgId}] [firestore → local] MKDIR ${docPath}`);
442
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
443
+ }
444
+ lastKnownUpdate.set(trackingKey, remoteUpdated);
445
+ return;
446
+ }
447
+ if (fs.existsSync(localFilePath)) {
448
+ const baseline = lastSyncedHash.get(trackingKey);
449
+ if (baseline !== undefined) {
450
+ // We have a record of what we last synced: decide by content,
451
+ // not mtime. (Obsidian keeps the mtime fresh, which made the old
452
+ // mtime guard wrongly skip web edits.)
453
+ const localHash = computeContentHash(fs.readFileSync(localFilePath));
454
+ if (localHash === data['contentHash']) {
455
+ // Local already matches the incoming server content; nothing
456
+ // to write.
457
+ lastKnownUpdate.set(trackingKey, remoteUpdated);
458
+ setBaseline(trackingKey, data['contentHash']);
459
+ return;
460
+ }
461
+ if (localHash !== baseline) {
462
+ // Local changed since our last sync AND the server changed:
463
+ // a genuine conflict. Keep the local copy rather than clobber
464
+ // it. (Writing a "conflicted copy" is the next increment.)
465
+ console.log(` [${orgId}] [conflict] ${docPath}: local and server both changed; keeping local`);
466
+ return;
467
+ }
468
+ // localHash === baseline: local is untouched since last sync, so
469
+ // the server is authoritative -> fall through and pull.
470
+ }
471
+ else {
472
+ // No baseline at all: a local file this daemon has never synced
473
+ // (e.g. created locally while offline, or a folder adopted from
474
+ // elsewhere). We can't tell by content who is newer, so fall
475
+ // back to the mtime heuristic. Conflict copies (next increment)
476
+ // will replace this last-resort with a no-loss outcome.
477
+ const localMtime = fs.statSync(localFilePath).mtime.toISOString();
478
+ if (localMtime > remoteUpdated)
479
+ return;
480
+ }
481
+ }
482
+ const action = change.type === 'added' ? 'CREATE' : 'UPDATE';
483
+ console.log(` [${orgId}] [firestore → local] ${action} ${docPath} (${data['scope'] || 'team'})`);
484
+ ensureDir(localFilePath);
409
485
  suppressLocal.add(localFilePath);
410
- fs.unlinkSync(localFilePath);
486
+ const ext = extFromPath(docPath);
487
+ // Extension is used as a fallback authority when the stored
488
+ // mimeType claims non-text but the extension maps to a text mime
489
+ // in EXT_TO_MIME. This heals files uploaded before their extension
490
+ // was registered as text (e.g. .base files uploaded prior to the
491
+ // text/yaml mapping landing): on the next pull the daemon writes
492
+ // them as UTF-8 so text-aware features (line diffing, conflict
493
+ // markers) can apply.
494
+ const extDerivedMime = mimeTypeFromExt(ext);
495
+ const isText = isTextMimeType(data['mimeType']) || isTextMimeType(extDerivedMime);
496
+ // Yjs path (DOU-178 PR 3): if a binding for this file is already
497
+ // up and carries non-empty ytext, the debounced flush owns the
498
+ // disk write and the legacy blob pull is skipped. Bindings that
499
+ // are still being set up, or that loaded empty ytext (pre-Yjs
500
+ // files), fall through to the blob path; PR 4 retires the blob
501
+ // path entirely.
502
+ if (isText) {
503
+ const binding = bindings.get(change.doc.id);
504
+ if (binding && binding.ytext.length > 0) {
505
+ lastKnownUpdate.set(trackingKey, remoteUpdated);
506
+ setBaseline(trackingKey, data['contentHash']);
507
+ setTimeout(() => suppressLocal.delete(localFilePath), 1000);
508
+ return;
509
+ }
510
+ }
511
+ // Storage rules require the Firestore doc to exist before they
512
+ // authorize a blob upload, so writers must create the doc first
513
+ // and upload second. That leaves a window where the metadata is
514
+ // visible (this listener fires) but the blob is not yet uploaded.
515
+ // Retry on object-not-found with exponential backoff covers that
516
+ // race; on a persistent miss we fall through to the original
517
+ // catch upstream.
518
+ const blobReadArgs = { storage, orgId, fileId: change.doc.id, ext };
519
+ const retryOpts = { expectedHash: data['contentHash'] };
520
+ if (isText) {
521
+ const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
522
+ fs.writeFileSync(localFilePath, content || '', 'utf-8');
523
+ }
524
+ else {
525
+ const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
526
+ fs.writeFileSync(localFilePath, bytes);
527
+ }
528
+ lastKnownUpdate.set(trackingKey, remoteUpdated);
529
+ setBaseline(trackingKey, data['contentHash']);
411
530
  setTimeout(() => suppressLocal.delete(localFilePath), 1000);
412
- }
413
- lastKnownUpdate.delete(trackingKey);
414
- deleteBaseline(trackingKey);
415
- knownPaths.delete(fileId);
416
- await disposeBinding(change.doc.id, localFilePath);
417
- return;
418
- }
419
-
420
- // Record (or refresh) the per-fileId path so the next 'modified'
421
- // event can detect a rename / move by comparing against this.
422
- knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
423
-
424
- // Kick off Yjs binding setup for text files. Fire-and-forget so
425
- // a slow start (retrying past the rules-engine race on a fresh
426
- // doc) does not stall the legacy blob pull. Once the binding is
427
- // registered, future events take the Yjs path; until then the
428
- // blob path still keeps disk in sync.
429
- if (!isFolder) {
430
- const ext = extFromPath(docPath);
431
- const extDerivedMime = mimeTypeFromExt(ext);
432
- const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
433
- if (isText) ensureBinding(change.doc.id, localFilePath);
434
- }
435
-
436
- const remoteUpdated = data.updatedAt || data.createdAt || '';
437
- const lastKnown = lastKnownUpdate.get(trackingKey);
438
- if (lastKnown && lastKnown >= remoteUpdated) return;
439
-
440
- if (isFolder) {
441
- // Folder docs mirror to a local directory. No blob, no content
442
- // comparison; if the directory doesn't exist yet, create it.
443
- if (!fs.existsSync(localFilePath)) {
531
+ }
532
+ // Set up per-team resources: mkdir the local folder, register the
533
+ // Firestore listener for that team's files. On startup, called once
534
+ // per bootstrapped team after reconcile; thereafter via TeamRegistry
535
+ // when the live teams-collection listener reports add/modify/remove.
536
+ setupTeam = (teamId, teamData) => {
537
+ // The pre-migration JS coerced a missing name to the literal
538
+ // "undefined" string via `name + ' Teamspace'`. Preserve that
539
+ // faithfully (String(undefined) === 'undefined'): a team without
540
+ // a name still gets a folder + listener, just under an
541
+ // obviously-broken display name that surfaces the bug instead of
542
+ // silently hiding the team.
543
+ const teamFolder = teamFolderName(String(teamData.name));
544
+ const teamPath = path.join(LOCAL_PATH, teamFolder);
545
+ if (!fs.existsSync(teamPath))
546
+ fs.mkdirSync(teamPath, { recursive: true });
547
+ const q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId));
548
+ const unsubscribe = onSnapshot(q, (snapshot) => {
549
+ snapshot.docChanges().forEach((change) => {
550
+ track(handleFirestoreChange(change, (data) => path.join(LOCAL_PATH, teamFolder, data['path'])).catch((err) => console.error(` [${orgId}] Firestore change handler error:`, err?.message)));
551
+ });
552
+ }, (err) => console.error(` [${orgId}] Team "${teamFolder}" listener error:`, err));
553
+ teamUnsubscribers.set(teamId, unsubscribe);
554
+ console.log(` [${orgId}] Listening: ${teamFolder}`);
555
+ };
556
+ // Bootstrap team metadata synchronously so manifest reconcile can
557
+ // query team-scoped cloud files before any file onSnapshot handlers
558
+ // run (DOU-55). Populating teamIdsByName here also lets chokidar
559
+ // route local paths once the watcher starts. Per-team file listeners
560
+ // and the live teams-collection onSnapshot are wired after reconcile
561
+ // completes so their initial snapshots cannot race with startup cleanup.
562
+ const teamsSnap = await getDocs(collection(db, `orgs/${orgId}/teams`));
563
+ for (const teamDoc of teamsSnap.docs) {
564
+ teamRegistry.bootstrap(teamDoc.id, teamDoc.data());
565
+ }
566
+ console.log(` [${orgId}] Teams: ${teamRegistry.names().join(', ') || 'none'}`);
567
+ function trackingKeyFromDocData(data) {
568
+ return `${data['scope'] || 'team'}:${data['path']}`;
569
+ }
570
+ // Paths the daemon does not scan or sync. Add checks here as needed
571
+ // (extensions, .compoundignore, etc.).
572
+ function shouldSkipLocalPath(name, rel) {
573
+ if (name.startsWith('.') || name === 'node_modules')
574
+ return true;
575
+ if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
576
+ return true;
577
+ if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
578
+ return true;
579
+ return false;
580
+ }
581
+ function walkLocalSyncableFiles(absDir, files) {
582
+ let entries;
583
+ try {
584
+ entries = fs.readdirSync(absDir, { withFileTypes: true });
585
+ }
586
+ catch (err) {
587
+ const rel = path.relative(LOCAL_PATH, absDir) || '.';
588
+ console.warn(` [${orgId}] [local scan] could not read directory ${rel}: ${err?.message}`);
589
+ return;
590
+ }
591
+ for (const ent of entries) {
592
+ const abs = path.join(absDir, ent.name);
593
+ const rel = path.relative(LOCAL_PATH, abs);
594
+ if (shouldSkipLocalPath(ent.name, rel))
595
+ continue;
596
+ if (ent.isDirectory()) {
597
+ walkLocalSyncableFiles(abs, files);
598
+ }
599
+ else if (ent.isFile()) {
600
+ files.push(abs);
601
+ }
602
+ }
603
+ }
604
+ function collectLocalSyncableFiles() {
605
+ const files = [];
606
+ walkLocalSyncableFiles(LOCAL_PATH, files);
607
+ return files;
608
+ }
609
+ function readLocalFilePayload(filePath) {
610
+ const rel = path.relative(LOCAL_PATH, filePath);
611
+ const ext = extFromPath(rel);
612
+ const mimeType = mimeTypeFromExt(ext);
613
+ const data = isTextMimeType(mimeType)
614
+ ? fs.readFileSync(filePath, 'utf-8')
615
+ : fs.readFileSync(filePath);
616
+ return { data, mimeType, contentHash: computeContentHash(data) };
617
+ }
618
+ async function pushLocalFileAtPath(filePath) {
619
+ let stat;
620
+ try {
621
+ stat = fs.statSync(filePath);
622
+ }
623
+ catch {
624
+ return;
625
+ }
626
+ if (!stat.isFile() || stat.size > maxUploadBytes)
627
+ return;
628
+ const { data, mimeType } = readLocalFilePayload(filePath);
629
+ await pushToFirestore(filePath, data, mimeType, stat.mtime.toISOString());
630
+ }
631
+ // Sync local cleanup during startup manifest reconcile. Runs before
632
+ // Firestore listeners and Yjs bindings are wired, so no async cloud or
633
+ // binding teardown is needed here.
634
+ function deleteLocalReconcile(localFilePath, fileId, trackingKey) {
444
635
  suppressLocal.add(localFilePath);
445
- fs.mkdirSync(localFilePath, { recursive: true });
446
- console.log(` [${orgId}] [firestore local] MKDIR ${docPath}`);
447
- setTimeout(() => suppressLocal.delete(localFilePath), 1000);
448
- }
449
- lastKnownUpdate.set(trackingKey, remoteUpdated);
450
- return;
451
- }
452
-
453
- if (fs.existsSync(localFilePath)) {
454
- const baseline = lastSyncedHash.get(trackingKey);
455
- if (baseline !== undefined) {
456
- // We have a record of what we last synced: decide by content,
457
- // not mtime. (Obsidian keeps the mtime fresh, which made the old
458
- // mtime guard wrongly skip web edits.)
459
- const localHash = computeContentHash(fs.readFileSync(localFilePath));
460
- if (localHash === data.contentHash) {
461
- // Local already matches the incoming server content; nothing
462
- // to write.
463
- lastKnownUpdate.set(trackingKey, remoteUpdated);
464
- setBaseline(trackingKey, data.contentHash);
465
- return;
466
- }
467
- if (localHash !== baseline) {
468
- // Local changed since our last sync AND the server changed:
469
- // a genuine conflict. Keep the local copy rather than clobber
470
- // it. (Writing a "conflicted copy" is the next increment.)
471
- console.log(` [${orgId}] [conflict] ${docPath}: local and server both changed; keeping local`);
472
- return;
473
- }
474
- // localHash === baseline: local is untouched since last sync, so
475
- // the server is authoritative -> fall through and pull.
476
- } else {
477
- // No baseline at all: a local file this daemon has never synced
478
- // (e.g. created locally while offline, or a folder adopted from
479
- // elsewhere). We can't tell by content who is newer, so fall
480
- // back to the mtime heuristic. Conflict copies (next increment)
481
- // will replace this last-resort with a no-loss outcome.
482
- const localMtime = fs.statSync(localFilePath).mtime.toISOString();
483
- if (localMtime > remoteUpdated) return;
484
- }
485
- }
486
-
487
- const action = change.type === 'added' ? 'CREATE' : 'UPDATE';
488
- console.log(` [${orgId}] [firestore → local] ${action} ${docPath} (${data.scope || 'team'})`);
489
- ensureDir(localFilePath);
490
- suppressLocal.add(localFilePath);
491
-
492
- const ext = extFromPath(docPath);
493
- // Extension is used as a fallback authority when the stored
494
- // mimeType claims non-text but the extension maps to a text mime
495
- // in EXT_TO_MIME. This heals files uploaded before their extension
496
- // was registered as text (e.g. .base files uploaded prior to the
497
- // text/yaml mapping landing): on the next pull the daemon writes
498
- // them as UTF-8 so text-aware features (line diffing, conflict
499
- // markers) can apply.
500
- const extDerivedMime = mimeTypeFromExt(ext);
501
- const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
502
-
503
- // Yjs path (DOU-178 PR 3): if a binding for this file is already
504
- // up and carries non-empty ytext, the debounced flush owns the
505
- // disk write and the legacy blob pull is skipped. Bindings that
506
- // are still being set up, or that loaded empty ytext (pre-Yjs
507
- // files), fall through to the blob path; PR 4 retires the blob
508
- // path entirely.
509
- if (isText) {
510
- const binding = bindings.get(change.doc.id);
511
- if (binding && binding.ytext.length > 0) {
512
- lastKnownUpdate.set(trackingKey, remoteUpdated);
513
- setBaseline(trackingKey, data.contentHash);
636
+ if (fs.existsSync(localFilePath)) {
637
+ console.log(` [${orgId}] [manifest reconcile] DELETE local ${path.relative(LOCAL_PATH, localFilePath)}`);
638
+ fs.unlinkSync(localFilePath);
639
+ }
640
+ lastKnownUpdate.delete(trackingKey);
641
+ deleteBaseline(trackingKey);
642
+ manifestStore.removeEntry(trackingKey);
643
+ if (fileId) {
644
+ knownPaths.delete(fileId);
645
+ }
514
646
  setTimeout(() => suppressLocal.delete(localFilePath), 1000);
515
- return;
516
- }
517
- }
518
-
519
- // Storage rules require the Firestore doc to exist before they
520
- // authorize a blob upload, so writers must create the doc first
521
- // and upload second. That leaves a window where the metadata is
522
- // visible (this listener fires) but the blob is not yet uploaded.
523
- // Retry on object-not-found with exponential backoff covers that
524
- // race; on a persistent miss we fall through to the original
525
- // catch upstream.
526
- const blobReadArgs = { storage, orgId, fileId: change.doc.id, ext };
527
- const retryOpts = { expectedHash: data.contentHash };
528
- if (isText) {
529
- const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
530
- fs.writeFileSync(localFilePath, content || '', 'utf-8');
531
- } else {
532
- const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
533
- fs.writeFileSync(localFilePath, bytes);
534
- }
535
- lastKnownUpdate.set(trackingKey, remoteUpdated);
536
- setBaseline(trackingKey, data.contentHash);
537
- setTimeout(() => suppressLocal.delete(localFilePath), 1000);
538
- }
539
-
540
- // Set up per-team resources: mkdir the local folder, register the
541
- // Firestore listener for that team's files. Wired into the
542
- // TeamRegistry; called once per teamspace as it appears.
543
- setupTeam = (teamId, teamData) => {
544
- const teamFolder = teamFolderName(teamData.name);
545
- const teamPath = path.join(LOCAL_PATH, teamFolder);
546
- if (!fs.existsSync(teamPath)) fs.mkdirSync(teamPath, { recursive: true });
547
-
548
- const q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId));
549
- const unsubscribe = onSnapshot(q, snapshot => {
550
- snapshot.docChanges().forEach(change => {
551
- track(handleFirestoreChange(change, data =>
552
- path.join(LOCAL_PATH, teamFolder, data.path),
553
- ).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
554
- });
555
- }, err => console.error(` [${orgId}] Team "${teamFolder}" listener error:`, err));
556
-
557
- teamUnsubscribers.set(teamId, unsubscribe);
558
- console.log(` [${orgId}] Listening: ${teamFolder}`);
559
- };
560
-
561
- // Live listener on the teams collection. Initial snapshot bootstraps
562
- // the known teamspaces; subsequent changes pick up new/renamed/
563
- // removed teamspaces without restarting the daemon. We await the
564
- // first snapshot so the local watcher sees a populated teamIdsByName
565
- // before it tries to route any addDir/add events.
566
- let teamsUnsub = null;
567
- await new Promise((resolve, reject) => {
568
- let firstSnapshotResolved = false;
569
- teamsUnsub = onSnapshot(collection(db, `orgs/${orgId}/teams`), (snapshot) => {
570
- snapshot.docChanges().forEach((change) => {
571
- teamRegistry.applyChange({
572
- type: change.type,
573
- id: change.doc.id,
574
- data: change.doc.data(),
647
+ }
648
+ async function deleteRemoteReconcile(cloudData, trackingKey) {
649
+ const scope = cloudData['scope'] || 'team';
650
+ const teamId = cloudData['teamId'];
651
+ const docPath = cloudData['path'];
652
+ const result = await deleteFileFromCloud({
653
+ filesRef,
654
+ storage,
655
+ orgId,
656
+ userId,
657
+ scope,
658
+ teamId,
659
+ docPath,
575
660
  });
576
- });
577
- if (!firstSnapshotResolved) {
578
- firstSnapshotResolved = true;
579
- console.log(` [${orgId}] Teams: ${teamRegistry.names().join(', ') || 'none'}`);
580
- resolve();
581
- }
582
- }, (err) => {
583
- console.error(` [${orgId}] Teams listener error:`, err);
584
- if (!firstSnapshotResolved) {
585
- firstSnapshotResolved = true;
586
- reject(err);
587
- }
588
- });
589
- });
590
- unsubscribers.push(teamsUnsub);
591
-
592
- function startFirestoreListeners() {
593
- console.log(` [${orgId}] Starting Firestore listeners...`);
594
-
595
- if (userId) {
596
- const privateQ = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId));
597
- const privateUnsub = onSnapshot(privateQ, snapshot => {
598
- snapshot.docChanges().forEach(change => {
599
- const data = change.doc.data();
600
- const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data.path);
601
-
602
- track(handleFirestoreChange(change, () => realPath)
603
- .catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
604
-
605
- if (change.type !== 'removed') {
606
- updateSharedByMeSymlink(data, realPath);
607
- } else {
608
- const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, data.path);
609
- try { fs.lstatSync(symlinkPath); fs.unlinkSync(symlinkPath); } catch (e) { /* ok */ }
610
- }
661
+ if (result.action === 'deleted') {
662
+ console.log(` [${orgId}] [manifest reconcile] DELETE cloud ${docPath}`);
663
+ }
664
+ manifestStore.removeEntry(trackingKey);
665
+ }
666
+ function localPathForCloudData(data) {
667
+ const docPath = data['path'];
668
+ if (typeof docPath !== 'string' || !docPath)
669
+ return null;
670
+ if (data['scope'] === 'private') {
671
+ if (data['ownerId'] === userId) {
672
+ return path.join(LOCAL_PATH, PRIVATE_FOLDER, docPath);
673
+ }
674
+ return path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, docPath);
675
+ }
676
+ if (data['scope'] === 'team' && data['teamId']) {
677
+ const team = teamRegistry.get(data['teamId']);
678
+ if (!team?.name)
679
+ return null;
680
+ return path.join(LOCAL_PATH, teamFolderName(team.name), docPath);
681
+ }
682
+ return null;
683
+ }
684
+ async function fetchCloudFilesByKey() {
685
+ const cloudByKey = new Map();
686
+ function ingestDoc(d) {
687
+ const data = d.data();
688
+ if (data['type'] === 'folder' || !data['path'])
689
+ return;
690
+ const key = trackingKeyFromDocData(data);
691
+ cloudByKey.set(key, { fileId: d.id, data });
692
+ }
693
+ for (const [teamId] of teamRegistry.entries()) {
694
+ const teamSnap = await getDocs(query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId)));
695
+ teamSnap.docs.forEach(ingestDoc);
696
+ }
697
+ if (userId) {
698
+ const privateSnap = await getDocs(query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId)));
699
+ privateSnap.docs.forEach(ingestDoc);
700
+ const sharedSnap = await getDocs(query(filesRef, where('scope', '==', 'private'), where('sharedWith', 'array-contains', userId)));
701
+ sharedSnap.docs.forEach(ingestDoc);
702
+ }
703
+ return cloudByKey;
704
+ }
705
+ // Runs after team metadata is loaded but before any file listeners or
706
+ // chokidar. See docs/features/sync/sync-manifest-design.md.
707
+ async function reconcileManifestOnStartup() {
708
+ console.log(` [${orgId}] Reconciling local manifest...`);
709
+ const cloudByKey = await fetchCloudFilesByKey();
710
+ const localByKey = new Map();
711
+ for (const filePath of collectLocalSyncableFiles()) {
712
+ const { scope, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
713
+ if (!docPath)
714
+ continue;
715
+ localByKey.set(`${scope}:${docPath}`, filePath);
716
+ }
717
+ // Bootstrap manifest entries for files already in sync before DOU-55.
718
+ for (const [key, cloud] of cloudByKey) {
719
+ if (manifestStore.getEntry(key))
720
+ continue;
721
+ const localFilePath = localByKey.get(key);
722
+ if (!localFilePath || !fs.existsSync(localFilePath))
723
+ continue;
724
+ try {
725
+ const { contentHash } = readLocalFilePayload(localFilePath);
726
+ if (contentHash === cloud.data['contentHash']) {
727
+ manifestStore.upsertEntry(key, {
728
+ fileId: cloud.fileId,
729
+ contentHash,
730
+ lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
731
+ });
732
+ }
733
+ }
734
+ catch {
735
+ // skip unreadable locals
736
+ }
737
+ }
738
+ const allKeys = new Set([
739
+ ...Object.keys(manifestStore.getState().entries),
740
+ ...localByKey.keys(),
741
+ ...cloudByKey.keys(),
742
+ ]);
743
+ for (const trackingKey of allKeys) {
744
+ const entry = manifestStore.getEntry(trackingKey);
745
+ let localFilePath = localByKey.get(trackingKey) ?? null;
746
+ const cloud = cloudByKey.get(trackingKey) ?? null;
747
+ if (!localFilePath && cloud) {
748
+ localFilePath = localPathForCloudData(cloud.data);
749
+ }
750
+ const onDisk = !!(localFilePath && fs.existsSync(localFilePath));
751
+ if (entry && onDisk && !cloud) {
752
+ let localHash;
753
+ try {
754
+ ({ contentHash: localHash } = readLocalFilePayload(localFilePath));
755
+ }
756
+ catch {
757
+ localHash = null;
758
+ }
759
+ if (localHash === null) {
760
+ console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
761
+ }
762
+ else if (entry.contentHash && localHash !== entry.contentHash) {
763
+ await pushLocalFileAtPath(localFilePath);
764
+ }
765
+ else {
766
+ deleteLocalReconcile(localFilePath, entry.fileId, trackingKey);
767
+ }
768
+ }
769
+ else if (!entry && onDisk && !cloud) {
770
+ const baselineHash = lastSyncedHash.get(trackingKey);
771
+ let localHash;
772
+ try {
773
+ ({ contentHash: localHash } = readLocalFilePayload(localFilePath));
774
+ }
775
+ catch {
776
+ localHash = null;
777
+ }
778
+ if (localHash === null) {
779
+ console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
780
+ }
781
+ else if (baselineHash && localHash === baselineHash) {
782
+ deleteLocalReconcile(localFilePath, null, trackingKey);
783
+ }
784
+ else {
785
+ await pushLocalFileAtPath(localFilePath);
786
+ }
787
+ }
788
+ else if (entry && !onDisk && cloud) {
789
+ await deleteRemoteReconcile(cloud.data, trackingKey);
790
+ }
791
+ else if ((entry || cloud) && onDisk && cloud) {
792
+ manifestStore.upsertEntry(trackingKey, {
793
+ fileId: cloud.fileId,
794
+ contentHash: cloud.data['contentHash'],
795
+ lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
796
+ });
797
+ }
798
+ else if (!entry && onDisk && cloud) {
799
+ manifestStore.upsertEntry(trackingKey, {
800
+ fileId: cloud.fileId,
801
+ contentHash: cloud.data['contentHash'],
802
+ lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
803
+ });
804
+ }
805
+ }
806
+ manifestStore.flush();
807
+ console.log(` [${orgId}] Manifest reconcile complete.`);
808
+ }
809
+ await reconcileManifestOnStartup();
810
+ // Wire per-team file listeners and the live teams-collection watcher
811
+ // after reconcile so initial onSnapshot deliveries cannot race with
812
+ // startup cleanup (DOU-55). applyChange on the live listener picks
813
+ // up teamspaces created/renamed/removed after daemon start without
814
+ // restarting sync.
815
+ for (const [teamId, teamData] of teamRegistry.entries()) {
816
+ setupTeam(teamId, teamData);
817
+ }
818
+ const teamsUnsub = onSnapshot(collection(db, `orgs/${orgId}/teams`), (snapshot) => {
819
+ snapshot.docChanges().forEach((change) => {
820
+ teamRegistry.applyChange({
821
+ type: change.type,
822
+ id: change.doc.id,
823
+ data: change.doc.data(),
824
+ });
825
+ });
826
+ }, (err) => console.error(` [${orgId}] Teams listener error:`, err));
827
+ unsubscribers.push(teamsUnsub);
828
+ function startFirestoreListeners() {
829
+ console.log(` [${orgId}] Starting Firestore listeners...`);
830
+ if (userId) {
831
+ const privateQ = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId));
832
+ const privateUnsub = onSnapshot(privateQ, snapshot => {
833
+ snapshot.docChanges().forEach(change => {
834
+ const data = change.doc.data();
835
+ const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data['path']);
836
+ track(handleFirestoreChange(change, () => realPath)
837
+ .catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
838
+ if (change.type !== 'removed') {
839
+ updateSharedByMeSymlink(data, realPath);
840
+ }
841
+ else {
842
+ const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, data['path']);
843
+ try {
844
+ fs.lstatSync(symlinkPath);
845
+ fs.unlinkSync(symlinkPath);
846
+ }
847
+ catch (e) { /* ok */ }
848
+ }
849
+ });
850
+ }, err => console.error(` [${orgId}] Private files listener error:`, err));
851
+ unsubscribers.push(privateUnsub);
852
+ console.log(` [${orgId}] Listening: Private files + Shared by Me symlinks`);
853
+ // scope=='private' is required: Firestore rules only grant
854
+ // sharedWith-based reads on private files, so the query must
855
+ // constrain scope or the whole listen is denied (rules are not
856
+ // filters). See tests/rules.test.js "shared-with-me query".
857
+ const sharedQ = query(filesRef, where('scope', '==', 'private'), where('sharedWith', 'array-contains', userId));
858
+ const sharedUnsub = onSnapshot(sharedQ, snapshot => {
859
+ snapshot.docChanges().forEach(change => {
860
+ const data = change.doc.data();
861
+ if (data['ownerId'] === userId)
862
+ return;
863
+ track(handleFirestoreChange(change, () => path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data['path'])).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
864
+ });
865
+ }, err => console.error(` [${orgId}] Shared with me listener error:`, err));
866
+ unsubscribers.push(sharedUnsub);
867
+ console.log(` [${orgId}] Listening: Shared with Me`);
868
+ }
869
+ }
870
+ async function firestoreDocExists({ scope, teamId, docPath, contentHash, }) {
871
+ let q;
872
+ if (scope === 'private') {
873
+ q = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId), where('path', '==', docPath));
874
+ }
875
+ else if (scope === 'team' && teamId) {
876
+ q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId), where('path', '==', docPath));
877
+ }
878
+ else {
879
+ return false;
880
+ }
881
+ const snap = await getDocs(q);
882
+ if (snap.empty)
883
+ return false;
884
+ // Also require the doc's contentHash to match the baseline. A doc
885
+ // with the same path but a different hash means a web edit
886
+ // happened that the daemon didn't observe; treat as "missing" so
887
+ // the push side proceeds and the pull side can reconcile.
888
+ const firstDoc = snap.docs[0];
889
+ return firstDoc ? firstDoc.data()['contentHash'] === contentHash : false;
890
+ }
891
+ async function pushToFirestore(filePath, data, mimeType, localModifiedAt) {
892
+ const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
893
+ const trackingKey = `${scope}:${docPath}`;
894
+ // Skip if the local content is unchanged since the last sync. This
895
+ // is the push-side counterpart of the content baseline: an editor
896
+ // re-saving or touching a file (or the watcher re-emitting it on
897
+ // startup) must not push stale bytes over a newer server edit. Only
898
+ // a genuine content change proceeds.
899
+ //
900
+ // DOU-217: trust the baseline ONLY after verifying the Firestore
901
+ // doc actually exists. DOU-55 startup reconcile deletes local orphans
902
+ // when the manifest records a remote delete; this guard applies when
903
+ // pushToFirestore runs at runtime (chokidar add/change) and the
904
+ // baseline would otherwise suppress re-upload forever.
905
+ const contentHash = computeContentHash(data);
906
+ if (lastSyncedHash.get(trackingKey) === contentHash) {
907
+ if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
908
+ return;
909
+ }
910
+ console.log(` [${orgId}] [reconcile] dropping stale baseline for ${docPath} (Firestore doc missing); re-pushing`);
911
+ deleteBaseline(trackingKey);
912
+ }
913
+ const now = new Date().toISOString();
914
+ // Set the echo-suppression marker BEFORE writing. Our own write
915
+ // round-trips through the Firestore listener; without a marker
916
+ // already in place, that echo is treated as an incoming change and
917
+ // triggers a spurious self-pull, which (a) reads the blob before it
918
+ // is uploaded (object-not-found) and (b) adds the path to
919
+ // suppressLocal, swallowing a subsequent rename's unlink (leaving a
920
+ // duplicate doc). Pre-setting it suppresses the echo.
921
+ lastKnownUpdate.set(trackingKey, now);
922
+ const result = await pushFileToCloud({
923
+ filesRef,
924
+ storage,
925
+ orgId,
926
+ userId,
927
+ scope,
928
+ teamId,
929
+ docPath,
930
+ data,
931
+ mimeType,
932
+ now,
933
+ localModifiedAt,
611
934
  });
612
- }, err => console.error(` [${orgId}] Private files listener error:`, err));
613
- unsubscribers.push(privateUnsub);
614
- console.log(` [${orgId}] Listening: Private files + Shared by Me symlinks`);
615
-
616
- // scope=='private' is required: Firestore rules only grant
617
- // sharedWith-based reads on private files, so the query must
618
- // constrain scope or the whole listen is denied (rules are not
619
- // filters). See tests/rules.test.js "shared-with-me query".
620
- const sharedQ = query(
621
- filesRef,
622
- where('scope', '==', 'private'),
623
- where('sharedWith', 'array-contains', userId),
624
- );
625
- const sharedUnsub = onSnapshot(sharedQ, snapshot => {
626
- snapshot.docChanges().forEach(change => {
627
- const data = change.doc.data();
628
- if (data.ownerId === userId) return;
629
- track(handleFirestoreChange(change, () =>
630
- path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data.path),
631
- ).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
935
+ // Record the synced content baseline whenever local matches the
936
+ // server (created/updated, or already-equal noop), so the pull side
937
+ // can tell "local unchanged" from "local edited" by content.
938
+ if (shouldPersistLocalSyncState(result)) {
939
+ setBaseline(trackingKey, contentHash);
940
+ // Narrow to the variants that actually carry fileId (every
941
+ // variant shouldPersistLocalSyncState accepts: created, updated,
942
+ // noop). The skipped/skipped-stale variants are gated out above.
943
+ if (result.action === 'created' || result.action === 'updated' || result.action === 'noop') {
944
+ manifestStore.upsertEntry(trackingKey, {
945
+ fileId: result.fileId,
946
+ contentHash,
947
+ lastSyncedAt: now,
948
+ });
949
+ manifestStore.flush();
950
+ }
951
+ }
952
+ else if (result.action === 'skipped-stale') {
953
+ // The server copy is genuinely newer; clear the pre-set marker so
954
+ // its firestore -> local snapshot is allowed through to update the
955
+ // local file.
956
+ lastKnownUpdate.delete(trackingKey);
957
+ }
958
+ if (result.action === 'created') {
959
+ console.log(` [${orgId}] [local → firestore] CREATE ${docPath} (${scope}, ${mimeType})`);
960
+ }
961
+ else if (result.action === 'updated') {
962
+ console.log(` [${orgId}] [local → firestore] UPDATE ${docPath} (${scope})`);
963
+ }
964
+ else if (result.action === 'skipped-stale') {
965
+ console.log(` [${orgId}] [local → firestore] SKIP ${docPath} (server copy is newer)`);
966
+ }
967
+ if ((result.action === 'created' || result.action === 'updated') && result.storageError) {
968
+ console.error(` [${orgId}] [storage] upload failed for ${docPath}: ${result.storageError?.message}`);
969
+ }
970
+ }
971
+ async function deleteFromFirestore(filePath) {
972
+ const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
973
+ const trackingKey = `${scope}:${docPath}`;
974
+ lastKnownUpdate.delete(trackingKey);
975
+ const result = await deleteFileFromCloud({
976
+ filesRef,
977
+ storage,
978
+ orgId,
979
+ userId,
980
+ scope,
981
+ teamId,
982
+ docPath,
632
983
  });
633
- }, err => console.error(` [${orgId}] Shared with me listener error:`, err));
634
- unsubscribers.push(sharedUnsub);
635
- console.log(` [${orgId}] Listening: Shared with Me`);
636
- }
637
- }
638
-
639
- // DOU-217 helper: does a doc with this path + matching contentHash
640
- // exist in Firestore right now? Used to verify a baseline entry
641
- // before trusting it (so a stale baseline can't silently suppress
642
- // re-upload of a file whose Firestore doc was deleted out-of-band).
643
- async function firestoreDocExists({ scope, teamId, docPath, contentHash }) {
644
- let q;
645
- if (scope === 'private') {
646
- q = query(filesRef,
647
- where('scope', '==', 'private'),
648
- where('ownerId', '==', userId),
649
- where('path', '==', docPath));
650
- } else if (scope === 'team' && teamId) {
651
- q = query(filesRef,
652
- where('scope', '==', 'team'),
653
- where('teamId', '==', teamId),
654
- where('path', '==', docPath));
655
- } else {
656
- return false;
657
- }
658
- const snap = await getDocs(q);
659
- if (snap.empty) return false;
660
- // Also require the doc's contentHash to match the baseline. A doc
661
- // with the same path but a different hash means a web edit
662
- // happened that the daemon didn't observe; treat as "missing" so
663
- // the push side proceeds and the pull side can reconcile.
664
- return snap.docs[0].data().contentHash === contentHash;
665
- }
666
-
667
- async function pushToFirestore(filePath, data, mimeType, localModifiedAt) {
668
- const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
669
- const trackingKey = `${scope}:${docPath}`;
670
-
671
- // Skip if the local content is unchanged since the last sync. This
672
- // is the push-side counterpart of the content baseline: an editor
673
- // re-saving or touching a file (or the watcher re-emitting it on
674
- // startup) must not push stale bytes over a newer server edit. Only
675
- // a genuine content change proceeds.
676
- //
677
- // DOU-217: trust the baseline ONLY after verifying the Firestore
678
- // doc actually exists. The baseline is persisted across daemon
679
- // restarts, so a doc deleted out-of-band (web delete the daemon
680
- // missed while offline, manual cleanup, etc.) leaves a permanently
681
- // stale entry that would silently suppress re-upload forever.
682
- // Verifying via a single indexed read per touch is cheap; if the
683
- // doc is gone, clear the baseline and fall through to push.
684
- const contentHash = computeContentHash(data);
685
- if (lastSyncedHash.get(trackingKey) === contentHash) {
686
- if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
687
- return;
688
- }
689
- console.log(` [${orgId}] [reconcile] dropping stale baseline for ${docPath} (Firestore doc missing); re-pushing`);
690
- deleteBaseline(trackingKey);
691
- }
692
-
693
- const now = new Date().toISOString();
694
-
695
- // Set the echo-suppression marker BEFORE writing. Our own write
696
- // round-trips through the Firestore listener; without a marker
697
- // already in place, that echo is treated as an incoming change and
698
- // triggers a spurious self-pull, which (a) reads the blob before it
699
- // is uploaded (object-not-found) and (b) adds the path to
700
- // suppressLocal, swallowing a subsequent rename's unlink (leaving a
701
- // duplicate doc). Pre-setting it suppresses the echo.
702
- lastKnownUpdate.set(trackingKey, now);
703
-
704
- const result = await pushFileToCloud({
705
- filesRef,
706
- storage,
707
- orgId,
708
- userId,
709
- scope,
710
- teamId,
711
- docPath,
712
- data,
713
- mimeType,
714
- now,
715
- localModifiedAt,
716
- });
717
-
718
- // Record the synced content baseline whenever local matches the
719
- // server (created/updated, or already-equal noop), so the pull side
720
- // can tell "local unchanged" from "local edited" by content.
721
- if (result.action === 'created' || result.action === 'updated' || result.action === 'noop') {
722
- setBaseline(trackingKey, contentHash);
723
- } else if (result.action === 'skipped-stale') {
724
- // The server copy is genuinely newer; clear the pre-set marker so
725
- // its firestore -> local snapshot is allowed through to update the
726
- // local file.
727
- lastKnownUpdate.delete(trackingKey);
728
- }
729
-
730
- if (result.action === 'created') {
731
- console.log(` [${orgId}] [local firestore] CREATE ${docPath} (${scope}, ${mimeType})`);
732
- } else if (result.action === 'updated') {
733
- console.log(` [${orgId}] [local firestore] UPDATE ${docPath} (${scope})`);
734
- } else if (result.action === 'skipped-stale') {
735
- console.log(` [${orgId}] [local firestore] SKIP ${docPath} (server copy is newer)`);
736
- }
737
- if (result.storageError) {
738
- console.error(` [${orgId}] [storage] upload failed for ${docPath}: ${result.storageError.message}`);
739
- }
740
- }
741
-
742
- async function deleteFromFirestore(filePath) {
743
- const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
744
- const trackingKey = `${scope}:${docPath}`;
745
- lastKnownUpdate.delete(trackingKey);
746
-
747
- const result = await deleteFileFromCloud({
748
- filesRef,
749
- storage,
750
- orgId,
751
- userId,
752
- scope,
753
- teamId,
754
- docPath,
755
- });
756
-
757
- if (result.action === 'deleted') {
758
- console.log(` [${orgId}] [local firestore] DELETE ${docPath} (${scope})`);
759
- }
760
- }
761
-
762
- // Determine whether a local path corresponds to a syncable directory.
763
- // Excludes the scope-root directories themselves (Private, Shared
764
- // with Me, Shared by Me, and the per-team roots) since those are
765
- // managed by ensureFolderStructure, not the user.
766
- function isSyncableLocalDir(rel) {
767
- if (!rel) return false;
768
- if (rel === PRIVATE_FOLDER || rel === SHARED_WITH_ME_FOLDER || rel === SHARED_BY_ME_FOLDER) {
769
- return false;
770
- }
771
- if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return false;
772
- if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return false;
773
- const parts = rel.split(path.sep);
774
- // A bare team root (e.g. "Engineering Teamspace") corresponds to
775
- // the scope root, not a subdirectory. Skip these.
776
- if (parts.length === 1 && parts[0].endsWith(' Teamspace')) return false;
777
- return true;
778
- }
779
-
780
- async function pushDirToFirestore(dirPath) {
781
- const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
782
- if (!docPath) return; // scope root
783
- const trackingKey = `${scope}:${docPath}`;
784
- const now = new Date().toISOString();
785
- lastKnownUpdate.set(trackingKey, now);
786
-
787
- const result = await pushFolderToCloud({
788
- filesRef,
789
- scope,
790
- teamId,
791
- userId,
792
- docPath,
793
- now,
794
- });
795
-
796
- if (result.action === 'ensured' && result.folderId) {
797
- console.log(` [${orgId}] [local → firestore] FOLDER ${docPath} (${scope})`);
798
- }
799
- }
800
-
801
- async function deleteDirFromFirestore(dirPath) {
802
- const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
803
- if (!docPath) return;
804
- const trackingKey = `${scope}:${docPath}`;
805
- lastKnownUpdate.delete(trackingKey);
806
-
807
- const result = await deleteFolderFromCloud({
808
- filesRef,
809
- storage,
810
- orgId,
811
- scope,
812
- teamId,
813
- userId,
814
- docPath,
815
- });
816
-
817
- if (result.action === 'deleted') {
818
- // childCount > 0 means cascade did real work; useful in logs
819
- // when investigating future folder-delete reports.
820
- const childSuffix = result.childCount > 0 ? ` (+${result.childCount} children)` : '';
821
- console.log(` [${orgId}] [local → firestore] RMFOLDER ${docPath} (${scope})${childSuffix}`);
822
- }
823
- }
824
-
825
- function handleLocalChange(filePath) {
826
- if (suppressLocal.has(filePath)) return;
827
-
828
- const rel = path.relative(LOCAL_PATH, filePath);
829
- if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return;
830
- if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return;
831
-
832
- let stat;
833
- try {
834
- stat = fs.statSync(filePath);
835
- } catch (err) {
836
- return; // disappeared between event and read
837
- }
838
- if (!stat.isFile()) return;
839
- if (stat.size > maxUploadBytes) {
840
- console.error(` [${orgId}] [skip] ${rel} exceeds ${maxUploadBytes} byte cap (${stat.size} bytes)`);
841
- return;
842
- }
843
-
844
- const ext = extFromPath(rel);
845
- const mimeType = mimeTypeFromExt(ext);
846
- // Text files: read as a UTF-8 string so future text-aware features
847
- // (line-level diffing, conflict markers) can work. Binary files:
848
- // read as a Buffer so bytes round-trip without corruption.
849
- const data = isTextMimeType(mimeType)
850
- ? fs.readFileSync(filePath, 'utf-8')
851
- : fs.readFileSync(filePath);
852
- const localModifiedAt = stat.mtime.toISOString();
853
-
854
- // Yjs path (DOU-178 PR 3): route text edits into the binding so
855
- // the disk delta is published as a yupdate (preserving concurrent
856
- // web edits via fast-diff). The binding only exists once the
857
- // Firestore listener has seen the file; for the very first
858
- // local-only save the binding hasn't been created yet, and the
859
- // blob path is still the only writer. Subsequent saves go through
860
- // both paths until PR 4 retires the blob upload.
861
- if (isTextMimeType(mimeType)) {
862
- const binding = bindingsByPath.get(filePath);
863
- if (binding) binding.applyDiskText(data);
864
- }
865
-
866
- track(enqueueLocalOp(filePath, () =>
867
- pushToFirestore(filePath, data, mimeType, localModifiedAt).catch(err =>
868
- console.error(` [${orgId}] Push error for ${rel}: ${err.message}`)
869
- )
870
- ));
871
- }
872
-
873
- function handleLocalDirAdded(dirPath) {
874
- if (suppressLocal.has(dirPath)) return;
875
- const rel = path.relative(LOCAL_PATH, dirPath);
876
- if (!isSyncableLocalDir(rel)) return;
877
- track(pushDirToFirestore(dirPath).catch(err =>
878
- console.error(` [${orgId}] Folder push error for ${rel}: ${err.message}`)
879
- ));
880
- }
881
-
882
- function handleLocalDirRemoved(dirPath) {
883
- if (suppressLocal.has(dirPath)) return;
884
- const rel = path.relative(LOCAL_PATH, dirPath);
885
- if (!isSyncableLocalDir(rel)) return;
886
- track(deleteDirFromFirestore(dirPath).catch(err =>
887
- console.error(` [${orgId}] Folder delete error for ${rel}: ${err.message}`)
888
- ));
889
- }
890
-
891
- function startLocalWatcher() {
892
- console.log(` [${orgId}] Watching local files...`);
893
-
894
- watcher = chokidar.watch(LOCAL_PATH, {
895
- ignored: [
896
- // Dotfiles (the sync engine's own .compound-sync-state.json lives
897
- // here, plus things like .DS_Store, .git, etc. that the user
898
- // doesn't want syncing).
899
- /(^|[\/\\])\./,
900
- // node_modules: large dependency dirs, never appropriate to sync.
901
- /node_modules/,
902
- ],
903
- persistent: true,
904
- ignoreInitial: false,
905
- followSymlinks: false,
906
- awaitWriteFinish: {
907
- stabilityThreshold: 500,
908
- pollInterval: 100,
909
- },
910
- });
911
-
912
- // Reusable per-event trace logging, gated by COMPOUND_CHOKIDAR_DEBUG.
913
- // Writes to both stdout AND a file (default /tmp/chokidar-debug.log)
914
- // so traces survive a process kill or buffered stdout. Used by the
915
- // Test-on-demand workflow's `Dump diagnostic logs` step for
916
- // chokidar-related diagnostic runs. Zero cost when the env var is
917
- // unset.
918
- const chokidarDebug = process.env.COMPOUND_CHOKIDAR_DEBUG === '1';
919
- const chokidarLogPath = process.env.COMPOUND_CHOKIDAR_LOG || '/tmp/chokidar-debug.log';
920
- function logEvent(event, p) {
921
- if (chokidarDebug) {
922
- const rel = path.relative(LOCAL_PATH, p);
923
- const line = ` [${orgId}] [chokidar ${event}] ${rel} @ ${new Date().toISOString()}\n`;
924
- try { fs.appendFileSync(chokidarLogPath, line); } catch (_) {}
925
- console.log(line.trimEnd());
926
- }
927
- }
928
-
929
- watcher.on('add', filePath => { logEvent('add', filePath); handleLocalChange(filePath); });
930
- watcher.on('change', filePath => { logEvent('change', filePath); handleLocalChange(filePath); });
931
- watcher.on('unlink', filePath => {
932
- logEvent('unlink', filePath);
933
- if (suppressLocal.has(filePath)) return;
934
- const rel = path.relative(LOCAL_PATH, filePath);
935
- track(enqueueLocalOp(filePath, () =>
936
- deleteFromFirestore(filePath).catch(err =>
937
- console.error(` [${orgId}] Delete error for ${rel}: ${err.message}`)
938
- )
939
- ));
940
- });
941
- watcher.on('addDir', dirPath => { logEvent('addDir', dirPath); handleLocalDirAdded(dirPath); });
942
- watcher.on('unlinkDir', dirPath => { logEvent('unlinkDir', dirPath); handleLocalDirRemoved(dirPath); });
943
- if (chokidarDebug) {
944
- watcher.on('raw', (event, p, details) => {
945
- const line = ` [${orgId}] [chokidar raw ${event}] ${p} @ ${new Date().toISOString()} ${JSON.stringify(details)}\n`;
946
- try { fs.appendFileSync(chokidarLogPath, line); } catch (_) {}
947
- console.log(line.trimEnd());
948
- });
949
- }
950
-
951
- // Resolve only once chokidar has finished its initial scan and
952
- // registered inotify (or FSEvents) watches on every existing
953
- // subdirectory. Until 'ready' fires, files written to the watched
954
- // tree can be silently missed because the OS-level watch hasn't
955
- // attached yet. Callers awaiting startOrgSync get a guarantee
956
- // that any local writes after the await are seen by the daemon.
957
- return new Promise((resolve) => {
958
- watcher.on('ready', () => {
959
- console.log(` [${orgId}] Local watcher ready.`);
960
- resolve();
961
- });
962
- });
963
- }
964
-
965
- startFirestoreListeners();
966
- await startLocalWatcher();
967
-
968
- return {
969
- // Tear down all listeners and the watcher. Awaits every tracked
970
- // in-flight handler so callers/tests get a strong "no daemon work
971
- // is still running" guarantee on the resolved promise.
972
- async stop() {
973
- stoppedRef.value = true;
974
- for (const unsub of unsubscribers) {
975
- try { unsub(); } catch (e) { /* ignore */ }
976
- }
977
- for (const unsub of teamUnsubscribers.values()) {
978
- try { unsub(); } catch (e) { /* ignore */ }
979
- }
980
- teamUnsubscribers.clear();
981
-
982
- // Drain tracked promises in a loop, bounded by a 5s ceiling.
983
- // A handler currently in flight (e.g., a snapshot callback
984
- // already dispatched before unsub()) can chain follow-up work
985
- // that gets added back to pendingPromises. Loop until the set
986
- // quiesces so no daemon task outlives stop() in the common
987
- // case. The cap protects against tests where a handler is
988
- // genuinely stuck (e.g., emulator pause), so stop() never
989
- // hangs indefinitely; anything still pending past 5s is
990
- // abandoned and the next test's clearFirestore will sweep it.
991
- const drainDeadline = Date.now() + 5000;
992
- while (pendingPromises.size > 0 && Date.now() < drainDeadline) {
993
- const remaining = Math.max(0, drainDeadline - Date.now());
994
- await Promise.race([
995
- Promise.all([...pendingPromises]),
996
- new Promise((r) => setTimeout(r, remaining)),
997
- ]);
998
- }
999
-
1000
- await Promise.all(
1001
- [...bindings.values()].map((b) => b.stop().catch(() => {})),
1002
- );
1003
- bindings.clear();
1004
- bindingsByPath.clear();
1005
-
1006
- if (watcher) await watcher.close();
1007
- },
1008
- };
984
+ if (result.action === 'deleted') {
985
+ console.log(` [${orgId}] [local → firestore] DELETE ${docPath} (${scope})`);
986
+ manifestStore.removeEntry(trackingKey);
987
+ }
988
+ }
989
+ // Determine whether a local path corresponds to a syncable directory.
990
+ // Excludes the scope-root directories themselves (Private, Shared
991
+ // with Me, Shared by Me, and the per-team roots) since those are
992
+ // managed by ensureFolderStructure, not the user.
993
+ function isSyncableLocalDir(rel) {
994
+ if (!rel)
995
+ return false;
996
+ if (rel === PRIVATE_FOLDER || rel === SHARED_WITH_ME_FOLDER || rel === SHARED_BY_ME_FOLDER) {
997
+ return false;
998
+ }
999
+ if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
1000
+ return false;
1001
+ if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
1002
+ return false;
1003
+ const parts = rel.split(path.sep);
1004
+ // A bare team root (e.g. "Engineering Teamspace") corresponds to
1005
+ // the scope root, not a subdirectory. Skip these.
1006
+ if (parts.length === 1 && parts[0]?.endsWith(' Teamspace'))
1007
+ return false;
1008
+ return true;
1009
+ }
1010
+ async function pushDirToFirestore(dirPath) {
1011
+ const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
1012
+ if (!docPath)
1013
+ return; // scope root
1014
+ const trackingKey = `${scope}:${docPath}`;
1015
+ const now = new Date().toISOString();
1016
+ lastKnownUpdate.set(trackingKey, now);
1017
+ const result = await pushFolderToCloud({
1018
+ filesRef,
1019
+ scope,
1020
+ teamId,
1021
+ userId,
1022
+ docPath,
1023
+ now,
1024
+ });
1025
+ if (result.action === 'ensured' && result.folderId) {
1026
+ console.log(` [${orgId}] [local firestore] FOLDER ${docPath} (${scope})`);
1027
+ }
1028
+ }
1029
+ async function deleteDirFromFirestore(dirPath) {
1030
+ const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
1031
+ if (!docPath)
1032
+ return;
1033
+ const trackingKey = `${scope}:${docPath}`;
1034
+ lastKnownUpdate.delete(trackingKey);
1035
+ const result = await deleteFolderFromCloud({
1036
+ filesRef,
1037
+ storage,
1038
+ orgId,
1039
+ scope,
1040
+ teamId,
1041
+ userId,
1042
+ docPath,
1043
+ });
1044
+ if (result.action === 'deleted') {
1045
+ // childCount > 0 means cascade did real work; useful in logs
1046
+ // when investigating future folder-delete reports.
1047
+ const childSuffix = result.childCount > 0 ? ` (+${result.childCount} children)` : '';
1048
+ console.log(` [${orgId}] [local firestore] RMFOLDER ${docPath} (${scope})${childSuffix}`);
1049
+ }
1050
+ }
1051
+ function handleLocalChange(filePath) {
1052
+ if (suppressLocal.has(filePath))
1053
+ return;
1054
+ const rel = path.relative(LOCAL_PATH, filePath);
1055
+ if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
1056
+ return;
1057
+ if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
1058
+ return;
1059
+ let stat;
1060
+ try {
1061
+ stat = fs.statSync(filePath);
1062
+ }
1063
+ catch {
1064
+ return; // disappeared between event and read
1065
+ }
1066
+ if (!stat.isFile())
1067
+ return;
1068
+ if (stat.size > maxUploadBytes) {
1069
+ console.error(` [${orgId}] [skip] ${rel} exceeds ${maxUploadBytes} byte cap (${stat.size} bytes)`);
1070
+ return;
1071
+ }
1072
+ const ext = extFromPath(rel);
1073
+ const mimeType = mimeTypeFromExt(ext);
1074
+ // Text files: read as a UTF-8 string so future text-aware features
1075
+ // (line-level diffing, conflict markers) can work. Binary files:
1076
+ // read as a Buffer so bytes round-trip without corruption.
1077
+ const data = isTextMimeType(mimeType)
1078
+ ? fs.readFileSync(filePath, 'utf-8')
1079
+ : fs.readFileSync(filePath);
1080
+ const localModifiedAt = stat.mtime.toISOString();
1081
+ // Yjs path (DOU-178 PR 3): route text edits into the binding so
1082
+ // the disk delta is published as a yupdate (preserving concurrent
1083
+ // web edits via fast-diff). The binding only exists once the
1084
+ // Firestore listener has seen the file; for the very first
1085
+ // local-only save the binding hasn't been created yet, and the
1086
+ // blob path is still the only writer. Subsequent saves go through
1087
+ // both paths until PR 4 retires the blob upload.
1088
+ if (typeof data === 'string' && isTextMimeType(mimeType)) {
1089
+ const binding = bindingsByPath.get(filePath);
1090
+ if (binding)
1091
+ binding.applyDiskText(data);
1092
+ }
1093
+ track(enqueueLocalOp(filePath, () => pushToFirestore(filePath, data, mimeType, localModifiedAt).catch((err) => console.error(` [${orgId}] Push error for ${rel}: ${err?.message}`))));
1094
+ }
1095
+ function handleLocalDirAdded(dirPath) {
1096
+ if (suppressLocal.has(dirPath))
1097
+ return;
1098
+ const rel = path.relative(LOCAL_PATH, dirPath);
1099
+ if (!isSyncableLocalDir(rel))
1100
+ return;
1101
+ track(pushDirToFirestore(dirPath).catch((err) => console.error(` [${orgId}] Folder push error for ${rel}: ${err?.message}`)));
1102
+ }
1103
+ function handleLocalDirRemoved(dirPath) {
1104
+ if (suppressLocal.has(dirPath))
1105
+ return;
1106
+ const rel = path.relative(LOCAL_PATH, dirPath);
1107
+ if (!isSyncableLocalDir(rel))
1108
+ return;
1109
+ track(deleteDirFromFirestore(dirPath).catch((err) => console.error(` [${orgId}] Folder delete error for ${rel}: ${err?.message}`)));
1110
+ }
1111
+ function startLocalWatcher() {
1112
+ console.log(` [${orgId}] Watching local files...`);
1113
+ watcher = chokidar.watch(LOCAL_PATH, {
1114
+ ignored: [
1115
+ // Dotfiles (the sync engine's own .compound-sync-state.json lives
1116
+ // here, plus things like .DS_Store, .git, etc. that the user
1117
+ // doesn't want syncing).
1118
+ /(^|[\/\\])\./,
1119
+ // node_modules: large dependency dirs, never appropriate to sync.
1120
+ /node_modules/,
1121
+ ],
1122
+ persistent: true,
1123
+ ignoreInitial: false,
1124
+ followSymlinks: false,
1125
+ awaitWriteFinish: {
1126
+ stabilityThreshold: 500,
1127
+ pollInterval: 100,
1128
+ },
1129
+ });
1130
+ // Reusable per-event trace logging, gated by COMPOUND_CHOKIDAR_DEBUG.
1131
+ // Writes to both stdout AND a file (default /tmp/chokidar-debug.log)
1132
+ // so traces survive a process kill or buffered stdout. Used by the
1133
+ // Test-on-demand workflow's `Dump diagnostic logs` step for
1134
+ // chokidar-related diagnostic runs. Zero cost when the env var is
1135
+ // unset.
1136
+ const chokidarDebug = process.env['COMPOUND_CHOKIDAR_DEBUG'] === '1';
1137
+ const chokidarLogPath = process.env['COMPOUND_CHOKIDAR_LOG'] || '/tmp/chokidar-debug.log';
1138
+ function logEvent(event, p) {
1139
+ if (chokidarDebug) {
1140
+ const rel = path.relative(LOCAL_PATH, p);
1141
+ const line = ` [${orgId}] [chokidar ${event}] ${rel} @ ${new Date().toISOString()}\n`;
1142
+ try {
1143
+ fs.appendFileSync(chokidarLogPath, line);
1144
+ }
1145
+ catch (_) { }
1146
+ console.log(line.trimEnd());
1147
+ }
1148
+ }
1149
+ // Local non-null alias so TS doesn't have to re-narrow `watcher`
1150
+ // on every .on() call below.
1151
+ const w = watcher;
1152
+ w.on('add', (filePath) => { logEvent('add', filePath); handleLocalChange(filePath); });
1153
+ w.on('change', (filePath) => { logEvent('change', filePath); handleLocalChange(filePath); });
1154
+ w.on('unlink', (filePath) => {
1155
+ logEvent('unlink', filePath);
1156
+ if (suppressLocal.has(filePath))
1157
+ return;
1158
+ const rel = path.relative(LOCAL_PATH, filePath);
1159
+ track(enqueueLocalOp(filePath, () => deleteFromFirestore(filePath).catch((err) => console.error(` [${orgId}] Delete error for ${rel}: ${err?.message}`))));
1160
+ });
1161
+ w.on('addDir', (dirPath) => { logEvent('addDir', dirPath); handleLocalDirAdded(dirPath); });
1162
+ w.on('unlinkDir', (dirPath) => { logEvent('unlinkDir', dirPath); handleLocalDirRemoved(dirPath); });
1163
+ if (chokidarDebug) {
1164
+ w.on('raw', (event, p, details) => {
1165
+ const line = ` [${orgId}] [chokidar raw ${event}] ${p} @ ${new Date().toISOString()} ${JSON.stringify(details)}\n`;
1166
+ try {
1167
+ fs.appendFileSync(chokidarLogPath, line);
1168
+ }
1169
+ catch { /* ignore */ }
1170
+ console.log(line.trimEnd());
1171
+ });
1172
+ }
1173
+ // Resolve only once chokidar has finished its initial scan and
1174
+ // registered inotify (or FSEvents) watches on every existing
1175
+ // subdirectory. Until 'ready' fires, files written to the watched
1176
+ // tree can be silently missed because the OS-level watch hasn't
1177
+ // attached yet. Callers awaiting startOrgSync get a guarantee
1178
+ // that any local writes after the await are seen by the daemon.
1179
+ return new Promise((resolve) => {
1180
+ w.on('ready', () => {
1181
+ console.log(` [${orgId}] Local watcher ready.`);
1182
+ resolve();
1183
+ });
1184
+ });
1185
+ }
1186
+ startFirestoreListeners();
1187
+ await startLocalWatcher();
1188
+ return {
1189
+ // Tear down all listeners and the watcher. Awaits every tracked
1190
+ // in-flight handler so callers/tests get a strong "no daemon work
1191
+ // is still running" guarantee on the resolved promise.
1192
+ async stop() {
1193
+ stoppedRef.value = true;
1194
+ manifestStore.flush();
1195
+ for (const unsub of unsubscribers) {
1196
+ try {
1197
+ unsub();
1198
+ }
1199
+ catch (e) { /* ignore */ }
1200
+ }
1201
+ for (const unsub of teamUnsubscribers.values()) {
1202
+ try {
1203
+ unsub();
1204
+ }
1205
+ catch (e) { /* ignore */ }
1206
+ }
1207
+ teamUnsubscribers.clear();
1208
+ // Drain tracked promises in a loop, bounded by a 5s ceiling.
1209
+ // A handler currently in flight (e.g., a snapshot callback
1210
+ // already dispatched before unsub()) can chain follow-up work
1211
+ // that gets added back to pendingPromises. Loop until the set
1212
+ // quiesces so no daemon task outlives stop() in the common
1213
+ // case. The cap protects against tests where a handler is
1214
+ // genuinely stuck (e.g., emulator pause), so stop() never
1215
+ // hangs indefinitely; anything still pending past 5s is
1216
+ // abandoned and the next test's clearFirestore will sweep it.
1217
+ const drainDeadline = Date.now() + 5000;
1218
+ while (pendingPromises.size > 0 && Date.now() < drainDeadline) {
1219
+ const remaining = Math.max(0, drainDeadline - Date.now());
1220
+ await Promise.race([
1221
+ Promise.all([...pendingPromises]),
1222
+ new Promise((r) => setTimeout(r, remaining)),
1223
+ ]);
1224
+ }
1225
+ await Promise.all([...bindings.values()].map((b) => b.stop().catch(() => { })));
1226
+ bindings.clear();
1227
+ bindingsByPath.clear();
1228
+ if (watcher)
1229
+ await watcher.close();
1230
+ },
1231
+ };
1009
1232
  }
1233
+ //# sourceMappingURL=org-sync.js.map