@doubling/compound-sync 1.12.4 → 1.12.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/auth-persistence.d.ts +22 -0
- package/auth-persistence.js +79 -83
- package/config.d.ts +13 -0
- package/config.js +55 -54
- package/files.d.ts +221 -0
- package/files.js +373 -438
- package/folder-chain.d.ts +44 -0
- package/folder-chain.js +52 -59
- package/manifest.d.ts +38 -0
- package/manifest.js +87 -89
- package/org-sync.d.ts +16 -0
- package/org-sync.js +1156 -1203
- package/package.json +23 -2
- package/paths.d.ts +1 -1
- package/pre-mint-app-check.d.ts +6 -0
- package/pre-mint-app-check.js +46 -39
- package/push-outcome.d.ts +9 -0
- package/push-outcome.js +6 -5
- package/setup-auth-with-pre-mint.d.ts +24 -0
- package/setup-auth-with-pre-mint.js +48 -55
- package/storage-helpers.d.ts +14 -0
- package/storage-helpers.js +43 -107
- package/sync-state.d.ts +4 -0
- package/sync-state.js +30 -16
- package/sync.d.ts +18 -0
- package/sync.js +464 -515
- package/team-registry.d.ts +41 -0
- package/team-registry.js +90 -81
- package/yjs-binding-state.d.ts +4 -0
- package/yjs-binding-state.js +25 -22
- package/yjs-file-binding.d.ts +32 -0
- package/yjs-file-binding.js +217 -204
- package/yjs-firestore-update-store.d.ts +13 -0
- package/yjs-firestore-update-store.js +99 -103
- package/yjs-provider.d.ts +43 -0
- package/yjs-provider.js +79 -75
package/org-sync.js
CHANGED
|
@@ -11,1270 +11,1223 @@
|
|
|
11
11
|
// caller. Behavior is otherwise unchanged (including the known
|
|
12
12
|
// mtime-based decision quirks DOU-154 will rework, those stay so the
|
|
13
13
|
// scenario tests can pin them red first).
|
|
14
|
-
|
|
15
14
|
import chokidar from 'chokidar';
|
|
16
|
-
import fs from 'fs';
|
|
17
|
-
import path from 'path';
|
|
18
|
-
import { collection, query, where, onSnapshot, getDocs } from 'firebase/firestore';
|
|
19
|
-
import {
|
|
20
|
-
|
|
21
|
-
deleteFileFromCloud,
|
|
22
|
-
pushFolderToCloud,
|
|
23
|
-
deleteFolderFromCloud,
|
|
24
|
-
readBlob,
|
|
25
|
-
readBlobAsText,
|
|
26
|
-
extFromPath,
|
|
27
|
-
isTextMimeType,
|
|
28
|
-
mimeTypeFromExt,
|
|
29
|
-
computeContentHash,
|
|
30
|
-
} from './files.js';
|
|
31
|
-
import {
|
|
32
|
-
scopeFromLocalPath,
|
|
33
|
-
teamFolderName,
|
|
34
|
-
PRIVATE_FOLDER,
|
|
35
|
-
SHARED_WITH_ME_FOLDER,
|
|
36
|
-
SHARED_BY_ME_FOLDER,
|
|
37
|
-
} from './paths.js';
|
|
15
|
+
import fs from 'node:fs';
|
|
16
|
+
import path from 'node:path';
|
|
17
|
+
import { collection, query, where, onSnapshot, getDocs, } from 'firebase/firestore';
|
|
18
|
+
import { pushFileToCloud, deleteFileFromCloud, pushFolderToCloud, deleteFolderFromCloud, readBlob, readBlobAsText, extFromPath, isTextMimeType, mimeTypeFromExt, computeContentHash, } from './files.js';
|
|
19
|
+
import { scopeFromLocalPath, teamFolderName, PRIVATE_FOLDER, SHARED_WITH_ME_FOLDER, SHARED_BY_ME_FOLDER, } from './paths.js';
|
|
38
20
|
import { TeamRegistry } from './team-registry.js';
|
|
39
21
|
import { loadSyncState, saveSyncState } from './sync-state.js';
|
|
40
22
|
import { createManifestStore } from './manifest.js';
|
|
41
23
|
import { shouldPersistLocalSyncState } from './push-outcome.js';
|
|
42
24
|
import { FirestoreUpdateStore } from './yjs-firestore-update-store.js';
|
|
43
25
|
import { YjsFileBinding } from './yjs-file-binding.js';
|
|
44
|
-
import {
|
|
45
|
-
loadBindingState, saveBindingState, deleteBindingState,
|
|
46
|
-
} from './yjs-binding-state.js';
|
|
47
|
-
|
|
26
|
+
import { loadBindingState, saveBindingState, deleteBindingState, } from './yjs-binding-state.js';
|
|
48
27
|
const DEFAULT_MAX_UPLOAD_BYTES = 25 * 1024 * 1024;
|
|
49
|
-
|
|
50
|
-
// Retry a blob download until either the bytes appear at all (covers
|
|
51
|
-
// the create race: doc written before blob upload) AND the content
|
|
52
|
-
// hash matches the metadata's `contentHash` (covers the update race:
|
|
53
|
-
// the blob still has the old bytes when the new doc is already
|
|
54
|
-
// visible). Backs off 100/200/400/800 ms (1.5s total) across both
|
|
55
|
-
// failure modes. Other errors throw immediately.
|
|
56
28
|
async function readBlobWithRetry(read, { expectedHash = null, maxAttempts = 5 } = {}) {
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
29
|
+
let delay = 100;
|
|
30
|
+
let lastErr;
|
|
31
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
32
|
+
try {
|
|
33
|
+
const content = await read();
|
|
34
|
+
if (!expectedHash || computeContentHash(content) === expectedHash) {
|
|
35
|
+
return content;
|
|
36
|
+
}
|
|
37
|
+
lastErr = new Error(`blob hash does not match metadata contentHash (attempt ${attempt}/${maxAttempts})`);
|
|
38
|
+
}
|
|
39
|
+
catch (err) {
|
|
40
|
+
if (err?.code !== 'storage/object-not-found')
|
|
41
|
+
throw err;
|
|
42
|
+
lastErr = err;
|
|
43
|
+
}
|
|
44
|
+
if (attempt < maxAttempts) {
|
|
45
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
46
|
+
delay *= 2;
|
|
47
|
+
}
|
|
75
48
|
}
|
|
76
|
-
|
|
77
|
-
throw lastErr;
|
|
49
|
+
throw lastErr;
|
|
78
50
|
}
|
|
79
|
-
|
|
80
51
|
// Start the per-org sync loop. Returns { stop } to unsubscribe all
|
|
81
52
|
// listeners and close the watcher (used by tests and by graceful
|
|
82
53
|
// shutdown). `localPath` must already be absolute/expanded.
|
|
83
|
-
export async function startOrgSync({
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
// surfacing as "Firebase app was deleted" errors in the daemon log
|
|
146
|
-
// and (suspected) accumulating gRPC stream state that the next
|
|
147
|
-
// listener attach receives as a single oversize Listen message.
|
|
148
|
-
const pendingBindings = new Set();
|
|
149
|
-
const pendingPromises = new Set();
|
|
150
|
-
const stoppedRef = { value: false };
|
|
151
|
-
|
|
152
|
-
// Helper: register a promise with pendingPromises so stop() awaits
|
|
153
|
-
// it, and auto-remove on settle. Returns the same promise so call
|
|
154
|
-
// sites can keep their chaining shape.
|
|
155
|
-
function track(p) {
|
|
156
|
-
pendingPromises.add(p);
|
|
157
|
-
p.finally(() => pendingPromises.delete(p));
|
|
158
|
-
return p;
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
// DOU-254: per-local-path operation queue. chokidar can fire `add`
|
|
162
|
-
// then `unlink` for the same path within a few hundred ms (a test
|
|
163
|
-
// writes a file, waits only for the Firestore doc to appear, then
|
|
164
|
-
// deletes locally). Without serialization, the push (setDoc +
|
|
165
|
-
// uploadBlob) and the delete (preRecheck + deleteBlob + deleteDoc)
|
|
166
|
-
// race against each other on the same fileId, and uploadBlob can
|
|
167
|
-
// land AFTER deleteBlob, leaving an orphaned blob after the doc is
|
|
168
|
-
// gone. Chain ops keyed by local path so the next op starts only
|
|
169
|
-
// after the previous one settles.
|
|
170
|
-
const localOpQueues = new Map();
|
|
171
|
-
function enqueueLocalOp(filePath, op) {
|
|
172
|
-
const prev = localOpQueues.get(filePath) || Promise.resolve();
|
|
173
|
-
// Swallow the previous op's rejection so a failed push doesn't
|
|
174
|
-
// skip the subsequent delete (or vice versa); each op handles
|
|
175
|
-
// its own errors via the .catch() at the call site.
|
|
176
|
-
const next = prev.catch(() => {}).then(op);
|
|
177
|
-
localOpQueues.set(filePath, next);
|
|
178
|
-
next.finally(() => {
|
|
179
|
-
if (localOpQueues.get(filePath) === next) localOpQueues.delete(filePath);
|
|
180
|
-
});
|
|
181
|
-
return next;
|
|
182
|
-
}
|
|
183
|
-
|
|
184
|
-
// Fire-and-forget binding setup. Returns nothing because the
|
|
185
|
-
// common shape "ensureBinding then read its ytext" runs straight
|
|
186
|
-
// into a race: the daemon's onSnapshot fires on Alice's own
|
|
187
|
-
// optimistic write before the file doc is server-committed, and
|
|
188
|
-
// the Firestore rules engine evaluates the yupdates/yjs nested
|
|
189
|
-
// rule via a server-side get() against the parent files/{id}
|
|
190
|
-
// doc. That get() sees no doc and returns null, so binding.start
|
|
191
|
-
// fails with permission-denied; the right behavior is to retry
|
|
192
|
-
// until the commit propagates, not to block handleFirestoreChange
|
|
193
|
-
// for seconds. Callers fall back to the blob path while the
|
|
194
|
-
// binding is still being set up; once it lands in
|
|
195
|
-
// bindings/bindingsByPath, subsequent events take the Yjs path.
|
|
196
|
-
function ensureBinding(fileId, localFilePath) {
|
|
197
|
-
if (bindings.has(fileId) || pendingBindings.has(fileId)) return;
|
|
198
|
-
pendingBindings.add(fileId);
|
|
199
|
-
const onDiskWrite = (text) => {
|
|
200
|
-
suppressLocal.add(localFilePath);
|
|
201
|
-
ensureDir(localFilePath);
|
|
202
|
-
fs.writeFileSync(localFilePath, text, 'utf-8');
|
|
203
|
-
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
204
|
-
};
|
|
205
|
-
// DOU-209: persist the Y.Doc binary state after every stable
|
|
206
|
-
// change so the next daemon restart has the right baseline to
|
|
207
|
-
// merge offline disk edits without losing offline web edits.
|
|
208
|
-
const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
|
|
209
|
-
// DOU-209: feed the binding's startup-merge with the daemon's
|
|
210
|
-
// last view (initialDocState) and the current disk content. If
|
|
211
|
-
// they diverge, the binding applies the diff as Yjs ops so disk-
|
|
212
|
-
// side offline edits survive the cloud-side replay.
|
|
213
|
-
const initialDocState = loadBindingState(LOCAL_PATH, fileId);
|
|
214
|
-
let currentDiskText = null;
|
|
215
|
-
try {
|
|
216
|
-
if (fs.existsSync(localFilePath)) {
|
|
217
|
-
currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
|
|
218
|
-
}
|
|
219
|
-
} catch (err) {
|
|
220
|
-
console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err && err.message}`);
|
|
54
|
+
export async function startOrgSync({ db, storage, userId, orgId, localPath, maxUploadBytes = DEFAULT_MAX_UPLOAD_BYTES, statePath, }) {
|
|
55
|
+
const LOCAL_PATH = localPath;
|
|
56
|
+
const filesRef = collection(db, `orgs/${orgId}/files`);
|
|
57
|
+
// Where the per-folder content baseline is persisted. Default: a
|
|
58
|
+
// dotfile in the synced folder (ignored by the watcher), so it travels
|
|
59
|
+
// with the folder and needs no extra wiring from the caller.
|
|
60
|
+
const STATE_PATH = statePath ?? path.join(LOCAL_PATH, '.compound-sync-state.json');
|
|
61
|
+
const manifestStore = createManifestStore(LOCAL_PATH);
|
|
62
|
+
// Teardown handles: every onSnapshot unsub plus the chokidar watcher.
|
|
63
|
+
const unsubscribers = [];
|
|
64
|
+
let watcher = null;
|
|
65
|
+
const lastKnownUpdate = new Map();
|
|
66
|
+
// Content hash of each file as of the last successful sync (push or
|
|
67
|
+
// pull), keyed by trackingKey. This "baseline" decides pull/push
|
|
68
|
+
// direction by CONTENT rather than filesystem mtime (which an editor
|
|
69
|
+
// like Obsidian bumps without a content change). Loaded from
|
|
70
|
+
// STATE_PATH so it survives restarts; persisted on every change.
|
|
71
|
+
const lastSyncedHash = new Map(Object.entries(loadSyncState(STATE_PATH)));
|
|
72
|
+
function persistBaseline() {
|
|
73
|
+
saveSyncState(STATE_PATH, Object.fromEntries(lastSyncedHash));
|
|
74
|
+
}
|
|
75
|
+
function setBaseline(key, hash) { lastSyncedHash.set(key, hash); persistBaseline(); }
|
|
76
|
+
function deleteBaseline(key) { lastSyncedHash.delete(key); persistBaseline(); }
|
|
77
|
+
const suppressLocal = new Set();
|
|
78
|
+
// Per-file Yjs bindings, created on first sight of a text file
|
|
79
|
+
// (DOU-178 PR 3). The binding owns the live Y.Doc that mirrors
|
|
80
|
+
// orgs/.../yupdates and the debounced disk flush. We keep two
|
|
81
|
+
// indexes: by fileId (so handleFirestoreChange can dispose on
|
|
82
|
+
// 'removed' and re-use across events) and by local path (so the
|
|
83
|
+
// chokidar handler can route disk edits into the binding without
|
|
84
|
+
// re-resolving the fileId).
|
|
85
|
+
const bindings = new Map();
|
|
86
|
+
const bindingsByPath = new Map();
|
|
87
|
+
// DOU-205: per-fileId record of the most recently observed Firestore
|
|
88
|
+
// metadata (local path + tracking key). Used to detect renames /
|
|
89
|
+
// moves on subsequent 'modified' snapshot events: when the file
|
|
90
|
+
// doc's path changes server-side (via the web rename UI), the local
|
|
91
|
+
// path computed from the new data also changes, and this map lets
|
|
92
|
+
// the handler relocate the local file in place (fs.renameSync,
|
|
93
|
+
// which preserves Obsidian's inode-level tracking) rather than
|
|
94
|
+
// leaving the old file orphaned at its previous location.
|
|
95
|
+
const knownPaths = new Map();
|
|
96
|
+
// Track which fileIds already have a binding (or one being set
|
|
97
|
+
// up) so we never double-create. `stoppedRef` is read by the
|
|
98
|
+
// background retry loops so they can abandon work after sync.stop.
|
|
99
|
+
// `pendingPromises` lets stop() await any in-flight setup OR any
|
|
100
|
+
// fire-and-forget snapshot/chokidar handler work before returning.
|
|
101
|
+
// Without this, the next test's clearFirestore + terminate can
|
|
102
|
+
// race a handler that's still flushing storage from this test,
|
|
103
|
+
// surfacing as "Firebase app was deleted" errors in the daemon log
|
|
104
|
+
// and (suspected) accumulating gRPC stream state that the next
|
|
105
|
+
// listener attach receives as a single oversize Listen message.
|
|
106
|
+
const pendingBindings = new Set();
|
|
107
|
+
const pendingPromises = new Set();
|
|
108
|
+
const stoppedRef = { value: false };
|
|
109
|
+
// Helper: register a promise with pendingPromises so stop() awaits
|
|
110
|
+
// it, and auto-remove on settle. Returns the same promise so call
|
|
111
|
+
// sites can keep their chaining shape.
|
|
112
|
+
function track(p) {
|
|
113
|
+
pendingPromises.add(p);
|
|
114
|
+
p.finally(() => pendingPromises.delete(p));
|
|
115
|
+
return p;
|
|
221
116
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
117
|
+
// DOU-254: per-local-path operation queue. chokidar can fire `add`
|
|
118
|
+
// then `unlink` for the same path within a few hundred ms (a test
|
|
119
|
+
// writes a file, waits only for the Firestore doc to appear, then
|
|
120
|
+
// deletes locally). Without serialization, the push (setDoc +
|
|
121
|
+
// uploadBlob) and the delete (preRecheck + deleteBlob + deleteDoc)
|
|
122
|
+
// race against each other on the same fileId, and uploadBlob can
|
|
123
|
+
// land AFTER deleteBlob, leaving an orphaned blob after the doc is
|
|
124
|
+
// gone. Chain ops keyed by local path so the next op starts only
|
|
125
|
+
// after the previous one settles.
|
|
126
|
+
const localOpQueues = new Map();
|
|
127
|
+
function enqueueLocalOp(filePath, op) {
|
|
128
|
+
const prev = localOpQueues.get(filePath) ?? Promise.resolve();
|
|
129
|
+
// Swallow the previous op's rejection so a failed push doesn't
|
|
130
|
+
// skip the subsequent delete (or vice versa); each op handles
|
|
131
|
+
// its own errors via the .catch() at the call site.
|
|
132
|
+
const next = prev.catch(() => undefined).then(op);
|
|
133
|
+
localOpQueues.set(filePath, next);
|
|
134
|
+
next.finally(() => {
|
|
135
|
+
if (localOpQueues.get(filePath) === next)
|
|
136
|
+
localOpQueues.delete(filePath);
|
|
232
137
|
});
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
138
|
+
return next;
|
|
139
|
+
}
|
|
140
|
+
// Fire-and-forget binding setup. Returns nothing because the
|
|
141
|
+
// common shape "ensureBinding then read its ytext" runs straight
|
|
142
|
+
// into a race: the daemon's onSnapshot fires on Alice's own
|
|
143
|
+
// optimistic write before the file doc is server-committed, and
|
|
144
|
+
// the Firestore rules engine evaluates the yupdates/yjs nested
|
|
145
|
+
// rule via a server-side get() against the parent files/{id}
|
|
146
|
+
// doc. That get() sees no doc and returns null, so binding.start
|
|
147
|
+
// fails with permission-denied; the right behavior is to retry
|
|
148
|
+
// until the commit propagates, not to block handleFirestoreChange
|
|
149
|
+
// for seconds. Callers fall back to the blob path while the
|
|
150
|
+
// binding is still being set up; once it lands in
|
|
151
|
+
// bindings/bindingsByPath, subsequent events take the Yjs path.
|
|
152
|
+
function ensureBinding(fileId, localFilePath) {
|
|
153
|
+
if (bindings.has(fileId) || pendingBindings.has(fileId))
|
|
237
154
|
return;
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
155
|
+
pendingBindings.add(fileId);
|
|
156
|
+
const onDiskWrite = (text) => {
|
|
157
|
+
suppressLocal.add(localFilePath);
|
|
158
|
+
ensureDir(localFilePath);
|
|
159
|
+
fs.writeFileSync(localFilePath, text, 'utf-8');
|
|
160
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
161
|
+
};
|
|
162
|
+
// DOU-209: persist the Y.Doc binary state after every stable
|
|
163
|
+
// change so the next daemon restart has the right baseline to
|
|
164
|
+
// merge offline disk edits without losing offline web edits.
|
|
165
|
+
const onDocStateChange = (encoded) => saveBindingState(LOCAL_PATH, fileId, encoded);
|
|
166
|
+
// DOU-209: feed the binding's startup-merge with the daemon's
|
|
167
|
+
// last view (initialDocState) and the current disk content. If
|
|
168
|
+
// they diverge, the binding applies the diff as Yjs ops so disk-
|
|
169
|
+
// side offline edits survive the cloud-side replay.
|
|
170
|
+
const initialDocState = loadBindingState(LOCAL_PATH, fileId);
|
|
171
|
+
let currentDiskText = null;
|
|
172
|
+
try {
|
|
173
|
+
if (fs.existsSync(localFilePath)) {
|
|
174
|
+
currentDiskText = fs.readFileSync(localFilePath, 'utf-8');
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
console.error(` [${orgId}] could not read ${localFilePath} for binding setup: ${err?.message}`);
|
|
179
|
+
}
|
|
180
|
+
const setupPromise = (async () => {
|
|
181
|
+
// Retry binding.start on permission-denied. See note above
|
|
182
|
+
// about the optimistic-write race with the nested-rule get().
|
|
183
|
+
// YjsProvider rejects double-start, so each retry needs a
|
|
184
|
+
// fresh binding (and a fresh provider underneath it).
|
|
185
|
+
for (let attempt = 0; attempt < 8; attempt++) {
|
|
186
|
+
if (stoppedRef.value)
|
|
187
|
+
return;
|
|
188
|
+
const store = new FirestoreUpdateStore(db, orgId, fileId);
|
|
189
|
+
const binding = new YjsFileBinding(store, {
|
|
190
|
+
onDiskWrite, onDocStateChange, initialDocState, currentDiskText,
|
|
191
|
+
});
|
|
192
|
+
try {
|
|
193
|
+
await binding.start();
|
|
194
|
+
if (stoppedRef.value) {
|
|
195
|
+
await binding.stop().catch(() => undefined);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
bindings.set(fileId, binding);
|
|
199
|
+
bindingsByPath.set(localFilePath, binding);
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
catch (err) {
|
|
203
|
+
if (stoppedRef.value)
|
|
204
|
+
return;
|
|
205
|
+
const code = err?.code;
|
|
206
|
+
if (code !== 'permission-denied' || attempt === 7) {
|
|
207
|
+
console.error(` [${orgId}] Yjs binding setup failed for ${fileId}: ${err?.message}`);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
await new Promise((r) => setTimeout(r, 150 * (attempt + 1)));
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
})().finally(() => {
|
|
214
|
+
pendingBindings.delete(fileId);
|
|
215
|
+
pendingPromises.delete(setupPromise);
|
|
216
|
+
});
|
|
217
|
+
pendingPromises.add(setupPromise);
|
|
218
|
+
}
|
|
219
|
+
async function disposeBinding(fileId, localFilePath) {
|
|
220
|
+
const binding = bindings.get(fileId);
|
|
221
|
+
if (!binding)
|
|
246
222
|
return;
|
|
247
|
-
|
|
248
|
-
|
|
223
|
+
bindings.delete(fileId);
|
|
224
|
+
if (localFilePath)
|
|
225
|
+
bindingsByPath.delete(localFilePath);
|
|
226
|
+
await binding.stop();
|
|
227
|
+
// DOU-209: drop the per-file Y.Doc state so deleted files don't
|
|
228
|
+
// accumulate stale binding-state blobs under .compound-yjs-binding/.
|
|
229
|
+
deleteBindingState(LOCAL_PATH, fileId);
|
|
230
|
+
}
|
|
231
|
+
// Per-team Firestore-listener unsubscribe handles, keyed by teamId.
|
|
232
|
+
const teamUnsubscribers = new Map();
|
|
233
|
+
// Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
|
|
234
|
+
// which is defined later in this function. Wired below.
|
|
235
|
+
let setupTeam = () => { };
|
|
236
|
+
function teardownTeam(teamId, teamData) {
|
|
237
|
+
const unsub = teamUnsubscribers.get(teamId);
|
|
238
|
+
if (unsub) {
|
|
239
|
+
try {
|
|
240
|
+
unsub();
|
|
241
|
+
}
|
|
242
|
+
catch { /* ignore */ }
|
|
243
|
+
teamUnsubscribers.delete(teamId);
|
|
244
|
+
const folder = teamData && teamData.name ? teamFolderName(teamData.name) : teamId;
|
|
245
|
+
console.log(` [${orgId}] Stopped listening: ${folder}`);
|
|
249
246
|
}
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
247
|
+
}
|
|
248
|
+
const teamRegistry = new TeamRegistry({
|
|
249
|
+
onSetup: (teamId, teamData) => setupTeam(teamId, teamData),
|
|
250
|
+
onTeardown: teardownTeam,
|
|
254
251
|
});
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
// DOU-209: drop the per-file Y.Doc state so deleted files don't
|
|
265
|
-
// accumulate stale binding-state blobs under .compound-yjs-binding/.
|
|
266
|
-
deleteBindingState(LOCAL_PATH, fileId);
|
|
267
|
-
}
|
|
268
|
-
|
|
269
|
-
// Per-team Firestore-listener unsubscribe handles, keyed by teamId.
|
|
270
|
-
const teamUnsubscribers = new Map();
|
|
271
|
-
// Hoisted forward-decl: setupTeam closes over handleFirestoreChange,
|
|
272
|
-
// which is defined later in this function. Wired below.
|
|
273
|
-
let setupTeam = () => {};
|
|
274
|
-
function teardownTeam(teamId, teamData) {
|
|
275
|
-
const unsub = teamUnsubscribers.get(teamId);
|
|
276
|
-
if (unsub) {
|
|
277
|
-
try { unsub(); } catch (e) { /* ignore */ }
|
|
278
|
-
teamUnsubscribers.delete(teamId);
|
|
279
|
-
const folder = teamData && teamData.name ? teamFolderName(teamData.name) : teamId;
|
|
280
|
-
console.log(` [${orgId}] Stopped listening: ${folder}`);
|
|
252
|
+
// scopeFromLocalPath expects a Map<teamName, teamId>; the registry
|
|
253
|
+
// exposes one matching that shape via teamIdsByNameMap().
|
|
254
|
+
const teamIdsByName = teamRegistry.teamIdsByNameMap();
|
|
255
|
+
console.log('');
|
|
256
|
+
console.log(` [${orgId}] Local: ${LOCAL_PATH}`);
|
|
257
|
+
function ensureDir(filePath) {
|
|
258
|
+
const dir = path.dirname(filePath);
|
|
259
|
+
if (!fs.existsSync(dir))
|
|
260
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
281
261
|
}
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
try {
|
|
307
|
-
if (fs.existsSync(symlinkPath) || fs.lstatSync(symlinkPath).isSymbolicLink()) {
|
|
308
|
-
fs.unlinkSync(symlinkPath);
|
|
262
|
+
function updateSharedByMeSymlink(fileDoc, realPath) {
|
|
263
|
+
const sharedWith = fileDoc['sharedWith'] ?? [];
|
|
264
|
+
const docPath = fileDoc['path'];
|
|
265
|
+
// Bail out on a missing / non-string path: path.join would resolve
|
|
266
|
+
// to the Shared-by-Me root, and a subsequent symlinkSync /
|
|
267
|
+
// unlinkSync there could clobber the entire folder.
|
|
268
|
+
if (typeof docPath !== 'string' || !docPath)
|
|
269
|
+
return;
|
|
270
|
+
const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, docPath);
|
|
271
|
+
// Check whether `symlinkPath` exists as anything (regular file,
|
|
272
|
+
// directory, or symlink including a dangling one). Uses lstatSync,
|
|
273
|
+
// returns null on ENOENT, rethrows other errors. Beats
|
|
274
|
+
// `existsSync || lstatSync(...).isSymbolicLink()` which leaned on
|
|
275
|
+
// a throw-and-catch for the not-found case and could mask real
|
|
276
|
+
// filesystem errors.
|
|
277
|
+
function lstatOrNull(p) {
|
|
278
|
+
try {
|
|
279
|
+
return fs.lstatSync(p);
|
|
280
|
+
}
|
|
281
|
+
catch (e) {
|
|
282
|
+
if (e?.code === 'ENOENT')
|
|
283
|
+
return null;
|
|
284
|
+
throw e;
|
|
285
|
+
}
|
|
309
286
|
}
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
287
|
+
if (sharedWith.length > 0) {
|
|
288
|
+
ensureDir(symlinkPath);
|
|
289
|
+
const relativeTo = path.relative(path.dirname(symlinkPath), realPath);
|
|
290
|
+
const existing = lstatOrNull(symlinkPath);
|
|
291
|
+
if (existing) {
|
|
292
|
+
fs.unlinkSync(symlinkPath);
|
|
293
|
+
}
|
|
294
|
+
try {
|
|
295
|
+
fs.symlinkSync(relativeTo, symlinkPath);
|
|
296
|
+
console.log(` [${orgId}] [symlink] ${docPath} → Shared by Me/`);
|
|
297
|
+
}
|
|
298
|
+
catch (e) {
|
|
299
|
+
const code = e?.code;
|
|
300
|
+
if (code !== 'EEXIST')
|
|
301
|
+
console.error(` [${orgId}] Symlink error: ${e?.message}`);
|
|
302
|
+
}
|
|
322
303
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
function ensureFolderStructure() {
|
|
330
|
-
const privatePath = path.join(LOCAL_PATH, PRIVATE_FOLDER);
|
|
331
|
-
if (!fs.existsSync(privatePath)) fs.mkdirSync(privatePath, { recursive: true });
|
|
332
|
-
|
|
333
|
-
const sharedWithMePath = path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER);
|
|
334
|
-
if (!fs.existsSync(sharedWithMePath)) fs.mkdirSync(sharedWithMePath, { recursive: true });
|
|
335
|
-
|
|
336
|
-
const sharedByMePath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER);
|
|
337
|
-
if (!fs.existsSync(sharedByMePath)) fs.mkdirSync(sharedByMePath, { recursive: true });
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
ensureFolderStructure();
|
|
341
|
-
|
|
342
|
-
async function handleFirestoreChange(change, getLocalPath) {
|
|
343
|
-
const data = change.doc.data();
|
|
344
|
-
const docPath = data.path;
|
|
345
|
-
if (!docPath) return;
|
|
346
|
-
|
|
347
|
-
const localFilePath = getLocalPath(data);
|
|
348
|
-
const trackingKey = `${data.scope || 'team'}:${docPath}`;
|
|
349
|
-
const isFolder = data.type === 'folder';
|
|
350
|
-
|
|
351
|
-
// DOU-205: rename / move detection. On a 'modified' snapshot for
|
|
352
|
-
// a fileId we've seen before, if the computed local path differs
|
|
353
|
-
// from what we last recorded, the server's path field has
|
|
354
|
-
// changed. Move the local file in place (fs.renameSync preserves
|
|
355
|
-
// Obsidian's inode tracking so notes don't lose their metadata,
|
|
356
|
-
// backlinks, or recent-files entries) and migrate the tracking
|
|
357
|
-
// maps from the old key to the new. We do this before the
|
|
358
|
-
// freshness check below because the rename happens irrespective
|
|
359
|
-
// of whether the content changed in the same update.
|
|
360
|
-
const fileId = change.doc.id;
|
|
361
|
-
const prevKnown = knownPaths.get(fileId);
|
|
362
|
-
if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
|
|
363
|
-
suppressLocal.add(prevKnown.localPath);
|
|
364
|
-
suppressLocal.add(localFilePath);
|
|
365
|
-
try {
|
|
366
|
-
if (fs.existsSync(prevKnown.localPath)) {
|
|
367
|
-
ensureDir(localFilePath);
|
|
368
|
-
fs.renameSync(prevKnown.localPath, localFilePath);
|
|
369
|
-
console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
|
|
304
|
+
else {
|
|
305
|
+
const existing = lstatOrNull(symlinkPath);
|
|
306
|
+
if (existing && existing.isSymbolicLink()) {
|
|
307
|
+
fs.unlinkSync(symlinkPath);
|
|
308
|
+
console.log(` [${orgId}] [symlink] removed Shared by Me/${docPath}`);
|
|
309
|
+
}
|
|
370
310
|
}
|
|
371
|
-
} catch (err) {
|
|
372
|
-
console.error(` [${orgId}] [firestore → local] RENAME failed: ${err && err.message}`);
|
|
373
|
-
}
|
|
374
|
-
if (lastKnownUpdate.has(prevKnown.trackingKey)) {
|
|
375
|
-
lastKnownUpdate.set(trackingKey, lastKnownUpdate.get(prevKnown.trackingKey));
|
|
376
|
-
lastKnownUpdate.delete(prevKnown.trackingKey);
|
|
377
|
-
}
|
|
378
|
-
if (lastSyncedHash.has(prevKnown.trackingKey)) {
|
|
379
|
-
setBaseline(trackingKey, lastSyncedHash.get(prevKnown.trackingKey));
|
|
380
|
-
deleteBaseline(prevKnown.trackingKey);
|
|
381
|
-
}
|
|
382
|
-
const manifestEntry = manifestStore.getEntry(prevKnown.trackingKey);
|
|
383
|
-
if (manifestEntry) {
|
|
384
|
-
manifestStore.removeEntry(prevKnown.trackingKey);
|
|
385
|
-
manifestStore.upsertEntry(trackingKey, manifestEntry);
|
|
386
|
-
}
|
|
387
|
-
const binding = bindings.get(fileId);
|
|
388
|
-
if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
|
|
389
|
-
bindingsByPath.delete(prevKnown.localPath);
|
|
390
|
-
bindingsByPath.set(localFilePath, binding);
|
|
391
|
-
}
|
|
392
|
-
setTimeout(() => {
|
|
393
|
-
suppressLocal.delete(prevKnown.localPath);
|
|
394
|
-
suppressLocal.delete(localFilePath);
|
|
395
|
-
}, 1000);
|
|
396
311
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
} catch (err) {
|
|
410
|
-
suppressLocal.delete(localFilePath);
|
|
411
|
-
if (err.code !== 'ENOENT' && err.code !== 'ENOTEMPTY') {
|
|
412
|
-
console.error(` [${orgId}] [firestore → local] RMDIR ${docPath} failed: ${err.message}`);
|
|
413
|
-
}
|
|
414
|
-
}
|
|
415
|
-
} else if (fs.existsSync(localFilePath)) {
|
|
416
|
-
console.log(` [${orgId}] [firestore → local] DELETE ${docPath}`);
|
|
417
|
-
suppressLocal.add(localFilePath);
|
|
418
|
-
fs.unlinkSync(localFilePath);
|
|
419
|
-
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
420
|
-
}
|
|
421
|
-
lastKnownUpdate.delete(trackingKey);
|
|
422
|
-
deleteBaseline(trackingKey);
|
|
423
|
-
manifestStore.removeEntry(trackingKey);
|
|
424
|
-
knownPaths.delete(fileId);
|
|
425
|
-
await disposeBinding(change.doc.id, localFilePath);
|
|
426
|
-
return;
|
|
312
|
+
// Static (user-level) folders. Team folders are mkdir'd by setupTeam,
|
|
313
|
+
// which runs after manifest reconcile (see startup ordering below).
|
|
314
|
+
function ensureFolderStructure() {
|
|
315
|
+
const privatePath = path.join(LOCAL_PATH, PRIVATE_FOLDER);
|
|
316
|
+
if (!fs.existsSync(privatePath))
|
|
317
|
+
fs.mkdirSync(privatePath, { recursive: true });
|
|
318
|
+
const sharedWithMePath = path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER);
|
|
319
|
+
if (!fs.existsSync(sharedWithMePath))
|
|
320
|
+
fs.mkdirSync(sharedWithMePath, { recursive: true });
|
|
321
|
+
const sharedByMePath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER);
|
|
322
|
+
if (!fs.existsSync(sharedByMePath))
|
|
323
|
+
fs.mkdirSync(sharedByMePath, { recursive: true });
|
|
427
324
|
}
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
325
|
+
ensureFolderStructure();
|
|
326
|
+
async function handleFirestoreChange(change, getLocalPath) {
|
|
327
|
+
const data = change.doc.data();
|
|
328
|
+
const docPath = data['path'];
|
|
329
|
+
if (typeof docPath !== 'string' || !docPath)
|
|
330
|
+
return;
|
|
331
|
+
const localFilePath = getLocalPath(data);
|
|
332
|
+
const trackingKey = `${data['scope'] || 'team'}:${docPath}`;
|
|
333
|
+
const isFolder = data['type'] === 'folder';
|
|
334
|
+
// DOU-205: rename / move detection. On a 'modified' snapshot for
|
|
335
|
+
// a fileId we've seen before, if the computed local path differs
|
|
336
|
+
// from what we last recorded, the server's path field has
|
|
337
|
+
// changed. Move the local file in place (fs.renameSync preserves
|
|
338
|
+
// Obsidian's inode tracking so notes don't lose their metadata,
|
|
339
|
+
// backlinks, or recent-files entries) and migrate the tracking
|
|
340
|
+
// maps from the old key to the new. We do this before the
|
|
341
|
+
// freshness check below because the rename happens irrespective
|
|
342
|
+
// of whether the content changed in the same update.
|
|
343
|
+
const fileId = change.doc.id;
|
|
344
|
+
const prevKnown = knownPaths.get(fileId);
|
|
345
|
+
if (change.type === 'modified' && prevKnown && prevKnown.localPath !== localFilePath) {
|
|
346
|
+
suppressLocal.add(prevKnown.localPath);
|
|
347
|
+
suppressLocal.add(localFilePath);
|
|
348
|
+
try {
|
|
349
|
+
if (fs.existsSync(prevKnown.localPath)) {
|
|
350
|
+
ensureDir(localFilePath);
|
|
351
|
+
fs.renameSync(prevKnown.localPath, localFilePath);
|
|
352
|
+
console.log(` [${orgId}] [firestore → local] RENAME ${prevKnown.localPath} -> ${localFilePath}`);
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
catch (err) {
|
|
356
|
+
console.error(` [${orgId}] [firestore → local] RENAME failed: ${err?.message}`);
|
|
357
|
+
}
|
|
358
|
+
const prevLastKnown = lastKnownUpdate.get(prevKnown.trackingKey);
|
|
359
|
+
if (prevLastKnown !== undefined) {
|
|
360
|
+
lastKnownUpdate.set(trackingKey, prevLastKnown);
|
|
361
|
+
lastKnownUpdate.delete(prevKnown.trackingKey);
|
|
362
|
+
}
|
|
363
|
+
const prevBaseline = lastSyncedHash.get(prevKnown.trackingKey);
|
|
364
|
+
if (prevBaseline !== undefined) {
|
|
365
|
+
setBaseline(trackingKey, prevBaseline);
|
|
366
|
+
deleteBaseline(prevKnown.trackingKey);
|
|
367
|
+
}
|
|
368
|
+
const manifestEntry = manifestStore.getEntry(prevKnown.trackingKey);
|
|
369
|
+
if (manifestEntry) {
|
|
370
|
+
manifestStore.removeEntry(prevKnown.trackingKey);
|
|
371
|
+
manifestStore.upsertEntry(trackingKey, manifestEntry);
|
|
372
|
+
}
|
|
373
|
+
const binding = bindings.get(fileId);
|
|
374
|
+
if (binding && bindingsByPath.get(prevKnown.localPath) === binding) {
|
|
375
|
+
bindingsByPath.delete(prevKnown.localPath);
|
|
376
|
+
bindingsByPath.set(localFilePath, binding);
|
|
377
|
+
}
|
|
378
|
+
setTimeout(() => {
|
|
379
|
+
suppressLocal.delete(prevKnown.localPath);
|
|
380
|
+
suppressLocal.delete(localFilePath);
|
|
381
|
+
}, 1000);
|
|
382
|
+
}
|
|
383
|
+
if (change.type === 'removed') {
|
|
384
|
+
if (isFolder) {
|
|
385
|
+
// Best-effort directory removal. The daemon's own file-delete
|
|
386
|
+
// events should have cleared all children by the time we get
|
|
387
|
+
// here; if not, rmdirSync throws ENOTEMPTY and we keep the
|
|
388
|
+
// directory rather than risk losing data the user hasn't seen.
|
|
389
|
+
try {
|
|
390
|
+
suppressLocal.add(localFilePath);
|
|
391
|
+
fs.rmdirSync(localFilePath);
|
|
392
|
+
console.log(` [${orgId}] [firestore → local] RMDIR ${docPath}`);
|
|
393
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
394
|
+
}
|
|
395
|
+
catch (err) {
|
|
396
|
+
suppressLocal.delete(localFilePath);
|
|
397
|
+
const code = err?.code;
|
|
398
|
+
if (code !== 'ENOENT' && code !== 'ENOTEMPTY') {
|
|
399
|
+
console.error(` [${orgId}] [firestore → local] RMDIR ${docPath} failed: ${err?.message}`);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
}
|
|
403
|
+
else if (fs.existsSync(localFilePath)) {
|
|
404
|
+
console.log(` [${orgId}] [firestore → local] DELETE ${docPath}`);
|
|
405
|
+
suppressLocal.add(localFilePath);
|
|
406
|
+
fs.unlinkSync(localFilePath);
|
|
407
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
408
|
+
}
|
|
409
|
+
lastKnownUpdate.delete(trackingKey);
|
|
410
|
+
deleteBaseline(trackingKey);
|
|
411
|
+
manifestStore.removeEntry(trackingKey);
|
|
412
|
+
knownPaths.delete(fileId);
|
|
413
|
+
await disposeBinding(change.doc.id, localFilePath);
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
// Record (or refresh) the per-fileId path so the next 'modified'
|
|
417
|
+
// event can detect a rename / move by comparing against this.
|
|
418
|
+
knownPaths.set(fileId, { localPath: localFilePath, trackingKey });
|
|
419
|
+
// Kick off Yjs binding setup for text files. Fire-and-forget so
|
|
420
|
+
// a slow start (retrying past the rules-engine race on a fresh
|
|
421
|
+
// doc) does not stall the legacy blob pull. Once the binding is
|
|
422
|
+
// registered, future events take the Yjs path; until then the
|
|
423
|
+
// blob path still keeps disk in sync.
|
|
424
|
+
if (!isFolder) {
|
|
425
|
+
const ext = extFromPath(docPath);
|
|
426
|
+
const extDerivedMime = mimeTypeFromExt(ext);
|
|
427
|
+
const isText = isTextMimeType(data['mimeType']) || isTextMimeType(extDerivedMime);
|
|
428
|
+
if (isText)
|
|
429
|
+
ensureBinding(change.doc.id, localFilePath);
|
|
430
|
+
}
|
|
431
|
+
const remoteUpdated = data['updatedAt'] || data['createdAt'] || '';
|
|
432
|
+
const lastKnown = lastKnownUpdate.get(trackingKey);
|
|
433
|
+
if (lastKnown && lastKnown >= remoteUpdated)
|
|
434
|
+
return;
|
|
435
|
+
if (isFolder) {
|
|
436
|
+
// Folder docs mirror to a local directory. No blob, no content
|
|
437
|
+
// comparison; if the directory doesn't exist yet, create it.
|
|
438
|
+
if (!fs.existsSync(localFilePath)) {
|
|
439
|
+
suppressLocal.add(localFilePath);
|
|
440
|
+
fs.mkdirSync(localFilePath, { recursive: true });
|
|
441
|
+
console.log(` [${orgId}] [firestore → local] MKDIR ${docPath}`);
|
|
442
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
443
|
+
}
|
|
444
|
+
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
445
|
+
return;
|
|
446
|
+
}
|
|
447
|
+
if (fs.existsSync(localFilePath)) {
|
|
448
|
+
const baseline = lastSyncedHash.get(trackingKey);
|
|
449
|
+
if (baseline !== undefined) {
|
|
450
|
+
// We have a record of what we last synced: decide by content,
|
|
451
|
+
// not mtime. (Obsidian keeps the mtime fresh, which made the old
|
|
452
|
+
// mtime guard wrongly skip web edits.)
|
|
453
|
+
const localHash = computeContentHash(fs.readFileSync(localFilePath));
|
|
454
|
+
if (localHash === data['contentHash']) {
|
|
455
|
+
// Local already matches the incoming server content; nothing
|
|
456
|
+
// to write.
|
|
457
|
+
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
458
|
+
setBaseline(trackingKey, data['contentHash']);
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
if (localHash !== baseline) {
|
|
462
|
+
// Local changed since our last sync AND the server changed:
|
|
463
|
+
// a genuine conflict. Keep the local copy rather than clobber
|
|
464
|
+
// it. (Writing a "conflicted copy" is the next increment.)
|
|
465
|
+
console.log(` [${orgId}] [conflict] ${docPath}: local and server both changed; keeping local`);
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
// localHash === baseline: local is untouched since last sync, so
|
|
469
|
+
// the server is authoritative -> fall through and pull.
|
|
470
|
+
}
|
|
471
|
+
else {
|
|
472
|
+
// No baseline at all: a local file this daemon has never synced
|
|
473
|
+
// (e.g. created locally while offline, or a folder adopted from
|
|
474
|
+
// elsewhere). We can't tell by content who is newer, so fall
|
|
475
|
+
// back to the mtime heuristic. Conflict copies (next increment)
|
|
476
|
+
// will replace this last-resort with a no-loss outcome.
|
|
477
|
+
const localMtime = fs.statSync(localFilePath).mtime.toISOString();
|
|
478
|
+
if (localMtime > remoteUpdated)
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
const action = change.type === 'added' ? 'CREATE' : 'UPDATE';
|
|
483
|
+
console.log(` [${orgId}] [firestore → local] ${action} ${docPath} (${data['scope'] || 'team'})`);
|
|
484
|
+
ensureDir(localFilePath);
|
|
453
485
|
suppressLocal.add(localFilePath);
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
//
|
|
466
|
-
//
|
|
467
|
-
//
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
486
|
+
const ext = extFromPath(docPath);
|
|
487
|
+
// Extension is used as a fallback authority when the stored
|
|
488
|
+
// mimeType claims non-text but the extension maps to a text mime
|
|
489
|
+
// in EXT_TO_MIME. This heals files uploaded before their extension
|
|
490
|
+
// was registered as text (e.g. .base files uploaded prior to the
|
|
491
|
+
// text/yaml mapping landing): on the next pull the daemon writes
|
|
492
|
+
// them as UTF-8 so text-aware features (line diffing, conflict
|
|
493
|
+
// markers) can apply.
|
|
494
|
+
const extDerivedMime = mimeTypeFromExt(ext);
|
|
495
|
+
const isText = isTextMimeType(data['mimeType']) || isTextMimeType(extDerivedMime);
|
|
496
|
+
// Yjs path (DOU-178 PR 3): if a binding for this file is already
|
|
497
|
+
// up and carries non-empty ytext, the debounced flush owns the
|
|
498
|
+
// disk write and the legacy blob pull is skipped. Bindings that
|
|
499
|
+
// are still being set up, or that loaded empty ytext (pre-Yjs
|
|
500
|
+
// files), fall through to the blob path; PR 4 retires the blob
|
|
501
|
+
// path entirely.
|
|
502
|
+
if (isText) {
|
|
503
|
+
const binding = bindings.get(change.doc.id);
|
|
504
|
+
if (binding && binding.ytext.length > 0) {
|
|
505
|
+
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
506
|
+
setBaseline(trackingKey, data['contentHash']);
|
|
507
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
508
|
+
return;
|
|
509
|
+
}
|
|
475
510
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
511
|
+
// Storage rules require the Firestore doc to exist before they
|
|
512
|
+
// authorize a blob upload, so writers must create the doc first
|
|
513
|
+
// and upload second. That leaves a window where the metadata is
|
|
514
|
+
// visible (this listener fires) but the blob is not yet uploaded.
|
|
515
|
+
// Retry on object-not-found with exponential backoff covers that
|
|
516
|
+
// race; on a persistent miss we fall through to the original
|
|
517
|
+
// catch upstream.
|
|
518
|
+
const blobReadArgs = { storage, orgId, fileId: change.doc.id, ext };
|
|
519
|
+
const retryOpts = { expectedHash: data['contentHash'] };
|
|
520
|
+
if (isText) {
|
|
521
|
+
const content = await readBlobWithRetry(() => readBlobAsText(blobReadArgs), retryOpts);
|
|
522
|
+
fs.writeFileSync(localFilePath, content || '', 'utf-8');
|
|
523
|
+
}
|
|
524
|
+
else {
|
|
525
|
+
const bytes = await readBlobWithRetry(() => readBlob(blobReadArgs), retryOpts);
|
|
526
|
+
fs.writeFileSync(localFilePath, bytes);
|
|
482
527
|
}
|
|
483
|
-
// localHash === baseline: local is untouched since last sync, so
|
|
484
|
-
// the server is authoritative -> fall through and pull.
|
|
485
|
-
} else {
|
|
486
|
-
// No baseline at all: a local file this daemon has never synced
|
|
487
|
-
// (e.g. created locally while offline, or a folder adopted from
|
|
488
|
-
// elsewhere). We can't tell by content who is newer, so fall
|
|
489
|
-
// back to the mtime heuristic. Conflict copies (next increment)
|
|
490
|
-
// will replace this last-resort with a no-loss outcome.
|
|
491
|
-
const localMtime = fs.statSync(localFilePath).mtime.toISOString();
|
|
492
|
-
if (localMtime > remoteUpdated) return;
|
|
493
|
-
}
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
const action = change.type === 'added' ? 'CREATE' : 'UPDATE';
|
|
497
|
-
console.log(` [${orgId}] [firestore → local] ${action} ${docPath} (${data.scope || 'team'})`);
|
|
498
|
-
ensureDir(localFilePath);
|
|
499
|
-
suppressLocal.add(localFilePath);
|
|
500
|
-
|
|
501
|
-
const ext = extFromPath(docPath);
|
|
502
|
-
// Extension is used as a fallback authority when the stored
|
|
503
|
-
// mimeType claims non-text but the extension maps to a text mime
|
|
504
|
-
// in EXT_TO_MIME. This heals files uploaded before their extension
|
|
505
|
-
// was registered as text (e.g. .base files uploaded prior to the
|
|
506
|
-
// text/yaml mapping landing): on the next pull the daemon writes
|
|
507
|
-
// them as UTF-8 so text-aware features (line diffing, conflict
|
|
508
|
-
// markers) can apply.
|
|
509
|
-
const extDerivedMime = mimeTypeFromExt(ext);
|
|
510
|
-
const isText = isTextMimeType(data.mimeType) || isTextMimeType(extDerivedMime);
|
|
511
|
-
|
|
512
|
-
// Yjs path (DOU-178 PR 3): if a binding for this file is already
|
|
513
|
-
// up and carries non-empty ytext, the debounced flush owns the
|
|
514
|
-
// disk write and the legacy blob pull is skipped. Bindings that
|
|
515
|
-
// are still being set up, or that loaded empty ytext (pre-Yjs
|
|
516
|
-
// files), fall through to the blob path; PR 4 retires the blob
|
|
517
|
-
// path entirely.
|
|
518
|
-
if (isText) {
|
|
519
|
-
const binding = bindings.get(change.doc.id);
|
|
520
|
-
if (binding && binding.ytext.length > 0) {
|
|
521
528
|
lastKnownUpdate.set(trackingKey, remoteUpdated);
|
|
522
|
-
setBaseline(trackingKey, data
|
|
529
|
+
setBaseline(trackingKey, data['contentHash']);
|
|
523
530
|
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
524
|
-
return;
|
|
525
|
-
}
|
|
526
531
|
}
|
|
527
|
-
|
|
528
|
-
//
|
|
529
|
-
//
|
|
530
|
-
//
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
532
|
+
// Set up per-team resources: mkdir the local folder, register the
|
|
533
|
+
// Firestore listener for that team's files. On startup, called once
|
|
534
|
+
// per bootstrapped team after reconcile; thereafter via TeamRegistry
|
|
535
|
+
// when the live teams-collection listener reports add/modify/remove.
|
|
536
|
+
setupTeam = (teamId, teamData) => {
|
|
537
|
+
// The pre-migration JS coerced a missing name to the literal
|
|
538
|
+
// "undefined" string via `name + ' Teamspace'`. Preserve that
|
|
539
|
+
// faithfully (String(undefined) === 'undefined'): a team without
|
|
540
|
+
// a name still gets a folder + listener, just under an
|
|
541
|
+
// obviously-broken display name that surfaces the bug instead of
|
|
542
|
+
// silently hiding the team.
|
|
543
|
+
const teamFolder = teamFolderName(String(teamData.name));
|
|
544
|
+
const teamPath = path.join(LOCAL_PATH, teamFolder);
|
|
545
|
+
if (!fs.existsSync(teamPath))
|
|
546
|
+
fs.mkdirSync(teamPath, { recursive: true });
|
|
547
|
+
const q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId));
|
|
548
|
+
const unsubscribe = onSnapshot(q, (snapshot) => {
|
|
549
|
+
snapshot.docChanges().forEach((change) => {
|
|
550
|
+
track(handleFirestoreChange(change, (data) => path.join(LOCAL_PATH, teamFolder, data['path'])).catch((err) => console.error(` [${orgId}] Firestore change handler error:`, err?.message)));
|
|
551
|
+
});
|
|
552
|
+
}, (err) => console.error(` [${orgId}] Team "${teamFolder}" listener error:`, err));
|
|
553
|
+
teamUnsubscribers.set(teamId, unsubscribe);
|
|
554
|
+
console.log(` [${orgId}] Listening: ${teamFolder}`);
|
|
555
|
+
};
|
|
556
|
+
// Bootstrap team metadata synchronously so manifest reconcile can
|
|
557
|
+
// query team-scoped cloud files before any file onSnapshot handlers
|
|
558
|
+
// run (DOU-55). Populating teamIdsByName here also lets chokidar
|
|
559
|
+
// route local paths once the watcher starts. Per-team file listeners
|
|
560
|
+
// and the live teams-collection onSnapshot are wired after reconcile
|
|
561
|
+
// completes so their initial snapshots cannot race with startup cleanup.
|
|
562
|
+
const teamsSnap = await getDocs(collection(db, `orgs/${orgId}/teams`));
|
|
563
|
+
for (const teamDoc of teamsSnap.docs) {
|
|
564
|
+
teamRegistry.bootstrap(teamDoc.id, teamDoc.data());
|
|
543
565
|
}
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
}
|
|
548
|
-
|
|
549
|
-
// Set up per-team resources: mkdir the local folder, register the
|
|
550
|
-
// Firestore listener for that team's files. On startup, called once
|
|
551
|
-
// per bootstrapped team after reconcile; thereafter via TeamRegistry
|
|
552
|
-
// when the live teams-collection listener reports add/modify/remove.
|
|
553
|
-
setupTeam = (teamId, teamData) => {
|
|
554
|
-
const teamFolder = teamFolderName(teamData.name);
|
|
555
|
-
const teamPath = path.join(LOCAL_PATH, teamFolder);
|
|
556
|
-
if (!fs.existsSync(teamPath)) fs.mkdirSync(teamPath, { recursive: true });
|
|
557
|
-
|
|
558
|
-
const q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId));
|
|
559
|
-
const unsubscribe = onSnapshot(q, snapshot => {
|
|
560
|
-
snapshot.docChanges().forEach(change => {
|
|
561
|
-
track(handleFirestoreChange(change, data =>
|
|
562
|
-
path.join(LOCAL_PATH, teamFolder, data.path),
|
|
563
|
-
).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
|
|
564
|
-
});
|
|
565
|
-
}, err => console.error(` [${orgId}] Team "${teamFolder}" listener error:`, err));
|
|
566
|
-
|
|
567
|
-
teamUnsubscribers.set(teamId, unsubscribe);
|
|
568
|
-
console.log(` [${orgId}] Listening: ${teamFolder}`);
|
|
569
|
-
};
|
|
570
|
-
|
|
571
|
-
// Bootstrap team metadata synchronously so manifest reconcile can
|
|
572
|
-
// query team-scoped cloud files before any file onSnapshot handlers
|
|
573
|
-
// run (DOU-55). Populating teamIdsByName here also lets chokidar
|
|
574
|
-
// route local paths once the watcher starts. Per-team file listeners
|
|
575
|
-
// and the live teams-collection onSnapshot are wired after reconcile
|
|
576
|
-
// completes so their initial snapshots cannot race with startup cleanup.
|
|
577
|
-
const teamsSnap = await getDocs(collection(db, `orgs/${orgId}/teams`));
|
|
578
|
-
for (const teamDoc of teamsSnap.docs) {
|
|
579
|
-
teamRegistry.bootstrap(teamDoc.id, teamDoc.data());
|
|
580
|
-
}
|
|
581
|
-
console.log(` [${orgId}] Teams: ${teamRegistry.names().join(', ') || 'none'}`);
|
|
582
|
-
|
|
583
|
-
function trackingKeyFromDocData(data) {
|
|
584
|
-
return `${data.scope || 'team'}:${data.path}`;
|
|
585
|
-
}
|
|
586
|
-
|
|
587
|
-
// Paths the daemon does not scan or sync. Add checks here as needed
|
|
588
|
-
// (extensions, .compoundignore, etc.).
|
|
589
|
-
function shouldSkipLocalPath(name, rel) {
|
|
590
|
-
if (name.startsWith('.') || name === 'node_modules') return true;
|
|
591
|
-
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return true;
|
|
592
|
-
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return true;
|
|
593
|
-
return false;
|
|
594
|
-
}
|
|
595
|
-
|
|
596
|
-
function walkLocalSyncableFiles(absDir, files) {
|
|
597
|
-
let entries;
|
|
598
|
-
try {
|
|
599
|
-
entries = fs.readdirSync(absDir, { withFileTypes: true });
|
|
600
|
-
} catch (err) {
|
|
601
|
-
const rel = path.relative(LOCAL_PATH, absDir) || '.';
|
|
602
|
-
console.warn(` [${orgId}] [local scan] could not read directory ${rel}: ${err.message}`);
|
|
603
|
-
return;
|
|
566
|
+
console.log(` [${orgId}] Teams: ${teamRegistry.names().join(', ') || 'none'}`);
|
|
567
|
+
function trackingKeyFromDocData(data) {
|
|
568
|
+
return `${data['scope'] || 'team'}:${data['path']}`;
|
|
604
569
|
}
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
570
|
+
// Paths the daemon does not scan or sync. Add checks here as needed
|
|
571
|
+
// (extensions, .compoundignore, etc.).
|
|
572
|
+
function shouldSkipLocalPath(name, rel) {
|
|
573
|
+
if (name.startsWith('.') || name === 'node_modules')
|
|
574
|
+
return true;
|
|
575
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
|
|
576
|
+
return true;
|
|
577
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
|
|
578
|
+
return true;
|
|
579
|
+
return false;
|
|
614
580
|
}
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
} catch {
|
|
638
|
-
return;
|
|
581
|
+
function walkLocalSyncableFiles(absDir, files) {
|
|
582
|
+
let entries;
|
|
583
|
+
try {
|
|
584
|
+
entries = fs.readdirSync(absDir, { withFileTypes: true });
|
|
585
|
+
}
|
|
586
|
+
catch (err) {
|
|
587
|
+
const rel = path.relative(LOCAL_PATH, absDir) || '.';
|
|
588
|
+
console.warn(` [${orgId}] [local scan] could not read directory ${rel}: ${err?.message}`);
|
|
589
|
+
return;
|
|
590
|
+
}
|
|
591
|
+
for (const ent of entries) {
|
|
592
|
+
const abs = path.join(absDir, ent.name);
|
|
593
|
+
const rel = path.relative(LOCAL_PATH, abs);
|
|
594
|
+
if (shouldSkipLocalPath(ent.name, rel))
|
|
595
|
+
continue;
|
|
596
|
+
if (ent.isDirectory()) {
|
|
597
|
+
walkLocalSyncableFiles(abs, files);
|
|
598
|
+
}
|
|
599
|
+
else if (ent.isFile()) {
|
|
600
|
+
files.push(abs);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
639
603
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
// Sync local cleanup during startup manifest reconcile. Runs before
|
|
646
|
-
// Firestore listeners and Yjs bindings are wired, so no async cloud or
|
|
647
|
-
// binding teardown is needed here.
|
|
648
|
-
function deleteLocalReconcile(localFilePath, fileId, trackingKey) {
|
|
649
|
-
suppressLocal.add(localFilePath);
|
|
650
|
-
if (fs.existsSync(localFilePath)) {
|
|
651
|
-
console.log(` [${orgId}] [manifest reconcile] DELETE local ${path.relative(LOCAL_PATH, localFilePath)}`);
|
|
652
|
-
fs.unlinkSync(localFilePath);
|
|
604
|
+
function collectLocalSyncableFiles() {
|
|
605
|
+
const files = [];
|
|
606
|
+
walkLocalSyncableFiles(LOCAL_PATH, files);
|
|
607
|
+
return files;
|
|
653
608
|
}
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
609
|
+
function readLocalFilePayload(filePath) {
|
|
610
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
611
|
+
const ext = extFromPath(rel);
|
|
612
|
+
const mimeType = mimeTypeFromExt(ext);
|
|
613
|
+
const data = isTextMimeType(mimeType)
|
|
614
|
+
? fs.readFileSync(filePath, 'utf-8')
|
|
615
|
+
: fs.readFileSync(filePath);
|
|
616
|
+
return { data, mimeType, contentHash: computeContentHash(data) };
|
|
659
617
|
}
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
scope,
|
|
673
|
-
teamId,
|
|
674
|
-
docPath,
|
|
675
|
-
});
|
|
676
|
-
if (result.action === 'deleted') {
|
|
677
|
-
console.log(` [${orgId}] [manifest reconcile] DELETE cloud ${docPath}`);
|
|
618
|
+
async function pushLocalFileAtPath(filePath) {
|
|
619
|
+
let stat;
|
|
620
|
+
try {
|
|
621
|
+
stat = fs.statSync(filePath);
|
|
622
|
+
}
|
|
623
|
+
catch {
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
if (!stat.isFile() || stat.size > maxUploadBytes)
|
|
627
|
+
return;
|
|
628
|
+
const { data, mimeType } = readLocalFilePayload(filePath);
|
|
629
|
+
await pushToFirestore(filePath, data, mimeType, stat.mtime.toISOString());
|
|
678
630
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
631
|
+
// Sync local cleanup during startup manifest reconcile. Runs before
|
|
632
|
+
// Firestore listeners and Yjs bindings are wired, so no async cloud or
|
|
633
|
+
// binding teardown is needed here.
|
|
634
|
+
function deleteLocalReconcile(localFilePath, fileId, trackingKey) {
|
|
635
|
+
suppressLocal.add(localFilePath);
|
|
636
|
+
if (fs.existsSync(localFilePath)) {
|
|
637
|
+
console.log(` [${orgId}] [manifest reconcile] DELETE local ${path.relative(LOCAL_PATH, localFilePath)}`);
|
|
638
|
+
fs.unlinkSync(localFilePath);
|
|
639
|
+
}
|
|
640
|
+
lastKnownUpdate.delete(trackingKey);
|
|
641
|
+
deleteBaseline(trackingKey);
|
|
642
|
+
manifestStore.removeEntry(trackingKey);
|
|
643
|
+
if (fileId) {
|
|
644
|
+
knownPaths.delete(fileId);
|
|
645
|
+
}
|
|
646
|
+
setTimeout(() => suppressLocal.delete(localFilePath), 1000);
|
|
689
647
|
}
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
648
|
+
async function deleteRemoteReconcile(cloudData, trackingKey) {
|
|
649
|
+
const scope = cloudData['scope'] || 'team';
|
|
650
|
+
const teamId = cloudData['teamId'];
|
|
651
|
+
const docPath = cloudData['path'];
|
|
652
|
+
const result = await deleteFileFromCloud({
|
|
653
|
+
filesRef,
|
|
654
|
+
storage,
|
|
655
|
+
orgId,
|
|
656
|
+
userId,
|
|
657
|
+
scope,
|
|
658
|
+
teamId,
|
|
659
|
+
docPath,
|
|
660
|
+
});
|
|
661
|
+
if (result.action === 'deleted') {
|
|
662
|
+
console.log(` [${orgId}] [manifest reconcile] DELETE cloud ${docPath}`);
|
|
663
|
+
}
|
|
664
|
+
manifestStore.removeEntry(trackingKey);
|
|
694
665
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
666
|
+
function localPathForCloudData(data) {
|
|
667
|
+
const docPath = data['path'];
|
|
668
|
+
if (typeof docPath !== 'string' || !docPath)
|
|
669
|
+
return null;
|
|
670
|
+
if (data['scope'] === 'private') {
|
|
671
|
+
if (data['ownerId'] === userId) {
|
|
672
|
+
return path.join(LOCAL_PATH, PRIVATE_FOLDER, docPath);
|
|
673
|
+
}
|
|
674
|
+
return path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, docPath);
|
|
675
|
+
}
|
|
676
|
+
if (data['scope'] === 'team' && data['teamId']) {
|
|
677
|
+
const team = teamRegistry.get(data['teamId']);
|
|
678
|
+
if (!team?.name)
|
|
679
|
+
return null;
|
|
680
|
+
return path.join(LOCAL_PATH, teamFolderName(team.name), docPath);
|
|
681
|
+
}
|
|
682
|
+
return null;
|
|
706
683
|
}
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
684
|
+
async function fetchCloudFilesByKey() {
|
|
685
|
+
const cloudByKey = new Map();
|
|
686
|
+
function ingestDoc(d) {
|
|
687
|
+
const data = d.data();
|
|
688
|
+
if (data['type'] === 'folder' || !data['path'])
|
|
689
|
+
return;
|
|
690
|
+
const key = trackingKeyFromDocData(data);
|
|
691
|
+
cloudByKey.set(key, { fileId: d.id, data });
|
|
692
|
+
}
|
|
693
|
+
for (const [teamId] of teamRegistry.entries()) {
|
|
694
|
+
const teamSnap = await getDocs(query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId)));
|
|
695
|
+
teamSnap.docs.forEach(ingestDoc);
|
|
696
|
+
}
|
|
697
|
+
if (userId) {
|
|
698
|
+
const privateSnap = await getDocs(query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId)));
|
|
699
|
+
privateSnap.docs.forEach(ingestDoc);
|
|
700
|
+
const sharedSnap = await getDocs(query(filesRef, where('scope', '==', 'private'), where('sharedWith', 'array-contains', userId)));
|
|
701
|
+
sharedSnap.docs.forEach(ingestDoc);
|
|
702
|
+
}
|
|
703
|
+
return cloudByKey;
|
|
715
704
|
}
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
705
|
+
// Runs after team metadata is loaded but before any file listeners or
|
|
706
|
+
// chokidar. See docs/features/sync/sync-manifest-design.md.
|
|
707
|
+
async function reconcileManifestOnStartup() {
|
|
708
|
+
console.log(` [${orgId}] Reconciling local manifest...`);
|
|
709
|
+
const cloudByKey = await fetchCloudFilesByKey();
|
|
710
|
+
const localByKey = new Map();
|
|
711
|
+
for (const filePath of collectLocalSyncableFiles()) {
|
|
712
|
+
const { scope, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
713
|
+
if (!docPath)
|
|
714
|
+
continue;
|
|
715
|
+
localByKey.set(`${scope}:${docPath}`, filePath);
|
|
716
|
+
}
|
|
717
|
+
// Bootstrap manifest entries for files already in sync before DOU-55.
|
|
718
|
+
for (const [key, cloud] of cloudByKey) {
|
|
719
|
+
if (manifestStore.getEntry(key))
|
|
720
|
+
continue;
|
|
721
|
+
const localFilePath = localByKey.get(key);
|
|
722
|
+
if (!localFilePath || !fs.existsSync(localFilePath))
|
|
723
|
+
continue;
|
|
724
|
+
try {
|
|
725
|
+
const { contentHash } = readLocalFilePayload(localFilePath);
|
|
726
|
+
if (contentHash === cloud.data['contentHash']) {
|
|
727
|
+
manifestStore.upsertEntry(key, {
|
|
728
|
+
fileId: cloud.fileId,
|
|
729
|
+
contentHash,
|
|
730
|
+
lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
|
|
731
|
+
});
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
catch {
|
|
735
|
+
// skip unreadable locals
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
const allKeys = new Set([
|
|
739
|
+
...Object.keys(manifestStore.getState().entries),
|
|
740
|
+
...localByKey.keys(),
|
|
741
|
+
...cloudByKey.keys(),
|
|
742
|
+
]);
|
|
743
|
+
for (const trackingKey of allKeys) {
|
|
744
|
+
const entry = manifestStore.getEntry(trackingKey);
|
|
745
|
+
let localFilePath = localByKey.get(trackingKey) ?? null;
|
|
746
|
+
const cloud = cloudByKey.get(trackingKey) ?? null;
|
|
747
|
+
if (!localFilePath && cloud) {
|
|
748
|
+
localFilePath = localPathForCloudData(cloud.data);
|
|
749
|
+
}
|
|
750
|
+
const onDisk = !!(localFilePath && fs.existsSync(localFilePath));
|
|
751
|
+
if (entry && onDisk && !cloud) {
|
|
752
|
+
let localHash;
|
|
753
|
+
try {
|
|
754
|
+
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
755
|
+
}
|
|
756
|
+
catch {
|
|
757
|
+
localHash = null;
|
|
758
|
+
}
|
|
759
|
+
if (localHash === null) {
|
|
760
|
+
console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
|
|
761
|
+
}
|
|
762
|
+
else if (entry.contentHash && localHash !== entry.contentHash) {
|
|
763
|
+
await pushLocalFileAtPath(localFilePath);
|
|
764
|
+
}
|
|
765
|
+
else {
|
|
766
|
+
deleteLocalReconcile(localFilePath, entry.fileId, trackingKey);
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
else if (!entry && onDisk && !cloud) {
|
|
770
|
+
const baselineHash = lastSyncedHash.get(trackingKey);
|
|
771
|
+
let localHash;
|
|
772
|
+
try {
|
|
773
|
+
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
774
|
+
}
|
|
775
|
+
catch {
|
|
776
|
+
localHash = null;
|
|
777
|
+
}
|
|
778
|
+
if (localHash === null) {
|
|
779
|
+
console.warn(` [${orgId}] [manifest reconcile] skip unreadable local file ${trackingKey}`);
|
|
780
|
+
}
|
|
781
|
+
else if (baselineHash && localHash === baselineHash) {
|
|
782
|
+
deleteLocalReconcile(localFilePath, null, trackingKey);
|
|
783
|
+
}
|
|
784
|
+
else {
|
|
785
|
+
await pushLocalFileAtPath(localFilePath);
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
else if (entry && !onDisk && cloud) {
|
|
789
|
+
await deleteRemoteReconcile(cloud.data, trackingKey);
|
|
790
|
+
}
|
|
791
|
+
else if ((entry || cloud) && onDisk && cloud) {
|
|
792
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
793
|
+
fileId: cloud.fileId,
|
|
794
|
+
contentHash: cloud.data['contentHash'],
|
|
795
|
+
lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
|
|
796
|
+
});
|
|
797
|
+
}
|
|
798
|
+
else if (!entry && onDisk && cloud) {
|
|
799
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
800
|
+
fileId: cloud.fileId,
|
|
801
|
+
contentHash: cloud.data['contentHash'],
|
|
802
|
+
lastSyncedAt: cloud.data['updatedAt'] || cloud.data['createdAt'] || new Date().toISOString(),
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
manifestStore.flush();
|
|
807
|
+
console.log(` [${orgId}] Manifest reconcile complete.`);
|
|
731
808
|
}
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
const cloudByKey = await fetchCloudFilesByKey();
|
|
741
|
-
|
|
742
|
-
const localByKey = new Map();
|
|
743
|
-
for (const filePath of collectLocalSyncableFiles()) {
|
|
744
|
-
const { scope, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
745
|
-
if (!docPath) continue;
|
|
746
|
-
localByKey.set(`${scope}:${docPath}`, filePath);
|
|
809
|
+
await reconcileManifestOnStartup();
|
|
810
|
+
// Wire per-team file listeners and the live teams-collection watcher
|
|
811
|
+
// after reconcile so initial onSnapshot deliveries cannot race with
|
|
812
|
+
// startup cleanup (DOU-55). applyChange on the live listener picks
|
|
813
|
+
// up teamspaces created/renamed/removed after daemon start without
|
|
814
|
+
// restarting sync.
|
|
815
|
+
for (const [teamId, teamData] of teamRegistry.entries()) {
|
|
816
|
+
setupTeam(teamId, teamData);
|
|
747
817
|
}
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
818
|
+
const teamsUnsub = onSnapshot(collection(db, `orgs/${orgId}/teams`), (snapshot) => {
|
|
819
|
+
snapshot.docChanges().forEach((change) => {
|
|
820
|
+
teamRegistry.applyChange({
|
|
821
|
+
type: change.type,
|
|
822
|
+
id: change.doc.id,
|
|
823
|
+
data: change.doc.data(),
|
|
824
|
+
});
|
|
825
|
+
});
|
|
826
|
+
}, (err) => console.error(` [${orgId}] Teams listener error:`, err));
|
|
827
|
+
unsubscribers.push(teamsUnsub);
|
|
828
|
+
function startFirestoreListeners() {
|
|
829
|
+
console.log(` [${orgId}] Starting Firestore listeners...`);
|
|
830
|
+
if (userId) {
|
|
831
|
+
const privateQ = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId));
|
|
832
|
+
const privateUnsub = onSnapshot(privateQ, snapshot => {
|
|
833
|
+
snapshot.docChanges().forEach(change => {
|
|
834
|
+
const data = change.doc.data();
|
|
835
|
+
const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data['path']);
|
|
836
|
+
track(handleFirestoreChange(change, () => realPath)
|
|
837
|
+
.catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
|
|
838
|
+
if (change.type !== 'removed') {
|
|
839
|
+
updateSharedByMeSymlink(data, realPath);
|
|
840
|
+
}
|
|
841
|
+
else {
|
|
842
|
+
const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, data['path']);
|
|
843
|
+
try {
|
|
844
|
+
fs.lstatSync(symlinkPath);
|
|
845
|
+
fs.unlinkSync(symlinkPath);
|
|
846
|
+
}
|
|
847
|
+
catch (e) { /* ok */ }
|
|
848
|
+
}
|
|
849
|
+
});
|
|
850
|
+
}, err => console.error(` [${orgId}] Private files listener error:`, err));
|
|
851
|
+
unsubscribers.push(privateUnsub);
|
|
852
|
+
console.log(` [${orgId}] Listening: Private files + Shared by Me symlinks`);
|
|
853
|
+
// scope=='private' is required: Firestore rules only grant
|
|
854
|
+
// sharedWith-based reads on private files, so the query must
|
|
855
|
+
// constrain scope or the whole listen is denied (rules are not
|
|
856
|
+
// filters). See tests/rules.test.js "shared-with-me query".
|
|
857
|
+
const sharedQ = query(filesRef, where('scope', '==', 'private'), where('sharedWith', 'array-contains', userId));
|
|
858
|
+
const sharedUnsub = onSnapshot(sharedQ, snapshot => {
|
|
859
|
+
snapshot.docChanges().forEach(change => {
|
|
860
|
+
const data = change.doc.data();
|
|
861
|
+
if (data['ownerId'] === userId)
|
|
862
|
+
return;
|
|
863
|
+
track(handleFirestoreChange(change, () => path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data['path'])).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
|
|
864
|
+
});
|
|
865
|
+
}, err => console.error(` [${orgId}] Shared with me listener error:`, err));
|
|
866
|
+
unsubscribers.push(sharedUnsub);
|
|
867
|
+
console.log(` [${orgId}] Listening: Shared with Me`);
|
|
762
868
|
}
|
|
763
|
-
} catch {
|
|
764
|
-
// skip unreadable locals
|
|
765
|
-
}
|
|
766
869
|
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
...cloudByKey.keys(),
|
|
772
|
-
]);
|
|
773
|
-
|
|
774
|
-
for (const trackingKey of allKeys) {
|
|
775
|
-
const entry = manifestStore.getEntry(trackingKey);
|
|
776
|
-
let localFilePath = localByKey.get(trackingKey) ?? null;
|
|
777
|
-
const cloud = cloudByKey.get(trackingKey) ?? null;
|
|
778
|
-
if (!localFilePath && cloud) {
|
|
779
|
-
localFilePath = localPathForCloudData(cloud.data);
|
|
780
|
-
}
|
|
781
|
-
const onDisk = !!(localFilePath && fs.existsSync(localFilePath));
|
|
782
|
-
|
|
783
|
-
if (entry && onDisk && !cloud) {
|
|
784
|
-
let localHash;
|
|
785
|
-
try {
|
|
786
|
-
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
787
|
-
} catch {
|
|
788
|
-
localHash = null;
|
|
870
|
+
async function firestoreDocExists({ scope, teamId, docPath, contentHash, }) {
|
|
871
|
+
let q;
|
|
872
|
+
if (scope === 'private') {
|
|
873
|
+
q = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId), where('path', '==', docPath));
|
|
789
874
|
}
|
|
790
|
-
if (
|
|
791
|
-
|
|
792
|
-
} else if (entry.contentHash && localHash !== entry.contentHash) {
|
|
793
|
-
await pushLocalFileAtPath(localFilePath);
|
|
794
|
-
} else {
|
|
795
|
-
deleteLocalReconcile(localFilePath, entry.fileId, trackingKey);
|
|
875
|
+
else if (scope === 'team' && teamId) {
|
|
876
|
+
q = query(filesRef, where('scope', '==', 'team'), where('teamId', '==', teamId), where('path', '==', docPath));
|
|
796
877
|
}
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
let localHash;
|
|
800
|
-
try {
|
|
801
|
-
({ contentHash: localHash } = readLocalFilePayload(localFilePath));
|
|
802
|
-
} catch {
|
|
803
|
-
localHash = null;
|
|
878
|
+
else {
|
|
879
|
+
return false;
|
|
804
880
|
}
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
881
|
+
const snap = await getDocs(q);
|
|
882
|
+
if (snap.empty)
|
|
883
|
+
return false;
|
|
884
|
+
// Also require the doc's contentHash to match the baseline. A doc
|
|
885
|
+
// with the same path but a different hash means a web edit
|
|
886
|
+
// happened that the daemon didn't observe; treat as "missing" so
|
|
887
|
+
// the push side proceeds and the pull side can reconcile.
|
|
888
|
+
const firstDoc = snap.docs[0];
|
|
889
|
+
return firstDoc ? firstDoc.data()['contentHash'] === contentHash : false;
|
|
890
|
+
}
|
|
891
|
+
async function pushToFirestore(filePath, data, mimeType, localModifiedAt) {
|
|
892
|
+
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
893
|
+
const trackingKey = `${scope}:${docPath}`;
|
|
894
|
+
// Skip if the local content is unchanged since the last sync. This
|
|
895
|
+
// is the push-side counterpart of the content baseline: an editor
|
|
896
|
+
// re-saving or touching a file (or the watcher re-emitting it on
|
|
897
|
+
// startup) must not push stale bytes over a newer server edit. Only
|
|
898
|
+
// a genuine content change proceeds.
|
|
899
|
+
//
|
|
900
|
+
// DOU-217: trust the baseline ONLY after verifying the Firestore
|
|
901
|
+
// doc actually exists. DOU-55 startup reconcile deletes local orphans
|
|
902
|
+
// when the manifest records a remote delete; this guard applies when
|
|
903
|
+
// pushToFirestore runs at runtime (chokidar add/change) and the
|
|
904
|
+
// baseline would otherwise suppress re-upload forever.
|
|
905
|
+
const contentHash = computeContentHash(data);
|
|
906
|
+
if (lastSyncedHash.get(trackingKey) === contentHash) {
|
|
907
|
+
if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
console.log(` [${orgId}] [reconcile] dropping stale baseline for ${docPath} (Firestore doc missing); re-pushing`);
|
|
911
|
+
deleteBaseline(trackingKey);
|
|
811
912
|
}
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
913
|
+
const now = new Date().toISOString();
|
|
914
|
+
// Set the echo-suppression marker BEFORE writing. Our own write
|
|
915
|
+
// round-trips through the Firestore listener; without a marker
|
|
916
|
+
// already in place, that echo is treated as an incoming change and
|
|
917
|
+
// triggers a spurious self-pull, which (a) reads the blob before it
|
|
918
|
+
// is uploaded (object-not-found) and (b) adds the path to
|
|
919
|
+
// suppressLocal, swallowing a subsequent rename's unlink (leaving a
|
|
920
|
+
// duplicate doc). Pre-setting it suppresses the echo.
|
|
921
|
+
lastKnownUpdate.set(trackingKey, now);
|
|
922
|
+
const result = await pushFileToCloud({
|
|
923
|
+
filesRef,
|
|
924
|
+
storage,
|
|
925
|
+
orgId,
|
|
926
|
+
userId,
|
|
927
|
+
scope,
|
|
928
|
+
teamId,
|
|
929
|
+
docPath,
|
|
930
|
+
data,
|
|
931
|
+
mimeType,
|
|
932
|
+
now,
|
|
933
|
+
localModifiedAt,
|
|
825
934
|
});
|
|
826
|
-
|
|
935
|
+
// Record the synced content baseline whenever local matches the
|
|
936
|
+
// server (created/updated, or already-equal noop), so the pull side
|
|
937
|
+
// can tell "local unchanged" from "local edited" by content.
|
|
938
|
+
if (shouldPersistLocalSyncState(result)) {
|
|
939
|
+
setBaseline(trackingKey, contentHash);
|
|
940
|
+
// Narrow to the variants that actually carry fileId (every
|
|
941
|
+
// variant shouldPersistLocalSyncState accepts: created, updated,
|
|
942
|
+
// noop). The skipped/skipped-stale variants are gated out above.
|
|
943
|
+
if (result.action === 'created' || result.action === 'updated' || result.action === 'noop') {
|
|
944
|
+
manifestStore.upsertEntry(trackingKey, {
|
|
945
|
+
fileId: result.fileId,
|
|
946
|
+
contentHash,
|
|
947
|
+
lastSyncedAt: now,
|
|
948
|
+
});
|
|
949
|
+
manifestStore.flush();
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
else if (result.action === 'skipped-stale') {
|
|
953
|
+
// The server copy is genuinely newer; clear the pre-set marker so
|
|
954
|
+
// its firestore -> local snapshot is allowed through to update the
|
|
955
|
+
// local file.
|
|
956
|
+
lastKnownUpdate.delete(trackingKey);
|
|
957
|
+
}
|
|
958
|
+
if (result.action === 'created') {
|
|
959
|
+
console.log(` [${orgId}] [local → firestore] CREATE ${docPath} (${scope}, ${mimeType})`);
|
|
960
|
+
}
|
|
961
|
+
else if (result.action === 'updated') {
|
|
962
|
+
console.log(` [${orgId}] [local → firestore] UPDATE ${docPath} (${scope})`);
|
|
963
|
+
}
|
|
964
|
+
else if (result.action === 'skipped-stale') {
|
|
965
|
+
console.log(` [${orgId}] [local → firestore] SKIP ${docPath} (server copy is newer)`);
|
|
966
|
+
}
|
|
967
|
+
if ((result.action === 'created' || result.action === 'updated') && result.storageError) {
|
|
968
|
+
console.error(` [${orgId}] [storage] upload failed for ${docPath}: ${result.storageError?.message}`);
|
|
969
|
+
}
|
|
827
970
|
}
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
for (const [teamId, teamData] of teamRegistry.entries()) {
|
|
841
|
-
setupTeam(teamId, teamData);
|
|
842
|
-
}
|
|
843
|
-
const teamsUnsub = onSnapshot(collection(db, `orgs/${orgId}/teams`), (snapshot) => {
|
|
844
|
-
snapshot.docChanges().forEach((change) => {
|
|
845
|
-
teamRegistry.applyChange({
|
|
846
|
-
type: change.type,
|
|
847
|
-
id: change.doc.id,
|
|
848
|
-
data: change.doc.data(),
|
|
849
|
-
});
|
|
850
|
-
});
|
|
851
|
-
}, (err) => console.error(` [${orgId}] Teams listener error:`, err));
|
|
852
|
-
unsubscribers.push(teamsUnsub);
|
|
853
|
-
|
|
854
|
-
function startFirestoreListeners() {
|
|
855
|
-
console.log(` [${orgId}] Starting Firestore listeners...`);
|
|
856
|
-
|
|
857
|
-
if (userId) {
|
|
858
|
-
const privateQ = query(filesRef, where('scope', '==', 'private'), where('ownerId', '==', userId));
|
|
859
|
-
const privateUnsub = onSnapshot(privateQ, snapshot => {
|
|
860
|
-
snapshot.docChanges().forEach(change => {
|
|
861
|
-
const data = change.doc.data();
|
|
862
|
-
const realPath = path.join(LOCAL_PATH, PRIVATE_FOLDER, data.path);
|
|
863
|
-
|
|
864
|
-
track(handleFirestoreChange(change, () => realPath)
|
|
865
|
-
.catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
|
|
866
|
-
|
|
867
|
-
if (change.type !== 'removed') {
|
|
868
|
-
updateSharedByMeSymlink(data, realPath);
|
|
869
|
-
} else {
|
|
870
|
-
const symlinkPath = path.join(LOCAL_PATH, SHARED_BY_ME_FOLDER, data.path);
|
|
871
|
-
try { fs.lstatSync(symlinkPath); fs.unlinkSync(symlinkPath); } catch (e) { /* ok */ }
|
|
872
|
-
}
|
|
971
|
+
async function deleteFromFirestore(filePath) {
|
|
972
|
+
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
973
|
+
const trackingKey = `${scope}:${docPath}`;
|
|
974
|
+
lastKnownUpdate.delete(trackingKey);
|
|
975
|
+
const result = await deleteFileFromCloud({
|
|
976
|
+
filesRef,
|
|
977
|
+
storage,
|
|
978
|
+
orgId,
|
|
979
|
+
userId,
|
|
980
|
+
scope,
|
|
981
|
+
teamId,
|
|
982
|
+
docPath,
|
|
873
983
|
});
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
// scope=='private' is required: Firestore rules only grant
|
|
879
|
-
// sharedWith-based reads on private files, so the query must
|
|
880
|
-
// constrain scope or the whole listen is denied (rules are not
|
|
881
|
-
// filters). See tests/rules.test.js "shared-with-me query".
|
|
882
|
-
const sharedQ = query(
|
|
883
|
-
filesRef,
|
|
884
|
-
where('scope', '==', 'private'),
|
|
885
|
-
where('sharedWith', 'array-contains', userId),
|
|
886
|
-
);
|
|
887
|
-
const sharedUnsub = onSnapshot(sharedQ, snapshot => {
|
|
888
|
-
snapshot.docChanges().forEach(change => {
|
|
889
|
-
const data = change.doc.data();
|
|
890
|
-
if (data.ownerId === userId) return;
|
|
891
|
-
track(handleFirestoreChange(change, () =>
|
|
892
|
-
path.join(LOCAL_PATH, SHARED_WITH_ME_FOLDER, data.path),
|
|
893
|
-
).catch(err => console.error(` [${orgId}] Firestore change handler error:`, err && err.message)));
|
|
894
|
-
});
|
|
895
|
-
}, err => console.error(` [${orgId}] Shared with me listener error:`, err));
|
|
896
|
-
unsubscribers.push(sharedUnsub);
|
|
897
|
-
console.log(` [${orgId}] Listening: Shared with Me`);
|
|
898
|
-
}
|
|
899
|
-
}
|
|
900
|
-
|
|
901
|
-
// DOU-217 helper: does a doc with this path + matching contentHash
|
|
902
|
-
// exist in Firestore right now? Startup reconcile (DOU-55) handles
|
|
903
|
-
// manifest-present orphans; this check covers the runtime push path
|
|
904
|
-
// when chokidar re-emits a file whose baseline still matches but the
|
|
905
|
-
// Firestore doc is gone (e.g. corrupt/missing manifest).
|
|
906
|
-
async function firestoreDocExists({ scope, teamId, docPath, contentHash }) {
|
|
907
|
-
let q;
|
|
908
|
-
if (scope === 'private') {
|
|
909
|
-
q = query(filesRef,
|
|
910
|
-
where('scope', '==', 'private'),
|
|
911
|
-
where('ownerId', '==', userId),
|
|
912
|
-
where('path', '==', docPath));
|
|
913
|
-
} else if (scope === 'team' && teamId) {
|
|
914
|
-
q = query(filesRef,
|
|
915
|
-
where('scope', '==', 'team'),
|
|
916
|
-
where('teamId', '==', teamId),
|
|
917
|
-
where('path', '==', docPath));
|
|
918
|
-
} else {
|
|
919
|
-
return false;
|
|
984
|
+
if (result.action === 'deleted') {
|
|
985
|
+
console.log(` [${orgId}] [local → firestore] DELETE ${docPath} (${scope})`);
|
|
986
|
+
manifestStore.removeEntry(trackingKey);
|
|
987
|
+
}
|
|
920
988
|
}
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
//
|
|
924
|
-
//
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
// doc actually exists. DOU-55 startup reconcile deletes local orphans
|
|
942
|
-
// when the manifest records a remote delete; this guard applies when
|
|
943
|
-
// pushToFirestore runs at runtime (chokidar add/change) and the
|
|
944
|
-
// baseline would otherwise suppress re-upload forever.
|
|
945
|
-
const contentHash = computeContentHash(data);
|
|
946
|
-
if (lastSyncedHash.get(trackingKey) === contentHash) {
|
|
947
|
-
if (await firestoreDocExists({ scope, teamId, docPath, contentHash })) {
|
|
948
|
-
return;
|
|
949
|
-
}
|
|
950
|
-
console.log(` [${orgId}] [reconcile] dropping stale baseline for ${docPath} (Firestore doc missing); re-pushing`);
|
|
951
|
-
deleteBaseline(trackingKey);
|
|
989
|
+
// Determine whether a local path corresponds to a syncable directory.
|
|
990
|
+
// Excludes the scope-root directories themselves (Private, Shared
|
|
991
|
+
// with Me, Shared by Me, and the per-team roots) since those are
|
|
992
|
+
// managed by ensureFolderStructure, not the user.
|
|
993
|
+
function isSyncableLocalDir(rel) {
|
|
994
|
+
if (!rel)
|
|
995
|
+
return false;
|
|
996
|
+
if (rel === PRIVATE_FOLDER || rel === SHARED_WITH_ME_FOLDER || rel === SHARED_BY_ME_FOLDER) {
|
|
997
|
+
return false;
|
|
998
|
+
}
|
|
999
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
|
|
1000
|
+
return false;
|
|
1001
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
|
|
1002
|
+
return false;
|
|
1003
|
+
const parts = rel.split(path.sep);
|
|
1004
|
+
// A bare team root (e.g. "Engineering Teamspace") corresponds to
|
|
1005
|
+
// the scope root, not a subdirectory. Skip these.
|
|
1006
|
+
if (parts.length === 1 && parts[0]?.endsWith(' Teamspace'))
|
|
1007
|
+
return false;
|
|
1008
|
+
return true;
|
|
952
1009
|
}
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
storage,
|
|
968
|
-
orgId,
|
|
969
|
-
userId,
|
|
970
|
-
scope,
|
|
971
|
-
teamId,
|
|
972
|
-
docPath,
|
|
973
|
-
data,
|
|
974
|
-
mimeType,
|
|
975
|
-
now,
|
|
976
|
-
localModifiedAt,
|
|
977
|
-
});
|
|
978
|
-
|
|
979
|
-
// Record the synced content baseline whenever local matches the
|
|
980
|
-
// server (created/updated, or already-equal noop), so the pull side
|
|
981
|
-
// can tell "local unchanged" from "local edited" by content.
|
|
982
|
-
if (shouldPersistLocalSyncState(result)) {
|
|
983
|
-
setBaseline(trackingKey, contentHash);
|
|
984
|
-
if (result.fileId) {
|
|
985
|
-
manifestStore.upsertEntry(trackingKey, {
|
|
986
|
-
fileId: result.fileId,
|
|
987
|
-
contentHash,
|
|
988
|
-
lastSyncedAt: now,
|
|
1010
|
+
async function pushDirToFirestore(dirPath) {
|
|
1011
|
+
const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
|
|
1012
|
+
if (!docPath)
|
|
1013
|
+
return; // scope root
|
|
1014
|
+
const trackingKey = `${scope}:${docPath}`;
|
|
1015
|
+
const now = new Date().toISOString();
|
|
1016
|
+
lastKnownUpdate.set(trackingKey, now);
|
|
1017
|
+
const result = await pushFolderToCloud({
|
|
1018
|
+
filesRef,
|
|
1019
|
+
scope,
|
|
1020
|
+
teamId,
|
|
1021
|
+
userId,
|
|
1022
|
+
docPath,
|
|
1023
|
+
now,
|
|
989
1024
|
});
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
// The server copy is genuinely newer; clear the pre-set marker so
|
|
994
|
-
// its firestore -> local snapshot is allowed through to update the
|
|
995
|
-
// local file.
|
|
996
|
-
lastKnownUpdate.delete(trackingKey);
|
|
997
|
-
}
|
|
998
|
-
|
|
999
|
-
if (result.action === 'created') {
|
|
1000
|
-
console.log(` [${orgId}] [local → firestore] CREATE ${docPath} (${scope}, ${mimeType})`);
|
|
1001
|
-
} else if (result.action === 'updated') {
|
|
1002
|
-
console.log(` [${orgId}] [local → firestore] UPDATE ${docPath} (${scope})`);
|
|
1003
|
-
} else if (result.action === 'skipped-stale') {
|
|
1004
|
-
console.log(` [${orgId}] [local → firestore] SKIP ${docPath} (server copy is newer)`);
|
|
1005
|
-
}
|
|
1006
|
-
if (result.storageError) {
|
|
1007
|
-
console.error(` [${orgId}] [storage] upload failed for ${docPath}: ${result.storageError.message}`);
|
|
1008
|
-
}
|
|
1009
|
-
}
|
|
1010
|
-
|
|
1011
|
-
async function deleteFromFirestore(filePath) {
|
|
1012
|
-
const { scope, teamId, docPath } = scopeFromLocalPath(filePath, LOCAL_PATH, teamIdsByName);
|
|
1013
|
-
const trackingKey = `${scope}:${docPath}`;
|
|
1014
|
-
lastKnownUpdate.delete(trackingKey);
|
|
1015
|
-
|
|
1016
|
-
const result = await deleteFileFromCloud({
|
|
1017
|
-
filesRef,
|
|
1018
|
-
storage,
|
|
1019
|
-
orgId,
|
|
1020
|
-
userId,
|
|
1021
|
-
scope,
|
|
1022
|
-
teamId,
|
|
1023
|
-
docPath,
|
|
1024
|
-
});
|
|
1025
|
-
|
|
1026
|
-
if (result.action === 'deleted') {
|
|
1027
|
-
console.log(` [${orgId}] [local → firestore] DELETE ${docPath} (${scope})`);
|
|
1028
|
-
manifestStore.removeEntry(trackingKey);
|
|
1029
|
-
}
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
// Determine whether a local path corresponds to a syncable directory.
|
|
1033
|
-
// Excludes the scope-root directories themselves (Private, Shared
|
|
1034
|
-
// with Me, Shared by Me, and the per-team roots) since those are
|
|
1035
|
-
// managed by ensureFolderStructure, not the user.
|
|
1036
|
-
function isSyncableLocalDir(rel) {
|
|
1037
|
-
if (!rel) return false;
|
|
1038
|
-
if (rel === PRIVATE_FOLDER || rel === SHARED_WITH_ME_FOLDER || rel === SHARED_BY_ME_FOLDER) {
|
|
1039
|
-
return false;
|
|
1040
|
-
}
|
|
1041
|
-
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep)) return false;
|
|
1042
|
-
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep)) return false;
|
|
1043
|
-
const parts = rel.split(path.sep);
|
|
1044
|
-
// A bare team root (e.g. "Engineering Teamspace") corresponds to
|
|
1045
|
-
// the scope root, not a subdirectory. Skip these.
|
|
1046
|
-
if (parts.length === 1 && parts[0].endsWith(' Teamspace')) return false;
|
|
1047
|
-
return true;
|
|
1048
|
-
}
|
|
1049
|
-
|
|
1050
|
-
async function pushDirToFirestore(dirPath) {
|
|
1051
|
-
const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
|
|
1052
|
-
if (!docPath) return; // scope root
|
|
1053
|
-
const trackingKey = `${scope}:${docPath}`;
|
|
1054
|
-
const now = new Date().toISOString();
|
|
1055
|
-
lastKnownUpdate.set(trackingKey, now);
|
|
1056
|
-
|
|
1057
|
-
const result = await pushFolderToCloud({
|
|
1058
|
-
filesRef,
|
|
1059
|
-
scope,
|
|
1060
|
-
teamId,
|
|
1061
|
-
userId,
|
|
1062
|
-
docPath,
|
|
1063
|
-
now,
|
|
1064
|
-
});
|
|
1065
|
-
|
|
1066
|
-
if (result.action === 'ensured' && result.folderId) {
|
|
1067
|
-
console.log(` [${orgId}] [local → firestore] FOLDER ${docPath} (${scope})`);
|
|
1068
|
-
}
|
|
1069
|
-
}
|
|
1070
|
-
|
|
1071
|
-
async function deleteDirFromFirestore(dirPath) {
|
|
1072
|
-
const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
|
|
1073
|
-
if (!docPath) return;
|
|
1074
|
-
const trackingKey = `${scope}:${docPath}`;
|
|
1075
|
-
lastKnownUpdate.delete(trackingKey);
|
|
1076
|
-
|
|
1077
|
-
const result = await deleteFolderFromCloud({
|
|
1078
|
-
filesRef,
|
|
1079
|
-
storage,
|
|
1080
|
-
orgId,
|
|
1081
|
-
scope,
|
|
1082
|
-
teamId,
|
|
1083
|
-
userId,
|
|
1084
|
-
docPath,
|
|
1085
|
-
});
|
|
1086
|
-
|
|
1087
|
-
if (result.action === 'deleted') {
|
|
1088
|
-
// childCount > 0 means cascade did real work; useful in logs
|
|
1089
|
-
// when investigating future folder-delete reports.
|
|
1090
|
-
const childSuffix = result.childCount > 0 ? ` (+${result.childCount} children)` : '';
|
|
1091
|
-
console.log(` [${orgId}] [local → firestore] RMFOLDER ${docPath} (${scope})${childSuffix}`);
|
|
1025
|
+
if (result.action === 'ensured' && result.folderId) {
|
|
1026
|
+
console.log(` [${orgId}] [local → firestore] FOLDER ${docPath} (${scope})`);
|
|
1027
|
+
}
|
|
1092
1028
|
}
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1029
|
+
async function deleteDirFromFirestore(dirPath) {
|
|
1030
|
+
const { scope, teamId, docPath } = scopeFromLocalPath(dirPath, LOCAL_PATH, teamIdsByName);
|
|
1031
|
+
if (!docPath)
|
|
1032
|
+
return;
|
|
1033
|
+
const trackingKey = `${scope}:${docPath}`;
|
|
1034
|
+
lastKnownUpdate.delete(trackingKey);
|
|
1035
|
+
const result = await deleteFolderFromCloud({
|
|
1036
|
+
filesRef,
|
|
1037
|
+
storage,
|
|
1038
|
+
orgId,
|
|
1039
|
+
scope,
|
|
1040
|
+
teamId,
|
|
1041
|
+
userId,
|
|
1042
|
+
docPath,
|
|
1043
|
+
});
|
|
1044
|
+
if (result.action === 'deleted') {
|
|
1045
|
+
// childCount > 0 means cascade did real work; useful in logs
|
|
1046
|
+
// when investigating future folder-delete reports.
|
|
1047
|
+
const childSuffix = result.childCount > 0 ? ` (+${result.childCount} children)` : '';
|
|
1048
|
+
console.log(` [${orgId}] [local → firestore] RMFOLDER ${docPath} (${scope})${childSuffix}`);
|
|
1049
|
+
}
|
|
1107
1050
|
}
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1051
|
+
function handleLocalChange(filePath) {
|
|
1052
|
+
if (suppressLocal.has(filePath))
|
|
1053
|
+
return;
|
|
1054
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
1055
|
+
if (rel.startsWith(SHARED_WITH_ME_FOLDER + path.sep))
|
|
1056
|
+
return;
|
|
1057
|
+
if (rel.startsWith(SHARED_BY_ME_FOLDER + path.sep))
|
|
1058
|
+
return;
|
|
1059
|
+
let stat;
|
|
1060
|
+
try {
|
|
1061
|
+
stat = fs.statSync(filePath);
|
|
1062
|
+
}
|
|
1063
|
+
catch {
|
|
1064
|
+
return; // disappeared between event and read
|
|
1065
|
+
}
|
|
1066
|
+
if (!stat.isFile())
|
|
1067
|
+
return;
|
|
1068
|
+
if (stat.size > maxUploadBytes) {
|
|
1069
|
+
console.error(` [${orgId}] [skip] ${rel} exceeds ${maxUploadBytes} byte cap (${stat.size} bytes)`);
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
const ext = extFromPath(rel);
|
|
1073
|
+
const mimeType = mimeTypeFromExt(ext);
|
|
1074
|
+
// Text files: read as a UTF-8 string so future text-aware features
|
|
1075
|
+
// (line-level diffing, conflict markers) can work. Binary files:
|
|
1076
|
+
// read as a Buffer so bytes round-trip without corruption.
|
|
1077
|
+
const data = isTextMimeType(mimeType)
|
|
1078
|
+
? fs.readFileSync(filePath, 'utf-8')
|
|
1079
|
+
: fs.readFileSync(filePath);
|
|
1080
|
+
const localModifiedAt = stat.mtime.toISOString();
|
|
1081
|
+
// Yjs path (DOU-178 PR 3): route text edits into the binding so
|
|
1082
|
+
// the disk delta is published as a yupdate (preserving concurrent
|
|
1083
|
+
// web edits via fast-diff). The binding only exists once the
|
|
1084
|
+
// Firestore listener has seen the file; for the very first
|
|
1085
|
+
// local-only save the binding hasn't been created yet, and the
|
|
1086
|
+
// blob path is still the only writer. Subsequent saves go through
|
|
1087
|
+
// both paths until PR 4 retires the blob upload.
|
|
1088
|
+
if (typeof data === 'string' && isTextMimeType(mimeType)) {
|
|
1089
|
+
const binding = bindingsByPath.get(filePath);
|
|
1090
|
+
if (binding)
|
|
1091
|
+
binding.applyDiskText(data);
|
|
1092
|
+
}
|
|
1093
|
+
track(enqueueLocalOp(filePath, () => pushToFirestore(filePath, data, mimeType, localModifiedAt).catch((err) => console.error(` [${orgId}] Push error for ${rel}: ${err?.message}`))));
|
|
1112
1094
|
}
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
? fs.readFileSync(filePath, 'utf-8')
|
|
1121
|
-
: fs.readFileSync(filePath);
|
|
1122
|
-
const localModifiedAt = stat.mtime.toISOString();
|
|
1123
|
-
|
|
1124
|
-
// Yjs path (DOU-178 PR 3): route text edits into the binding so
|
|
1125
|
-
// the disk delta is published as a yupdate (preserving concurrent
|
|
1126
|
-
// web edits via fast-diff). The binding only exists once the
|
|
1127
|
-
// Firestore listener has seen the file; for the very first
|
|
1128
|
-
// local-only save the binding hasn't been created yet, and the
|
|
1129
|
-
// blob path is still the only writer. Subsequent saves go through
|
|
1130
|
-
// both paths until PR 4 retires the blob upload.
|
|
1131
|
-
if (isTextMimeType(mimeType)) {
|
|
1132
|
-
const binding = bindingsByPath.get(filePath);
|
|
1133
|
-
if (binding) binding.applyDiskText(data);
|
|
1095
|
+
function handleLocalDirAdded(dirPath) {
|
|
1096
|
+
if (suppressLocal.has(dirPath))
|
|
1097
|
+
return;
|
|
1098
|
+
const rel = path.relative(LOCAL_PATH, dirPath);
|
|
1099
|
+
if (!isSyncableLocalDir(rel))
|
|
1100
|
+
return;
|
|
1101
|
+
track(pushDirToFirestore(dirPath).catch((err) => console.error(` [${orgId}] Folder push error for ${rel}: ${err?.message}`)));
|
|
1134
1102
|
}
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
function handleLocalDirAdded(dirPath) {
|
|
1144
|
-
if (suppressLocal.has(dirPath)) return;
|
|
1145
|
-
const rel = path.relative(LOCAL_PATH, dirPath);
|
|
1146
|
-
if (!isSyncableLocalDir(rel)) return;
|
|
1147
|
-
track(pushDirToFirestore(dirPath).catch(err =>
|
|
1148
|
-
console.error(` [${orgId}] Folder push error for ${rel}: ${err.message}`)
|
|
1149
|
-
));
|
|
1150
|
-
}
|
|
1151
|
-
|
|
1152
|
-
function handleLocalDirRemoved(dirPath) {
|
|
1153
|
-
if (suppressLocal.has(dirPath)) return;
|
|
1154
|
-
const rel = path.relative(LOCAL_PATH, dirPath);
|
|
1155
|
-
if (!isSyncableLocalDir(rel)) return;
|
|
1156
|
-
track(deleteDirFromFirestore(dirPath).catch(err =>
|
|
1157
|
-
console.error(` [${orgId}] Folder delete error for ${rel}: ${err.message}`)
|
|
1158
|
-
));
|
|
1159
|
-
}
|
|
1160
|
-
|
|
1161
|
-
function startLocalWatcher() {
|
|
1162
|
-
console.log(` [${orgId}] Watching local files...`);
|
|
1163
|
-
|
|
1164
|
-
watcher = chokidar.watch(LOCAL_PATH, {
|
|
1165
|
-
ignored: [
|
|
1166
|
-
// Dotfiles (the sync engine's own .compound-sync-state.json lives
|
|
1167
|
-
// here, plus things like .DS_Store, .git, etc. that the user
|
|
1168
|
-
// doesn't want syncing).
|
|
1169
|
-
/(^|[\/\\])\./,
|
|
1170
|
-
// node_modules: large dependency dirs, never appropriate to sync.
|
|
1171
|
-
/node_modules/,
|
|
1172
|
-
],
|
|
1173
|
-
persistent: true,
|
|
1174
|
-
ignoreInitial: false,
|
|
1175
|
-
followSymlinks: false,
|
|
1176
|
-
awaitWriteFinish: {
|
|
1177
|
-
stabilityThreshold: 500,
|
|
1178
|
-
pollInterval: 100,
|
|
1179
|
-
},
|
|
1180
|
-
});
|
|
1181
|
-
|
|
1182
|
-
// Reusable per-event trace logging, gated by COMPOUND_CHOKIDAR_DEBUG.
|
|
1183
|
-
// Writes to both stdout AND a file (default /tmp/chokidar-debug.log)
|
|
1184
|
-
// so traces survive a process kill or buffered stdout. Used by the
|
|
1185
|
-
// Test-on-demand workflow's `Dump diagnostic logs` step for
|
|
1186
|
-
// chokidar-related diagnostic runs. Zero cost when the env var is
|
|
1187
|
-
// unset.
|
|
1188
|
-
const chokidarDebug = process.env.COMPOUND_CHOKIDAR_DEBUG === '1';
|
|
1189
|
-
const chokidarLogPath = process.env.COMPOUND_CHOKIDAR_LOG || '/tmp/chokidar-debug.log';
|
|
1190
|
-
function logEvent(event, p) {
|
|
1191
|
-
if (chokidarDebug) {
|
|
1192
|
-
const rel = path.relative(LOCAL_PATH, p);
|
|
1193
|
-
const line = ` [${orgId}] [chokidar ${event}] ${rel} @ ${new Date().toISOString()}\n`;
|
|
1194
|
-
try { fs.appendFileSync(chokidarLogPath, line); } catch (_) {}
|
|
1195
|
-
console.log(line.trimEnd());
|
|
1196
|
-
}
|
|
1103
|
+
function handleLocalDirRemoved(dirPath) {
|
|
1104
|
+
if (suppressLocal.has(dirPath))
|
|
1105
|
+
return;
|
|
1106
|
+
const rel = path.relative(LOCAL_PATH, dirPath);
|
|
1107
|
+
if (!isSyncableLocalDir(rel))
|
|
1108
|
+
return;
|
|
1109
|
+
track(deleteDirFromFirestore(dirPath).catch((err) => console.error(` [${orgId}] Folder delete error for ${rel}: ${err?.message}`)));
|
|
1197
1110
|
}
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1111
|
+
function startLocalWatcher() {
|
|
1112
|
+
console.log(` [${orgId}] Watching local files...`);
|
|
1113
|
+
watcher = chokidar.watch(LOCAL_PATH, {
|
|
1114
|
+
ignored: [
|
|
1115
|
+
// Dotfiles (the sync engine's own .compound-sync-state.json lives
|
|
1116
|
+
// here, plus things like .DS_Store, .git, etc. that the user
|
|
1117
|
+
// doesn't want syncing).
|
|
1118
|
+
/(^|[\/\\])\./,
|
|
1119
|
+
// node_modules: large dependency dirs, never appropriate to sync.
|
|
1120
|
+
/node_modules/,
|
|
1121
|
+
],
|
|
1122
|
+
persistent: true,
|
|
1123
|
+
ignoreInitial: false,
|
|
1124
|
+
followSymlinks: false,
|
|
1125
|
+
awaitWriteFinish: {
|
|
1126
|
+
stabilityThreshold: 500,
|
|
1127
|
+
pollInterval: 100,
|
|
1128
|
+
},
|
|
1129
|
+
});
|
|
1130
|
+
// Reusable per-event trace logging, gated by COMPOUND_CHOKIDAR_DEBUG.
|
|
1131
|
+
// Writes to both stdout AND a file (default /tmp/chokidar-debug.log)
|
|
1132
|
+
// so traces survive a process kill or buffered stdout. Used by the
|
|
1133
|
+
// Test-on-demand workflow's `Dump diagnostic logs` step for
|
|
1134
|
+
// chokidar-related diagnostic runs. Zero cost when the env var is
|
|
1135
|
+
// unset.
|
|
1136
|
+
const chokidarDebug = process.env['COMPOUND_CHOKIDAR_DEBUG'] === '1';
|
|
1137
|
+
const chokidarLogPath = process.env['COMPOUND_CHOKIDAR_LOG'] || '/tmp/chokidar-debug.log';
|
|
1138
|
+
function logEvent(event, p) {
|
|
1139
|
+
if (chokidarDebug) {
|
|
1140
|
+
const rel = path.relative(LOCAL_PATH, p);
|
|
1141
|
+
const line = ` [${orgId}] [chokidar ${event}] ${rel} @ ${new Date().toISOString()}\n`;
|
|
1142
|
+
try {
|
|
1143
|
+
fs.appendFileSync(chokidarLogPath, line);
|
|
1144
|
+
}
|
|
1145
|
+
catch (_) { }
|
|
1146
|
+
console.log(line.trimEnd());
|
|
1147
|
+
}
|
|
1148
|
+
}
|
|
1149
|
+
// Local non-null alias so TS doesn't have to re-narrow `watcher`
|
|
1150
|
+
// on every .on() call below.
|
|
1151
|
+
const w = watcher;
|
|
1152
|
+
w.on('add', (filePath) => { logEvent('add', filePath); handleLocalChange(filePath); });
|
|
1153
|
+
w.on('change', (filePath) => { logEvent('change', filePath); handleLocalChange(filePath); });
|
|
1154
|
+
w.on('unlink', (filePath) => {
|
|
1155
|
+
logEvent('unlink', filePath);
|
|
1156
|
+
if (suppressLocal.has(filePath))
|
|
1157
|
+
return;
|
|
1158
|
+
const rel = path.relative(LOCAL_PATH, filePath);
|
|
1159
|
+
track(enqueueLocalOp(filePath, () => deleteFromFirestore(filePath).catch((err) => console.error(` [${orgId}] Delete error for ${rel}: ${err?.message}`))));
|
|
1160
|
+
});
|
|
1161
|
+
w.on('addDir', (dirPath) => { logEvent('addDir', dirPath); handleLocalDirAdded(dirPath); });
|
|
1162
|
+
w.on('unlinkDir', (dirPath) => { logEvent('unlinkDir', dirPath); handleLocalDirRemoved(dirPath); });
|
|
1163
|
+
if (chokidarDebug) {
|
|
1164
|
+
w.on('raw', (event, p, details) => {
|
|
1165
|
+
const line = ` [${orgId}] [chokidar raw ${event}] ${p} @ ${new Date().toISOString()} ${JSON.stringify(details)}\n`;
|
|
1166
|
+
try {
|
|
1167
|
+
fs.appendFileSync(chokidarLogPath, line);
|
|
1168
|
+
}
|
|
1169
|
+
catch { /* ignore */ }
|
|
1170
|
+
console.log(line.trimEnd());
|
|
1171
|
+
});
|
|
1172
|
+
}
|
|
1173
|
+
// Resolve only once chokidar has finished its initial scan and
|
|
1174
|
+
// registered inotify (or FSEvents) watches on every existing
|
|
1175
|
+
// subdirectory. Until 'ready' fires, files written to the watched
|
|
1176
|
+
// tree can be silently missed because the OS-level watch hasn't
|
|
1177
|
+
// attached yet. Callers awaiting startOrgSync get a guarantee
|
|
1178
|
+
// that any local writes after the await are seen by the daemon.
|
|
1179
|
+
return new Promise((resolve) => {
|
|
1180
|
+
w.on('ready', () => {
|
|
1181
|
+
console.log(` [${orgId}] Local watcher ready.`);
|
|
1182
|
+
resolve();
|
|
1183
|
+
});
|
|
1184
|
+
});
|
|
1219
1185
|
}
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
Promise.all([...pendingPromises]),
|
|
1267
|
-
new Promise((r) => setTimeout(r, remaining)),
|
|
1268
|
-
]);
|
|
1269
|
-
}
|
|
1270
|
-
|
|
1271
|
-
await Promise.all(
|
|
1272
|
-
[...bindings.values()].map((b) => b.stop().catch(() => {})),
|
|
1273
|
-
);
|
|
1274
|
-
bindings.clear();
|
|
1275
|
-
bindingsByPath.clear();
|
|
1276
|
-
|
|
1277
|
-
if (watcher) await watcher.close();
|
|
1278
|
-
},
|
|
1279
|
-
};
|
|
1186
|
+
startFirestoreListeners();
|
|
1187
|
+
await startLocalWatcher();
|
|
1188
|
+
return {
|
|
1189
|
+
// Tear down all listeners and the watcher. Awaits every tracked
|
|
1190
|
+
// in-flight handler so callers/tests get a strong "no daemon work
|
|
1191
|
+
// is still running" guarantee on the resolved promise.
|
|
1192
|
+
async stop() {
|
|
1193
|
+
stoppedRef.value = true;
|
|
1194
|
+
manifestStore.flush();
|
|
1195
|
+
for (const unsub of unsubscribers) {
|
|
1196
|
+
try {
|
|
1197
|
+
unsub();
|
|
1198
|
+
}
|
|
1199
|
+
catch (e) { /* ignore */ }
|
|
1200
|
+
}
|
|
1201
|
+
for (const unsub of teamUnsubscribers.values()) {
|
|
1202
|
+
try {
|
|
1203
|
+
unsub();
|
|
1204
|
+
}
|
|
1205
|
+
catch (e) { /* ignore */ }
|
|
1206
|
+
}
|
|
1207
|
+
teamUnsubscribers.clear();
|
|
1208
|
+
// Drain tracked promises in a loop, bounded by a 5s ceiling.
|
|
1209
|
+
// A handler currently in flight (e.g., a snapshot callback
|
|
1210
|
+
// already dispatched before unsub()) can chain follow-up work
|
|
1211
|
+
// that gets added back to pendingPromises. Loop until the set
|
|
1212
|
+
// quiesces so no daemon task outlives stop() in the common
|
|
1213
|
+
// case. The cap protects against tests where a handler is
|
|
1214
|
+
// genuinely stuck (e.g., emulator pause), so stop() never
|
|
1215
|
+
// hangs indefinitely; anything still pending past 5s is
|
|
1216
|
+
// abandoned and the next test's clearFirestore will sweep it.
|
|
1217
|
+
const drainDeadline = Date.now() + 5000;
|
|
1218
|
+
while (pendingPromises.size > 0 && Date.now() < drainDeadline) {
|
|
1219
|
+
const remaining = Math.max(0, drainDeadline - Date.now());
|
|
1220
|
+
await Promise.race([
|
|
1221
|
+
Promise.all([...pendingPromises]),
|
|
1222
|
+
new Promise((r) => setTimeout(r, remaining)),
|
|
1223
|
+
]);
|
|
1224
|
+
}
|
|
1225
|
+
await Promise.all([...bindings.values()].map((b) => b.stop().catch(() => { })));
|
|
1226
|
+
bindings.clear();
|
|
1227
|
+
bindingsByPath.clear();
|
|
1228
|
+
if (watcher)
|
|
1229
|
+
await watcher.close();
|
|
1230
|
+
},
|
|
1231
|
+
};
|
|
1280
1232
|
}
|
|
1233
|
+
//# sourceMappingURL=org-sync.js.map
|