@geml/logseq-sync 2.0.9 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +27 -28
- package/README.md +396 -377
- package/core/src/bridge.mjs +8 -8
- package/core/src/discovery.mjs +204 -204
- package/core/src/sync-engine.mjs +623 -560
- package/docs/how-it-works.svg +42 -42
- package/package.json +58 -58
- package/watcher/bin/create-graph.mjs +51 -51
- package/watcher/bin/create_graph_headless.cljs +22 -22
- package/watcher/bin/live-roundtrip.mjs +131 -131
- package/watcher/bin/logseq-sync.mjs +994 -940
package/core/src/sync-engine.mjs
CHANGED
|
@@ -1,560 +1,623 @@
|
|
|
1
|
-
// Core Sync Engine for Logseq GEML
|
|
2
|
-
// Keeps a local folder of .geml files in sync with a Logseq DB graph,
|
|
3
|
-
// ensuring only changed files are written so Git diffs stay clean.
|
|
4
|
-
|
|
5
|
-
import {
|
|
6
|
-
readdirSync,
|
|
7
|
-
readFileSync,
|
|
8
|
-
writeFileSync,
|
|
9
|
-
unlinkSync,
|
|
10
|
-
mkdirSync,
|
|
11
|
-
rmdirSync,
|
|
12
|
-
renameSync,
|
|
13
|
-
existsSync,
|
|
14
|
-
} from "node:fs";
|
|
15
|
-
import { join, dirname, relative, resolve, sep, isAbsolute } from "node:path";
|
|
16
|
-
import { execFileSync } from "node:child_process";
|
|
17
|
-
import { randomUUID, createHash } from "node:crypto";
|
|
18
|
-
import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
|
|
19
|
-
import { gemlToOgMarkdown } from "./og-markdown.mjs";
|
|
20
|
-
|
|
21
|
-
const MANIFEST_FILE = ".geml-manifest.json";
|
|
22
|
-
// The Markdown tree needs a ledger of its own: `--markdown` takes ANY
|
|
23
|
-
// directory, so it is the tree most likely to be pointed at a graph someone
|
|
24
|
-
// already has, and it must know which .md files are its own writes before it
|
|
25
|
-
// overwrites one. A separate file because markdownDir may BE targetDir.
|
|
26
|
-
const MD_MANIFEST_FILE = ".geml-md-manifest.json";
|
|
27
|
-
|
|
28
|
-
const sha256 = (s) => createHash("sha256").update(s).digest("hex");
|
|
29
|
-
|
|
30
|
-
/**
|
|
31
|
-
* Read the sync manifest in either of its two shapes.
|
|
32
|
-
* v1 was a sorted array of paths — enough to know which files the sync owns.
|
|
33
|
-
* v2 ({ version: 2, files: { rel: sha256 } }) also records the content the
|
|
34
|
-
* sync last wrote or saw, which is what lets two-way sync tell an external
|
|
35
|
-
* edit from its own echo: a file whose hash matches the manifest is the
|
|
36
|
-
* watcher's own last write, not something a person or agent changed.
|
|
37
|
-
* @returns {{ known: boolean, hashed: boolean, files: Map<string, string|null> }}
|
|
38
|
-
*/
|
|
39
|
-
function readManifest(targetDir, manifestFile = MANIFEST_FILE) {
|
|
40
|
-
const manifestPath = join(targetDir, manifestFile);
|
|
41
|
-
if (!existsSync(manifestPath)) return { known: false, hashed: false, files: new Map() };
|
|
42
|
-
try {
|
|
43
|
-
const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
44
|
-
if (Array.isArray(parsed)) {
|
|
45
|
-
return { known: true, hashed: false, files: new Map(parsed.map((p) => [p, null])) };
|
|
46
|
-
}
|
|
47
|
-
if (parsed && parsed.version === 2 && parsed.files && typeof parsed.files === "object") {
|
|
48
|
-
return { known: true, hashed: true, files: new Map(Object.entries(parsed.files)) };
|
|
49
|
-
}
|
|
50
|
-
} catch {}
|
|
51
|
-
return { known: false, hashed: false, files: new Map() };
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* What changed in the vault since the sync last touched it — the read side of
|
|
56
|
-
* the two-way bridge. Baselines come from the v2 manifest hashes; a v1
|
|
57
|
-
* manifest (or none) knows which files exist but not what they held, so it
|
|
58
|
-
* reports nothing rather than guessing: `baselineKnown: false` means "sync
|
|
59
|
-
* once first".
|
|
60
|
-
*
|
|
61
|
-
* With `graphFiles` (the current export, as ednToGemlFiles returns it) the
|
|
62
|
-
* vault-modified files are split further: one the GRAPH also moved since the
|
|
63
|
-
* last sync is a `conflict` — importing it would clobber the graph's edit,
|
|
64
|
-
* exporting over it would clobber the person's, so two-way sync does neither
|
|
65
|
-
* and a person merges.
|
|
66
|
-
* @param {string} targetDir
|
|
67
|
-
* @param {{ graphFiles?: Map<string, string> }} [opts]
|
|
68
|
-
* @returns {{ baselineKnown: boolean, modified: string[], added: string[], missing: string[], conflicts: string[] }}
|
|
69
|
-
*/
|
|
70
|
-
export function detectExternalEdits(targetDir, opts = {}) {
|
|
71
|
-
const manifest = readManifest(targetDir);
|
|
72
|
-
const onDisk = readGemlFilesFromDisk(targetDir);
|
|
73
|
-
const modified = [];
|
|
74
|
-
const added = [];
|
|
75
|
-
const missing = [];
|
|
76
|
-
const conflicts = [];
|
|
77
|
-
if (!manifest.hashed) return { baselineKnown: false, modified, added, missing, conflicts };
|
|
78
|
-
for (const [rel, hash] of manifest.files) {
|
|
79
|
-
const content = onDisk.get(rel);
|
|
80
|
-
if (content === undefined) missing.push(rel);
|
|
81
|
-
else if (hash !== null && sha256(content) !== hash) {
|
|
82
|
-
const graphContent = opts.graphFiles?.get(rel);
|
|
83
|
-
const graphMoved =
|
|
84
|
-
graphContent !== undefined && sha256(normalizeEol(graphContent)) !== hash;
|
|
85
|
-
(graphMoved ? conflicts : modified).push(rel);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
for (const rel of onDisk.keys()) {
|
|
89
|
-
if (!manifest.files.has(rel)) added.push(rel);
|
|
90
|
-
}
|
|
91
|
-
return {
|
|
92
|
-
baselineKnown: true,
|
|
93
|
-
modified: modified.sort(),
|
|
94
|
-
added: added.sort(),
|
|
95
|
-
missing: missing.sort(),
|
|
96
|
-
conflicts: conflicts.sort(),
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
|
|
102
|
-
*/
|
|
103
|
-
export function normalizeEol(str) {
|
|
104
|
-
return typeof str === "string" ? str.replace(/\r\n?/g, "\n") : str;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
/**
|
|
108
|
-
* Atomically write a file via a temporary file in the same directory.
|
|
109
|
-
*/
|
|
110
|
-
export function atomicWriteFileSync(filePath, content) {
|
|
111
|
-
const dir = dirname(filePath);
|
|
112
|
-
mkdirSync(dir, { recursive: true });
|
|
113
|
-
const tmpPath = join(dir, `.tmp-${Date.now()}-${process.pid}-${randomUUID()}`);
|
|
114
|
-
writeFileSync(tmpPath, content, "utf8");
|
|
115
|
-
renameSync(tmpPath, filePath);
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
/**
|
|
119
|
-
* Remove empty parent directories recursively up to stopDir.
|
|
120
|
-
*/
|
|
121
|
-
function cleanEmptyParents(dir, stopDir) {
|
|
122
|
-
let current = resolve(dir);
|
|
123
|
-
const stop = resolve(stopDir);
|
|
124
|
-
while (current && current !== stop && current.startsWith(stop)) {
|
|
125
|
-
try {
|
|
126
|
-
const remaining = readdirSync(current);
|
|
127
|
-
if (remaining.length === 0) {
|
|
128
|
-
rmdirSync(current);
|
|
129
|
-
current = dirname(current);
|
|
130
|
-
} else {
|
|
131
|
-
break;
|
|
132
|
-
}
|
|
133
|
-
} catch {
|
|
134
|
-
break;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
/**
|
|
140
|
-
* Scan a directory recursively for all .geml files.
|
|
141
|
-
* @param {string} dir Root directory to scan.
|
|
142
|
-
* @param {string} [baseDir] Base directory for computing relative paths.
|
|
143
|
-
* @returns {Map<string, string>} Map of relative path (POSIX style) -> file content (normalized LF).
|
|
144
|
-
*/
|
|
145
|
-
export function readGemlFilesFromDisk(dir, baseDir = dir) {
|
|
146
|
-
const files = new Map();
|
|
147
|
-
if (!existsSync(dir)) return files;
|
|
148
|
-
|
|
149
|
-
const entries = readdirSync(dir, { withFileTypes: true });
|
|
150
|
-
for (const entry of entries) {
|
|
151
|
-
const fullPath = join(dir, entry.name);
|
|
152
|
-
if (entry.isDirectory()) {
|
|
153
|
-
// Ignore .git, node_modules, and hidden directories
|
|
154
|
-
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
155
|
-
const subFiles = readGemlFilesFromDisk(fullPath, baseDir);
|
|
156
|
-
for (const [rel, content] of subFiles) {
|
|
157
|
-
files.set(rel, content);
|
|
158
|
-
}
|
|
159
|
-
} else if (entry.isFile() && entry.name.endsWith(".geml")) {
|
|
160
|
-
const rel = relative(baseDir, fullPath).split(sep).join("/");
|
|
161
|
-
files.set(rel, normalizeEol(readFileSync(fullPath, "utf8")));
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
return files;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Count the Markdown pages of an OG graph laid out under dir.
|
|
169
|
-
* readGemlFilesFromDisk answers "what of ours is here"; this answers "is this
|
|
170
|
-
* already somebody's graph", which is a different question and the one the
|
|
171
|
-
* empty-export guard actually asks.
|
|
172
|
-
*/
|
|
173
|
-
function countMarkdownPages(dir) {
|
|
174
|
-
let count = 0;
|
|
175
|
-
for (const sub of ["pages", "journals"]) {
|
|
176
|
-
const root = join(dir, sub);
|
|
177
|
-
if (!existsSync(root)) continue;
|
|
178
|
-
const stack = [root];
|
|
179
|
-
while (stack.length > 0) {
|
|
180
|
-
const current = stack.pop();
|
|
181
|
-
let entries;
|
|
182
|
-
try {
|
|
183
|
-
entries = readdirSync(current, { withFileTypes: true });
|
|
184
|
-
} catch {
|
|
185
|
-
continue;
|
|
186
|
-
}
|
|
187
|
-
for (const entry of entries) {
|
|
188
|
-
if (entry.isDirectory()) {
|
|
189
|
-
if (!entry.name.startsWith(".") && entry.name !== "node_modules") stack.push(join(current, entry.name));
|
|
190
|
-
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
191
|
-
count += 1;
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
return count;
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
/**
|
|
200
|
-
* Incrementally sync a Map of GEML files to disk.
|
|
201
|
-
* Only writes files whose content has changed or do not yet exist.
|
|
202
|
-
* Detects files on disk that are absent from the export (e.g. deleted pages or journals),
|
|
203
|
-
* and reports them without destructive deletion by default.
|
|
204
|
-
*
|
|
205
|
-
* Safety note: @logseq/cli 0.4.3 does not include journals in export-edn.
|
|
206
|
-
* Manifest-based orphan tracking ensures user-authored files outside previous syncs are never deleted.
|
|
207
|
-
* Destructive deletion requires explicit `opts.deleteOrphans === true`.
|
|
208
|
-
*
|
|
209
|
-
* @param {Map<string, string>} gemlFiles Map of relative path (POSIX) -> gemlText.
|
|
210
|
-
* @param {string} targetDir Local destination directory.
|
|
211
|
-
* @param {object} [opts]
|
|
212
|
-
* @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
|
|
213
|
-
* @param {string[]} [opts.preserve] Files NOT to overwrite even when the graph
|
|
214
|
-
* differs — the conflicted files of a two-way cycle. Their manifest entry
|
|
215
|
-
* keeps its previous hash, so they stay flagged until a person resolves them.
|
|
216
|
-
* @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files that exist
|
|
217
|
-
* on disk but that no manifest ever claimed. Off by default: a vault IS a
|
|
218
|
-
* graph, so people point this at one they already have, and a file we never
|
|
219
|
-
* wrote is theirs, not our own echo.
|
|
220
|
-
* @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[], preserved: string[], unmanaged: string[] }}
|
|
221
|
-
*/
|
|
222
|
-
export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
|
|
223
|
-
const deleteOrphans = opts.deleteOrphans ?? false;
|
|
224
|
-
const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
|
|
225
|
-
const preserve = new Set(opts.preserve ?? []);
|
|
226
|
-
const written = [];
|
|
227
|
-
const unchanged = [];
|
|
228
|
-
const orphaned = [];
|
|
229
|
-
const deleted = [];
|
|
230
|
-
const preserved = [];
|
|
231
|
-
const unmanaged = [];
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
const
|
|
239
|
-
const
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
const
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
existingContent !==
|
|
253
|
-
|
|
254
|
-
!
|
|
255
|
-
) {
|
|
256
|
-
// On disk, different from the graph, and no manifest ever claimed it:
|
|
257
|
-
// someone else's file, not our own echo. A file that already MATCHES
|
|
258
|
-
// what we would write falls through and is adopted — identical bytes
|
|
259
|
-
// mean there is nothing of theirs to lose.
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
}
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
if (
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
if (
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
const
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
const
|
|
458
|
-
|
|
459
|
-
//
|
|
460
|
-
//
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
}
|
|
1
|
+
// Core Sync Engine for Logseq GEML
|
|
2
|
+
// Keeps a local folder of .geml files in sync with a Logseq DB graph,
|
|
3
|
+
// ensuring only changed files are written so Git diffs stay clean.
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
readdirSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
writeFileSync,
|
|
9
|
+
unlinkSync,
|
|
10
|
+
mkdirSync,
|
|
11
|
+
rmdirSync,
|
|
12
|
+
renameSync,
|
|
13
|
+
existsSync,
|
|
14
|
+
} from "node:fs";
|
|
15
|
+
import { join, dirname, relative, resolve, sep, isAbsolute } from "node:path";
|
|
16
|
+
import { execFileSync } from "node:child_process";
|
|
17
|
+
import { randomUUID, createHash } from "node:crypto";
|
|
18
|
+
import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
|
|
19
|
+
import { gemlToOgMarkdown } from "./og-markdown.mjs";
|
|
20
|
+
|
|
21
|
+
const MANIFEST_FILE = ".geml-manifest.json";
|
|
22
|
+
// The Markdown tree needs a ledger of its own: `--markdown` takes ANY
|
|
23
|
+
// directory, so it is the tree most likely to be pointed at a graph someone
|
|
24
|
+
// already has, and it must know which .md files are its own writes before it
|
|
25
|
+
// overwrites one. A separate file because markdownDir may BE targetDir.
|
|
26
|
+
const MD_MANIFEST_FILE = ".geml-md-manifest.json";
|
|
27
|
+
|
|
28
|
+
const sha256 = (s) => createHash("sha256").update(s).digest("hex");
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Read the sync manifest in either of its two shapes.
|
|
32
|
+
* v1 was a sorted array of paths — enough to know which files the sync owns.
|
|
33
|
+
* v2 ({ version: 2, files: { rel: sha256 } }) also records the content the
|
|
34
|
+
* sync last wrote or saw, which is what lets two-way sync tell an external
|
|
35
|
+
* edit from its own echo: a file whose hash matches the manifest is the
|
|
36
|
+
* watcher's own last write, not something a person or agent changed.
|
|
37
|
+
* @returns {{ known: boolean, hashed: boolean, files: Map<string, string|null> }}
|
|
38
|
+
*/
|
|
39
|
+
function readManifest(targetDir, manifestFile = MANIFEST_FILE) {
|
|
40
|
+
const manifestPath = join(targetDir, manifestFile);
|
|
41
|
+
if (!existsSync(manifestPath)) return { known: false, hashed: false, files: new Map() };
|
|
42
|
+
try {
|
|
43
|
+
const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
|
|
44
|
+
if (Array.isArray(parsed)) {
|
|
45
|
+
return { known: true, hashed: false, files: new Map(parsed.map((p) => [p, null])) };
|
|
46
|
+
}
|
|
47
|
+
if (parsed && parsed.version === 2 && parsed.files && typeof parsed.files === "object") {
|
|
48
|
+
return { known: true, hashed: true, files: new Map(Object.entries(parsed.files)) };
|
|
49
|
+
}
|
|
50
|
+
} catch {}
|
|
51
|
+
return { known: false, hashed: false, files: new Map() };
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* What changed in the vault since the sync last touched it — the read side of
|
|
56
|
+
* the two-way bridge. Baselines come from the v2 manifest hashes; a v1
|
|
57
|
+
* manifest (or none) knows which files exist but not what they held, so it
|
|
58
|
+
* reports nothing rather than guessing: `baselineKnown: false` means "sync
|
|
59
|
+
* once first".
|
|
60
|
+
*
|
|
61
|
+
* With `graphFiles` (the current export, as ednToGemlFiles returns it) the
|
|
62
|
+
* vault-modified files are split further: one the GRAPH also moved since the
|
|
63
|
+
* last sync is a `conflict` — importing it would clobber the graph's edit,
|
|
64
|
+
* exporting over it would clobber the person's, so two-way sync does neither
|
|
65
|
+
* and a person merges.
|
|
66
|
+
* @param {string} targetDir
|
|
67
|
+
* @param {{ graphFiles?: Map<string, string> }} [opts]
|
|
68
|
+
* @returns {{ baselineKnown: boolean, modified: string[], added: string[], missing: string[], conflicts: string[] }}
|
|
69
|
+
*/
|
|
70
|
+
export function detectExternalEdits(targetDir, opts = {}) {
|
|
71
|
+
const manifest = readManifest(targetDir);
|
|
72
|
+
const onDisk = readGemlFilesFromDisk(targetDir);
|
|
73
|
+
const modified = [];
|
|
74
|
+
const added = [];
|
|
75
|
+
const missing = [];
|
|
76
|
+
const conflicts = [];
|
|
77
|
+
if (!manifest.hashed) return { baselineKnown: false, modified, added, missing, conflicts };
|
|
78
|
+
for (const [rel, hash] of manifest.files) {
|
|
79
|
+
const content = onDisk.get(rel);
|
|
80
|
+
if (content === undefined) missing.push(rel);
|
|
81
|
+
else if (hash !== null && sha256(content) !== hash) {
|
|
82
|
+
const graphContent = opts.graphFiles?.get(rel);
|
|
83
|
+
const graphMoved =
|
|
84
|
+
graphContent !== undefined && sha256(normalizeEol(graphContent)) !== hash;
|
|
85
|
+
(graphMoved ? conflicts : modified).push(rel);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
for (const rel of onDisk.keys()) {
|
|
89
|
+
if (!manifest.files.has(rel)) added.push(rel);
|
|
90
|
+
}
|
|
91
|
+
return {
|
|
92
|
+
baselineKnown: true,
|
|
93
|
+
modified: modified.sort(),
|
|
94
|
+
added: added.sort(),
|
|
95
|
+
missing: missing.sort(),
|
|
96
|
+
conflicts: conflicts.sort(),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
|
|
102
|
+
*/
|
|
103
|
+
export function normalizeEol(str) {
|
|
104
|
+
return typeof str === "string" ? str.replace(/\r\n?/g, "\n") : str;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Atomically write a file via a temporary file in the same directory.
|
|
109
|
+
*/
|
|
110
|
+
export function atomicWriteFileSync(filePath, content) {
|
|
111
|
+
const dir = dirname(filePath);
|
|
112
|
+
mkdirSync(dir, { recursive: true });
|
|
113
|
+
const tmpPath = join(dir, `.tmp-${Date.now()}-${process.pid}-${randomUUID()}`);
|
|
114
|
+
writeFileSync(tmpPath, content, "utf8");
|
|
115
|
+
renameSync(tmpPath, filePath);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Remove empty parent directories recursively up to stopDir.
|
|
120
|
+
*/
|
|
121
|
+
function cleanEmptyParents(dir, stopDir) {
|
|
122
|
+
let current = resolve(dir);
|
|
123
|
+
const stop = resolve(stopDir);
|
|
124
|
+
while (current && current !== stop && current.startsWith(stop)) {
|
|
125
|
+
try {
|
|
126
|
+
const remaining = readdirSync(current);
|
|
127
|
+
if (remaining.length === 0) {
|
|
128
|
+
rmdirSync(current);
|
|
129
|
+
current = dirname(current);
|
|
130
|
+
} else {
|
|
131
|
+
break;
|
|
132
|
+
}
|
|
133
|
+
} catch {
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Scan a directory recursively for all .geml files.
|
|
141
|
+
* @param {string} dir Root directory to scan.
|
|
142
|
+
* @param {string} [baseDir] Base directory for computing relative paths.
|
|
143
|
+
* @returns {Map<string, string>} Map of relative path (POSIX style) -> file content (normalized LF).
|
|
144
|
+
*/
|
|
145
|
+
export function readGemlFilesFromDisk(dir, baseDir = dir) {
|
|
146
|
+
const files = new Map();
|
|
147
|
+
if (!existsSync(dir)) return files;
|
|
148
|
+
|
|
149
|
+
const entries = readdirSync(dir, { withFileTypes: true });
|
|
150
|
+
for (const entry of entries) {
|
|
151
|
+
const fullPath = join(dir, entry.name);
|
|
152
|
+
if (entry.isDirectory()) {
|
|
153
|
+
// Ignore .git, node_modules, and hidden directories
|
|
154
|
+
if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
|
|
155
|
+
const subFiles = readGemlFilesFromDisk(fullPath, baseDir);
|
|
156
|
+
for (const [rel, content] of subFiles) {
|
|
157
|
+
files.set(rel, content);
|
|
158
|
+
}
|
|
159
|
+
} else if (entry.isFile() && entry.name.endsWith(".geml")) {
|
|
160
|
+
const rel = relative(baseDir, fullPath).split(sep).join("/");
|
|
161
|
+
files.set(rel, normalizeEol(readFileSync(fullPath, "utf8")));
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return files;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/**
|
|
168
|
+
* Count the Markdown pages of an OG graph laid out under dir.
|
|
169
|
+
* readGemlFilesFromDisk answers "what of ours is here"; this answers "is this
|
|
170
|
+
* already somebody's graph", which is a different question and the one the
|
|
171
|
+
* empty-export guard actually asks.
|
|
172
|
+
*/
|
|
173
|
+
function countMarkdownPages(dir) {
|
|
174
|
+
let count = 0;
|
|
175
|
+
for (const sub of ["pages", "journals"]) {
|
|
176
|
+
const root = join(dir, sub);
|
|
177
|
+
if (!existsSync(root)) continue;
|
|
178
|
+
const stack = [root];
|
|
179
|
+
while (stack.length > 0) {
|
|
180
|
+
const current = stack.pop();
|
|
181
|
+
let entries;
|
|
182
|
+
try {
|
|
183
|
+
entries = readdirSync(current, { withFileTypes: true });
|
|
184
|
+
} catch {
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
187
|
+
for (const entry of entries) {
|
|
188
|
+
if (entry.isDirectory()) {
|
|
189
|
+
if (!entry.name.startsWith(".") && entry.name !== "node_modules") stack.push(join(current, entry.name));
|
|
190
|
+
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
191
|
+
count += 1;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return count;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Incrementally sync a Map of GEML files to disk.
|
|
201
|
+
* Only writes files whose content has changed or do not yet exist.
|
|
202
|
+
* Detects files on disk that are absent from the export (e.g. deleted pages or journals),
|
|
203
|
+
* and reports them without destructive deletion by default.
|
|
204
|
+
*
|
|
205
|
+
* Safety note: @logseq/cli 0.4.3 does not include journals in export-edn.
|
|
206
|
+
* Manifest-based orphan tracking ensures user-authored files outside previous syncs are never deleted.
|
|
207
|
+
* Destructive deletion requires explicit `opts.deleteOrphans === true`.
|
|
208
|
+
*
|
|
209
|
+
* @param {Map<string, string>} gemlFiles Map of relative path (POSIX) -> gemlText.
|
|
210
|
+
* @param {string} targetDir Local destination directory.
|
|
211
|
+
* @param {object} [opts]
|
|
212
|
+
* @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
|
|
213
|
+
* @param {string[]} [opts.preserve] Files NOT to overwrite even when the graph
|
|
214
|
+
* differs — the conflicted files of a two-way cycle. Their manifest entry
|
|
215
|
+
* keeps its previous hash, so they stay flagged until a person resolves them.
|
|
216
|
+
* @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files that exist
|
|
217
|
+
* on disk but that no manifest ever claimed. Off by default: a vault IS a
|
|
218
|
+
* graph, so people point this at one they already have, and a file we never
|
|
219
|
+
* wrote is theirs, not our own echo.
|
|
220
|
+
* @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[], preserved: string[], unmanaged: string[], overwritten: string[] }}
|
|
221
|
+
*/
|
|
222
|
+
export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
|
|
223
|
+
const deleteOrphans = opts.deleteOrphans ?? false;
|
|
224
|
+
const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
|
|
225
|
+
const preserve = new Set(opts.preserve ?? []);
|
|
226
|
+
const written = [];
|
|
227
|
+
const unchanged = [];
|
|
228
|
+
const orphaned = [];
|
|
229
|
+
const deleted = [];
|
|
230
|
+
const preserved = [];
|
|
231
|
+
const unmanaged = [];
|
|
232
|
+
const overwritten = []; // unmanaged, and taken anyway because the caller asked
|
|
233
|
+
|
|
234
|
+
mkdirSync(targetDir, { recursive: true });
|
|
235
|
+
const existingFiles = readGemlFilesFromDisk(targetDir);
|
|
236
|
+
|
|
237
|
+
// Load previous sync manifest to know which files belong to sync vs user-authored files
|
|
238
|
+
const manifestPath = join(targetDir, MANIFEST_FILE);
|
|
239
|
+
const previous = readManifest(targetDir);
|
|
240
|
+
const lastManifest = new Set(previous.files.keys());
|
|
241
|
+
|
|
242
|
+
// Write new or updated files atomically with CRLF normalization
|
|
243
|
+
for (const [rel, newContent] of gemlFiles) {
|
|
244
|
+
const fullPath = join(targetDir, rel);
|
|
245
|
+
const normNew = normalizeEol(newContent);
|
|
246
|
+
const existingContent = existingFiles.get(rel);
|
|
247
|
+
|
|
248
|
+
if (preserve.has(rel)) {
|
|
249
|
+
if (existingContent !== normNew) preserved.push(rel);
|
|
250
|
+
else unchanged.push(rel);
|
|
251
|
+
} else if (
|
|
252
|
+
existingContent !== undefined &&
|
|
253
|
+
existingContent !== normNew &&
|
|
254
|
+
!lastManifest.has(rel)
|
|
255
|
+
) {
|
|
256
|
+
// On disk, different from the graph, and no manifest ever claimed it:
|
|
257
|
+
// someone else's file, not our own echo. A file that already MATCHES
|
|
258
|
+
// what we would write falls through and is adopted — identical bytes
|
|
259
|
+
// mean there is nothing of theirs to lose.
|
|
260
|
+
//
|
|
261
|
+
// Overwriting is a CHOICE, and it is named either way. A mode that
|
|
262
|
+
// discards somebody's edit without saying which file it was cannot be
|
|
263
|
+
// trusted with a graph — the list is the input the person's next step
|
|
264
|
+
// needs, and it is gone the moment it is not printed.
|
|
265
|
+
if (!overwriteUnmanaged) {
|
|
266
|
+
unmanaged.push(rel);
|
|
267
|
+
} else {
|
|
268
|
+
atomicWriteFileSync(fullPath, normNew);
|
|
269
|
+
written.push(rel);
|
|
270
|
+
overwritten.push(rel);
|
|
271
|
+
}
|
|
272
|
+
} else if (existingContent === undefined || existingContent !== normNew) {
|
|
273
|
+
atomicWriteFileSync(fullPath, normNew);
|
|
274
|
+
written.push(rel);
|
|
275
|
+
} else {
|
|
276
|
+
unchanged.push(rel);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
// Detect files present on disk but absent from current graph export
|
|
281
|
+
for (const [rel] of existingFiles) {
|
|
282
|
+
if (!gemlFiles.has(rel)) {
|
|
283
|
+
orphaned.push(rel);
|
|
284
|
+
// Safe deletion: only delete if explicit AND file was generated by previous sync
|
|
285
|
+
// User-authored files in targetDir not in manifest are NEVER deleted.
|
|
286
|
+
if (deleteOrphans && lastManifest.has(rel)) {
|
|
287
|
+
const fullPath = join(targetDir, rel);
|
|
288
|
+
if (existsSync(fullPath)) {
|
|
289
|
+
unlinkSync(fullPath);
|
|
290
|
+
cleanEmptyParents(dirname(fullPath), targetDir);
|
|
291
|
+
deleted.push(rel);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// Save updated manifest of managed sync files: all current gemlFiles, plus
|
|
298
|
+
// any existing files on disk that were in lastManifest and not deleted.
|
|
299
|
+
// v2 records each file's content hash AS OF THIS SYNC — the baseline
|
|
300
|
+
// detectExternalEdits() compares against, so the watcher's own writes never
|
|
301
|
+
// read as someone else's edits.
|
|
302
|
+
const currentManifest = new Set(gemlFiles.keys());
|
|
303
|
+
// A held file stays unowned: recording a hash for it would make the next run
|
|
304
|
+
// read the person's content as the sync's own last write and clobber it.
|
|
305
|
+
for (const rel of unmanaged) currentManifest.delete(rel);
|
|
306
|
+
for (const rel of lastManifest) {
|
|
307
|
+
if (existingFiles.has(rel) && !deleted.includes(rel)) {
|
|
308
|
+
currentManifest.add(rel);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
const manifestFiles = {};
|
|
312
|
+
for (const rel of [...currentManifest].sort()) {
|
|
313
|
+
if (preserved.includes(rel)) {
|
|
314
|
+
// A conflicted file keeps its OLD baseline: recording what sits on disk
|
|
315
|
+
// now would make the person's unmerged edit read as "already synced" on
|
|
316
|
+
// the next cycle, and the conflict would be silently forgotten.
|
|
317
|
+
manifestFiles[rel] = previous.files.get(rel) ?? null;
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
const content = gemlFiles.has(rel) ? normalizeEol(gemlFiles.get(rel)) : existingFiles.get(rel);
|
|
321
|
+
manifestFiles[rel] = content === undefined ? null : sha256(content);
|
|
322
|
+
}
|
|
323
|
+
atomicWriteFileSync(manifestPath, JSON.stringify({ version: 2, files: manifestFiles }, null, 1) + "\n");
|
|
324
|
+
|
|
325
|
+
return { written, orphaned, unchanged, deleted, preserved, unmanaged, overwritten };
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/**
|
|
329
|
+
* Execute Git commands to commit changes scoped strictly to the synced files.
|
|
330
|
+
* Protects parent repository from having unrelated files swept into the commit.
|
|
331
|
+
*
|
|
332
|
+
* @param {string} targetDir Directory where the sync target lives.
|
|
333
|
+
* @param {string} commitMessage Commit message.
|
|
334
|
+
* @param {string[]} pathsToCommit Relative paths within targetDir that were written or deleted.
|
|
335
|
+
* @param {function} [gitRunner] Optional custom git runner `(args) => Promise<{ stdout, stderr, exitCode }>`.
|
|
336
|
+
* @returns {Promise<{ committed: boolean, changes: boolean, output: string }>}
|
|
337
|
+
*/
|
|
338
|
+
export async function gitAutoCommit(targetDir, commitMessage = "logseq-geml sync", pathsToCommit = [], gitRunner = null,
|
|
339
|
+
// Where the manifest sits RELATIVE TO THE REPO. It used to be assumed to be
|
|
340
|
+
// the repo root, which held only while the vault and the GEML tree were the
|
|
341
|
+
// same directory; once the tree moved into `.logseq-sync-vault-with-geml/`
|
|
342
|
+
// that assumption staged a pathspec matching nothing and lost the commit.
|
|
343
|
+
manifestPath = MANIFEST_FILE) {
|
|
344
|
+
const defaultRunner = async (args) => {
|
|
345
|
+
try {
|
|
346
|
+
const stdout = execFileSync("git", args, {
|
|
347
|
+
cwd: targetDir,
|
|
348
|
+
encoding: "utf8",
|
|
349
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
350
|
+
});
|
|
351
|
+
return { stdout, stderr: "", exitCode: 0 };
|
|
352
|
+
} catch (err) {
|
|
353
|
+
return {
|
|
354
|
+
stdout: err.stdout ? String(err.stdout) : "",
|
|
355
|
+
stderr: err.stderr ? String(err.stderr) : err.message,
|
|
356
|
+
exitCode: err.status || 1,
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
};
|
|
360
|
+
|
|
361
|
+
const run = gitRunner || defaultRunner;
|
|
362
|
+
|
|
363
|
+
// Check if targetDir is inside a git repository
|
|
364
|
+
const revRes = await run(["rev-parse", "--show-toplevel"]);
|
|
365
|
+
if (revRes.exitCode !== 0) {
|
|
366
|
+
return {
|
|
367
|
+
committed: false,
|
|
368
|
+
changes: false,
|
|
369
|
+
output: `Not a git repository: ${revRes.stderr.trim()}`,
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
if (!pathsToCommit || pathsToCommit.length === 0) {
|
|
374
|
+
return {
|
|
375
|
+
committed: false,
|
|
376
|
+
changes: false,
|
|
377
|
+
output: "No synced paths to commit.",
|
|
378
|
+
};
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
// Always include the manifest file in staged paths
|
|
382
|
+
const allPaths = [...new Set([...pathsToCommit, manifestPath])];
|
|
383
|
+
|
|
384
|
+
// Stage ONLY the specified paths (never a bare git add -A)
|
|
385
|
+
// Split into existing files vs deleted files
|
|
386
|
+
const toAdd = allPaths.filter((p) => existsSync(join(targetDir, p)));
|
|
387
|
+
const toRemove = allPaths.filter((p) => !existsSync(join(targetDir, p)));
|
|
388
|
+
|
|
389
|
+
if (toAdd.length > 0) {
|
|
390
|
+
const addRes = await run(["add", "--", ...toAdd]);
|
|
391
|
+
if (addRes.exitCode !== 0) {
|
|
392
|
+
return { committed: false, changes: true, output: `git add failed: ${addRes.stderr}` };
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
if (toRemove.length > 0) {
|
|
397
|
+
const rmRes = await run(["add", "-u", "--", ...toRemove]);
|
|
398
|
+
if (rmRes.exitCode !== 0) {
|
|
399
|
+
return { committed: false, changes: true, output: `git update index failed: ${rmRes.stderr}` };
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Verify whether our target paths have staged changes
|
|
404
|
+
const statusRes = await run(["status", "--porcelain", "--", ...allPaths]);
|
|
405
|
+
if (statusRes.exitCode !== 0 || statusRes.stdout.trim().length === 0) {
|
|
406
|
+
return {
|
|
407
|
+
committed: false,
|
|
408
|
+
changes: false,
|
|
409
|
+
output: "No changes in target paths to commit.",
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
// Commit with pathspec: commits ONLY changes matching our synced paths,
|
|
414
|
+
// leaving any other staged or unstaged changes in parent repository untouched!
|
|
415
|
+
const commitRes = await run(["commit", "-m", commitMessage, "--", ...allPaths]);
|
|
416
|
+
if (commitRes.exitCode !== 0) {
|
|
417
|
+
return {
|
|
418
|
+
committed: false,
|
|
419
|
+
changes: true,
|
|
420
|
+
output: `git commit failed: ${commitRes.stderr}`,
|
|
421
|
+
};
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
return {
|
|
425
|
+
committed: true,
|
|
426
|
+
changes: true,
|
|
427
|
+
output: commitRes.stdout.trim(),
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/**
|
|
432
|
+
* Full Sync Pipeline from EDN string to disk.
|
|
433
|
+
*
|
|
434
|
+
* @param {string} ednText EDN string (from logseq export-edn).
|
|
435
|
+
* @param {string} targetDir Destination folder.
|
|
436
|
+
* @param {object} [opts]
|
|
437
|
+
* @param {boolean} [opts.autoCommit=false]
|
|
438
|
+
* @param {string} [opts.commitMessage]
|
|
439
|
+
* @param {boolean} [opts.allowEmptyGraph=false] Refuse 0-page export over non-empty targetDir unless true.
|
|
440
|
+
* @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files no manifest
|
|
441
|
+
* ever claimed, in BOTH trees. Off by default — see writeGemlFilesToDisk.
|
|
442
|
+
* @param {function} [opts.gitRunner]
|
|
443
|
+
* @returns {Promise<{ written: string[], deleted: string[], orphaned: string[], unchanged: string[], unmanaged: string[], markdownWritten: string[], markdownUnmanaged: string[], gitResult?: any }>}
|
|
444
|
+
*/
|
|
445
|
+
export async function syncEdnToDisk(ednText, targetDir, opts = {}) {
|
|
446
|
+
const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
|
|
447
|
+
|
|
448
|
+
// Guard 1: Refuse empty or truncated EDN input
|
|
449
|
+
if (!ednText || typeof ednText !== "string" || ednText.trim().length === 0) {
|
|
450
|
+
throw new Error("EDN input is empty or truncated; refusing to sync to prevent data loss.");
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
const gemlFiles = ednToGemlFiles(ednText);
|
|
454
|
+
|
|
455
|
+
// Guard 2: Refuse 0-page export if targetDir already has existing pages
|
|
456
|
+
const pageCount = [...gemlFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length;
|
|
457
|
+
const existingFiles = readGemlFilesFromDisk(targetDir);
|
|
458
|
+
// Counting only .geml left this guard blind to exactly what it guards
|
|
459
|
+
// against: a directory that is ALREADY an OG graph holds its pages as
|
|
460
|
+
// Markdown, and readGemlFilesFromDisk does not see one of them.
|
|
461
|
+
const markdownDir = opts.markdownDir ?? null;
|
|
462
|
+
const existingPageCount =
|
|
463
|
+
[...existingFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length +
|
|
464
|
+
countMarkdownPages(targetDir) +
|
|
465
|
+
(markdownDir && resolve(markdownDir) !== resolve(targetDir) ? countMarkdownPages(markdownDir) : 0);
|
|
466
|
+
|
|
467
|
+
if (pageCount === 0 && existingPageCount > 0 && !opts.allowEmptyGraph) {
|
|
468
|
+
throw new Error(
|
|
469
|
+
`Refusing to sync empty graph (0 pages) over directory with existing pages (${existingPageCount} pages). Pass allowEmptyGraph: true to force.`
|
|
470
|
+
);
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
const diffResult = writeGemlFilesToDisk(gemlFiles, targetDir, opts);
|
|
474
|
+
|
|
475
|
+
// A parallel Markdown tree in LOGSEQ'S OWN dialect — bullets, `id::`,
|
|
476
|
+
// `((uuid))` refs — so the directory opens as a graph in the file version of
|
|
477
|
+
// the app. Generic GEML-to-Markdown is `geml <file> --to md`, the parser's
|
|
478
|
+
// job; the only reason this integration writes Markdown at all is Logseq,
|
|
479
|
+
// and writing anything else here would throw away the uuids and the outline
|
|
480
|
+
// depth that make an OG graph an OG graph.
|
|
481
|
+
//
|
|
482
|
+
// Deliberately lossy and one-way: the GEML tree stays the one that
|
|
483
|
+
// round-trips, and `restore` never reads this. The parser library is
|
|
484
|
+
// injected, so core keeps its single dependency.
|
|
485
|
+
const markdownWritten = [];
|
|
486
|
+
const markdownUnmanaged = [];
|
|
487
|
+
const markdownOverwritten = [];
|
|
488
|
+
if (markdownDir && opts.lib) {
|
|
489
|
+
const mdPrevious = readManifest(markdownDir, MD_MANIFEST_FILE);
|
|
490
|
+
const mdManifest = {};
|
|
491
|
+
for (const [rel, content] of gemlFiles) {
|
|
492
|
+
// Only pages and journals are a graph; the index and the ontology carry
|
|
493
|
+
// machine bookkeeping OG has no page for.
|
|
494
|
+
if (!rel.startsWith("pages/") && !rel.startsWith("journals/")) continue;
|
|
495
|
+
const mdRel = rel.replace(/\.geml$/, ".md");
|
|
496
|
+
const full = join(markdownDir, mdRel);
|
|
497
|
+
let md;
|
|
498
|
+
try {
|
|
499
|
+
md = normalizeEol(gemlToOgMarkdown(content, opts.lib));
|
|
500
|
+
} catch {
|
|
501
|
+
continue; // one unconvertible document must not fail the sync
|
|
502
|
+
}
|
|
503
|
+
if (md === "") continue; // nothing OG can hold — write no file
|
|
504
|
+
const onDisk = existsSync(full) ? normalizeEol(readFileSync(full, "utf8")) : null;
|
|
505
|
+
// "Ours" means the bytes still MATCH WHAT WE RECORDED WRITING, not merely
|
|
506
|
+
// that the path appears in the ledger. Membership alone answers "did we
|
|
507
|
+
// ever write this file", and the question is "is this still our file" —
|
|
508
|
+
// so an edit to a page we had written was read as our own echo and
|
|
509
|
+
// overwritten, which is the exact loss this ledger exists to prevent.
|
|
510
|
+
//
|
|
511
|
+
// The GEML tree can afford the looser test: a `.geml` edit is recoverable
|
|
512
|
+
// through the two-way bridge. A Markdown edit is not — the mapping is
|
|
513
|
+
// lossy and one-way, and importing it back is out of scope by design — so
|
|
514
|
+
// overwriting one destroys it permanently.
|
|
515
|
+
//
|
|
516
|
+
// A v1 manifest has no hashes: it can say "we wrote this" and not "this is
|
|
517
|
+
// still what we wrote". Unknown counts as ours, so upgrading does not hold
|
|
518
|
+
// the entire tree on the first run after it.
|
|
519
|
+
const recorded = mdPrevious.files.get(mdRel);
|
|
520
|
+
const stillOurs =
|
|
521
|
+
mdPrevious.files.has(mdRel) &&
|
|
522
|
+
(!mdPrevious.hashed || recorded === null || recorded === sha256(onDisk ?? ""));
|
|
523
|
+
if (onDisk !== null && onDisk !== md && !stillOurs && overwriteUnmanaged) {
|
|
524
|
+
// Named, not silent — same rule as the GEML tree, and it matters more
|
|
525
|
+
// here: a Markdown edit cannot come back through the bridge.
|
|
526
|
+
markdownOverwritten.push(mdRel);
|
|
527
|
+
}
|
|
528
|
+
if (onDisk !== null && onDisk !== md && !stillOurs && !overwriteUnmanaged) {
|
|
529
|
+
// `--markdown` takes any directory, and the vault root now IS a graph
|
|
530
|
+
// people open, so somebody's own pages/*.md is precisely what this
|
|
531
|
+
// lands on. Held, named, and left exactly as they wrote it.
|
|
532
|
+
markdownUnmanaged.push(mdRel);
|
|
533
|
+
continue;
|
|
534
|
+
}
|
|
535
|
+
if (onDisk !== md) {
|
|
536
|
+
mkdirSync(dirname(full), { recursive: true });
|
|
537
|
+
atomicWriteFileSync(full, md);
|
|
538
|
+
markdownWritten.push(mdRel);
|
|
539
|
+
}
|
|
540
|
+
mdManifest[mdRel] = sha256(md);
|
|
541
|
+
}
|
|
542
|
+
// A page the graph stopped exporting keeps its entry while the file is
|
|
543
|
+
// still there: dropping it would make our own past write read as a
|
|
544
|
+
// stranger's on the next run, and the sync would refuse to touch it.
|
|
545
|
+
for (const [mdRel, hash] of mdPrevious.files) {
|
|
546
|
+
if (!(mdRel in mdManifest) && existsSync(join(markdownDir, mdRel))) mdManifest[mdRel] = hash;
|
|
547
|
+
}
|
|
548
|
+
const mdManifestPath = join(markdownDir, MD_MANIFEST_FILE);
|
|
549
|
+
const mdManifestFiles = {};
|
|
550
|
+
for (const mdRel of Object.keys(mdManifest).sort()) mdManifestFiles[mdRel] = mdManifest[mdRel];
|
|
551
|
+
const mdManifestText = JSON.stringify({ version: 2, files: mdManifestFiles }, null, 1) + "\n";
|
|
552
|
+
const mdManifestExists = existsSync(mdManifestPath);
|
|
553
|
+
// Only when it actually changed: this file lives inside someone's OG graph,
|
|
554
|
+
// and rewriting it every poll would have Logseq re-reading it forever.
|
|
555
|
+
if (
|
|
556
|
+
(Object.keys(mdManifestFiles).length > 0 || mdManifestExists) &&
|
|
557
|
+
(!mdManifestExists || readFileSync(mdManifestPath, "utf8") !== mdManifestText)
|
|
558
|
+
) {
|
|
559
|
+
atomicWriteFileSync(mdManifestPath, mdManifestText);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
let gitResult = null;
|
|
564
|
+
// WHERE THE GEML TREE LIVES AND WHAT GIT COVERS ARE NOT THE SAME DIRECTORY.
|
|
565
|
+
// They used to be, because the vault WAS the GEML tree. With Markdown at the
|
|
566
|
+
// vault root and GEML in a dot directory beneath it, a repo rooted at the
|
|
567
|
+
// GEML tree would silently drop every Markdown file from the commit — the
|
|
568
|
+
// half of the vault a person actually reads. `gitDir` defaults to targetDir,
|
|
569
|
+
// so a caller that has not moved anything keeps exactly its old behaviour.
|
|
570
|
+
const gitDir = opts.gitDir ?? targetDir;
|
|
571
|
+
const inGit = (abs) => {
|
|
572
|
+
const rel = relative(gitDir, abs);
|
|
573
|
+
return rel && !rel.startsWith("..") && !isAbsolute(rel) ? rel : null;
|
|
574
|
+
};
|
|
575
|
+
const pathsModified = [];
|
|
576
|
+
for (const rel of [...diffResult.written, ...diffResult.deleted]) {
|
|
577
|
+
const p = inGit(join(targetDir, rel));
|
|
578
|
+
if (p) pathsModified.push(p);
|
|
579
|
+
}
|
|
580
|
+
for (const rel of markdownWritten) {
|
|
581
|
+
const p = inGit(join(markdownDir, rel));
|
|
582
|
+
if (p) pathsModified.push(p);
|
|
583
|
+
}
|
|
584
|
+
// The Markdown ledger is versioned WITH the tree it describes. Leaving it out
|
|
585
|
+
// is not a tidiness question: the .md files are committed, so a clone restores
|
|
586
|
+
// them and not the record of who wrote them, and the next sync reads every
|
|
587
|
+
// changed page as a stranger's and holds it — the Markdown tree stops updating
|
|
588
|
+
// and says only that it is protecting files nobody edited. Measured on a
|
|
589
|
+
// simulated clone before it was fixed, not feared.
|
|
590
|
+
if (markdownDir && opts.lib) {
|
|
591
|
+
const p = inGit(join(markdownDir, MD_MANIFEST_FILE));
|
|
592
|
+
if (p) pathsModified.push(p);
|
|
593
|
+
}
|
|
594
|
+
|
|
595
|
+
if (opts.autoCommit && pathsModified.length > 0) {
|
|
596
|
+
const msg = opts.commitMessage || `logseq-geml: synced ${diffResult.written.length} modified, ${diffResult.deleted.length} deleted`;
|
|
597
|
+
gitResult = await gitAutoCommit(gitDir, msg, pathsModified, opts.gitRunner, inGit(join(targetDir, MANIFEST_FILE)) ?? MANIFEST_FILE);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
return {
|
|
601
|
+
...diffResult,
|
|
602
|
+
markdownWritten,
|
|
603
|
+
markdownUnmanaged,
|
|
604
|
+
markdownOverwritten,
|
|
605
|
+
gitResult,
|
|
606
|
+
};
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
/**
|
|
610
|
+
* Full Sync Pipeline from disk back to EDN string.
|
|
611
|
+
*
|
|
612
|
+
* @param {string} targetDir Local folder containing .geml files.
|
|
613
|
+
* @param {object} lib Parser library containing { parse, addressedUnits, sliceUnit }.
|
|
614
|
+
* @param {{ exclude?: string[] }} [opts] Files to leave OUT of the import —
|
|
615
|
+
* the conflicted files of a two-way cycle: absent from the EDN means the
|
|
616
|
+
* graph's version stays untouched (import merges by uuid, it never deletes).
|
|
617
|
+
* @returns {string} EDN string ready for logseq import-edn.
|
|
618
|
+
*/
|
|
619
|
+
export function syncDiskToEdn(targetDir, lib, opts = {}) {
|
|
620
|
+
const files = readGemlFilesFromDisk(targetDir);
|
|
621
|
+
for (const rel of opts.exclude ?? []) files.delete(rel);
|
|
622
|
+
return gemlFilesToEdn(files, lib);
|
|
623
|
+
}
|