@geml/logseq-sync 2.0.9 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,560 +1,623 @@
1
- // Core Sync Engine for Logseq GEML
2
- // Keeps a local folder of .geml files in sync with a Logseq DB graph,
3
- // ensuring only changed files are written so Git diffs stay clean.
4
-
5
- import {
6
- readdirSync,
7
- readFileSync,
8
- writeFileSync,
9
- unlinkSync,
10
- mkdirSync,
11
- rmdirSync,
12
- renameSync,
13
- existsSync,
14
- } from "node:fs";
15
- import { join, dirname, relative, resolve, sep, isAbsolute } from "node:path";
16
- import { execFileSync } from "node:child_process";
17
- import { randomUUID, createHash } from "node:crypto";
18
- import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
19
- import { gemlToOgMarkdown } from "./og-markdown.mjs";
20
-
21
- const MANIFEST_FILE = ".geml-manifest.json";
22
- // The Markdown tree needs a ledger of its own: `--markdown` takes ANY
23
- // directory, so it is the tree most likely to be pointed at a graph someone
24
- // already has, and it must know which .md files are its own writes before it
25
- // overwrites one. A separate file because markdownDir may BE targetDir.
26
- const MD_MANIFEST_FILE = ".geml-md-manifest.json";
27
-
28
- const sha256 = (s) => createHash("sha256").update(s).digest("hex");
29
-
30
- /**
31
- * Read the sync manifest in either of its two shapes.
32
- * v1 was a sorted array of paths — enough to know which files the sync owns.
33
- * v2 ({ version: 2, files: { rel: sha256 } }) also records the content the
34
- * sync last wrote or saw, which is what lets two-way sync tell an external
35
- * edit from its own echo: a file whose hash matches the manifest is the
36
- * watcher's own last write, not something a person or agent changed.
37
- * @returns {{ known: boolean, hashed: boolean, files: Map<string, string|null> }}
38
- */
39
- function readManifest(targetDir, manifestFile = MANIFEST_FILE) {
40
- const manifestPath = join(targetDir, manifestFile);
41
- if (!existsSync(manifestPath)) return { known: false, hashed: false, files: new Map() };
42
- try {
43
- const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
44
- if (Array.isArray(parsed)) {
45
- return { known: true, hashed: false, files: new Map(parsed.map((p) => [p, null])) };
46
- }
47
- if (parsed && parsed.version === 2 && parsed.files && typeof parsed.files === "object") {
48
- return { known: true, hashed: true, files: new Map(Object.entries(parsed.files)) };
49
- }
50
- } catch {}
51
- return { known: false, hashed: false, files: new Map() };
52
- }
53
-
54
- /**
55
- * What changed in the vault since the sync last touched it — the read side of
56
- * the two-way bridge. Baselines come from the v2 manifest hashes; a v1
57
- * manifest (or none) knows which files exist but not what they held, so it
58
- * reports nothing rather than guessing: `baselineKnown: false` means "sync
59
- * once first".
60
- *
61
- * With `graphFiles` (the current export, as ednToGemlFiles returns it) the
62
- * vault-modified files are split further: one the GRAPH also moved since the
63
- * last sync is a `conflict` — importing it would clobber the graph's edit,
64
- * exporting over it would clobber the person's, so two-way sync does neither
65
- * and a person merges.
66
- * @param {string} targetDir
67
- * @param {{ graphFiles?: Map<string, string> }} [opts]
68
- * @returns {{ baselineKnown: boolean, modified: string[], added: string[], missing: string[], conflicts: string[] }}
69
- */
70
- export function detectExternalEdits(targetDir, opts = {}) {
71
- const manifest = readManifest(targetDir);
72
- const onDisk = readGemlFilesFromDisk(targetDir);
73
- const modified = [];
74
- const added = [];
75
- const missing = [];
76
- const conflicts = [];
77
- if (!manifest.hashed) return { baselineKnown: false, modified, added, missing, conflicts };
78
- for (const [rel, hash] of manifest.files) {
79
- const content = onDisk.get(rel);
80
- if (content === undefined) missing.push(rel);
81
- else if (hash !== null && sha256(content) !== hash) {
82
- const graphContent = opts.graphFiles?.get(rel);
83
- const graphMoved =
84
- graphContent !== undefined && sha256(normalizeEol(graphContent)) !== hash;
85
- (graphMoved ? conflicts : modified).push(rel);
86
- }
87
- }
88
- for (const rel of onDisk.keys()) {
89
- if (!manifest.files.has(rel)) added.push(rel);
90
- }
91
- return {
92
- baselineKnown: true,
93
- modified: modified.sort(),
94
- added: added.sort(),
95
- missing: missing.sort(),
96
- conflicts: conflicts.sort(),
97
- };
98
- }
99
-
100
- /**
101
- * Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
102
- */
103
- export function normalizeEol(str) {
104
- return typeof str === "string" ? str.replace(/\r\n?/g, "\n") : str;
105
- }
106
-
107
- /**
108
- * Atomically write a file via a temporary file in the same directory.
109
- */
110
- export function atomicWriteFileSync(filePath, content) {
111
- const dir = dirname(filePath);
112
- mkdirSync(dir, { recursive: true });
113
- const tmpPath = join(dir, `.tmp-${Date.now()}-${process.pid}-${randomUUID()}`);
114
- writeFileSync(tmpPath, content, "utf8");
115
- renameSync(tmpPath, filePath);
116
- }
117
-
118
- /**
119
- * Remove empty parent directories recursively up to stopDir.
120
- */
121
- function cleanEmptyParents(dir, stopDir) {
122
- let current = resolve(dir);
123
- const stop = resolve(stopDir);
124
- while (current && current !== stop && current.startsWith(stop)) {
125
- try {
126
- const remaining = readdirSync(current);
127
- if (remaining.length === 0) {
128
- rmdirSync(current);
129
- current = dirname(current);
130
- } else {
131
- break;
132
- }
133
- } catch {
134
- break;
135
- }
136
- }
137
- }
138
-
139
- /**
140
- * Scan a directory recursively for all .geml files.
141
- * @param {string} dir Root directory to scan.
142
- * @param {string} [baseDir] Base directory for computing relative paths.
143
- * @returns {Map<string, string>} Map of relative path (POSIX style) -> file content (normalized LF).
144
- */
145
- export function readGemlFilesFromDisk(dir, baseDir = dir) {
146
- const files = new Map();
147
- if (!existsSync(dir)) return files;
148
-
149
- const entries = readdirSync(dir, { withFileTypes: true });
150
- for (const entry of entries) {
151
- const fullPath = join(dir, entry.name);
152
- if (entry.isDirectory()) {
153
- // Ignore .git, node_modules, and hidden directories
154
- if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
155
- const subFiles = readGemlFilesFromDisk(fullPath, baseDir);
156
- for (const [rel, content] of subFiles) {
157
- files.set(rel, content);
158
- }
159
- } else if (entry.isFile() && entry.name.endsWith(".geml")) {
160
- const rel = relative(baseDir, fullPath).split(sep).join("/");
161
- files.set(rel, normalizeEol(readFileSync(fullPath, "utf8")));
162
- }
163
- }
164
- return files;
165
- }
166
-
167
- /**
168
- * Count the Markdown pages of an OG graph laid out under dir.
169
- * readGemlFilesFromDisk answers "what of ours is here"; this answers "is this
170
- * already somebody's graph", which is a different question and the one the
171
- * empty-export guard actually asks.
172
- */
173
- function countMarkdownPages(dir) {
174
- let count = 0;
175
- for (const sub of ["pages", "journals"]) {
176
- const root = join(dir, sub);
177
- if (!existsSync(root)) continue;
178
- const stack = [root];
179
- while (stack.length > 0) {
180
- const current = stack.pop();
181
- let entries;
182
- try {
183
- entries = readdirSync(current, { withFileTypes: true });
184
- } catch {
185
- continue;
186
- }
187
- for (const entry of entries) {
188
- if (entry.isDirectory()) {
189
- if (!entry.name.startsWith(".") && entry.name !== "node_modules") stack.push(join(current, entry.name));
190
- } else if (entry.isFile() && entry.name.endsWith(".md")) {
191
- count += 1;
192
- }
193
- }
194
- }
195
- }
196
- return count;
197
- }
198
-
199
- /**
200
- * Incrementally sync a Map of GEML files to disk.
201
- * Only writes files whose content has changed or do not yet exist.
202
- * Detects files on disk that are absent from the export (e.g. deleted pages or journals),
203
- * and reports them without destructive deletion by default.
204
- *
205
- * Safety note: @logseq/cli 0.4.3 does not include journals in export-edn.
206
- * Manifest-based orphan tracking ensures user-authored files outside previous syncs are never deleted.
207
- * Destructive deletion requires explicit `opts.deleteOrphans === true`.
208
- *
209
- * @param {Map<string, string>} gemlFiles Map of relative path (POSIX) -> gemlText.
210
- * @param {string} targetDir Local destination directory.
211
- * @param {object} [opts]
212
- * @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
213
- * @param {string[]} [opts.preserve] Files NOT to overwrite even when the graph
214
- * differs — the conflicted files of a two-way cycle. Their manifest entry
215
- * keeps its previous hash, so they stay flagged until a person resolves them.
216
- * @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files that exist
217
- * on disk but that no manifest ever claimed. Off by default: a vault IS a
218
- * graph, so people point this at one they already have, and a file we never
219
- * wrote is theirs, not our own echo.
220
- * @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[], preserved: string[], unmanaged: string[] }}
221
- */
222
- export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
223
- const deleteOrphans = opts.deleteOrphans ?? false;
224
- const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
225
- const preserve = new Set(opts.preserve ?? []);
226
- const written = [];
227
- const unchanged = [];
228
- const orphaned = [];
229
- const deleted = [];
230
- const preserved = [];
231
- const unmanaged = [];
232
-
233
- mkdirSync(targetDir, { recursive: true });
234
- const existingFiles = readGemlFilesFromDisk(targetDir);
235
-
236
- // Load previous sync manifest to know which files belong to sync vs user-authored files
237
- const manifestPath = join(targetDir, MANIFEST_FILE);
238
- const previous = readManifest(targetDir);
239
- const lastManifest = new Set(previous.files.keys());
240
-
241
- // Write new or updated files atomically with CRLF normalization
242
- for (const [rel, newContent] of gemlFiles) {
243
- const fullPath = join(targetDir, rel);
244
- const normNew = normalizeEol(newContent);
245
- const existingContent = existingFiles.get(rel);
246
-
247
- if (preserve.has(rel)) {
248
- if (existingContent !== normNew) preserved.push(rel);
249
- else unchanged.push(rel);
250
- } else if (
251
- existingContent !== undefined &&
252
- existingContent !== normNew &&
253
- !lastManifest.has(rel) &&
254
- !overwriteUnmanaged
255
- ) {
256
- // On disk, different from the graph, and no manifest ever claimed it:
257
- // someone else's file, not our own echo. A file that already MATCHES
258
- // what we would write falls through and is adopted — identical bytes
259
- // mean there is nothing of theirs to lose.
260
- unmanaged.push(rel);
261
- } else if (existingContent === undefined || existingContent !== normNew) {
262
- atomicWriteFileSync(fullPath, normNew);
263
- written.push(rel);
264
- } else {
265
- unchanged.push(rel);
266
- }
267
- }
268
-
269
- // Detect files present on disk but absent from current graph export
270
- for (const [rel] of existingFiles) {
271
- if (!gemlFiles.has(rel)) {
272
- orphaned.push(rel);
273
- // Safe deletion: only delete if explicit AND file was generated by previous sync
274
- // User-authored files in targetDir not in manifest are NEVER deleted.
275
- if (deleteOrphans && lastManifest.has(rel)) {
276
- const fullPath = join(targetDir, rel);
277
- if (existsSync(fullPath)) {
278
- unlinkSync(fullPath);
279
- cleanEmptyParents(dirname(fullPath), targetDir);
280
- deleted.push(rel);
281
- }
282
- }
283
- }
284
- }
285
-
286
- // Save updated manifest of managed sync files: all current gemlFiles, plus
287
- // any existing files on disk that were in lastManifest and not deleted.
288
- // v2 records each file's content hash AS OF THIS SYNC — the baseline
289
- // detectExternalEdits() compares against, so the watcher's own writes never
290
- // read as someone else's edits.
291
- const currentManifest = new Set(gemlFiles.keys());
292
- // A held file stays unowned: recording a hash for it would make the next run
293
- // read the person's content as the sync's own last write and clobber it.
294
- for (const rel of unmanaged) currentManifest.delete(rel);
295
- for (const rel of lastManifest) {
296
- if (existingFiles.has(rel) && !deleted.includes(rel)) {
297
- currentManifest.add(rel);
298
- }
299
- }
300
- const manifestFiles = {};
301
- for (const rel of [...currentManifest].sort()) {
302
- if (preserved.includes(rel)) {
303
- // A conflicted file keeps its OLD baseline: recording what sits on disk
304
- // now would make the person's unmerged edit read as "already synced" on
305
- // the next cycle, and the conflict would be silently forgotten.
306
- manifestFiles[rel] = previous.files.get(rel) ?? null;
307
- continue;
308
- }
309
- const content = gemlFiles.has(rel) ? normalizeEol(gemlFiles.get(rel)) : existingFiles.get(rel);
310
- manifestFiles[rel] = content === undefined ? null : sha256(content);
311
- }
312
- atomicWriteFileSync(manifestPath, JSON.stringify({ version: 2, files: manifestFiles }, null, 1) + "\n");
313
-
314
- return { written, orphaned, unchanged, deleted, preserved, unmanaged };
315
- }
316
-
317
- /**
318
- * Execute Git commands to commit changes scoped strictly to the synced files.
319
- * Protects parent repository from having unrelated files swept into the commit.
320
- *
321
- * @param {string} targetDir Directory where the sync target lives.
322
- * @param {string} commitMessage Commit message.
323
- * @param {string[]} pathsToCommit Relative paths within targetDir that were written or deleted.
324
- * @param {function} [gitRunner] Optional custom git runner `(args) => Promise<{ stdout, stderr, exitCode }>`.
325
- * @returns {Promise<{ committed: boolean, changes: boolean, output: string }>}
326
- */
327
- export async function gitAutoCommit(targetDir, commitMessage = "logseq-geml sync", pathsToCommit = [], gitRunner = null) {
328
- const defaultRunner = async (args) => {
329
- try {
330
- const stdout = execFileSync("git", args, {
331
- cwd: targetDir,
332
- encoding: "utf8",
333
- stdio: ["ignore", "pipe", "pipe"],
334
- });
335
- return { stdout, stderr: "", exitCode: 0 };
336
- } catch (err) {
337
- return {
338
- stdout: err.stdout ? String(err.stdout) : "",
339
- stderr: err.stderr ? String(err.stderr) : err.message,
340
- exitCode: err.status || 1,
341
- };
342
- }
343
- };
344
-
345
- const run = gitRunner || defaultRunner;
346
-
347
- // Check if targetDir is inside a git repository
348
- const revRes = await run(["rev-parse", "--show-toplevel"]);
349
- if (revRes.exitCode !== 0) {
350
- return {
351
- committed: false,
352
- changes: false,
353
- output: `Not a git repository: ${revRes.stderr.trim()}`,
354
- };
355
- }
356
-
357
- if (!pathsToCommit || pathsToCommit.length === 0) {
358
- return {
359
- committed: false,
360
- changes: false,
361
- output: "No synced paths to commit.",
362
- };
363
- }
364
-
365
- // Always include the manifest file in staged paths
366
- const allPaths = [...new Set([...pathsToCommit, MANIFEST_FILE])];
367
-
368
- // Stage ONLY the specified paths (never a bare git add -A)
369
- // Split into existing files vs deleted files
370
- const toAdd = allPaths.filter((p) => existsSync(join(targetDir, p)));
371
- const toRemove = allPaths.filter((p) => !existsSync(join(targetDir, p)));
372
-
373
- if (toAdd.length > 0) {
374
- const addRes = await run(["add", "--", ...toAdd]);
375
- if (addRes.exitCode !== 0) {
376
- return { committed: false, changes: true, output: `git add failed: ${addRes.stderr}` };
377
- }
378
- }
379
-
380
- if (toRemove.length > 0) {
381
- const rmRes = await run(["add", "-u", "--", ...toRemove]);
382
- if (rmRes.exitCode !== 0) {
383
- return { committed: false, changes: true, output: `git update index failed: ${rmRes.stderr}` };
384
- }
385
- }
386
-
387
- // Verify whether our target paths have staged changes
388
- const statusRes = await run(["status", "--porcelain", "--", ...allPaths]);
389
- if (statusRes.exitCode !== 0 || statusRes.stdout.trim().length === 0) {
390
- return {
391
- committed: false,
392
- changes: false,
393
- output: "No changes in target paths to commit.",
394
- };
395
- }
396
-
397
- // Commit with pathspec: commits ONLY changes matching our synced paths,
398
- // leaving any other staged or unstaged changes in parent repository untouched!
399
- const commitRes = await run(["commit", "-m", commitMessage, "--", ...allPaths]);
400
- if (commitRes.exitCode !== 0) {
401
- return {
402
- committed: false,
403
- changes: true,
404
- output: `git commit failed: ${commitRes.stderr}`,
405
- };
406
- }
407
-
408
- return {
409
- committed: true,
410
- changes: true,
411
- output: commitRes.stdout.trim(),
412
- };
413
- }
414
-
415
- /**
416
- * Full Sync Pipeline from EDN string to disk.
417
- *
418
- * @param {string} ednText EDN string (from logseq export-edn).
419
- * @param {string} targetDir Destination folder.
420
- * @param {object} [opts]
421
- * @param {boolean} [opts.autoCommit=false]
422
- * @param {string} [opts.commitMessage]
423
- * @param {boolean} [opts.allowEmptyGraph=false] Refuse 0-page export over non-empty targetDir unless true.
424
- * @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files no manifest
425
- * ever claimed, in BOTH trees. Off by default — see writeGemlFilesToDisk.
426
- * @param {function} [opts.gitRunner]
427
- * @returns {Promise<{ written: string[], deleted: string[], orphaned: string[], unchanged: string[], unmanaged: string[], markdownWritten: string[], markdownUnmanaged: string[], gitResult?: any }>}
428
- */
429
- export async function syncEdnToDisk(ednText, targetDir, opts = {}) {
430
- const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
431
-
432
- // Guard 1: Refuse empty or truncated EDN input
433
- if (!ednText || typeof ednText !== "string" || ednText.trim().length === 0) {
434
- throw new Error("EDN input is empty or truncated; refusing to sync to prevent data loss.");
435
- }
436
-
437
- const gemlFiles = ednToGemlFiles(ednText);
438
-
439
- // Guard 2: Refuse 0-page export if targetDir already has existing pages
440
- const pageCount = [...gemlFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length;
441
- const existingFiles = readGemlFilesFromDisk(targetDir);
442
- // Counting only .geml left this guard blind to exactly what it guards
443
- // against: a directory that is ALREADY an OG graph holds its pages as
444
- // Markdown, and readGemlFilesFromDisk does not see one of them.
445
- const markdownDir = opts.markdownDir ?? null;
446
- const existingPageCount =
447
- [...existingFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length +
448
- countMarkdownPages(targetDir) +
449
- (markdownDir && resolve(markdownDir) !== resolve(targetDir) ? countMarkdownPages(markdownDir) : 0);
450
-
451
- if (pageCount === 0 && existingPageCount > 0 && !opts.allowEmptyGraph) {
452
- throw new Error(
453
- `Refusing to sync empty graph (0 pages) over directory with existing pages (${existingPageCount} pages). Pass allowEmptyGraph: true to force.`
454
- );
455
- }
456
-
457
- const diffResult = writeGemlFilesToDisk(gemlFiles, targetDir, opts);
458
-
459
- // A parallel Markdown tree in LOGSEQ'S OWN dialect bullets, `id::`,
460
- // `((uuid))` refs so the directory opens as a graph in the file version of
461
- // the app. Generic GEML-to-Markdown is `geml <file> --to md`, the parser's
462
- // job; the only reason this integration writes Markdown at all is Logseq,
463
- // and writing anything else here would throw away the uuids and the outline
464
- // depth that make an OG graph an OG graph.
465
- //
466
- // Deliberately lossy and one-way: the GEML tree stays the one that
467
- // round-trips, and `restore` never reads this. The parser library is
468
- // injected, so core keeps its single dependency.
469
- const markdownWritten = [];
470
- const markdownUnmanaged = [];
471
- if (markdownDir && opts.lib) {
472
- const mdPrevious = readManifest(markdownDir, MD_MANIFEST_FILE);
473
- const mdManifest = {};
474
- for (const [rel, content] of gemlFiles) {
475
- // Only pages and journals are a graph; the index and the ontology carry
476
- // machine bookkeeping OG has no page for.
477
- if (!rel.startsWith("pages/") && !rel.startsWith("journals/")) continue;
478
- const mdRel = rel.replace(/\.geml$/, ".md");
479
- const full = join(markdownDir, mdRel);
480
- let md;
481
- try {
482
- md = normalizeEol(gemlToOgMarkdown(content, opts.lib));
483
- } catch {
484
- continue; // one unconvertible document must not fail the sync
485
- }
486
- if (md === "") continue; // nothing OG can hold — write no file
487
- const onDisk = existsSync(full) ? normalizeEol(readFileSync(full, "utf8")) : null;
488
- if (onDisk !== null && onDisk !== md && !mdPrevious.files.has(mdRel) && !overwriteUnmanaged) {
489
- // The GEML tree's rule, and this tree needs it more: `--markdown` takes
490
- // any directory, so somebody's own pages/*.md is precisely what it
491
- // lands on. Held, named, and left exactly as they wrote it.
492
- markdownUnmanaged.push(mdRel);
493
- continue;
494
- }
495
- if (onDisk !== md) {
496
- mkdirSync(dirname(full), { recursive: true });
497
- atomicWriteFileSync(full, md);
498
- markdownWritten.push(mdRel);
499
- }
500
- mdManifest[mdRel] = sha256(md);
501
- }
502
- // A page the graph stopped exporting keeps its entry while the file is
503
- // still there: dropping it would make our own past write read as a
504
- // stranger's on the next run, and the sync would refuse to touch it.
505
- for (const [mdRel, hash] of mdPrevious.files) {
506
- if (!(mdRel in mdManifest) && existsSync(join(markdownDir, mdRel))) mdManifest[mdRel] = hash;
507
- }
508
- const mdManifestPath = join(markdownDir, MD_MANIFEST_FILE);
509
- const mdManifestFiles = {};
510
- for (const mdRel of Object.keys(mdManifest).sort()) mdManifestFiles[mdRel] = mdManifest[mdRel];
511
- const mdManifestText = JSON.stringify({ version: 2, files: mdManifestFiles }, null, 1) + "\n";
512
- const mdManifestExists = existsSync(mdManifestPath);
513
- // Only when it actually changed: this file lives inside someone's OG graph,
514
- // and rewriting it every poll would have Logseq re-reading it forever.
515
- if (
516
- (Object.keys(mdManifestFiles).length > 0 || mdManifestExists) &&
517
- (!mdManifestExists || readFileSync(mdManifestPath, "utf8") !== mdManifestText)
518
- ) {
519
- atomicWriteFileSync(mdManifestPath, mdManifestText);
520
- }
521
- }
522
-
523
- let gitResult = null;
524
- const pathsModified = [...diffResult.written, ...diffResult.deleted];
525
- for (const rel of markdownWritten) {
526
- const abs = join(markdownDir, rel);
527
- const insideVault = relative(targetDir, abs);
528
- if (insideVault && !insideVault.startsWith("..") && !isAbsolute(insideVault)) {
529
- pathsModified.push(insideVault);
530
- }
531
- }
532
-
533
- if (opts.autoCommit && pathsModified.length > 0) {
534
- const msg = opts.commitMessage || `logseq-geml: synced ${diffResult.written.length} modified, ${diffResult.deleted.length} deleted`;
535
- gitResult = await gitAutoCommit(targetDir, msg, pathsModified, opts.gitRunner);
536
- }
537
-
538
- return {
539
- ...diffResult,
540
- markdownWritten,
541
- markdownUnmanaged,
542
- gitResult,
543
- };
544
- }
545
-
546
- /**
547
- * Full Sync Pipeline from disk back to EDN string.
548
- *
549
- * @param {string} targetDir Local folder containing .geml files.
550
- * @param {object} lib Parser library containing { parse, addressedUnits, sliceUnit }.
551
- * @param {{ exclude?: string[] }} [opts] Files to leave OUT of the import —
552
- * the conflicted files of a two-way cycle: absent from the EDN means the
553
- * graph's version stays untouched (import merges by uuid, it never deletes).
554
- * @returns {string} EDN string ready for logseq import-edn.
555
- */
556
- export function syncDiskToEdn(targetDir, lib, opts = {}) {
557
- const files = readGemlFilesFromDisk(targetDir);
558
- for (const rel of opts.exclude ?? []) files.delete(rel);
559
- return gemlFilesToEdn(files, lib);
560
- }
1
+ // Core Sync Engine for Logseq GEML
2
+ // Keeps a local folder of .geml files in sync with a Logseq DB graph,
3
+ // ensuring only changed files are written so Git diffs stay clean.
4
+
5
+ import {
6
+ readdirSync,
7
+ readFileSync,
8
+ writeFileSync,
9
+ unlinkSync,
10
+ mkdirSync,
11
+ rmdirSync,
12
+ renameSync,
13
+ existsSync,
14
+ } from "node:fs";
15
+ import { join, dirname, relative, resolve, sep, isAbsolute } from "node:path";
16
+ import { execFileSync } from "node:child_process";
17
+ import { randomUUID, createHash } from "node:crypto";
18
+ import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
19
+ import { gemlToOgMarkdown } from "./og-markdown.mjs";
20
+
21
+ const MANIFEST_FILE = ".geml-manifest.json";
22
+ // The Markdown tree needs a ledger of its own: `--markdown` takes ANY
23
+ // directory, so it is the tree most likely to be pointed at a graph someone
24
+ // already has, and it must know which .md files are its own writes before it
25
+ // overwrites one. A separate file because markdownDir may BE targetDir.
26
+ const MD_MANIFEST_FILE = ".geml-md-manifest.json";
27
+
28
+ const sha256 = (s) => createHash("sha256").update(s).digest("hex");
29
+
30
+ /**
31
+ * Read the sync manifest in either of its two shapes.
32
+ * v1 was a sorted array of paths — enough to know which files the sync owns.
33
+ * v2 ({ version: 2, files: { rel: sha256 } }) also records the content the
34
+ * sync last wrote or saw, which is what lets two-way sync tell an external
35
+ * edit from its own echo: a file whose hash matches the manifest is the
36
+ * watcher's own last write, not something a person or agent changed.
37
+ * @returns {{ known: boolean, hashed: boolean, files: Map<string, string|null> }}
38
+ */
39
+ function readManifest(targetDir, manifestFile = MANIFEST_FILE) {
40
+ const manifestPath = join(targetDir, manifestFile);
41
+ if (!existsSync(manifestPath)) return { known: false, hashed: false, files: new Map() };
42
+ try {
43
+ const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
44
+ if (Array.isArray(parsed)) {
45
+ return { known: true, hashed: false, files: new Map(parsed.map((p) => [p, null])) };
46
+ }
47
+ if (parsed && parsed.version === 2 && parsed.files && typeof parsed.files === "object") {
48
+ return { known: true, hashed: true, files: new Map(Object.entries(parsed.files)) };
49
+ }
50
+ } catch {}
51
+ return { known: false, hashed: false, files: new Map() };
52
+ }
53
+
54
+ /**
55
+ * What changed in the vault since the sync last touched it — the read side of
56
+ * the two-way bridge. Baselines come from the v2 manifest hashes; a v1
57
+ * manifest (or none) knows which files exist but not what they held, so it
58
+ * reports nothing rather than guessing: `baselineKnown: false` means "sync
59
+ * once first".
60
+ *
61
+ * With `graphFiles` (the current export, as ednToGemlFiles returns it) the
62
+ * vault-modified files are split further: one the GRAPH also moved since the
63
+ * last sync is a `conflict` — importing it would clobber the graph's edit,
64
+ * exporting over it would clobber the person's, so two-way sync does neither
65
+ * and a person merges.
66
+ * @param {string} targetDir
67
+ * @param {{ graphFiles?: Map<string, string> }} [opts]
68
+ * @returns {{ baselineKnown: boolean, modified: string[], added: string[], missing: string[], conflicts: string[] }}
69
+ */
70
+ export function detectExternalEdits(targetDir, opts = {}) {
71
+ const manifest = readManifest(targetDir);
72
+ const onDisk = readGemlFilesFromDisk(targetDir);
73
+ const modified = [];
74
+ const added = [];
75
+ const missing = [];
76
+ const conflicts = [];
77
+ if (!manifest.hashed) return { baselineKnown: false, modified, added, missing, conflicts };
78
+ for (const [rel, hash] of manifest.files) {
79
+ const content = onDisk.get(rel);
80
+ if (content === undefined) missing.push(rel);
81
+ else if (hash !== null && sha256(content) !== hash) {
82
+ const graphContent = opts.graphFiles?.get(rel);
83
+ const graphMoved =
84
+ graphContent !== undefined && sha256(normalizeEol(graphContent)) !== hash;
85
+ (graphMoved ? conflicts : modified).push(rel);
86
+ }
87
+ }
88
+ for (const rel of onDisk.keys()) {
89
+ if (!manifest.files.has(rel)) added.push(rel);
90
+ }
91
+ return {
92
+ baselineKnown: true,
93
+ modified: modified.sort(),
94
+ added: added.sort(),
95
+ missing: missing.sort(),
96
+ conflicts: conflicts.sort(),
97
+ };
98
+ }
99
+
100
+ /**
101
+ * Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
102
+ */
103
+ export function normalizeEol(str) {
104
+ return typeof str === "string" ? str.replace(/\r\n?/g, "\n") : str;
105
+ }
106
+
107
+ /**
108
+ * Atomically write a file via a temporary file in the same directory.
109
+ */
110
+ export function atomicWriteFileSync(filePath, content) {
111
+ const dir = dirname(filePath);
112
+ mkdirSync(dir, { recursive: true });
113
+ const tmpPath = join(dir, `.tmp-${Date.now()}-${process.pid}-${randomUUID()}`);
114
+ writeFileSync(tmpPath, content, "utf8");
115
+ renameSync(tmpPath, filePath);
116
+ }
117
+
118
+ /**
119
+ * Remove empty parent directories recursively up to stopDir.
120
+ */
121
+ function cleanEmptyParents(dir, stopDir) {
122
+ let current = resolve(dir);
123
+ const stop = resolve(stopDir);
124
+ while (current && current !== stop && current.startsWith(stop)) {
125
+ try {
126
+ const remaining = readdirSync(current);
127
+ if (remaining.length === 0) {
128
+ rmdirSync(current);
129
+ current = dirname(current);
130
+ } else {
131
+ break;
132
+ }
133
+ } catch {
134
+ break;
135
+ }
136
+ }
137
+ }
138
+
139
+ /**
140
+ * Scan a directory recursively for all .geml files.
141
+ * @param {string} dir Root directory to scan.
142
+ * @param {string} [baseDir] Base directory for computing relative paths.
143
+ * @returns {Map<string, string>} Map of relative path (POSIX style) -> file content (normalized LF).
144
+ */
145
+ export function readGemlFilesFromDisk(dir, baseDir = dir) {
146
+ const files = new Map();
147
+ if (!existsSync(dir)) return files;
148
+
149
+ const entries = readdirSync(dir, { withFileTypes: true });
150
+ for (const entry of entries) {
151
+ const fullPath = join(dir, entry.name);
152
+ if (entry.isDirectory()) {
153
+ // Ignore .git, node_modules, and hidden directories
154
+ if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
155
+ const subFiles = readGemlFilesFromDisk(fullPath, baseDir);
156
+ for (const [rel, content] of subFiles) {
157
+ files.set(rel, content);
158
+ }
159
+ } else if (entry.isFile() && entry.name.endsWith(".geml")) {
160
+ const rel = relative(baseDir, fullPath).split(sep).join("/");
161
+ files.set(rel, normalizeEol(readFileSync(fullPath, "utf8")));
162
+ }
163
+ }
164
+ return files;
165
+ }
166
+
167
+ /**
168
+ * Count the Markdown pages of an OG graph laid out under dir.
169
+ * readGemlFilesFromDisk answers "what of ours is here"; this answers "is this
170
+ * already somebody's graph", which is a different question and the one the
171
+ * empty-export guard actually asks.
172
+ */
173
+ function countMarkdownPages(dir) {
174
+ let count = 0;
175
+ for (const sub of ["pages", "journals"]) {
176
+ const root = join(dir, sub);
177
+ if (!existsSync(root)) continue;
178
+ const stack = [root];
179
+ while (stack.length > 0) {
180
+ const current = stack.pop();
181
+ let entries;
182
+ try {
183
+ entries = readdirSync(current, { withFileTypes: true });
184
+ } catch {
185
+ continue;
186
+ }
187
+ for (const entry of entries) {
188
+ if (entry.isDirectory()) {
189
+ if (!entry.name.startsWith(".") && entry.name !== "node_modules") stack.push(join(current, entry.name));
190
+ } else if (entry.isFile() && entry.name.endsWith(".md")) {
191
+ count += 1;
192
+ }
193
+ }
194
+ }
195
+ }
196
+ return count;
197
+ }
198
+
199
+ /**
200
+ * Incrementally sync a Map of GEML files to disk.
201
+ * Only writes files whose content has changed or do not yet exist.
202
+ * Detects files on disk that are absent from the export (e.g. deleted pages or journals),
203
+ * and reports them without destructive deletion by default.
204
+ *
205
+ * Safety note: @logseq/cli 0.4.3 does not include journals in export-edn.
206
+ * Manifest-based orphan tracking ensures user-authored files outside previous syncs are never deleted.
207
+ * Destructive deletion requires explicit `opts.deleteOrphans === true`.
208
+ *
209
+ * @param {Map<string, string>} gemlFiles Map of relative path (POSIX) -> gemlText.
210
+ * @param {string} targetDir Local destination directory.
211
+ * @param {object} [opts]
212
+ * @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
213
+ * @param {string[]} [opts.preserve] Files NOT to overwrite even when the graph
214
+ * differs — the conflicted files of a two-way cycle. Their manifest entry
215
+ * keeps its previous hash, so they stay flagged until a person resolves them.
216
+ * @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files that exist
217
+ * on disk but that no manifest ever claimed. Off by default: a vault IS a
218
+ * graph, so people point this at one they already have, and a file we never
219
+ * wrote is theirs, not our own echo.
220
+ * @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[], preserved: string[], unmanaged: string[], overwritten: string[] }}
221
+ */
222
+ export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
223
+ const deleteOrphans = opts.deleteOrphans ?? false;
224
+ const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
225
+ const preserve = new Set(opts.preserve ?? []);
226
+ const written = [];
227
+ const unchanged = [];
228
+ const orphaned = [];
229
+ const deleted = [];
230
+ const preserved = [];
231
+ const unmanaged = [];
232
+ const overwritten = []; // unmanaged, and taken anyway because the caller asked
233
+
234
+ mkdirSync(targetDir, { recursive: true });
235
+ const existingFiles = readGemlFilesFromDisk(targetDir);
236
+
237
+ // Load previous sync manifest to know which files belong to sync vs user-authored files
238
+ const manifestPath = join(targetDir, MANIFEST_FILE);
239
+ const previous = readManifest(targetDir);
240
+ const lastManifest = new Set(previous.files.keys());
241
+
242
+ // Write new or updated files atomically with CRLF normalization
243
+ for (const [rel, newContent] of gemlFiles) {
244
+ const fullPath = join(targetDir, rel);
245
+ const normNew = normalizeEol(newContent);
246
+ const existingContent = existingFiles.get(rel);
247
+
248
+ if (preserve.has(rel)) {
249
+ if (existingContent !== normNew) preserved.push(rel);
250
+ else unchanged.push(rel);
251
+ } else if (
252
+ existingContent !== undefined &&
253
+ existingContent !== normNew &&
254
+ !lastManifest.has(rel)
255
+ ) {
256
+ // On disk, different from the graph, and no manifest ever claimed it:
257
+ // someone else's file, not our own echo. A file that already MATCHES
258
+ // what we would write falls through and is adopted — identical bytes
259
+ // mean there is nothing of theirs to lose.
260
+ //
261
+ // Overwriting is a CHOICE, and it is named either way. A mode that
262
+ // discards somebody's edit without saying which file it was cannot be
263
+ // trusted with a graph — the list is the input the person's next step
264
+ // needs, and it is gone the moment it is not printed.
265
+ if (!overwriteUnmanaged) {
266
+ unmanaged.push(rel);
267
+ } else {
268
+ atomicWriteFileSync(fullPath, normNew);
269
+ written.push(rel);
270
+ overwritten.push(rel);
271
+ }
272
+ } else if (existingContent === undefined || existingContent !== normNew) {
273
+ atomicWriteFileSync(fullPath, normNew);
274
+ written.push(rel);
275
+ } else {
276
+ unchanged.push(rel);
277
+ }
278
+ }
279
+
280
+ // Detect files present on disk but absent from current graph export
281
+ for (const [rel] of existingFiles) {
282
+ if (!gemlFiles.has(rel)) {
283
+ orphaned.push(rel);
284
+ // Safe deletion: only delete if explicit AND file was generated by previous sync
285
+ // User-authored files in targetDir not in manifest are NEVER deleted.
286
+ if (deleteOrphans && lastManifest.has(rel)) {
287
+ const fullPath = join(targetDir, rel);
288
+ if (existsSync(fullPath)) {
289
+ unlinkSync(fullPath);
290
+ cleanEmptyParents(dirname(fullPath), targetDir);
291
+ deleted.push(rel);
292
+ }
293
+ }
294
+ }
295
+ }
296
+
297
+ // Save updated manifest of managed sync files: all current gemlFiles, plus
298
+ // any existing files on disk that were in lastManifest and not deleted.
299
+ // v2 records each file's content hash AS OF THIS SYNC — the baseline
300
+ // detectExternalEdits() compares against, so the watcher's own writes never
301
+ // read as someone else's edits.
302
+ const currentManifest = new Set(gemlFiles.keys());
303
+ // A held file stays unowned: recording a hash for it would make the next run
304
+ // read the person's content as the sync's own last write and clobber it.
305
+ for (const rel of unmanaged) currentManifest.delete(rel);
306
+ for (const rel of lastManifest) {
307
+ if (existingFiles.has(rel) && !deleted.includes(rel)) {
308
+ currentManifest.add(rel);
309
+ }
310
+ }
311
+ const manifestFiles = {};
312
+ for (const rel of [...currentManifest].sort()) {
313
+ if (preserved.includes(rel)) {
314
+ // A conflicted file keeps its OLD baseline: recording what sits on disk
315
+ // now would make the person's unmerged edit read as "already synced" on
316
+ // the next cycle, and the conflict would be silently forgotten.
317
+ manifestFiles[rel] = previous.files.get(rel) ?? null;
318
+ continue;
319
+ }
320
+ const content = gemlFiles.has(rel) ? normalizeEol(gemlFiles.get(rel)) : existingFiles.get(rel);
321
+ manifestFiles[rel] = content === undefined ? null : sha256(content);
322
+ }
323
+ atomicWriteFileSync(manifestPath, JSON.stringify({ version: 2, files: manifestFiles }, null, 1) + "\n");
324
+
325
+ return { written, orphaned, unchanged, deleted, preserved, unmanaged, overwritten };
326
+ }
327
+
328
+ /**
329
+ * Execute Git commands to commit changes scoped strictly to the synced files.
330
+ * Protects parent repository from having unrelated files swept into the commit.
331
+ *
332
+ * @param {string} targetDir Directory where the sync target lives.
333
+ * @param {string} commitMessage Commit message.
334
+ * @param {string[]} pathsToCommit Relative paths within targetDir that were written or deleted.
335
+ * @param {function} [gitRunner] Optional custom git runner `(args) => Promise<{ stdout, stderr, exitCode }>`.
336
+ * @returns {Promise<{ committed: boolean, changes: boolean, output: string }>}
337
+ */
338
+ export async function gitAutoCommit(targetDir, commitMessage = "logseq-geml sync", pathsToCommit = [], gitRunner = null,
339
+ // Where the manifest sits RELATIVE TO THE REPO. It used to be assumed to be
340
+ // the repo root, which held only while the vault and the GEML tree were the
341
+ // same directory; once the tree moved into `.logseq-sync-vault-with-geml/`
342
+ // that assumption staged a pathspec matching nothing and lost the commit.
343
+ manifestPath = MANIFEST_FILE) {
344
+ const defaultRunner = async (args) => {
345
+ try {
346
+ const stdout = execFileSync("git", args, {
347
+ cwd: targetDir,
348
+ encoding: "utf8",
349
+ stdio: ["ignore", "pipe", "pipe"],
350
+ });
351
+ return { stdout, stderr: "", exitCode: 0 };
352
+ } catch (err) {
353
+ return {
354
+ stdout: err.stdout ? String(err.stdout) : "",
355
+ stderr: err.stderr ? String(err.stderr) : err.message,
356
+ exitCode: err.status || 1,
357
+ };
358
+ }
359
+ };
360
+
361
+ const run = gitRunner || defaultRunner;
362
+
363
+ // Check if targetDir is inside a git repository
364
+ const revRes = await run(["rev-parse", "--show-toplevel"]);
365
+ if (revRes.exitCode !== 0) {
366
+ return {
367
+ committed: false,
368
+ changes: false,
369
+ output: `Not a git repository: ${revRes.stderr.trim()}`,
370
+ };
371
+ }
372
+
373
+ if (!pathsToCommit || pathsToCommit.length === 0) {
374
+ return {
375
+ committed: false,
376
+ changes: false,
377
+ output: "No synced paths to commit.",
378
+ };
379
+ }
380
+
381
+ // Always include the manifest file in staged paths
382
+ const allPaths = [...new Set([...pathsToCommit, manifestPath])];
383
+
384
+ // Stage ONLY the specified paths (never a bare git add -A)
385
+ // Split into existing files vs deleted files
386
+ const toAdd = allPaths.filter((p) => existsSync(join(targetDir, p)));
387
+ const toRemove = allPaths.filter((p) => !existsSync(join(targetDir, p)));
388
+
389
+ if (toAdd.length > 0) {
390
+ const addRes = await run(["add", "--", ...toAdd]);
391
+ if (addRes.exitCode !== 0) {
392
+ return { committed: false, changes: true, output: `git add failed: ${addRes.stderr}` };
393
+ }
394
+ }
395
+
396
+ if (toRemove.length > 0) {
397
+ const rmRes = await run(["add", "-u", "--", ...toRemove]);
398
+ if (rmRes.exitCode !== 0) {
399
+ return { committed: false, changes: true, output: `git update index failed: ${rmRes.stderr}` };
400
+ }
401
+ }
402
+
403
+ // Verify whether our target paths have staged changes
404
+ const statusRes = await run(["status", "--porcelain", "--", ...allPaths]);
405
+ if (statusRes.exitCode !== 0 || statusRes.stdout.trim().length === 0) {
406
+ return {
407
+ committed: false,
408
+ changes: false,
409
+ output: "No changes in target paths to commit.",
410
+ };
411
+ }
412
+
413
+ // Commit with pathspec: commits ONLY changes matching our synced paths,
414
+ // leaving any other staged or unstaged changes in parent repository untouched!
415
+ const commitRes = await run(["commit", "-m", commitMessage, "--", ...allPaths]);
416
+ if (commitRes.exitCode !== 0) {
417
+ return {
418
+ committed: false,
419
+ changes: true,
420
+ output: `git commit failed: ${commitRes.stderr}`,
421
+ };
422
+ }
423
+
424
+ return {
425
+ committed: true,
426
+ changes: true,
427
+ output: commitRes.stdout.trim(),
428
+ };
429
+ }
430
+
431
+ /**
432
+ * Full Sync Pipeline from EDN string to disk.
433
+ *
434
+ * @param {string} ednText EDN string (from logseq export-edn).
435
+ * @param {string} targetDir Destination folder.
436
+ * @param {object} [opts]
437
+ * @param {boolean} [opts.autoCommit=false]
438
+ * @param {string} [opts.commitMessage]
439
+ * @param {boolean} [opts.allowEmptyGraph=false] Refuse 0-page export over non-empty targetDir unless true.
440
+ * @param {boolean} [opts.overwriteUnmanaged=false] Overwrite files no manifest
441
+ * ever claimed, in BOTH trees. Off by default — see writeGemlFilesToDisk.
442
+ * @param {function} [opts.gitRunner]
443
+ * @returns {Promise<{ written: string[], deleted: string[], orphaned: string[], unchanged: string[], unmanaged: string[], markdownWritten: string[], markdownUnmanaged: string[], gitResult?: any }>}
444
+ */
445
+ export async function syncEdnToDisk(ednText, targetDir, opts = {}) {
446
+ const overwriteUnmanaged = opts.overwriteUnmanaged ?? false;
447
+
448
+ // Guard 1: Refuse empty or truncated EDN input
449
+ if (!ednText || typeof ednText !== "string" || ednText.trim().length === 0) {
450
+ throw new Error("EDN input is empty or truncated; refusing to sync to prevent data loss.");
451
+ }
452
+
453
+ const gemlFiles = ednToGemlFiles(ednText);
454
+
455
+ // Guard 2: Refuse 0-page export if targetDir already has existing pages
456
+ const pageCount = [...gemlFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length;
457
+ const existingFiles = readGemlFilesFromDisk(targetDir);
458
+ // Counting only .geml left this guard blind to exactly what it guards
459
+ // against: a directory that is ALREADY an OG graph holds its pages as
460
+ // Markdown, and readGemlFilesFromDisk does not see one of them.
461
+ const markdownDir = opts.markdownDir ?? null;
462
+ const existingPageCount =
463
+ [...existingFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length +
464
+ countMarkdownPages(targetDir) +
465
+ (markdownDir && resolve(markdownDir) !== resolve(targetDir) ? countMarkdownPages(markdownDir) : 0);
466
+
467
+ if (pageCount === 0 && existingPageCount > 0 && !opts.allowEmptyGraph) {
468
+ throw new Error(
469
+ `Refusing to sync empty graph (0 pages) over directory with existing pages (${existingPageCount} pages). Pass allowEmptyGraph: true to force.`
470
+ );
471
+ }
472
+
473
+ const diffResult = writeGemlFilesToDisk(gemlFiles, targetDir, opts);
474
+
475
+ // A parallel Markdown tree in LOGSEQ'S OWN dialect bullets, `id::`,
476
+ // `((uuid))` refs so the directory opens as a graph in the file version of
477
+ // the app. Generic GEML-to-Markdown is `geml <file> --to md`, the parser's
478
+ // job; the only reason this integration writes Markdown at all is Logseq,
479
+ // and writing anything else here would throw away the uuids and the outline
480
+ // depth that make an OG graph an OG graph.
481
+ //
482
+ // Deliberately lossy and one-way: the GEML tree stays the one that
483
+ // round-trips, and `restore` never reads this. The parser library is
484
+ // injected, so core keeps its single dependency.
485
+ const markdownWritten = [];
486
+ const markdownUnmanaged = [];
487
+ const markdownOverwritten = [];
488
+ if (markdownDir && opts.lib) {
489
+ const mdPrevious = readManifest(markdownDir, MD_MANIFEST_FILE);
490
+ const mdManifest = {};
491
+ for (const [rel, content] of gemlFiles) {
492
+ // Only pages and journals are a graph; the index and the ontology carry
493
+ // machine bookkeeping OG has no page for.
494
+ if (!rel.startsWith("pages/") && !rel.startsWith("journals/")) continue;
495
+ const mdRel = rel.replace(/\.geml$/, ".md");
496
+ const full = join(markdownDir, mdRel);
497
+ let md;
498
+ try {
499
+ md = normalizeEol(gemlToOgMarkdown(content, opts.lib));
500
+ } catch {
501
+ continue; // one unconvertible document must not fail the sync
502
+ }
503
+ if (md === "") continue; // nothing OG can hold write no file
504
+ const onDisk = existsSync(full) ? normalizeEol(readFileSync(full, "utf8")) : null;
505
+ // "Ours" means the bytes still MATCH WHAT WE RECORDED WRITING, not merely
506
+ // that the path appears in the ledger. Membership alone answers "did we
507
+ // ever write this file", and the question is "is this still our file" —
508
+ // so an edit to a page we had written was read as our own echo and
509
+ // overwritten, which is the exact loss this ledger exists to prevent.
510
+ //
511
+ // The GEML tree can afford the looser test: a `.geml` edit is recoverable
512
+ // through the two-way bridge. A Markdown edit is not — the mapping is
513
+ // lossy and one-way, and importing it back is out of scope by design so
514
+ // overwriting one destroys it permanently.
515
+ //
516
+ // A v1 manifest has no hashes: it can say "we wrote this" and not "this is
517
+ // still what we wrote". Unknown counts as ours, so upgrading does not hold
518
+ // the entire tree on the first run after it.
519
+ const recorded = mdPrevious.files.get(mdRel);
520
+ const stillOurs =
521
+ mdPrevious.files.has(mdRel) &&
522
+ (!mdPrevious.hashed || recorded === null || recorded === sha256(onDisk ?? ""));
523
+ if (onDisk !== null && onDisk !== md && !stillOurs && overwriteUnmanaged) {
524
+ // Named, not silent — same rule as the GEML tree, and it matters more
525
+ // here: a Markdown edit cannot come back through the bridge.
526
+ markdownOverwritten.push(mdRel);
527
+ }
528
+ if (onDisk !== null && onDisk !== md && !stillOurs && !overwriteUnmanaged) {
529
+ // `--markdown` takes any directory, and the vault root now IS a graph
530
+ // people open, so somebody's own pages/*.md is precisely what this
531
+ // lands on. Held, named, and left exactly as they wrote it.
532
+ markdownUnmanaged.push(mdRel);
533
+ continue;
534
+ }
535
+ if (onDisk !== md) {
536
+ mkdirSync(dirname(full), { recursive: true });
537
+ atomicWriteFileSync(full, md);
538
+ markdownWritten.push(mdRel);
539
+ }
540
+ mdManifest[mdRel] = sha256(md);
541
+ }
542
+ // A page the graph stopped exporting keeps its entry while the file is
543
+ // still there: dropping it would make our own past write read as a
544
+ // stranger's on the next run, and the sync would refuse to touch it.
545
+ for (const [mdRel, hash] of mdPrevious.files) {
546
+ if (!(mdRel in mdManifest) && existsSync(join(markdownDir, mdRel))) mdManifest[mdRel] = hash;
547
+ }
548
+ const mdManifestPath = join(markdownDir, MD_MANIFEST_FILE);
549
+ const mdManifestFiles = {};
550
+ for (const mdRel of Object.keys(mdManifest).sort()) mdManifestFiles[mdRel] = mdManifest[mdRel];
551
+ const mdManifestText = JSON.stringify({ version: 2, files: mdManifestFiles }, null, 1) + "\n";
552
+ const mdManifestExists = existsSync(mdManifestPath);
553
+ // Only when it actually changed: this file lives inside someone's OG graph,
554
+ // and rewriting it every poll would have Logseq re-reading it forever.
555
+ if (
556
+ (Object.keys(mdManifestFiles).length > 0 || mdManifestExists) &&
557
+ (!mdManifestExists || readFileSync(mdManifestPath, "utf8") !== mdManifestText)
558
+ ) {
559
+ atomicWriteFileSync(mdManifestPath, mdManifestText);
560
+ }
561
+ }
562
+
563
+ let gitResult = null;
564
+ // WHERE THE GEML TREE LIVES AND WHAT GIT COVERS ARE NOT THE SAME DIRECTORY.
565
+ // They used to be, because the vault WAS the GEML tree. With Markdown at the
566
+ // vault root and GEML in a dot directory beneath it, a repo rooted at the
567
+ // GEML tree would silently drop every Markdown file from the commit — the
568
+ // half of the vault a person actually reads. `gitDir` defaults to targetDir,
569
+ // so a caller that has not moved anything keeps exactly its old behaviour.
570
+ const gitDir = opts.gitDir ?? targetDir;
571
+ const inGit = (abs) => {
572
+ const rel = relative(gitDir, abs);
573
+ return rel && !rel.startsWith("..") && !isAbsolute(rel) ? rel : null;
574
+ };
575
+ const pathsModified = [];
576
+ for (const rel of [...diffResult.written, ...diffResult.deleted]) {
577
+ const p = inGit(join(targetDir, rel));
578
+ if (p) pathsModified.push(p);
579
+ }
580
+ for (const rel of markdownWritten) {
581
+ const p = inGit(join(markdownDir, rel));
582
+ if (p) pathsModified.push(p);
583
+ }
584
+ // The Markdown ledger is versioned WITH the tree it describes. Leaving it out
585
+ // is not a tidiness question: the .md files are committed, so a clone restores
586
+ // them and not the record of who wrote them, and the next sync reads every
587
+ // changed page as a stranger's and holds it — the Markdown tree stops updating
588
+ // and says only that it is protecting files nobody edited. Measured on a
589
+ // simulated clone before it was fixed, not feared.
590
+ if (markdownDir && opts.lib) {
591
+ const p = inGit(join(markdownDir, MD_MANIFEST_FILE));
592
+ if (p) pathsModified.push(p);
593
+ }
594
+
595
+ if (opts.autoCommit && pathsModified.length > 0) {
596
+ const msg = opts.commitMessage || `logseq-geml: synced ${diffResult.written.length} modified, ${diffResult.deleted.length} deleted`;
597
+ gitResult = await gitAutoCommit(gitDir, msg, pathsModified, opts.gitRunner, inGit(join(targetDir, MANIFEST_FILE)) ?? MANIFEST_FILE);
598
+ }
599
+
600
+ return {
601
+ ...diffResult,
602
+ markdownWritten,
603
+ markdownUnmanaged,
604
+ markdownOverwritten,
605
+ gitResult,
606
+ };
607
+ }
608
+
609
+ /**
610
+ * Full Sync Pipeline from disk back to EDN string.
611
+ *
612
+ * @param {string} targetDir Local folder containing .geml files.
613
+ * @param {object} lib Parser library containing { parse, addressedUnits, sliceUnit }.
614
+ * @param {{ exclude?: string[] }} [opts] Files to leave OUT of the import —
615
+ * the conflicted files of a two-way cycle: absent from the EDN means the
616
+ * graph's version stays untouched (import merges by uuid, it never deletes).
617
+ * @returns {string} EDN string ready for logseq import-edn.
618
+ */
619
+ export function syncDiskToEdn(targetDir, lib, opts = {}) {
620
+ const files = readGemlFilesFromDisk(targetDir);
621
+ for (const rel of opts.exclude ?? []) files.delete(rel);
622
+ return gemlFilesToEdn(files, lib);
623
+ }