@geml/logseq-sync 2.0.5 → 2.0.6

package/README.md

@@ -20,7 +20,8 @@ Two settings, and only the first one usually needs touching:
 - 📦 **A plain-text copy that stays yours** — every page a readable file, not a
   database dump, in a folder you chose
 - 🔁 **Continuous, not one-shot** — edit in Logseq, and seconds later the file
-  on disk has caught up
+  on disk has caught up; with `--two-way`, edit the file and the graph
+  catches up the same way
 - ↩️ **A way back** — `logseq-sync restore` imports the vault into a graph,
   merging by block uuid. Files you can read are worth more when they are also
   files you can return
@@ -150,6 +151,7 @@ found and what is missing, and exits non-zero when the setup cannot sync:
 | `--once` | sync once and exit, instead of watching |
 | `--git-commit` | commit, creating the vault repository if there is none |
 | `--no-git-commit` | never touch git |
+| `--two-way` | also import vault edits back, every cycle — conflicts held, deletions never imported (needs the app CLI) |
 | `--mirror` | delete vault files for pages removed from the graph |
 | `--markdown <dir>` | also write a lossy Markdown copy there, for other tools |
 | `--interval <seconds>` | heartbeat between signals (default 10) |
@@ -199,11 +201,58 @@ reads them back from. *Debounce (seconds)* — quiet
 period after the last change before the watcher is signalled (default 5; syncs
 feed git commits, so this is deliberately calmer than UI-style debounce).
 
+### Editing the vault from outside
+
+The vault is ordinary text, and that is the point: agents, scripts and plain
+`sed` all work on it, and none of them needs to know Logseq exists. With
+`--two-way` running, an edit imports on the next cycle; without it, run
+`logseq-sync restore` when you are ready.
+
+**An agent (Claude, or anything speaking MCP)** gets addressed, validated
+block edits from the [`geml` MCP server](https://github.com/geml-spec/geml):
+
+```sh
+npm i -g @geml/geml
+geml mcp --root <your-vault-dir> --no-history
+```
+
+`--no-history` matters here: git is this vault's history, and without the flag
+every MCP write also saves a `.gemlhistory` sidecar revision beside the file.
+(If you want those too, drop the flag — the sync ignores sidecars either way
+and never commits them.)
+
+**A one-liner** reads or edits one block by its address — every block carries
+its uuid:
+
+```sh
+geml find "that phrase" <vault-dir>              # → pages/foo.geml  #<uuid>
+geml get  <vault-dir>/pages/foo.geml '#<uuid>'
+printf 'new text' | geml set <vault-dir>/pages/foo.geml '#<uuid>' --in - -o <same-file>
+```
+
+**Bulk refactoring** is whatever your shell already does — the result is
+re-imported by uuid, so identity survives the edit:
+
+```sh
+grep -rl "old-tag" <vault-dir>/pages | xargs sed -i 's/old-tag/new-tag/g'
+logseq-sync restore <vault-dir> --yes            # or let --two-way pick it up
+```
+
+A `geml check <file>` after a bulk edit is cheap insurance: it names a mangled
+block before the import carries it into the graph, while the diff is still in
+front of you.
+
 ## Honesty corner
 
-- The **continuous** direction is graph → files. Going back is a deliberate
-  command (`restore`), not a background loop — two live writers over one graph
-  is a merge problem this does not pretend to have solved.
+- The **default** continuous direction is graph → files; going back is a
+  deliberate command (`restore`). `--two-way` makes the return trip continuous
+  too — every cycle imports what changed in the vault — under three rules that
+  say what it does NOT pretend to solve: a file changed on **both** sides
+  since the last sync is a conflict, held exactly as you left it (not
+  imported, not overwritten, named in the toolbar status until you merge it);
+  deletions are **never** imported; and a graph backup is taken before the
+  first import and every tenth after. The sync tells its own writes from
+  yours by content hash, so nothing echoes.
 - Files the sync did not write are never touched: a manifest tracks what it
   owns, and `--mirror` only ever removes files from that list.
 - **The app's lock is the thing to know about.** A running Logseq holds

package/core/src/sync-engine.mjs

@@ -14,11 +14,83 @@ import {
 } from "node:fs";
 import { join, dirname, relative, resolve, sep, isAbsolute } from "node:path";
 import { execFileSync } from "node:child_process";
-import { randomUUID } from "node:crypto";
+import { randomUUID, createHash } from "node:crypto";
 import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
 
 const MANIFEST_FILE = ".geml-manifest.json";
 
+const sha256 = (s) => createHash("sha256").update(s).digest("hex");
+
+/**
+ * Read the sync manifest in either of its two shapes.
+ * v1 was a sorted array of paths — enough to know which files the sync owns.
+ * v2 ({ version: 2, files: { rel: sha256 } }) also records the content the
+ * sync last wrote or saw, which is what lets two-way sync tell an external
+ * edit from its own echo: a file whose hash matches the manifest is the
+ * watcher's own last write, not something a person or agent changed.
+ * @returns {{ known: boolean, hashed: boolean, files: Map<string, string|null> }}
+ */
+function readManifest(targetDir) {
+  const manifestPath = join(targetDir, MANIFEST_FILE);
+  if (!existsSync(manifestPath)) return { known: false, hashed: false, files: new Map() };
+  try {
+    const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
+    if (Array.isArray(parsed)) {
+      return { known: true, hashed: false, files: new Map(parsed.map((p) => [p, null])) };
+    }
+    if (parsed && parsed.version === 2 && parsed.files && typeof parsed.files === "object") {
+      return { known: true, hashed: true, files: new Map(Object.entries(parsed.files)) };
+    }
+  } catch {}
+  return { known: false, hashed: false, files: new Map() };
+}
+
+/**
+ * What changed in the vault since the sync last touched it — the read side of
+ * the two-way bridge. Baselines come from the v2 manifest hashes; a v1
+ * manifest (or none) knows which files exist but not what they held, so it
+ * reports nothing rather than guessing: `baselineKnown: false` means "sync
+ * once first".
+ *
+ * With `graphFiles` (the current export, as ednToGemlFiles returns it) the
+ * vault-modified files are split further: one the GRAPH also moved since the
+ * last sync is a `conflict` — importing it would clobber the graph's edit,
+ * exporting over it would clobber the person's, so two-way sync does neither
+ * and a person merges.
+ * @param {string} targetDir
+ * @param {{ graphFiles?: Map<string, string> }} [opts]
+ * @returns {{ baselineKnown: boolean, modified: string[], added: string[], missing: string[], conflicts: string[] }}
+ */
+export function detectExternalEdits(targetDir, opts = {}) {
+  const manifest = readManifest(targetDir);
+  const onDisk = readGemlFilesFromDisk(targetDir);
+  const modified = [];
+  const added = [];
+  const missing = [];
+  const conflicts = [];
+  if (!manifest.hashed) return { baselineKnown: false, modified, added, missing, conflicts };
+  for (const [rel, hash] of manifest.files) {
+    const content = onDisk.get(rel);
+    if (content === undefined) missing.push(rel);
+    else if (hash !== null && sha256(content) !== hash) {
+      const graphContent = opts.graphFiles?.get(rel);
+      const graphMoved =
+        graphContent !== undefined && sha256(normalizeEol(graphContent)) !== hash;
+      (graphMoved ? conflicts : modified).push(rel);
+    }
+  }
+  for (const rel of onDisk.keys()) {
+    if (!manifest.files.has(rel)) added.push(rel);
+  }
+  return {
+    baselineKnown: true,
+    modified: modified.sort(),
+    added: added.sort(),
+    missing: missing.sort(),
+    conflicts: conflicts.sort(),
+  };
+}
+
 /**
  * Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
  */
@@ -100,27 +172,27 @@ export function readGemlFilesFromDisk(dir, baseDir = dir) {
  * @param {string} targetDir Local destination directory.
  * @param {object} [opts]
  * @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
- * @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[] }}
+ * @param {string[]} [opts.preserve] Files NOT to overwrite even when the graph
+ *   differs — the conflicted files of a two-way cycle. Their manifest entry
+ *   keeps its previous hash, so they stay flagged until a person resolves them.
+ * @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[], preserved: string[] }}
  */
 export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
   const deleteOrphans = opts.deleteOrphans ?? false;
+  const preserve = new Set(opts.preserve ?? []);
   const written = [];
   const unchanged = [];
   const orphaned = [];
   const deleted = [];
+  const preserved = [];
 
   mkdirSync(targetDir, { recursive: true });
   const existingFiles = readGemlFilesFromDisk(targetDir);
 
   // Load previous sync manifest to know which files belong to sync vs user-authored files
   const manifestPath = join(targetDir, MANIFEST_FILE);
-  let lastManifest = new Set();
-  if (existsSync(manifestPath)) {
-    try {
-      const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
-      if (Array.isArray(parsed)) lastManifest = new Set(parsed);
-    } catch {}
-  }
+  const previous = readManifest(targetDir);
+  const lastManifest = new Set(previous.files.keys());
 
   // Write new or updated files atomically with CRLF normalization
   for (const [rel, newContent] of gemlFiles) {
@@ -128,7 +200,10 @@ export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
     const normNew = normalizeEol(newContent);
     const existingContent = existingFiles.get(rel);
 
-    if (existingContent === undefined || existingContent !== normNew) {
+    if (preserve.has(rel)) {
+      if (existingContent !== normNew) preserved.push(rel);
+      else unchanged.push(rel);
+    } else if (existingContent === undefined || existingContent !== normNew) {
       atomicWriteFileSync(fullPath, normNew);
       written.push(rel);
     } else {
@@ -153,17 +228,32 @@ export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
     }
   }
 
-  // Save updated manifest of managed sync files:
-  // includes all current gemlFiles, plus any existing files on disk that were in lastManifest and not deleted.
+  // Save updated manifest of managed sync files: all current gemlFiles, plus
+  // any existing files on disk that were in lastManifest and not deleted.
+  // v2 records each file's content hash AS OF THIS SYNC — the baseline
+  // detectExternalEdits() compares against, so the watcher's own writes never
+  // read as someone else's edits.
   const currentManifest = new Set(gemlFiles.keys());
   for (const rel of lastManifest) {
     if (existingFiles.has(rel) && !deleted.includes(rel)) {
       currentManifest.add(rel);
     }
   }
-  atomicWriteFileSync(manifestPath, JSON.stringify([...currentManifest].sort(), null, 1) + "\n");
+  const manifestFiles = {};
+  for (const rel of [...currentManifest].sort()) {
+    if (preserved.includes(rel)) {
+      // A conflicted file keeps its OLD baseline: recording what sits on disk
+      // now would make the person's unmerged edit read as "already synced" on
+      // the next cycle, and the conflict would be silently forgotten.
+      manifestFiles[rel] = previous.files.get(rel) ?? null;
+      continue;
+    }
+    const content = gemlFiles.has(rel) ? normalizeEol(gemlFiles.get(rel)) : existingFiles.get(rel);
+    manifestFiles[rel] = content === undefined ? null : sha256(content);
+  }
+  atomicWriteFileSync(manifestPath, JSON.stringify({ version: 2, files: manifestFiles }, null, 1) + "\n");
 
-  return { written, orphaned, unchanged, deleted };
+  return { written, orphaned, unchanged, deleted, preserved };
 }
 
 /**
@@ -344,12 +434,16 @@ export async function syncEdnToDisk(ednText, targetDir, opts = {}) {
 
 /**
  * Full Sync Pipeline from disk back to EDN string.
- * 
+ *
  * @param {string} targetDir Local folder containing .geml files.
  * @param {object} lib Parser library containing { parse, addressedUnits, sliceUnit }.
+ * @param {{ exclude?: string[] }} [opts] Files to leave OUT of the import —
+ *   the conflicted files of a two-way cycle: absent from the EDN means the
+ *   graph's version stays untouched (import merges by uuid, it never deletes).
  * @returns {string} EDN string ready for logseq import-edn.
  */
-export function syncDiskToEdn(targetDir, lib) {
+export function syncDiskToEdn(targetDir, lib, opts = {}) {
   const files = readGemlFilesFromDisk(targetDir);
+  for (const rel of opts.exclude ?? []) files.delete(rel);
   return gemlFilesToEdn(files, lib);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@geml/logseq-sync",
-  "version": "2.0.5",
+  "version": "2.0.6",
   "publishConfig": {
     "access": "public"
   },

package/watcher/bin/logseq-sync.mjs

@@ -9,7 +9,8 @@ import {
 import { join, resolve, dirname, basename, sep } from "node:path";
 import { tmpdir, homedir } from "node:os";
 import { randomUUID, createHash } from "node:crypto";
-import { syncEdnToDisk, syncDiskToEdn, atomicWriteFileSync } from "../../core/src/sync-engine.mjs";
+import { syncEdnToDisk, syncDiskToEdn, atomicWriteFileSync, detectExternalEdits } from "../../core/src/sync-engine.mjs";
+import { ednToGemlFiles } from "../../core/src/mapping.mjs";
 import { STATUS_FILE } from "../../core/src/bridge.mjs";
 import { parse as parseGeml, addressedUnits, sliceUnit, gemlToMd } from "@geml/geml";
 
@@ -39,6 +40,12 @@ to override it.
 
 Flags:
   --once                 Sync once and exit (default: keep watching)
+  --two-way              Also import vault edits back into the graph, checked
+                         on every cycle. A file changed on BOTH sides is a
+                         conflict: neither imported nor overwritten, reported
+                         until you merge it. Deletions are never imported.
+                         Takes a graph backup before the first import and
+                         every 10th after. Needs the app CLI.
   --git-commit           Commit, creating the vault repository if there is none
                          (default: commit only when the vault ALREADY is a repository)
   --no-git-commit        Never touch git
@@ -64,6 +71,7 @@ const args = process.argv.slice(2);
 const positional = [];
 const flags = {
   once: false,
+  twoWay: false,
   gitCommit: "auto",
   mirror: false,
   markdown: null,
@@ -102,6 +110,8 @@ for (let i = 0; i < args.length; i++) {
     flags.yes = true;
   } else if (arg === "--no-backup") {
     flags.backup = false;
+  } else if (arg === "--two-way") {
+    flags.twoWay = true;
   } else if (arg === "--mirror") {
     flags.mirror = true;
   } else if (arg === "--markdown") {
@@ -436,6 +446,14 @@ const signalPath = resolveSignalPath();
 const watchMode = !flags.once;
 const gitCommit = subcommand === "restore" ? false : resolveGitCommit();
 
+if (flags.twoWay && !appCli) {
+  console.error(
+    "Error: --two-way needs the Logseq desktop app's CLI (it performs the imports). " +
+      "Install Logseq, or pass --app-cli <path>. Run `logseq-sync doctor` for the full picture."
+  );
+  process.exit(2);
+}
+
 function isGitRepo(dir) {
   try {
     execFileSync("git", ["rev-parse", "--is-inside-work-tree"], {
@@ -599,21 +617,86 @@ async function restore() {
 
 let lastEdnHash = null;
 
+// ⑤'s bookkeeping: a graph backup before the session's first import, then
+// every BACKUP_EVERY imports after — enough that an import gone wrong always
+// has a recent restore point, without one backup per keystroke.
+let sessionBackupTaken = false;
+let importsSinceBackup = 0;
+const BACKUP_EVERY = 10;
+
+// Export the graph as EDN into tempPath — the one exporter, used once per
+// cycle, twice when a two-way import changed the graph mid-cycle.
+// With a token the CLI goes through the running app's HTTP API server and
+// exports whatever graph the app has OPEN — the graph name is not part of
+// that request, so -a REPLACES -g rather than joining it. Without a token
+// the CLI opens the named graph's sqlite directly, which only works while
+// the app does not hold the lock on it.
+function exportGraphEdn(tempPath) {
+  if (appCli) {
+    runAppCli(tempPath);
+  } else {
+    const exportSource = apiServerToken ? ["-a", apiServerToken] : ["-g", graphName];
+    runLogseqCli("export-edn", ...exportSource, "-f", tempPath);
+  }
+}
+
+// The import half of --two-way, run before the export lands on disk: whatever
+// a person or agent changed in the vault goes back into the graph first, so
+// the write that follows holds the merged state and re-baselines the
+// manifest. Deletions are reported, never imported (the vault's stance, now
+// in both directions); a file changed on BOTH sides since the last sync is a
+// conflict — importing it would clobber the graph's edit, exporting over it
+// would clobber the person's, so two-way does neither and says so until a
+// person merges.
+async function importExternalEdits(ednText) {
+  const graphFiles = ednToGemlFiles(ednText);
+  const edits = detectExternalEdits(targetDir, { graphFiles });
+  if (!edits.baselineKnown) {
+    // A v1 manifest (or none) has no content baseline — the sync about to run
+    // writes one, and the NEXT cycle can start importing.
+    return { imported: 0, conflicts: [], missing: [] };
+  }
+  const importable = [...edits.modified, ...edits.added];
+  const result = { imported: 0, conflicts: edits.conflicts, missing: edits.missing };
+  if (edits.missing.length > 0) {
+    console.log(
+      `  two-way: ${edits.missing.length} vault file(s) deleted on disk — deletions are never imported; ` +
+        `delete the page in Logseq if you mean it.`
+    );
+  }
+  if (importable.length === 0) return result;
+
+  if (!sessionBackupTaken || importsSinceBackup >= BACKUP_EVERY) {
+    appCliRun("graph", "backup", "create", "--graph", graphName);
+    sessionBackupTaken = true;
+    importsSinceBackup = 0;
+  }
+
+  const tmpEdn = join(tmpdir(), `geml-twoway-${process.pid}-${randomUUID()}.edn`);
+  try {
+    atomicWriteFileSync(
+      tmpEdn,
+      syncDiskToEdn(targetDir, { parse: parseGeml, addressedUnits, sliceUnit }, { exclude: edits.conflicts })
+    );
+    appCliRun("graph", "import", "--graph", graphName, "--type", "edn", "--input", tmpEdn);
+  } finally {
+    if (existsSync(tmpEdn)) { try { unlinkSync(tmpEdn); } catch {} }
+  }
+  importsSinceBackup += 1;
+  result.imported = importable.length;
+  console.log(
+    `[${new Date().toLocaleTimeString()}] two-way: imported ${importable.length} vault edit(s) into "${graphName}"` +
+      (edits.conflicts.length ? `; ${edits.conflicts.length} conflict(s) held` : "") +
+      `.`
+  );
+  return result;
+}
+
 async function performSync() {
   const tempEdnPath = join(tmpdir(), `logseq-export-${process.pid}-${Date.now()}-${randomUUID()}.edn`);
   try {
     // 1. Export from Logseq DB via official CLI.
-    // With a token the CLI goes through the running app's HTTP API server and
-    // exports whatever graph the app has OPEN — the graph name is not part of
-    // that request, so -a REPLACES -g rather than joining it. Without a token
-    // the CLI opens the named graph's sqlite directly, which only works while
-    // the app does not hold the lock on it.
-    if (appCli) {
-      runAppCli(tempEdnPath);
-    } else {
-      const exportSource = apiServerToken ? ["-a", apiServerToken] : ["-g", graphName];
-      runLogseqCli("export-edn", ...exportSource, "-f", tempEdnPath);
-    }
+    exportGraphEdn(tempEdnPath);
 
     if (!existsSync(tempEdnPath)) {
       throw new Error(`Export failed: ${tempEdnPath} was not created.`);
@@ -624,11 +707,26 @@ async function performSync() {
       throw new Error(`Export produced an empty (0 byte) EDN file.`);
     }
 
-    const ednText = readFileSync(tempEdnPath, "utf8");
+    let ednText = readFileSync(tempEdnPath, "utf8");
+
+    // 1.5 Two-way import, BEFORE the unchanged-export short-circuit below:
+    // the graph being unchanged says nothing about the vault.
+    let twoWay = null;
+    if (flags.twoWay) {
+      twoWay = await importExternalEdits(ednText);
+      if (twoWay.imported > 0) {
+        // The graph just absorbed the vault edits — export again, so the disk
+        // write and the manifest baseline hold the merged state.
+        exportGraphEdn(tempEdnPath);
+        ednText = readFileSync(tempEdnPath, "utf8");
+      }
+    }
+    const twoWayActivity =
+      twoWay !== null && (twoWay.imported > 0 || twoWay.conflicts.length > 0 || twoWay.missing.length > 0);
 
     // 2. Efficiency: In watch mode, skip disk scanning if export content is bit-for-bit identical
     const currentHash = createHash("sha256").update(ednText).digest("hex");
-    if (watchMode && currentHash === lastEdnHash) {
+    if (watchMode && currentHash === lastEdnHash && !twoWayActivity) {
       return;
     }
 
@@ -636,6 +734,7 @@ async function performSync() {
     const res = await syncEdnToDisk(ednText, targetDir, {
       autoCommit: gitCommit,
       deleteOrphans: flags.mirror,
+      preserve: twoWay?.conflicts ?? [],
       markdownDir: flags.markdown ? resolve(expandHome(flags.markdown)) : null,
       gemlToMd: gemlSourceToMd,
       commitMessage: flags.message || `logseq-geml: sync graph "${graphName}" (${new Date().toISOString()})`,
@@ -650,18 +749,29 @@ async function performSync() {
       unchanged: res.unchanged.length,
       orphaned: res.orphaned.length,
       deleted: res.deleted.length,
+      imported: twoWay?.imported ?? 0,
+      conflicts: twoWay?.conflicts ?? [],
     });
 
     const timestamp = new Date().toLocaleTimeString();
     const parts = [`${res.written.length} written`, `${res.unchanged.length} unchanged`];
+    if (twoWay && twoWay.imported > 0) {
+      parts.unshift(`${twoWay.imported} imported`);
+    }
     if (res.orphaned && res.orphaned.length > 0) {
       parts.push(`${res.orphaned.length} orphaned/absent from export (preserved safely)`);
     }
     if (res.deleted && res.deleted.length > 0) {
       parts.push(`${res.deleted.length} deleted`);
     }
+    if (twoWay && twoWay.conflicts.length > 0) {
+      console.error(
+        `  ⚠ conflict(s), changed in BOTH the vault and the graph since the last sync — ` +
+          `held as you left them, not imported, not overwritten: ${twoWay.conflicts.join(", ")}`
+      );
+    }
 
-    if (res.written.length > 0 || res.deleted.length > 0) {
+    if (res.written.length > 0 || res.deleted.length > 0 || twoWayActivity) {
       console.log(`[${timestamp}] Synced: ${parts.join(", ")}.`);
       if (res.gitResult && res.gitResult.committed) {
         console.log(`  Git: ${res.gitResult.output}`);