@geml/logseq-sync 2.0.0

package/watcher/bin/geml-sync.mjs

@@ -0,0 +1,290 @@
+#!/usr/bin/env node
+// CLI Companion for Logseq GEML Sync
+// Runs continuous or one-shot sync from a Logseq DB graph to a local Git-versioned GEML folder.
+//
+// Usage:
+//   node watcher/bin/geml-sync.mjs <graph-name> <target-dir> [flags]
+//
+// Flags:
+//   --watch               Run in continuous watch/sync loop
+//   --interval <seconds>  Poll interval for watch mode (positive integer, default: 10)
+//   --git-commit          Auto-commit changes to git (scoped strictly to sync folder)
+//   --message <text>      Custom git commit message template
+//   --signal <file>       Sync immediately when this file changes (the in-app
+//                         plugin writes it via logseq.FileStorage), and write
+//                         the sync result to geml-sync-status.json beside it
+//                         so the plugin can show it. Interval stays as fallback.
+
+import { execFileSync } from "node:child_process";
+import { readFileSync, unlinkSync, existsSync, statSync, mkdirSync, watch } from "node:fs";
+import { join, resolve, dirname, basename } from "node:path";
+import { tmpdir } from "node:os";
+import { randomUUID, createHash } from "node:crypto";
+import { syncEdnToDisk, atomicWriteFileSync } from "../../core/src/sync-engine.mjs";
+import { STATUS_FILE } from "../../core/src/bridge.mjs";
+
+const args = process.argv.slice(2);
+const positional = [];
+const flags = {
+  watch: false,
+  gitCommit: false,
+  interval: 10,
+  message: null,
+  signal: null,
+};
+
+for (let i = 0; i < args.length; i++) {
+  const arg = args[i];
+  if (arg === "--watch") {
+    flags.watch = true;
+  } else if (arg === "--git-commit") {
+    flags.gitCommit = true;
+  } else if (arg === "--interval") {
+    if (i + 1 >= args.length) {
+      console.error("Error: --interval requires a value.");
+      process.exit(2);
+    }
+    const rawVal = args[++i];
+    const val = Number(rawVal);
+    if (!Number.isInteger(val) || val <= 0) {
+      console.error(`Error: --interval must be a positive integer >= 1 (got "${rawVal}").`);
+      process.exit(2);
+    }
+    flags.interval = val;
+  } else if (arg === "--message") {
+    if (i + 1 >= args.length) {
+      console.error("Error: --message requires a value.");
+      process.exit(2);
+    }
+    flags.message = args[++i];
+  } else if (arg === "--signal") {
+    if (i + 1 >= args.length) {
+      console.error("Error: --signal requires a value.");
+      process.exit(2);
+    }
+    flags.signal = args[++i];
+  } else if (arg.startsWith("--")) {
+    console.error(`Error: Unknown flag "${arg}".`);
+    process.exit(2);
+  } else {
+    positional.push(arg);
+  }
+}
+
+if (positional.length < 2) {
+  console.error("Usage: node watcher/bin/geml-sync.mjs <graph-name> <target-dir> [--watch] [--interval <sec>] [--git-commit] [--message <text>] [--signal <file>]");
+  process.exit(2);
+}
+
+const [graphName, targetDirRaw] = positional;
+
+// Validate graph name to prevent command/path injection
+if (!/^[a-zA-Z0-9_.-]+$/.test(graphName)) {
+  console.error(`Error: Invalid graph name "${graphName}". Only alphanumeric characters, hyphens, and underscores are allowed.`);
+  process.exit(2);
+}
+
+const targetDir = resolve(targetDirRaw);
+const cliCwd = process.env.LOGSEQ_CLI_DIR ?? process.cwd();
+const signalPath = flags.signal ? resolve(flags.signal) : null;
+
+// The status file lands beside the signal file — the plugin's storage
+// directory — the one place logseq.FileStorage.getItem can read it back from.
+function writeStatus(status) {
+  if (!signalPath) return;
+  try {
+    atomicWriteFileSync(join(dirname(signalPath), STATUS_FILE), JSON.stringify(status, null, 1) + "\n");
+  } catch (err) {
+    console.error(`Could not write status file: ${err.message}`);
+  }
+}
+
+// Find @logseq/cli entry point or run via npx without shell: true
+function runLogseqCli(...cmdArgs) {
+  const directCliPath = resolve(cliCwd, "node_modules", "@logseq", "cli", "cli.mjs");
+  if (existsSync(directCliPath)) {
+    return execFileSync(process.execPath, [directCliPath, ...cmdArgs], {
+      cwd: cliCwd,
+      encoding: "utf8",
+      shell: false,
+      maxBuffer: 1 << 28,
+    });
+  }
+
+  // Fallback to npx (executable npx.cmd on Windows, npx on Unix) without shell: true
+  const npxCmd = process.platform === "win32" ? "npx.cmd" : "npx";
+  return execFileSync(npxCmd, ["-y", "@logseq/cli", ...cmdArgs], {
+    cwd: cliCwd,
+    encoding: "utf8",
+    shell: false,
+    maxBuffer: 1 << 28,
+  });
+}
+
+let lastEdnHash = null;
+
+async function performSync() {
+  const tempEdnPath = join(tmpdir(), `logseq-export-${process.pid}-${Date.now()}-${randomUUID()}.edn`);
+  try {
+    // 1. Export from Logseq DB via official CLI
+    runLogseqCli("export-edn", "-g", graphName, "-f", tempEdnPath);
+
+    if (!existsSync(tempEdnPath)) {
+      throw new Error(`Export failed: ${tempEdnPath} was not created.`);
+    }
+
+    const stat = statSync(tempEdnPath);
+    if (stat.size === 0) {
+      throw new Error(`Export produced an empty (0 byte) EDN file.`);
+    }
+
+    const ednText = readFileSync(tempEdnPath, "utf8");
+
+    // 2. Efficiency: In watch mode, skip disk scanning if export content is bit-for-bit identical
+    const currentHash = createHash("sha256").update(ednText).digest("hex");
+    if (flags.watch && currentHash === lastEdnHash) {
+      return;
+    }
+
+    // 3. Incremental sync to disk
+    const res = await syncEdnToDisk(ednText, targetDir, {
+      autoCommit: flags.gitCommit,
+      commitMessage: flags.message || `logseq-geml: sync graph "${graphName}" (${new Date().toISOString()})`,
+    });
+
+    lastEdnHash = currentHash;
+    writeStatus({
+      ok: true,
+      at: new Date().toISOString(),
+      graph: graphName,
+      written: res.written.length,
+      unchanged: res.unchanged.length,
+      orphaned: res.orphaned.length,
+      deleted: res.deleted.length,
+    });
+
+    const timestamp = new Date().toLocaleTimeString();
+    const parts = [`${res.written.length} written`, `${res.unchanged.length} unchanged`];
+    if (res.orphaned && res.orphaned.length > 0) {
+      parts.push(`${res.orphaned.length} orphaned/absent from export (preserved safely)`);
+    }
+    if (res.deleted && res.deleted.length > 0) {
+      parts.push(`${res.deleted.length} deleted`);
+    }
+
+    if (res.written.length > 0 || res.deleted.length > 0) {
+      console.log(`[${timestamp}] Synced: ${parts.join(", ")}.`);
+      if (res.gitResult && res.gitResult.committed) {
+        console.log(`  Git: ${res.gitResult.output}`);
+      }
+    } else if (!flags.watch) {
+      console.log(`[${timestamp}] Graph is up-to-date (${parts.join(", ")}).`);
+    }
+  } catch (err) {
+    writeStatus({ ok: false, at: new Date().toISOString(), graph: graphName, error: err.message });
+    throw err;
+  } finally {
+    if (existsSync(tempEdnPath)) {
+      try { unlinkSync(tempEdnPath); } catch {}
+    }
+  }
+}
+
+async function main() {
+  console.log(`Starting GEML Sync: Graph "${graphName}" ➔ ${targetDir}`);
+  if (flags.gitCommit) console.log("Git auto-commit: enabled (scoped to target paths)");
+
+  if (!flags.watch) {
+    // One-shot mode: fail loudly with non-zero exit code if sync fails
+    try {
+      await performSync();
+    } catch (err) {
+      console.error(`[${new Date().toLocaleTimeString()}] Sync failed:`, err.message);
+      process.exit(1);
+    }
+    return;
+  }
+
+  // Watch mode: sequential non-overlapping syncs. The interval loop is the
+  // heartbeat; a --signal file, when given, triggers a sync the moment the
+  // in-app plugin reports a change, instead of waiting out the interval.
+  console.log(`Watch mode active (polling every ${flags.interval}s). Press Ctrl+C to stop.`);
+
+  let running = true;
+  let timer = null;
+  let isSyncing = false;
+  let queued = false;
+  let fsWatcher = null;
+  let signalTimer = null;
+
+  const cleanup = () => {
+    running = false;
+    if (timer) clearTimeout(timer);
+    if (signalTimer) clearTimeout(signalTimer);
+    if (fsWatcher) fsWatcher.close();
+    console.log("\nWatch mode stopped.");
+    process.exit(0);
+  };
+
+  process.on("SIGINT", cleanup);
+  process.on("SIGTERM", cleanup);
+
+  async function requestSync() {
+    if (!running) return;
+    if (isSyncing) {
+      // A change arrived mid-sync: run once more when this one finishes,
+      // rather than dropping it or overlapping exports.
+      queued = true;
+      return;
+    }
+    isSyncing = true;
+    try {
+      await performSync();
+    } catch (err) {
+      console.error(`[${new Date().toLocaleTimeString()}] Sync error:`, err.message);
+    } finally {
+      isSyncing = false;
+    }
+    if (queued) {
+      queued = false;
+      await requestSync();
+    }
+  }
+
+  function scheduleNext() {
+    if (!running) return;
+    timer = setTimeout(async () => {
+      await requestSync();
+      scheduleNext();
+    }, flags.interval * 1000);
+  }
+
+  if (signalPath) {
+    const signalDir = dirname(signalPath);
+    mkdirSync(signalDir, { recursive: true });
+    try {
+      // Watch the directory, not the file: the plugin's storage write may
+      // replace the file, and a watch pinned to the old inode goes silent.
+      fsWatcher = watch(signalDir, (eventType, filename) => {
+        // A null filename is legal on some platforms; treat it as a hit.
+        if (filename && filename !== basename(signalPath)) return;
+        if (signalTimer) clearTimeout(signalTimer);
+        signalTimer = setTimeout(() => {
+          signalTimer = null;
+          requestSync();
+        }, 300);
+      });
+      console.log(`Signal file watched: ${signalPath}`);
+    } catch (err) {
+      console.error(`Signal watch failed (${err.message}); interval polling only.`);
+    }
+  }
+
+  await requestSync();
+  scheduleNext();
+}
+
+main().catch((err) => {
+  console.error("Fatal:", err);
+  process.exit(1);
+});

package/watcher/bin/live-roundtrip.mjs

@@ -0,0 +1,131 @@
+// The live half of the spike: run the round trip against a REAL DB graph via
+// the official @logseq/cli, with Logseq's own `validate` as the judge.
+//
+//   node bin/live-roundtrip.mjs <graph-name> [--edit]
+//
+// Stages (each printed, each gated):
+//   1. `logseq export-edn -g <graph>`            → out/export-1.edn
+//   2. ednToGemlFiles                            → out/geml/**.geml
+//   3. `geml check --root out/geml` on every doc (zero errors required)
+//   4. gemlFilesToEdn                            → out/import.edn
+//   5. STRUCTURAL identity export-1 ⇄ import.edn (EDN semantics) — the offline
+//      criterion, now on real data
+//   6. with --edit: `geml set` the first uuid block, rebuild import.edn, and
+//      `logseq import-edn` it back, then `logseq validate` — the semantics
+//      probe: does re-import by uuid update in place, or append?
+//
+// Import-back is opt-in (--edit) because import semantics on a whole-graph
+// re-import are exactly what this stage exists to LEARN — it may merge, it may
+// duplicate id-less blocks. The script never touches a graph unless told to.
+import { execFileSync } from "node:child_process";
+import { mkdirSync, writeFileSync, readFileSync, rmSync, readdirSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { parseEDNString } from "edn-data";
+import { ednToGemlFiles, gemlFilesToEdn } from "../../core/src/mapping.mjs";
+import { parse, addressedUnits, sliceUnit } from "../../../../geml-parser/dist/geml.js";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const out = join(here, "..", "out");
+const GEML = resolve(here, "..", "..", "..", "geml-parser", "dist", "geml.js");
+const lib = { parse, addressedUnits, sliceUnit };
+
+const [graph, ...flags] = process.argv.slice(2);
+if (!graph) { console.error("usage: node bin/live-roundtrip.mjs <graph-name> [--edit]"); process.exit(2); }
+const doEdit = flags.includes("--edit");
+
+// The CLI is driven through npx so nothing here depends on a global install;
+// LOGSEQ_CLI_DIR points at a directory whose node_modules has @logseq/cli.
+const cliCwd = process.env.LOGSEQ_CLI_DIR ?? join(here, "..");
+const logseq = (...args) =>
+  execFileSync("npx", ["-y", "@logseq/cli", ...args], { cwd: cliCwd, encoding: "utf8", shell: true, maxBuffer: 1 << 28 });
+const geml = (...args) =>
+  execFileSync(process.execPath, [GEML, ...args], { encoding: "utf8", maxBuffer: 1 << 28 });
+
+// EDN-semantics canonical form (same as the test suite).
+function canon(v) {
+  if (Array.isArray(v)) return v.map(canon);
+  if (v !== null && typeof v === "object") {
+    if (Array.isArray(v.map)) {
+      const entries = v.map.map(([k, val]) => [canon(k), canon(val)]);
+      entries.sort((a, b) => (JSON.stringify(a[0]) < JSON.stringify(b[0]) ? -1 : 1));
+      return { map: entries };
+    }
+    if (Array.isArray(v.set)) {
+      const items = v.set.map(canon);
+      items.sort((a, b) => (JSON.stringify(a) < JSON.stringify(b) ? -1 : 1));
+      return { set: items };
+    }
+    const o = {};
+    for (const k of Object.keys(v).sort()) o[k] = canon(v[k]);
+    return o;
+  }
+  return v;
+}
+const same = (a, b) => JSON.stringify(canon(parseEDNString(a))) === JSON.stringify(canon(parseEDNString(b)));
+
+rmSync(out, { recursive: true, force: true });
+mkdirSync(join(out, "geml"), { recursive: true });
+
+console.log(`1. export-edn from graph "${graph}"`);
+logseq("export-edn", "-g", graph, "-f", join(out, "export-1.edn"));
+const edn1 = readFileSync(join(out, "export-1.edn"), "utf8");
+console.log(`   ${edn1.length} bytes of EDN`);
+
+console.log("2. EDN -> GEML");
+const files = ednToGemlFiles(edn1);
+for (const [rel, text] of files) {
+  mkdirSync(dirname(join(out, "geml", rel)), { recursive: true });
+  writeFileSync(join(out, "geml", rel), text);
+}
+console.log(`   ${files.size} documents`);
+
+console.log("3. geml check on every document");
+let dirty = 0;
+for (const rel of files.keys()) {
+  try { geml("check", "--root", join(out, "geml"), join(out, "geml", rel)); }
+  catch (e) { dirty++; console.error(`   FAIL ${rel}\n${e.stdout ?? ""}${e.stderr ?? ""}`); }
+}
+if (dirty) { console.error(`   ${dirty} document(s) not clean — stopping`); process.exit(1); }
+console.log("   all clean");
+
+console.log("4. GEML -> EDN");
+const files2 = new Map();
+for (const rel of files.keys()) files2.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
+const edn2 = gemlFilesToEdn(files2, lib);
+writeFileSync(join(out, "import.edn"), edn2);
+
+console.log("5. structural identity, on the real graph");
+if (!same(edn1, edn2)) { console.error("   NOT identical — diff out/export-1.edn against out/import.edn"); process.exit(1); }
+console.log("   identical (EDN semantics)");
+
+if (!doEdit) { console.log("\nround trip holds. Re-run with --edit to probe import-back semantics."); process.exit(0); }
+
+console.log("6. edit one uuid block via `geml set`, import back, validate");
+const withUuid = [...files.keys()].map((rel) => {
+  const text = files2.get(rel);
+  const unit = [...addressedUnits(text)].map((a) => a.unit).find((u) => u.kind === "block" && u.id && /^[0-9a-f-]{36}$/.test(u.id));
+  return unit ? { rel, unit } : null;
+}).find(Boolean);
+if (!withUuid) { console.log("   no uuid-bearing block in this graph (nothing referenced) — skipping the edit probe"); process.exit(0); }
+
+const target = join(out, "geml", withUuid.rel);
+// Keep the block's own head line (type, id, level) — the edit is to the BODY.
+const src = files2.get(withUuid.rel);
+const head = sliceUnit(src, withUuid.unit.span, "head").trimEnd();
+const body = sliceUnit(src, withUuid.unit.span, "body").trimEnd();
+const fence = head.match(/^=+/)[0];
+writeFileSync(join(out, "edit.txt"), `${head}\n${body} — edited by geml\n${fence}\n`);
+geml("set", target, `#${withUuid.unit.id}`, "--in", join(out, "edit.txt"), "--root", join(out, "geml"));
+console.log(`   edited #${withUuid.unit.id} in ${withUuid.rel}`);
+
+const files3 = new Map();
+for (const rel of files.keys()) files3.set(rel, readFileSync(join(out, "geml", rel), "utf8"));
+writeFileSync(join(out, "import-edited.edn"), gemlFilesToEdn(files3, lib));
+logseq("import-edn", "-g", graph, "-f", join(out, "import-edited.edn"));
+console.log("   imported");
+console.log(logseq("validate", "-g", graph).trim());
+
+console.log("7. export again — inspect out/export-2.edn to judge merge semantics");
+logseq("export-edn", "-g", graph, "-f", join(out, "export-2.edn"));
+console.log("done — compare out/export-1.edn / out/export-2.edn");