@geml/logseq-sync 2.0.0

package/core/src/sync-engine.mjs

@@ -0,0 +1,324 @@
+// Core Sync Engine for Logseq GEML
+// Keeps a local folder of .geml files in sync with a Logseq DB graph,
+// ensuring only changed files are written so Git diffs stay clean.
+
+import {
+  readdirSync,
+  readFileSync,
+  writeFileSync,
+  unlinkSync,
+  mkdirSync,
+  rmdirSync,
+  renameSync,
+  existsSync,
+} from "node:fs";
+import { join, dirname, relative, resolve, sep } from "node:path";
+import { execFileSync } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import { ednToGemlFiles, gemlFilesToEdn } from "./mapping.mjs";
+
+const MANIFEST_FILE = ".geml-manifest.json";
+
+/**
+ * Normalize line endings to LF, handling CRLF (\r\n) and lone CR (\r).
+ */
+export function normalizeEol(str) {
+  return typeof str === "string" ? str.replace(/\r\n?/g, "\n") : str;
+}
+
+/**
+ * Atomically write a file via a temporary file in the same directory.
+ */
+export function atomicWriteFileSync(filePath, content) {
+  const dir = dirname(filePath);
+  mkdirSync(dir, { recursive: true });
+  const tmpPath = join(dir, `.tmp-${Date.now()}-${process.pid}-${randomUUID()}`);
+  writeFileSync(tmpPath, content, "utf8");
+  renameSync(tmpPath, filePath);
+}
+
+/**
+ * Remove empty parent directories recursively up to stopDir.
+ */
+function cleanEmptyParents(dir, stopDir) {
+  let current = resolve(dir);
+  const stop = resolve(stopDir);
+  while (current && current !== stop && current.startsWith(stop)) {
+    try {
+      const remaining = readdirSync(current);
+      if (remaining.length === 0) {
+        rmdirSync(current);
+        current = dirname(current);
+      } else {
+        break;
+      }
+    } catch {
+      break;
+    }
+  }
+}
+
+/**
+ * Scan a directory recursively for all .geml files.
+ * @param {string} dir Root directory to scan.
+ * @param {string} [baseDir] Base directory for computing relative paths.
+ * @returns {Map<string, string>} Map of relative path (POSIX style) -> file content (normalized LF).
+ */
+export function readGemlFilesFromDisk(dir, baseDir = dir) {
+  const files = new Map();
+  if (!existsSync(dir)) return files;
+
+  const entries = readdirSync(dir, { withFileTypes: true });
+  for (const entry of entries) {
+    const fullPath = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      // Ignore .git, node_modules, and hidden directories
+      if (entry.name.startsWith(".") || entry.name === "node_modules") continue;
+      const subFiles = readGemlFilesFromDisk(fullPath, baseDir);
+      for (const [rel, content] of subFiles) {
+        files.set(rel, content);
+      }
+    } else if (entry.isFile() && entry.name.endsWith(".geml")) {
+      const rel = relative(baseDir, fullPath).split(sep).join("/");
+      files.set(rel, normalizeEol(readFileSync(fullPath, "utf8")));
+    }
+  }
+  return files;
+}
+
+/**
+ * Incrementally sync a Map of GEML files to disk.
+ * Only writes files whose content has changed or do not yet exist.
+ * Detects files on disk that are absent from the export (e.g. deleted pages or journals),
+ * and reports them without destructive deletion by default.
+ * 
+ * Safety note: @logseq/cli 0.4.3 does not include journals in export-edn.
+ * Manifest-based orphan tracking ensures user-authored files outside previous syncs are never deleted.
+ * Destructive deletion requires explicit `opts.deleteOrphans === true`.
+ * 
+ * @param {Map<string, string>} gemlFiles Map of relative path (POSIX) -> gemlText.
+ * @param {string} targetDir Local destination directory.
+ * @param {object} [opts]
+ * @param {boolean} [opts.deleteOrphans=false] Whether to delete previous-sync .geml files no longer in graph.
+ * @returns {{ written: string[], orphaned: string[], unchanged: string[], deleted: string[] }}
+ */
+export function writeGemlFilesToDisk(gemlFiles, targetDir, opts = {}) {
+  const deleteOrphans = opts.deleteOrphans ?? false;
+  const written = [];
+  const unchanged = [];
+  const orphaned = [];
+  const deleted = [];
+
+  mkdirSync(targetDir, { recursive: true });
+  const existingFiles = readGemlFilesFromDisk(targetDir);
+
+  // Load previous sync manifest to know which files belong to sync vs user-authored files
+  const manifestPath = join(targetDir, MANIFEST_FILE);
+  let lastManifest = new Set();
+  if (existsSync(manifestPath)) {
+    try {
+      const parsed = JSON.parse(readFileSync(manifestPath, "utf8"));
+      if (Array.isArray(parsed)) lastManifest = new Set(parsed);
+    } catch {}
+  }
+
+  // Write new or updated files atomically with CRLF normalization
+  for (const [rel, newContent] of gemlFiles) {
+    const fullPath = join(targetDir, rel);
+    const normNew = normalizeEol(newContent);
+    const existingContent = existingFiles.get(rel);
+
+    if (existingContent === undefined || existingContent !== normNew) {
+      atomicWriteFileSync(fullPath, normNew);
+      written.push(rel);
+    } else {
+      unchanged.push(rel);
+    }
+  }
+
+  // Detect files present on disk but absent from current graph export
+  for (const [rel] of existingFiles) {
+    if (!gemlFiles.has(rel)) {
+      orphaned.push(rel);
+      // Safe deletion: only delete if explicit AND file was generated by previous sync
+      // User-authored files in targetDir not in manifest are NEVER deleted.
+      if (deleteOrphans && lastManifest.has(rel)) {
+        const fullPath = join(targetDir, rel);
+        if (existsSync(fullPath)) {
+          unlinkSync(fullPath);
+          cleanEmptyParents(dirname(fullPath), targetDir);
+          deleted.push(rel);
+        }
+      }
+    }
+  }
+
+  // Save updated manifest of managed sync files:
+  // includes all current gemlFiles, plus any existing files on disk that were in lastManifest and not deleted.
+  const currentManifest = new Set(gemlFiles.keys());
+  for (const rel of lastManifest) {
+    if (existingFiles.has(rel) && !deleted.includes(rel)) {
+      currentManifest.add(rel);
+    }
+  }
+  atomicWriteFileSync(manifestPath, JSON.stringify([...currentManifest].sort(), null, 1) + "\n");
+
+  return { written, orphaned, unchanged, deleted };
+}
+
+/**
+ * Execute Git commands to commit changes scoped strictly to the synced files.
+ * Protects parent repository from having unrelated files swept into the commit.
+ * 
+ * @param {string} targetDir Directory where the sync target lives.
+ * @param {string} commitMessage Commit message.
+ * @param {string[]} pathsToCommit Relative paths within targetDir that were written or deleted.
+ * @param {function} [gitRunner] Optional custom git runner `(args) => Promise<{ stdout, stderr, exitCode }>`.
+ * @returns {Promise<{ committed: boolean, changes: boolean, output: string }>}
+ */
+export async function gitAutoCommit(targetDir, commitMessage = "logseq-geml sync", pathsToCommit = [], gitRunner = null) {
+  const defaultRunner = async (args) => {
+    try {
+      const stdout = execFileSync("git", args, {
+        cwd: targetDir,
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "pipe"],
+      });
+      return { stdout, stderr: "", exitCode: 0 };
+    } catch (err) {
+      return {
+        stdout: err.stdout ? String(err.stdout) : "",
+        stderr: err.stderr ? String(err.stderr) : err.message,
+        exitCode: err.status || 1,
+      };
+    }
+  };
+
+  const run = gitRunner || defaultRunner;
+
+  // Check if targetDir is inside a git repository
+  const revRes = await run(["rev-parse", "--show-toplevel"]);
+  if (revRes.exitCode !== 0) {
+    return {
+      committed: false,
+      changes: false,
+      output: `Not a git repository: ${revRes.stderr.trim()}`,
+    };
+  }
+
+  if (!pathsToCommit || pathsToCommit.length === 0) {
+    return {
+      committed: false,
+      changes: false,
+      output: "No synced paths to commit.",
+    };
+  }
+
+  // Always include the manifest file in staged paths
+  const allPaths = [...new Set([...pathsToCommit, MANIFEST_FILE])];
+
+  // Stage ONLY the specified paths (never a bare git add -A)
+  // Split into existing files vs deleted files
+  const toAdd = allPaths.filter((p) => existsSync(join(targetDir, p)));
+  const toRemove = allPaths.filter((p) => !existsSync(join(targetDir, p)));
+
+  if (toAdd.length > 0) {
+    const addRes = await run(["add", "--", ...toAdd]);
+    if (addRes.exitCode !== 0) {
+      return { committed: false, changes: true, output: `git add failed: ${addRes.stderr}` };
+    }
+  }
+
+  if (toRemove.length > 0) {
+    const rmRes = await run(["add", "-u", "--", ...toRemove]);
+    if (rmRes.exitCode !== 0) {
+      return { committed: false, changes: true, output: `git update index failed: ${rmRes.stderr}` };
+    }
+  }
+
+  // Verify whether our target paths have staged changes
+  const statusRes = await run(["status", "--porcelain", "--", ...allPaths]);
+  if (statusRes.exitCode !== 0 || statusRes.stdout.trim().length === 0) {
+    return {
+      committed: false,
+      changes: false,
+      output: "No changes in target paths to commit.",
+    };
+  }
+
+  // Commit with pathspec: commits ONLY changes matching our synced paths,
+  // leaving any other staged or unstaged changes in parent repository untouched!
+  const commitRes = await run(["commit", "-m", commitMessage, "--", ...allPaths]);
+  if (commitRes.exitCode !== 0) {
+    return {
+      committed: false,
+      changes: true,
+      output: `git commit failed: ${commitRes.stderr}`,
+    };
+  }
+
+  return {
+    committed: true,
+    changes: true,
+    output: commitRes.stdout.trim(),
+  };
+}
+
+/**
+ * Full Sync Pipeline from EDN string to disk.
+ * 
+ * @param {string} ednText EDN string (from logseq export-edn).
+ * @param {string} targetDir Destination folder.
+ * @param {object} [opts]
+ * @param {boolean} [opts.autoCommit=false]
+ * @param {string} [opts.commitMessage]
+ * @param {boolean} [opts.allowEmptyGraph=false] Refuse 0-page export over non-empty targetDir unless true.
+ * @param {function} [opts.gitRunner]
+ * @returns {Promise<{ written: string[], deleted: string[], orphaned: string[], unchanged: string[], gitResult?: any }>}
+ */
+export async function syncEdnToDisk(ednText, targetDir, opts = {}) {
+  // Guard 1: Refuse empty or truncated EDN input
+  if (!ednText || typeof ednText !== "string" || ednText.trim().length === 0) {
+    throw new Error("EDN input is empty or truncated; refusing to sync to prevent data loss.");
+  }
+
+  const gemlFiles = ednToGemlFiles(ednText);
+
+  // Guard 2: Refuse 0-page export if targetDir already has existing pages
+  const pageCount = [...gemlFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length;
+  const existingFiles = readGemlFilesFromDisk(targetDir);
+  const existingPageCount = [...existingFiles.keys()].filter((k) => k.startsWith("pages/") || k.startsWith("journals/")).length;
+
+  if (pageCount === 0 && existingPageCount > 0 && !opts.allowEmptyGraph) {
+    throw new Error(
+      `Refusing to sync empty graph (0 pages) over directory with existing pages (${existingPageCount} pages). Pass allowEmptyGraph: true to force.`
+    );
+  }
+
+  const diffResult = writeGemlFilesToDisk(gemlFiles, targetDir, opts);
+
+  let gitResult = null;
+  const pathsModified = [...diffResult.written, ...diffResult.deleted];
+
+  if (opts.autoCommit && pathsModified.length > 0) {
+    const msg = opts.commitMessage || `logseq-geml: synced ${diffResult.written.length} modified, ${diffResult.deleted.length} deleted`;
+    gitResult = await gitAutoCommit(targetDir, msg, pathsModified, opts.gitRunner);
+  }
+
+  return {
+    ...diffResult,
+    gitResult,
+  };
+}
+
+/**
+ * Full Sync Pipeline from disk back to EDN string.
+ * 
+ * @param {string} targetDir Local folder containing .geml files.
+ * @param {object} lib Parser library containing { parse, addressedUnits, sliceUnit }.
+ * @returns {string} EDN string ready for logseq import-edn.
+ */
+export function syncDiskToEdn(targetDir, lib) {
+  const files = readGemlFilesFromDisk(targetDir);
+  return gemlFilesToEdn(files, lib);
+}

package/docs/how-it-works.svg

@@ -0,0 +1,42 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 250" font-family="ui-sans-serif,system-ui,Segoe UI,Helvetica,Arial,sans-serif">
+  <defs>
+    <marker id="arr" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
+      <path d="M0,0 L10,5 L0,10 z" fill="#8b949e"/>
+    </marker>
+  </defs>
+  <rect x="0" y="0" width="760" height="250" rx="12" fill="#f6f8fa" stroke="#d0d7de"/>
+
+  <!-- Logseq app box -->
+  <rect x="24" y="36" width="220" height="130" rx="10" fill="#ffffff" stroke="#a475f9" stroke-width="1.5"/>
+  <text x="134" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">Logseq 2.0 (DB graph)</text>
+  <rect x="44" y="80" width="180" height="66" rx="8" fill="#f3ecff" stroke="#a475f9"/>
+  <text x="134" y="104" text-anchor="middle" font-size="13" font-weight="600" fill="#3b2a63">Sync Vault with GEML plugin</text>
+  <text x="134" y="124" text-anchor="middle" font-size="11.5" fill="#57606a">DB.onChanged → debounce</text>
+  <text x="134" y="139" text-anchor="middle" font-size="11.5" fill="#57606a">status in toolbar ⇄</text>
+
+  <!-- signal file -->
+  <rect x="286" y="70" width="160" height="42" rx="8" fill="#fff8c5" stroke="#d4a72c"/>
+  <text x="366" y="88" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">dirty-marker file</text>
+  <text x="366" y="103" text-anchor="middle" font-size="10.5" fill="#7d8590">…/storages/logseq-plugin-sync-vault-with-geml/</text>
+
+  <!-- status file -->
+  <rect x="286" y="128" width="160" height="34" rx="8" fill="#ddf4ff" stroke="#54aeff"/>
+  <text x="366" y="149" text-anchor="middle" font-size="12" font-weight="600" fill="#57606a">geml-sync-status.json</text>
+
+  <!-- watcher box -->
+  <rect x="488" y="36" width="248" height="130" rx="10" fill="#ffffff" stroke="#2da44e" stroke-width="1.5"/>
+  <text x="612" y="62" text-anchor="middle" font-size="15" font-weight="700" fill="#57606a">geml-sync watcher (CLI)</text>
+  <text x="612" y="86" text-anchor="middle" font-size="11.5" fill="#57606a">export via official @logseq/cli</text>
+  <text x="612" y="103" text-anchor="middle" font-size="11.5" fill="#57606a">writes only files that changed</text>
+  <text x="612" y="120" text-anchor="middle" font-size="11.5" fill="#57606a">git commit scoped to the vault</text>
+  <text x="612" y="145" text-anchor="middle" font-size="11.5" font-weight="600" fill="#2da44e">your-vault/pages/*.geml · git</text>
+
+  <!-- arrows -->
+  <line x1="244" y1="91" x2="284" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
+  <line x1="446" y1="91" x2="486" y2="91" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
+  <line x1="486" y1="145" x2="446" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
+  <line x1="286" y1="145" x2="246" y2="145" stroke="#8b949e" stroke-width="1.6" marker-end="url(#arr)"/>
+
+  <text x="380" y="196" text-anchor="middle" font-size="12.5" fill="#57606a">The plugin is a doorbell, not the mover: the sandbox has no filesystem and no git,</text>
+  <text x="380" y="214" text-anchor="middle" font-size="12.5" fill="#57606a">so everything with side effects lives in the watcher — auditable, scoped, outside the app.</text>
+</svg>

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@geml/logseq-sync",
+  "version": "2.0.0",
+  "publishConfig": {
+    "access": "public"
+  },
+  "description": "Continuously sync a Logseq DB graph to a Git-friendly folder of readable plain-text GEML files — the watcher half of the Sync Vault with GEML plugin. Built on the official @logseq/cli export; writes only files that changed, commits scoped strictly to the vault.",
+  "type": "module",
+  "bin": {
+    "geml-sync": "watcher/bin/geml-sync.mjs"
+  },
+  "files": [
+    "core/src",
+    "watcher/bin",
+    "docs",
+    "README.md",
+    "LICENSE"
+  ],
+  "engines": {
+    "node": ">=22"
+  },
+  "keywords": [
+    "logseq",
+    "logseq-plugin",
+    "sync",
+    "git",
+    "plain-text",
+    "geml",
+    "vault",
+    "backup"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/geml-spec/geml.git",
+    "directory": "integrations/logseq"
+  },
+  "homepage": "https://github.com/geml-spec/logseq-plugin-sync-vault-with-geml#readme",
+  "bugs": {
+    "url": "https://github.com/geml-spec/logseq-plugin-sync-vault-with-geml/issues"
+  },
+  "license": "MIT",
+  "workspaces": [
+    "plugin"
+  ],
+  "scripts": {
+    "test": "node core/test/roundtrip.test.mjs && node core/test/sync.test.mjs && node watcher/test/cli-sync.test.mjs && node watcher/test/signal-sync.test.mjs && node plugin/test/core.test.mjs",
+    "sync": "node watcher/bin/geml-sync.mjs",
+    "build:plugin": "node plugin/build.mjs"
+  },
+  "dependencies": {
+    "edn-data": "^1.2.2"
+  }
+}

package/watcher/bin/create-graph.mjs

@@ -0,0 +1,51 @@
+// Create an empty Logseq DB graph WITHOUT the desktop app.
+//
+//   LOGSEQ_CLI_DIR=<dir whose node_modules holds @logseq/cli>  \
+//   node bin/create-graph.mjs <graph-name>   (name travels via GEML_GRAPH_NAME —
+//   nbb loadFile does not surface *command-line-args*)
+//
+// Why this exists: `@logseq/cli` (0.4.3) can export, import and validate a DB
+// graph, but cannot create one — creation lives in the desktop app, and on a
+// machine where the app cannot run (permissions, CI) that is a dead end. The
+// CLI package VENDORS the whole logseq.db stack though, and its `open-db!`
+// creates the sqlite tables on open; the only other thing the app does at
+// create time is transact `build-db-initial-data`. So this does exactly those
+// two steps, through the CLI's own vendored code — the resulting graph is one
+// `logseq list/show/validate` accepts as its own (verified: schema 65.22,
+// "Valid!").
+import { fileURLToPath, pathToFileURL } from "url";
+import { dirname, resolve } from "path";
+import { existsSync, readFileSync, copyFileSync, rmSync } from "fs";
+
+const here = fileURLToPath(dirname(import.meta.url));
+const cliDir = process.env.LOGSEQ_CLI_DIR ?? resolve(here, "..");
+const CLI = resolve(cliDir, "node_modules", "@logseq", "cli");
+if (!existsSync(CLI)) {
+  console.error(`@logseq/cli not found under ${cliDir}/node_modules — set LOGSEQ_CLI_DIR to a directory where it is installed.`);
+  console.error("Note: on Node 24 its better-sqlite3 needs an override to >=12.11.1 for a prebuilt binding.");
+  process.exit(2);
+}
+
+// nbb-logseq is resolved from the CLI's install, not from this package: ESM
+// import specifiers resolve relative to THIS file, which would demand a local
+// install of a runtime the CLI already carries.
+const nbbDir = resolve(cliDir, "node_modules", "@logseq", "nbb-logseq");
+const nbbPkg = JSON.parse(readFileSync(resolve(nbbDir, "package.json"), "utf8"));
+const entry = typeof nbbPkg.exports === "object"
+  ? (nbbPkg.exports["."]?.import ?? nbbPkg.exports["."]) : (nbbPkg.main ?? "index.mjs");
+const { loadFile, addClassPath } = await import(pathToFileURL(resolve(nbbDir, entry)).href);
+
+global.__dirname = here;
+addClassPath(resolve(CLI, "src"));
+addClassPath(resolve(CLI, "vendor/src"));
+// nbb resolves the stack's node `require`s (better-sqlite3) relative to the
+// LOADED FILE's directory, so the .cljs must sit beside a node_modules that
+// has them — copy it into the CLI dir for the duration of the run.
+process.env.GEML_GRAPH_NAME = process.argv[2] ?? "geml-spike";
+const staged = resolve(cliDir, ".geml-create-graph.cljs");
+copyFileSync(resolve(here, "create_graph_headless.cljs"), staged);
+try {
+  await loadFile(staged);
+} finally {
+  rmSync(staged, { force: true });
+}

package/watcher/bin/create_graph_headless.cljs

@@ -0,0 +1,22 @@
+(ns create-graph-headless
+  "Create an empty Logseq DB graph without the desktop app: the CLI package
+   vendors the whole logseq.db stack, and open-db! creates tables on open.
+   This does exactly what the app's create-graph does at the db level:
+   mkdir + open + transact build-db-initial-data.
+
+   open-db! creates the DATABASE but refuses a missing DIRECTORY (its error
+   says only 'Cannot open database because the directory does not exist'),
+   so the mkdir here is load-bearing."
+  (:require [logseq.db.common.sqlite-cli :as sqlite-cli]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [datascript.core :as d]
+            ["fs" :as fs]
+            ["os" :as os]
+            ["path" :as node-path]))
+
+(def graph-name (or (aget (.-env js/process) "GEML_GRAPH_NAME") "geml-spike"))
+(def graphs-dir (node-path/join (os/homedir) "logseq" "graphs"))
+(fs/mkdirSync (node-path/join graphs-dir graph-name) #js {:recursive true})
+(def conn (sqlite-cli/open-db! graphs-dir graph-name))
+(d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))
+(println "created" (node-path/join graphs-dir graph-name))