create-evda 0.1.8 → 0.1.9

package/lib/sync-lock.mjs

@@ -0,0 +1,143 @@
+// One sync at a time per record, across processes (HQ-1925).
+//
+// The Mac app syncs on its own, a launchd job or a person at a terminal can run `evda sync`, and the
+// MCP's evda_sync_devices syncs too. Every one of them reaches the relay through lib/pairing.mjs, so
+// the lock is taken there and nowhere else.
+//
+// The lock is `<record>/.evda/sync.lock`: the record's derived state, which sync-rules never sends
+// to a device, never under record/. It holds the holder's pid and that process's start time. A lock
+// is stale only when its holder is gone: the pid runs no process, or the process running under that
+// pid started at a different time (the pid was reused). Its age never counts, because a sync over a
+// slow network is still a live sync.
+//
+
+import { execFileSync } from "node:child_process";
+import { mkdirSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { randomBytes } from "node:crypto";
+import { join } from "node:path";
+import { EvdaError } from "./errors.mjs";
+
+export const SYNC_LOCK = join(".evda", "sync.lock");
+export const SYNC_BUSY = "A sync of this EVÐA is already running. Try again in a moment.";
+export const SYNC_WAIT_MS = 20_000;
+
+/** When the process with this pid started, as the system reports it, or null when no process has that pid. */
+export function processStart(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return null;
+  try {
+    process.kill(pid, 0);
+  } catch (error) {
+    if (error?.code !== "EPERM") return null; // EPERM: it runs, as someone else
+  }
+  if (process.platform === "win32") return ""; // no start time to read; the pid alone decides
+  try {
+    const out = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], {
+      env: { PATH: "/bin:/usr/bin", LC_ALL: "C" },
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    }).trim();
+    return out || null;
+  } catch {
+    return null; // ps exits 1 when the pid is gone between the two checks
+  }
+}
+
+/** Is the holder named in a lock file still the process that wrote it? */
+export function holderAlive(holder, { startOf = processStart } = {}) {
+  if (!holder || !Number.isInteger(holder.pid)) return false;
+  const now = startOf(holder.pid);
+  if (now === null) return false;
+  if (now === "" || !holder.start) return true; // no start time on this system: the pid decides
+  return now === holder.start;
+}
+
+function readHolder(raw) {
+  try {
+    const h = JSON.parse(raw);
+    return h && typeof h === "object" ? h : null;
+  } catch {
+    return null;
+  }
+}
+
+function readRaw(path) {
+  try {
+    return readFileSync(path, "utf8");
+  } catch (error) {
+    if (error?.code === "ENOENT") return null;
+    throw error;
+  }
+}
+
+export const SYNC_GUARD = join(".evda", "sync.guard");
+
+// Judging a lock stale and taking it must be one step, or two processes that both read the same
+// stale lock can each take it. That step runs inside an exclusive SQLite transaction on
+// .evda/sync.guard: SQLite holds it with an OS file lock, which the OS drops when a process dies, so
+// the guard itself can never be left stale. It is held only for the few file operations below,
+// never for a sync. node:sqlite is loaded here and not at the top, because pairing (the only caller)
+// already needs it and every other evda command must still run on a Node without it.
+async function guarded(dir, fn) {
+  const { DatabaseSync } = await import("node:sqlite");
+  const db = new DatabaseSync(join(dir, SYNC_GUARD));
+  try {
+    db.exec("PRAGMA busy_timeout = 10000");
+    db.exec("BEGIN EXCLUSIVE");
+    try {
+      return fn();
+    } finally {
+      db.exec("COMMIT");
+    }
+  } finally {
+    db.close();
+  }
+}
+
+const pause = (ms) => new Promise((ok) => setTimeout(ok, ms));
+
+/**
+ * Take the record's sync lock, waiting up to `waitMs` for a running sync to finish. Returns a
+ * release function. When the wait runs out, throws SYNC_BUSY (exit 2: nothing was written).
+ * `beforeReclaim` is for tests: it runs, synchronously, between judging a lock stale and taking it.
+ */
+export async function acquireSyncLock(dir, { waitMs, pollMs = 100, startOf = processStart, env = process.env, beforeReclaim } = {}) {
+  const fromEnv = env.EVDA_SYNC_WAIT_MS ? Number(env.EVDA_SYNC_WAIT_MS) : NaN;
+  const wait = waitMs ?? (Number.isFinite(fromEnv) && fromEnv >= 0 ? fromEnv : SYNC_WAIT_MS);
+  const path = join(dir, SYNC_LOCK);
+  mkdirSync(join(dir, ".evda"), { recursive: true });
+  const mine = JSON.stringify({ pid: process.pid, start: startOf(process.pid) || null, n: randomBytes(4).toString("hex") });
+  const deadline = Date.now() + wait;
+  for (;;) {
+    const took = await guarded(dir, () => {
+      const raw = readRaw(path);
+      if (raw !== null) {
+        if (holderAlive(readHolder(raw), { startOf })) return false;
+        if (beforeReclaim) beforeReclaim();
+        unlinkSync(path);
+      }
+      writeFileSync(path, mine, { flag: "wx" });
+      return true;
+    });
+    if (took) break;
+    if (Date.now() >= deadline) throw new EvdaError(SYNC_BUSY, { code: "SYNC_BUSY", exit: 2 });
+    await pause(pollMs);
+  }
+  let released = false;
+  return async function release() {
+    if (released) return;
+    released = true;
+    await guarded(dir, () => {
+      if (readRaw(path) === mine) unlinkSync(path);
+    });
+  };
+}
+
+/** Run `fn` holding the record's sync lock; the lock is released however `fn` ends. */
+export async function withSyncLock(dir, fn, opts = {}) {
+  const release = await acquireSyncLock(dir, opts);
+  try {
+    return await fn();
+  } finally {
+    await release();
+  }
+}

package/lib/sync-rules.mjs

@@ -21,7 +21,8 @@
 // And one kind cannot sync yet: a file larger than the relay's blob lane takes (64 MB, less the
 // seal), which fails the coverage check until the plan excludes it.
 
-import { lstatSync, readdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { lstatSync, readdirSync, statfsSync } from "node:fs";
 import { join } from "node:path";
 
 export const EXCLUDE_NAMES = [".DS_Store", "node_modules", ".git", ".Trash", "Icon\r"];
@@ -91,3 +92,155 @@ export function scanSkip(extra = []) {
     return isPlaceholder(name) || st.size > MAX_SYNC_BYTES;
   };
 }
+
+// ─── What travels (HQ-2017) ─────────────────────────────────────────────────────────────────────
+//
+// Only text travels by default. His decision, 2026-09-30: "I don't think it needs to carry media. We
+// don't need media to be the thing that travels. That can be one device." A pairing sealed every
+// voice memo, video and PDF of a 1.8 GB iCloud record into the device database (21 GB, the disk full).
+//
+// One rule, read by the scan (which files are read), the minter and the catch-up (what a Change
+// carries) and parity (what a shadow must hold), on the laptop's `evda sync` and on the shadow tick:
+//
+//   text       a name ending in one of TEXT_EXTENSIONS. Minted with its bytes, as before.
+//   carried    any other file under a path the manifest names in `sync.carry`. Its bytes ride the
+//              blob lane, as every binary did before.
+//   reference  everything else. Its Change names the file (path, name, size, sha256) and carries no
+//              bytes; nothing is sealed into the blob store and nothing is uploaded.
+//
+// Decided from the NAME alone, never from the bytes, because reading a file iCloud holds only in the
+// cloud is what downloads it.
+
+export const TEXT_EXTENSIONS = new Set([
+  "md", "markdown", "txt", "text", "canvas", "base", "json", "jsonl", "yaml", "yml", "csv", "tsv",
+  "html", "htm", "css", "js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "sh", "zsh", "toml", "xml",
+  "ini", "tex", "bib", "org", "rst", "sql", "log",
+]);
+
+const extOf = (rel) => {
+  const name = rel.slice(rel.lastIndexOf("/") + 1);
+  const dot = name.lastIndexOf(".");
+  return dot <= 0 ? "" : name.slice(dot + 1).toLowerCase();
+};
+
+/** Whether a path is text by its name. */
+export const isTextPath = (rel) => TEXT_EXTENSIONS.has(extOf(rel));
+
+/**
+ * The paths whose bytes travel whatever they are, from the manifest: `"sync": { "carry": [...] }`,
+ * each a path relative to the record covering itself and everything under it. Anything else in
+ * that key is ignored.
+ */
+export function carryList(manifest) {
+  const list = manifest?.sync?.carry;
+  return Array.isArray(list) ? list.filter((p) => typeof p === "string").map(norm).filter((p) => p !== "") : [];
+}
+
+/** `(rel) -> "text" | "carried" | "reference"`, for a carry list from `carryList`. */
+export function fileClass(carry = []) {
+  const paths = carry.map(norm).filter((p) => p !== "");
+  return (rel) => {
+    if (isTextPath(rel)) return "text";
+    const r = norm(rel);
+    return paths.some((p) => r === p || r.startsWith(`${p}/`)) ? "carried" : "reference";
+  };
+}
+
+// ─── Files iCloud holds only in the cloud (HQ-2017) ─────────────────────────────────────────────
+//
+// A dataless file is listed with its full size and has no bytes on this disk; opening it makes iCloud
+// download it. macOS marks it with SF_DATALESS in st_flags. Node's stat does not expose st_flags, so
+// a file is a candidate when it has a size and no blocks (the strong sign), and a candidate is
+// confirmed by reading its flags with stat(1), which does not open it. A file whose bytes are local
+// is never a candidate, so stat(1) runs only for the few that are.
+export const SF_DATALESS = 0x40000000;
+
+/**
+ * The paths among `rels` (relative to `root`) that are dataless, read without opening any of them.
+ * `statBin` is stat(1); `EVDA_STAT_BIN` replaces it, so a test can stand in for iCloud.
+ */
+export function datalessAmong(root, rels, { env = process.env, statOf = (rel) => lstatSync(join(root, rel)) } = {}) {
+  if (process.platform !== "darwin" && !env.EVDA_STAT_BIN) return new Set();
+  const candidates = [];
+  for (const rel of rels) {
+    let st;
+    try { st = statOf(rel); } catch { continue; }
+    if (st.isFile() && st.size > 0 && st.blocks === 0) candidates.push(rel);
+  }
+  const out = new Set();
+  const bin = env.EVDA_STAT_BIN || "/usr/bin/stat";
+  for (let i = 0; i < candidates.length; i += 200) {
+    const chunk = candidates.slice(i, i + 200);
+    let text = "";
+    try {
+      text = execFileSync(bin, ["-f", "%f", ...chunk.map((r) => join(root, r))], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    } catch (e) {
+      text = typeof e?.stdout === "string" ? e.stdout : "";
+    }
+    const flags = text.split("\n").filter((l) => l.trim() !== "");
+    chunk.forEach((rel, j) => {
+      const f = Number(flags[j]);
+      if (Number.isFinite(f) && (f & SF_DATALESS) !== 0) out.add(rel);
+    });
+  }
+  return out;
+}
+
+// ─── The owner's size limit (HQ-2017) ───────────────────────────────────────────────────────────
+//
+// `"sync": { "maxBytes": N }` in evda.json: a file larger than N bytes is never opened by a sync or a
+// tick, whatever its class, and is reported. Missing, zero, negative or not a number reads as
+// MAX_SYNC_BYTES; above it reads as MAX_SYNC_BYTES, since the relay refuses anything larger anyway.
+// On 2026-09-30 a sync read a 686 MB video whole, to seal it.
+
+/** The size limit a manifest sets, in bytes, clamped to the relay's. */
+export function maxSyncBytes(manifest) {
+  const n = manifest?.sync?.maxBytes;
+  if (typeof n !== "number" || !Number.isFinite(n) || n <= 0) return MAX_SYNC_BYTES;
+  return Math.min(Math.floor(n), MAX_SYNC_BYTES);
+}
+
+const mbOf = (n) => (n < 1024 ** 2 ? `${n} bytes` : `${(n / 1024 ** 2).toFixed(1)} MB`);
+
+/**
+ * What a scan does with each listed file, decided before a byte is read: a Map from path to
+ * "read" (text or carried: minted with its bytes), "reference" (hashed for its identity, minted
+ * without its bytes) or { skip, kind } (never opened, and reported). A dataless file is skipped, text
+ * or not: its bytes are not on this disk, and reading them is what downloads them. A file over
+ * `maxBytes` is skipped too, marked `kind: "size"`: reading it whole is the cost being avoided.
+ */
+export function scanClasses(root, carry = [], { env = process.env, maxBytes = MAX_SYNC_BYTES } = {}) {
+  const classOf = fileClass(carry);
+  const limit = Number.isFinite(maxBytes) && maxBytes > 0 ? Math.min(maxBytes, MAX_SYNC_BYTES) : MAX_SYNC_BYTES;
+  return async (stats) => {
+    const rels = stats.map((s) => s.path);
+    const dataless = datalessAmong(root, rels, { env });
+    const out = new Map();
+    for (const s of stats) {
+      const rel = s.path;
+      if (dataless.has(rel)) out.set(rel, { skip: "in iCloud only; not downloaded" });
+      else if (s.size > limit) out.set(rel, { skip: `${mbOf(s.size)}, over the ${mbOf(limit)} size limit; not read`, kind: "size" });
+      else out.set(rel, classOf(rel) === "reference" ? "reference" : "read");
+    }
+    return out;
+  };
+}
+
+// ─── The disk floor (HQ-2017) ───────────────────────────────────────────────────────────────────
+//
+// A sync or tick stops before the disk its state is on falls below a floor, and says why in one line.
+// The default is 5 GiB; `EVDA_DISK_FLOOR_BYTES` sets another (a test sets one above what is free).
+export const DISK_FLOOR_BYTES = 5 * 1024 ** 3;
+
+const gib = (n) => `${(n / 1024 ** 3).toFixed(1)} GB`;
+
+/** `(bytes) -> null | "<one line>"`: whether writing `bytes` more under `dir` keeps the floor. */
+export function diskRoom(dir, { env = process.env, freeOf = (d) => { const s = statfsSync(d); return Number(s.bavail) * Number(s.bsize); } } = {}) {
+  const set = Number(env.EVDA_DISK_FLOOR_BYTES);
+  const floor = env.EVDA_DISK_FLOOR_BYTES !== undefined && Number.isFinite(set) && set >= 0 ? set : DISK_FLOOR_BYTES;
+  return (bytes = 0) => {
+    const free = freeOf(dir);
+    if (free - bytes >= floor) return null;
+    return `Stopped: this disk has ${gib(free)} free, and syncing more would take it below the ${gib(floor)} floor. Nothing more was written; free some space and sync again.`;
+  };
+}