create-evda 0.1.8 → 0.1.10

package/lib/sync-rules.mjs

@@ -21,7 +21,8 @@
 // And one kind cannot sync yet: a file larger than the relay's blob lane takes (64 MB, less the
 // seal), which fails the coverage check until the plan excludes it.
 
-import { lstatSync, readdirSync } from "node:fs";
+import { execFileSync } from "node:child_process";
+import { lstatSync, readdirSync, statfsSync } from "node:fs";
 import { join } from "node:path";
 
 export const EXCLUDE_NAMES = [".DS_Store", "node_modules", ".git", ".Trash", "Icon\r"];
@@ -91,3 +92,155 @@ export function scanSkip(extra = []) {
     return isPlaceholder(name) || st.size > MAX_SYNC_BYTES;
   };
 }
+
+// ─── What travels (HQ-2017) ─────────────────────────────────────────────────────────────────────
+//
+// Only text travels by default. His decision, 2026-09-30: "I don't think it needs to carry media. We
+// don't need media to be the thing that travels. That can be one device." A pairing sealed every
+// voice memo, video and PDF of a 1.8 GB iCloud record into the device database (21 GB, the disk full).
+//
+// One rule, read by the scan (which files are read), the minter and the catch-up (what a Change
+// carries) and parity (what a shadow must hold), on the laptop's `evda sync` and on the shadow tick:
+//
+//   text       a name ending in one of TEXT_EXTENSIONS. Minted with its bytes, as before.
+//   carried    any other file under a path the manifest names in `sync.carry`. Its bytes ride the
+//              blob lane, as every binary did before.
+//   reference  everything else. Its Change names the file (path, name, size, sha256) and carries no
+//              bytes; nothing is sealed into the blob store and nothing is uploaded.
+//
+// Decided from the NAME alone, never from the bytes, because reading a file iCloud holds only in the
+// cloud is what downloads it.
+
+export const TEXT_EXTENSIONS = new Set([
+  "md", "markdown", "txt", "text", "canvas", "base", "json", "jsonl", "yaml", "yml", "csv", "tsv",
+  "html", "htm", "css", "js", "mjs", "cjs", "ts", "tsx", "jsx", "py", "sh", "zsh", "toml", "xml",
+  "ini", "tex", "bib", "org", "rst", "sql", "log",
+]);
+
+const extOf = (rel) => {
+  const name = rel.slice(rel.lastIndexOf("/") + 1);
+  const dot = name.lastIndexOf(".");
+  return dot <= 0 ? "" : name.slice(dot + 1).toLowerCase();
+};
+
+/** Whether a path is text by its name. */
+export const isTextPath = (rel) => TEXT_EXTENSIONS.has(extOf(rel));
+
+/**
+ * The paths whose bytes travel whatever they are, from the manifest: `"sync": { "carry": [...] }`,
+ * each a path relative to the record covering itself and everything under it. Anything else in
+ * that key is ignored.
+ */
+export function carryList(manifest) {
+  const list = manifest?.sync?.carry;
+  return Array.isArray(list) ? list.filter((p) => typeof p === "string").map(norm).filter((p) => p !== "") : [];
+}
+
+/** `(rel) -> "text" | "carried" | "reference"`, for a carry list from `carryList`. */
+export function fileClass(carry = []) {
+  const paths = carry.map(norm).filter((p) => p !== "");
+  return (rel) => {
+    if (isTextPath(rel)) return "text";
+    const r = norm(rel);
+    return paths.some((p) => r === p || r.startsWith(`${p}/`)) ? "carried" : "reference";
+  };
+}
+
+// ─── Files iCloud holds only in the cloud (HQ-2017) ─────────────────────────────────────────────
+//
+// A dataless file is listed with its full size and has no bytes on this disk; opening it makes iCloud
+// download it. macOS marks it with SF_DATALESS in st_flags. Node's stat does not expose st_flags, so
+// a file is a candidate when it has a size and no blocks (the strong sign), and a candidate is
+// confirmed by reading its flags with stat(1), which does not open it. A file whose bytes are local
+// is never a candidate, so stat(1) runs only for the few that are.
+export const SF_DATALESS = 0x40000000;
+
+/**
+ * The paths among `rels` (relative to `root`) that are dataless, read without opening any of them.
+ * `statBin` is stat(1); `EVDA_STAT_BIN` replaces it, so a test can stand in for iCloud.
+ */
+export function datalessAmong(root, rels, { env = process.env, statOf = (rel) => lstatSync(join(root, rel)) } = {}) {
+  if (process.platform !== "darwin" && !env.EVDA_STAT_BIN) return new Set();
+  const candidates = [];
+  for (const rel of rels) {
+    let st;
+    try { st = statOf(rel); } catch { continue; }
+    if (st.isFile() && st.size > 0 && st.blocks === 0) candidates.push(rel);
+  }
+  const out = new Set();
+  const bin = env.EVDA_STAT_BIN || "/usr/bin/stat";
+  for (let i = 0; i < candidates.length; i += 200) {
+    const chunk = candidates.slice(i, i + 200);
+    let text = "";
+    try {
+      text = execFileSync(bin, ["-f", "%f", ...chunk.map((r) => join(root, r))], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    } catch (e) {
+      text = typeof e?.stdout === "string" ? e.stdout : "";
+    }
+    const flags = text.split("\n").filter((l) => l.trim() !== "");
+    chunk.forEach((rel, j) => {
+      const f = Number(flags[j]);
+      if (Number.isFinite(f) && (f & SF_DATALESS) !== 0) out.add(rel);
+    });
+  }
+  return out;
+}
+
+// ─── The owner's size limit (HQ-2017) ───────────────────────────────────────────────────────────
+//
+// `"sync": { "maxBytes": N }` in evda.json: a file larger than N bytes is never opened by a sync or a
+// tick, whatever its class, and is reported. Missing, zero, negative or not a number reads as
+// MAX_SYNC_BYTES; above it reads as MAX_SYNC_BYTES, since the relay refuses anything larger anyway.
+// On 2026-09-30 a sync read a 686 MB video whole, to seal it.
+
+/** The size limit a manifest sets, in bytes, clamped to the relay's. */
+export function maxSyncBytes(manifest) {
+  const n = manifest?.sync?.maxBytes;
+  if (typeof n !== "number" || !Number.isFinite(n) || n <= 0) return MAX_SYNC_BYTES;
+  return Math.min(Math.floor(n), MAX_SYNC_BYTES);
+}
+
+const mbOf = (n) => (n < 1024 ** 2 ? `${n} bytes` : `${(n / 1024 ** 2).toFixed(1)} MB`);
+
+/**
+ * What a scan does with each listed file, decided before a byte is read: a Map from path to
+ * "read" (text or carried: minted with its bytes), "reference" (hashed for its identity, minted
+ * without its bytes) or { skip, kind } (never opened, and reported). A dataless file is skipped, text
+ * or not: its bytes are not on this disk, and reading them is what downloads them. A file over
+ * `maxBytes` is skipped too, marked `kind: "size"`: reading it whole is the cost being avoided.
+ */
+export function scanClasses(root, carry = [], { env = process.env, maxBytes = MAX_SYNC_BYTES } = {}) {
+  const classOf = fileClass(carry);
+  const limit = Number.isFinite(maxBytes) && maxBytes > 0 ? Math.min(maxBytes, MAX_SYNC_BYTES) : MAX_SYNC_BYTES;
+  return async (stats) => {
+    const rels = stats.map((s) => s.path);
+    const dataless = datalessAmong(root, rels, { env });
+    const out = new Map();
+    for (const s of stats) {
+      const rel = s.path;
+      if (dataless.has(rel)) out.set(rel, { skip: "in iCloud only; not downloaded" });
+      else if (s.size > limit) out.set(rel, { skip: `${mbOf(s.size)}, over the ${mbOf(limit)} size limit; not read`, kind: "size" });
+      else out.set(rel, classOf(rel) === "reference" ? "reference" : "read");
+    }
+    return out;
+  };
+}
+
+// ─── The disk floor (HQ-2017) ───────────────────────────────────────────────────────────────────
+//
+// A sync or tick stops before the disk its state is on falls below a floor, and says why in one line.
+// The default is 5 GiB; `EVDA_DISK_FLOOR_BYTES` sets another (a test sets one above what is free).
+export const DISK_FLOOR_BYTES = 5 * 1024 ** 3;
+
+const gib = (n) => `${(n / 1024 ** 3).toFixed(1)} GB`;
+
+/** `(bytes) -> null | "<one line>"`: whether writing `bytes` more under `dir` keeps the floor. */
+export function diskRoom(dir, { env = process.env, freeOf = (d) => { const s = statfsSync(d); return Number(s.bavail) * Number(s.bsize); } } = {}) {
+  const set = Number(env.EVDA_DISK_FLOOR_BYTES);
+  const floor = env.EVDA_DISK_FLOOR_BYTES !== undefined && Number.isFinite(set) && set >= 0 ? set : DISK_FLOOR_BYTES;
+  return (bytes = 0) => {
+    const free = freeOf(dir);
+    if (free - bytes >= floor) return null;
+    return `Stopped: this disk has ${gib(free)} free, and syncing more would take it below the ${gib(floor)} floor. Nothing more was written; free some space and sync again.`;
+  };
+}

package/lib/takeout.mjs

@@ -0,0 +1,169 @@
+// Google Takeout's sidecars (HQ-1368). Takeout writes a `.json` file beside each photo and each Keep
+// note, holding what the app knew about it: when a photo was taken, its caption, when a note was made
+// and last edited. Gathered as files of their own they became captures while their photo or note went
+// somewhere else (the 2026-09-27 trial). So a gather pairs each sidecar with the file it describes,
+// folds the sidecar's date and caption into that file's entry, and never places the sidecar itself.
+//
+// The names Takeout gives them, all paired here:
+//   IMG_1.jpg.json                                  the older name
+//   IMG_1.jpg.supplemental-metadata.json            the newer one
+//   IMG_1.jpg.supplemental-metad.json, .supp.json   cut short: Takeout caps a name near 51 characters
+//   <the first 40-odd characters of a long name>.json   the photo's own name cut short
+//   IMG_1.jpg(1).json, IMG_1.jpg.supplemental-metadata(1).json   the sidecar of IMG_1(1).jpg
+//   Note.json                                       a Keep note's, beside Note.html
+// A photo's `-edited` copy has no sidecar of its own and shares its original's.
+//
+// What is folded, and what is not: a photo's date taken (`photoTakenTime.timestamp`, UTC epoch seconds,
+// kept to the second) and whether it has a caption; a Keep note's created and edited times. Its place
+// (`geoData`) and the people Google tagged in it are never read into the proposal or the record. The
+// caption is the owner's words, so the proposal only says there is one; apply reads it again from the
+// sidecar, unchanged since the gather, and keeps it on the record's gathered list beside the photo.
+//
+//   pairSidecars(files, read)   { folded: Map<rel, facts>, sidecars: [{ rel, why, of }] }
+//   captionOf(buf)              the caption a photo sidecar holds, or null
+//
+// EVDA_GATHER_FALSIFY=sidecars-as-entries turns the pairing off, so the tests can show their checks fail
+// without it (TAKEOUT_FALSIFY_MODES).
+
+import { createHash } from "node:crypto";
+import { basename, dirname, extname } from "node:path";
+import { secretShape } from "./references.mjs";
+
+export const TAKEOUT_FALSIFY_MODES = ["sidecars-as-entries"];
+const falsified = (mode) => (process.env.EVDA_GATHER_FALSIFY ?? "").split(",").map((s) => s.trim()).includes(mode);
+const sha256 = (buf) => createHash("sha256").update(buf).digest("hex");
+
+// The files a sidecar describes: what Takeout exports from Photos, and Keep's notes.
+const MEDIA_EXT = new Set([".jpg", ".jpeg", ".png", ".gif", ".heic", ".heif", ".webp", ".tif", ".tiff", ".bmp", ".dng", ".raw", ".cr2", ".nef", ".arw", ".mp4", ".mov", ".m4v", ".avi", ".3gp", ".mkv", ".mpg", ".webm", ".mp"]);
+const NOTE_EXT = new Set([".html", ".htm", ".txt", ".md"]);
+const SUPPLEMENTAL = "supplemental-metadata";
+// Takeout's sidecar names run to about 51 characters; a photo's own name is cut from about 46.
+const CUT_AT = 30;
+const SIDECAR_MAX = 1024 * 1024;
+
+// The name a sidecar describes, from the sidecar's own name: { target, n, marked }, or null for a name
+// that is not `.json`. `n` is Takeout's "(1)" copy number; `marked` says the name carries the
+// supplemental-metadata marker, whole or cut short.
+export function sidecarTarget(name) {
+  if (!/\.json$/i.test(name)) return null;
+  let stem = name.slice(0, -5);
+  let n = null;
+  const copy = /\((\d+)\)$/.exec(stem);
+  if (copy) { n = copy[1]; stem = stem.slice(0, copy.index); }
+  let marked = false;
+  const tail = /\.([a-z-]*)$/i.exec(stem);
+  if (tail && SUPPLEMENTAL.startsWith(tail[1].toLowerCase())) {
+    const before = stem.slice(0, tail.index);
+    if (/\.[a-z0-9]{2,5}$/i.test(before) || tail[1].length >= 4) { stem = before; marked = true; }
+  }
+  return { target: stem, n, marked };
+}
+
+// What kind of Takeout file a parsed JSON value is: "photo", "keep", "album", or null.
+export function takeoutShape(v) {
+  if (!v || typeof v !== "object" || Array.isArray(v)) return null;
+  if (v.photoTakenTime || (v.creationTime && (v.title !== undefined || v.imageViews !== undefined))) return "photo";
+  if (v.createdTimestampUsec !== undefined || v.userEditedTimestampUsec !== undefined) return "keep";
+  if (v.albumData || (v.title !== undefined && v.date?.timestamp !== undefined && v.access !== undefined)) return "album";
+  return null;
+}
+
+// Seconds or microseconds since the epoch, as Takeout writes them, to an ISO time in UTC.
+const iso = (ms) => new Date(ms).toISOString().replace(/\.000Z$/, "Z");
+const seconds = (x) => (/^\d{1,12}$/.test(String(x ?? "").trim()) ? Number(String(x).trim()) : null);
+const usec = (x) => (/^\d{1,18}$/.test(String(x ?? "").trim()) ? Math.floor(Number(String(x).trim()) / 1000) : null);
+
+// The facts a sidecar folds into its file's entry. Nothing from geoData or people is read.
+function factsOf(v, shape) {
+  const out = {};
+  if (shape === "photo") {
+    const e = seconds(v.photoTakenTime?.timestamp);
+    if (e !== null) { out.takenEpoch = e; out.taken = iso(e * 1000); }
+    if (captionText(v)) out.caption = true;
+  } else if (shape === "keep") {
+    const c = usec(v.createdTimestampUsec);
+    const e = usec(v.userEditedTimestampUsec);
+    if (c !== null && c > 0) out.created = iso(c);
+    if (e !== null && e > 0) out.edited = iso(e);
+  }
+  return out;
+}
+
+// A caption, unless it is empty or shaped like a secret (then it is folded nowhere).
+function captionText(v) {
+  const t = typeof v?.description === "string" ? v.description.trim() : "";
+  if (!t || t.split("\n").some(secretShape)) return null;
+  return t;
+}
+
+export function captionOf(buf) {
+  try { return captionText(JSON.parse(buf.toString("utf8"))); } catch { return null; }
+}
+
+const withN = (name, n) => {
+  const ext = extname(name);
+  return ext ? `${name.slice(0, -ext.length)}(${n})${ext}` : `${name}(${n})`;
+};
+
+// Pair every sidecar in a walked source with the file it describes. `files` are the walk's
+// { rel, abs, size }, `read(abs)` their bytes. Returns { folded, sidecars }: `folded` maps a file's rel
+// to the facts its sidecar gave it, `sidecars` lists every sidecar with why it was not placed and, when
+// it has one, the file it describes (`of`).
+export function pairSidecars(files, read) {
+  const folded = new Map();
+  const sidecars = [];
+  if (falsified("sidecars-as-entries")) return { folded, sidecars };
+  const byDir = new Map();
+  for (const f of files) {
+    const d = dirname(f.rel);
+    byDir.set(d, [...(byDir.get(d) ?? []), f]);
+  }
+  for (const [dir, here] of byDir) {
+    const others = here.filter((f) => !/\.json$/i.test(f.rel));
+    const names = new Map(others.map((f) => [basename(f.rel), f]));
+    for (const f of here) {
+      const name = basename(f.rel);
+      const t = sidecarTarget(name);
+      if (!t) continue;
+      let v = null;
+      let bad = null;
+      if (f.size > SIDECAR_MAX) bad = "too large to be a sidecar";
+      else {
+        const text = read(f.abs).toString("utf8");
+        try { v = JSON.parse(text); } catch { bad = "not valid JSON"; }
+      }
+      const shape = v ? takeoutShape(v) : null;
+      const named = t.marked || MEDIA_EXT.has(extname(t.target).toLowerCase()) || NOTE_EXT.has(extname(t.target).toLowerCase());
+      // A .json that is neither shaped nor named like a sidecar is an ordinary file of the pile.
+      if (!shape && !named) continue;
+      if (shape === "album") { sidecars.push({ rel: f.rel, why: "Google Photos' details of the album this folder holds; not an entry of its own", sidecar: true }); continue; }
+      // The file it describes, in this folder.
+      let match = null;
+      if (t.n) match = names.get(withN(t.target, t.n)) ?? null;
+      else match = names.get(t.target) ?? null;
+      if (!match && !t.n) {
+        const same = others.filter((o) => basename(o.rel, extname(o.rel)) === t.target);
+        const notes = same.filter((o) => NOTE_EXT.has(extname(o.rel).toLowerCase()));
+        match = (notes.length === 1 ? notes[0] : same.length === 1 ? same[0] : null);
+      }
+      if (!match && !t.n && t.target.length >= CUT_AT) {
+        const cut = others.filter((o) => basename(o.rel).startsWith(t.target));
+        if (cut.length === 1) match = cut[0];
+      }
+      if (!match) {
+        sidecars.push({ rel: f.rel, why: bad ? `a Takeout sidecar that is ${bad}, and no file it describes is beside it; nothing read from it` : `a Takeout sidecar for ${t.n ? withN(t.target, t.n) : t.target}, which is not in the pile; placed nowhere`, sidecar: true, unmatched: true });
+        continue;
+      }
+      const facts = bad ? {} : factsOf(v, shape ?? "photo");
+      sidecars.push({ rel: f.rel, of: match.rel, why: bad ? `a Takeout sidecar for ${basename(match.rel)} that is ${bad}; nothing read from it, and ${basename(match.rel)} is placed without it` : `Google Takeout's details of ${basename(match.rel)}, folded into its entry`, sidecar: true, ...(bad ? { malformed: true } : {}) });
+      if (bad) continue;
+      const fold = { sidecar: f.rel, sha256: sha256(read(f.abs)), ...facts };
+      folded.set(match.rel, fold);
+      // A photo's edited copy has no sidecar of its own: it is the same moment.
+      const ext = extname(match.rel);
+      const edited = names.get(`${basename(match.rel, ext)}-edited${ext}`);
+      if (edited && !folded.has(edited.rel)) folded.set(edited.rel, { ...fold, sharedWith: match.rel });
+    }
+  }
+  return { folded, sidecars };
+}