create-evda 0.1.4 → 0.1.6

package/lib/filechanges.mjs

@@ -0,0 +1,622 @@
+// File Changes: what the person layer hands the substrate's scan and sync (Spec 08, sections 1 to 3).
+//
+// The substrate walks a folder it does not own and calls a `mint` callback for every new or edited
+// file (`atlas-local/src/vault-scan.ts`, `scanAndApply`), and its own comment says why the callback
+// is ours: it does not know what a person's markdown means. This module is that callback for plain
+// files, plus the three things the sync path needs from the person layer and must not read itself:
+//
+//   circleIndex    which circle each path belongs to, by the circle's opaque id
+//   grantView      who holds a circle (routing) and who may write into it (receipt)
+//   circleCrypto   a payload sealed to its circle, and opened only with the circle's key
+//   releasesOf     the circles a marked file is released into, beyond its home (HQ-1351)
+//   addressing     a file's address on the wire, and where it lands on a device that mounts the
+//                  circle from another holder (HQ-835)
+//
+// A file Change:
+//
+//   kind:    file.set | file.remove
+//   target:  <the file's address: its path in the HOME holder's record>
+//   eco:     <opaque circle id>
+//   payload: sealed to the circle's current epoch, opening to { path, sha256, baseSha256, text }
+//
+// A released file (Spec 07 section 3) is one Change per circle it reaches: one sealed to its home
+// circle, and one more sealed to each released circle, all with the same address and bytes. A
+// device that holds a circle from another holder (its manifest carries `mount`) writes the file at
+// `<mount>/<address>` and, when it edits the file there, mints the Change back at the address.
+//
+// This package has no runtime dependencies, so nothing here imports the substrate. Whoever composes
+// a device passes the pieces in: the Change log, the host's crypto, and a routing function.
+
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { readCircles, frontmatterOf, matchesWhere, heldBy } from "./circles.mjs";
+import { readGrants } from "./grants.mjs";
+import { adoptDeclaration, declarationOf, freezeCircle } from "./declarations.mjs";
+import { admitDevice, devicesOf, readDevices, recordOf, retireDevice } from "./devices.mjs";
+import { resolveRole } from "./roles.mjs";
+import { boundaries, insideAny } from "./records.mjs";
+import { EvdaError } from "./errors.mjs";
+
+export const FILE_SET = "file.set";
+export const FILE_REMOVE = "file.remove";
+export const SEALED_FORMAT = "evda-circle/1";
+export const PUBLIC_FORMAT = "evda-public/1";
+
+// Every circle in the record, with the prefixes it covers and its opaque id. Throws on a malformed
+// circle: syncing a record whose boundary declaration is broken is worse than not syncing it.
+//
+// One failure is not a broken declaration here: a member with nothing on disk. A device that holds
+// no grant on a circle was never sent its files, so on that device the member is absent by design.
+// The member is still indexed, so the device knows which circle the path belongs to.
+export function circleIndex(evda) {
+  const circles = readCircles(evda) ?? [];
+  const broken = circles.find((c) => c.problems.some((p) => p.level === "fail" && p.code !== "MISSING_MEMBER"));
+  if (broken) {
+    const first = broken.problems.find((p) => p.level === "fail" && p.code !== "MISSING_MEMBER");
+    throw new EvdaError(`circle "${broken.name}" is malformed: ${first.message}`, { code: "BAD_CIRCLE" });
+  }
+  const index = circles.map((c) => ({
+    name: c.name,
+    id: c.id,
+    public: c.manifest.public === true,
+    // A directory member covers everything under it, so its prefix ends in "/"; a file member covers
+    // exactly one path. An absent member could be either, so it covers both readings.
+    members: [
+      ...Object.values(c.members)
+        .filter((m) => m.kind === "path")
+        .map((m) => ({ rel: m.rel.replace(/\/+$/, ""), isDir: m.isDir })),
+      ...c.problems
+        .filter((p) => p.code === "MISSING_MEMBER")
+        .map((p) => ({ rel: p.member.rel.replace(/\/+$/, ""), isDir: null })),
+    ],
+    // The rule members: files gathered by their frontmatter, released out of their home.
+    rules: Object.values(c.members).filter((m) => m.kind === "rule").map((m) => m.where),
+    // Where a file living here may be released to: "never" or circle names.
+    release: c.release,
+    // Set when this circle is another holder's, held here: { holder, rel }.
+    mount: c.mount,
+    // Set when a held circle's grant was revoked: { since, by }. Read-only from then on.
+    frozen: c.frozen,
+  }));
+  // Where this record's circles stop (HQ-1371): every sub-record root when the record declares
+  // "subrecords": "bounded", none otherwise. A path inside one is in no circle of this record.
+  index.boundaries = boundaries(evda);
+  return index;
+}
+
+// The circles a file is released into beyond its home, by its frontmatter mark and its home's
+// release list (Spec 07 section 3, HQ-1351): the same test `released()` in circles.mjs runs for
+// doctor, for one file. A file in no circle, in a public circle, in a mounted circle, or in a home
+// that releases nothing is released nowhere. `text` is the file's contents; a file that is not
+// markdown carries no mark.
+export function releasesOf(index, rel, text) {
+  const home = circleForPath(index, rel);
+  if (!home || home.public || home.mount || home.release === "never" || !Array.isArray(home.release)) return [];
+  const front = typeof text === "string" && /\.md$/i.test(rel) ? frontmatterOf(text) : null;
+  if (!front) return [];
+  // A sealed or just-noted entry stays in its home circle whatever its mark says (HQ-1414), as
+  // released() refuses it for doctor and publish: a mark never carries either one to another holder.
+  if (heldBy(front, "sync")) return [];
+  return index.filter((c) => c !== home && !c.public && home.release.includes(c.name) && c.rules.some((where) => matchesWhere(front, where)));
+}
+
+// The conflict file's original, when `rel` is named as one: `<name> (conflict, <device>, <date>)<ext>`
+// (atlas-local `conflictPath`). Null otherwise.
+export function conflictOriginal(rel) {
+  const m = /^(.*) \(conflict, [^,/()]+, \d{4}-\d{2}-\d{2}\)(\.[^/.]*)?$/.exec(rel);
+  return m ? `${m[1]}${m[2] ?? ""}` : null;
+}
+
+// Every circle a local file reaches: its home, then each release (HQ-1351). A conflict file also goes
+// wherever its original goes (HQ-835): the disagreement came from a circle the original reaches, and
+// every holder of that circle is owed both texts, whatever the incoming text's own mark says.
+// `readText(rel)` reads a local file, or returns null; absent, only the file's own mark counts.
+export function circlesFor(index, rel, text, readText = null) {
+  const home = circleForPath(index, rel);
+  if (!home) return [];
+  const out = [home, ...releasesOf(index, rel, text)];
+  const original = conflictOriginal(rel);
+  if (original && readText) {
+    const originalText = readText(original);
+    if (typeof originalText === "string") {
+      for (const c of releasesOf(index, original, originalText)) if (!out.includes(c)) out.push(c);
+    }
+  }
+  return out;
+}
+
+// The circles a file's address has reached on this device and not yet been removed from: for each
+// circle it was logged in, whether its latest Change there is a file.set. Mounted circles are
+// another holder's namespace and are left out, since an address there is theirs.
+async function reachedBy(log, index, address) {
+  if (typeof log.forTargetIn !== "function") return [];
+  const latest = new Map();
+  for (const { eco, change } of await log.forTargetIn(address)) latest.set(eco, change);
+  return [...latest.entries()]
+    .filter(([, change]) => change.kind === FILE_SET)
+    .map(([eco]) => circleById(index, eco))
+    .filter((c) => c && !c.mount);
+}
+
+// Where a file is on the wire and on this device (HQ-835). The wire carries a file at its ADDRESS,
+// its path in the home holder's record. On the home holder's devices that is the path itself. On a
+// device holding the circle from another holder, the circle is mounted, and the file lands at
+// `<mount>/<address>`; an edit made there is minted back at the address.
+export function addressing(index) {
+  const mountOf = (eco) => circleById(index, eco)?.mount ?? null;
+  return {
+    // The local path for a Change's target, given the circle it arrived in.
+    local(eco, address) {
+      const mount = mountOf(eco);
+      return mount ? `${mount.rel}/${address}` : address;
+    },
+    // A local path's address, given the circle it is minted into.
+    address(circle, rel) {
+      if (!circle.mount) return rel;
+      const prefix = `${circle.mount.rel}/`;
+      if (!rel.startsWith(prefix)) throw new EvdaError(`${rel} is in mounted circle "${circle.name}" but not under its mount ${prefix}.`, { code: "BAD_MOUNT" });
+      return rel.slice(prefix.length);
+    },
+    // The opaque id of the circle a local path is homed in on THIS record, or null. A path under a
+    // mount is another holder's file, homed in their record, so it has no home here.
+    homeOf(rel) {
+      const c = circleForPath(index, rel);
+      return c && !c.mount ? c.id : null;
+    },
+  };
+}
+
+// A member at the record root ("" after normalising) is a directory member covering every path.
+const covers = (member, rel) =>
+  member.rel === "" ? member.isDir !== false
+  : member.isDir === null ? rel === member.rel || rel.startsWith(`${member.rel}/`)
+  : member.isDir ? rel.startsWith(`${member.rel}/`)
+  : rel === member.rel;
+
+// The member a path belongs to, by LONGEST matching path (HQ-1364): a circle declared at the record
+// root ("") claims everything, and a circle whose member is a longer prefix carves its subtree out of
+// it, so `02_Areas/Health & Energy/Therapy/x.md` is therapy, not health, not core. Two circles with
+// members of the SAME length covering one path is still refused: one access answer per entry.
+// Returns { circle, member } or null.
+export function memberForPath(index, rel) {
+  if (index.boundaries && insideAny(index.boundaries, rel)) return null;
+  let best = null;
+  let tie = null;
+  for (const c of index) {
+    for (const m of c.members) {
+      if (!covers(m, rel)) continue;
+      if (best === null || m.rel.length > best.member.rel.length) { best = { circle: c, member: m }; tie = null; }
+      else if (m.rel.length === best.member.rel.length && c !== best.circle) tie = c;
+    }
+  }
+  if (tie) {
+    const names = [...new Set([best.circle.name, tie.name])];
+    throw new EvdaError(`${rel} is covered by ${names.length} circles (${names.join(", ")}) at the same depth; an entry has one access answer.`, { code: "TWO_CIRCLES" });
+  }
+  return best;
+}
+
+// The one circle a path belongs to, or null.
+export function circleForPath(index, rel) {
+  return memberForPath(index, rel)?.circle ?? null;
+}
+
+export const circleById = (index, id) => index.find((c) => c.id === id) ?? null;
+
+// What a scan walks for the circles a device holds, one entry per path member: `prefix` is where the
+// listing starts (`dir/` for a directory member, "" for the record root, the path itself for a file
+// member) and `match(path)` says whether a listed path is this member's, which is true only when the
+// longest-prefix resolution lands on this very member. So a file member matches exactly one path
+// (`CLAUDE.md`, never `CLAUDE.md.bak`), and a root member matches everything no longer member of any
+// circle claims. A device scans only what it holds, so a circle it is not granted is never read,
+// hashed or minted on it.
+export function scanPrefixes(index, holds) {
+  const isMember = (m, path) => {
+    let hit;
+    try { hit = memberForPath(index, path); } catch { return false; }
+    return hit !== null && hit.member.rel === m.rel && hit.circle.id === m.circle.id;
+  };
+  return index.filter(holds).flatMap((c) =>
+    c.members
+      .filter((m) => m.isDir !== null)
+      .map((m) => ({ circle: c, prefix: m.rel === "" ? "" : m.isDir ? `${m.rel}/` : m.rel, match: (path) => isMember({ rel: m.rel, circle: c }, path) })),
+  );
+}
+
+// Who holds which circle, read from the grant files. The substrate's `CircleGrants`.
+//
+// A grant names a circle by its name, on the owner's devices; the wire names it by opaque id. This
+// is where the two meet, and nothing past it sees a name.
+//
+// A grant names a HOLDER by record id; the substrate routes to, and hears from, DEVICES by atlas id
+// (HQ-835). The device lists under the grants role (`lib/devices.mjs`) are where the two meet: a
+// record id resolves to its active devices for routing, and a device resolves to the record it
+// speaks for before any grant is looked up. A record with no list is one device of the same id.
+//
+// The view re-reads the record when a declaration, a revocation or a device notice from another
+// holder lands on it (`adoptDeclaration`, `freezeCircle`, `admitDevice`, `retireDevice`), because
+// those write the very files it is built from, and what follows in the same poll is judged against
+// the new view. `reload()` does the same on request.
+export function grantView(evda) {
+  const load = () => ({
+    index: circleIndex(evda),
+    grants: (readGrants(evda) ?? []).filter((g) => g.problems.every((p) => p.level !== "fail")),
+    devices: readDevices(evda) ?? { lists: new Map(), problems: [] },
+  });
+  let state = load();
+  const reload = () => {
+    state = load();
+  };
+  const named = (eco) => circleById(state.index, eco);
+  const onCircle = (name) => state.grants.filter((g) => g.circle === name);
+  // The holder a device speaks for; null for a retired device.
+  const holderOf = (atlasId) => recordOf(state.devices, atlasId);
+  return {
+    get index() {
+      return state.index;
+    },
+    reload,
+    // A public circle is held by the record's own devices and by no other holder, who reaches the
+    // public set through its export. When the record lists its own devices (grants/devices/<own
+    // id>.yaml), a public Change routes to them alone; a record that lists none routes it to every
+    // pinned peer, as before devices were listed.
+    async readers(eco) {
+      const c = named(eco);
+      if (!c) return [];
+      if (c.public) {
+        const own = evda.manifest?.id;
+        return typeof own === "string" && state.devices.lists.has(own) ? devicesOf(state.devices, own) : "all";
+      }
+      return [...new Set(onCircle(c.name).flatMap((g) => devicesOf(state.devices, g.principalId)))];
+    },
+    async mayAuthor(atlasId, eco) {
+      const c = named(eco);
+      if (!c) return false;
+      const holder = holderOf(atlasId);
+      if (holder === null) return false;
+      const writes = (g) => g.principalId === holder && (g.role === "owner" || g.role === "editor");
+      // A public circle takes no grant, so writing into one takes an OWNER grant on some circle of
+      // the record: a device of the record's own holder. Never a viewer, and never another holder,
+      // who holds an editor grant at most and must not be able to publish in this record's name.
+      return c.public ? state.grants.some((g) => g.principalId === holder && g.role === "owner") : onCircle(c.name).some(writes);
+    },
+    // A frozen circle is held in the sense of "kept", not "worked on": it is not scanned or minted into.
+    holds(atlasId) {
+      const holder = holderOf(atlasId);
+      return (c) => !c.frozen && (c.public || (holder !== null && onCircle(c.name).some((g) => g.principalId === holder)));
+    },
+    // The private circles, by opaque id: the ones with a key to deliver.
+    async circles() {
+      return state.index.filter((c) => !c.public).map((c) => c.id);
+    },
+    // Only an owner hands out a circle's key (Spec 08 §2). Checked by the sender before delivering
+    // and by the receiver before importing, so a viewer device can neither give a key nor be the
+    // source of one.
+    async mayShareKey(atlasId, eco) {
+      const c = named(eco);
+      if (!c || c.public) return false;
+      const holder = holderOf(atlasId);
+      return holder !== null && onCircle(c.name).some((g) => g.principalId === holder && g.role === "owner");
+    },
+    // The declaration of a circle this record homes, for the wire (HQ-835); null for a public
+    // circle, a circle held from someone else, or one that is not here.
+    async declaration(eco) {
+      const c = named(eco);
+      if (!c || c.public || c.mount) return null;
+      return declarationOf(evda, c.name);
+    },
+    // Hold a circle another holder declared, then read the record again. A refusal names why. The
+    // sender is a device; the declaration names its owner by record id.
+    async adoptDeclaration(from, declaration) {
+      const holder = holderOf(from);
+      if (holder === null) return `${from} is a retired device here and declares nothing`;
+      const r = adoptDeclaration(evda, holder, declaration);
+      if (!r.ok) return r.why;
+      reload();
+      return null;
+    },
+    // Freeze a held circle whose grant was removed, then read the record again.
+    async freezeCircle(from, eco, at) {
+      const holder = holderOf(from);
+      if (holder === null) return `${from} is a retired device here and revokes nothing`;
+      const r = freezeCircle(evda, holder, eco, at);
+      if (!r.ok) return r.why;
+      reload();
+      return null;
+    },
+    // The record a device speaks for, or null.
+    async recordOf(atlasId) {
+      return holderOf(atlasId);
+    },
+    // Whether a device speaks for another holder than this record, or for none: the fold's `foreign`,
+    // so an edit from another holder into a file this record homes is re-minted to every holder.
+    isForeign(atlasId) {
+      return holderOf(atlasId) !== evda.manifest?.id;
+    },
+    // The active devices of a record id.
+    async devicesOf(record) {
+      return devicesOf(state.devices, record);
+    },
+    // This record's own id: a device-list notice naming it comes from a sibling device.
+    async ownRecord() {
+      return typeof evda.manifest?.id === "string" && evda.manifest.id !== "" ? evda.manifest.id : null;
+    },
+    // The active devices of a record id, with the identity key listed for each. No list: none, since
+    // no key is known for a device nobody listed.
+    async deviceList(record) {
+      const list = state.devices.lists.get(record);
+      if (!list) return [];
+      return list.devices.filter((d) => typeof d.retiredAt !== "string" || d.retiredAt === "").map((d) => ({ atlasId: d.atlasId, identityPublicKey: d.identityPublicKey ?? "" }));
+    },
+    // A holder's device introduced a new one: list it, then read the record again.
+    async admitDevice(record, device, introducer, at) {
+      const r = admitDevice(evda, { record, device, introducer, at });
+      if (!r.ok) return r.why;
+      reload();
+      return null;
+    },
+    // A holder's device retired one of its devices: mark it, then read the record again.
+    async retireDevice(record, atlasId, by, at) {
+      const r = retireDevice(evda, { record, atlasId, by, at });
+      if (!r.ok) return r.why;
+      reload();
+      return null;
+    },
+  };
+}
+
+// Paths under an append-only role, where two devices' concurrent appends union instead of
+// conflicting: the daily notes and the ledger (Spec 08, section 3).
+export function unionPaths(evda) {
+  const roots = ["record/daily", "ledger"]
+    .map((role) => resolveRole(evda, role))
+    .filter((r) => r.mapped)
+    .map((r) => `${r.rel.replace(/\/+$/, "")}/`);
+  return (rel) => roots.some((root) => rel.startsWith(root));
+}
+
+// Seal a payload to its circle, and open one. `crypto` is the host's HostCrypto.
+//
+// A public circle needs no key, so its payload travels in the clear inside the device envelope. A
+// private circle's payload is sealed to the circle's current epoch; a device with no key for the
+// circle cannot open it, which is what makes a Change routed to the wrong device fail closed.
+export function circleCrypto(crypto) {
+  return {
+    async seal(circle, payload) {
+      if (circle.public) return { format: PUBLIC_FORMAT, clear: payload };
+      const key = await crypto.circleKey(circle.id);
+      if (!key) throw new EvdaError(`this device holds no key for circle "${circle.name}", so it cannot write into it.`, { code: "NO_CIRCLE_KEY" });
+      const bytes = new TextEncoder().encode(JSON.stringify(payload));
+      return { format: SEALED_FORMAT, epoch: key.epoch, sealed: Buffer.from(await crypto.seal(key.publicKey, bytes)).toString("base64") };
+    },
+    async open(circle, sealed) {
+      if (sealed?.format === PUBLIC_FORMAT) {
+        // A private circle's payload in the clear is refused, not read: it means a seal was skipped.
+        if (!circle.public) throw new Error(`circle payload arrived unsealed for a private circle`);
+        return sealed.clear;
+      }
+      if (sealed?.format !== SEALED_FORMAT || typeof sealed.sealed !== "string") throw new Error("not a circle payload");
+      const opened = await crypto.openCircle(circle.id, new Uint8Array(Buffer.from(sealed.sealed, "base64")));
+      return JSON.parse(new TextDecoder().decode(opened));
+    },
+  };
+}
+
+// The mint callbacks for `scanAndApply`.
+//
+//   log            the substrate's ChangeLog (`append`)
+//   index          circleIndex(evda)
+//   seal           (circle, payload) -> sealed payload, from circleCrypto
+//   notifyFor      (eco, changesetId) -> [{ targetAtlas, pushBatchId }], the routing
+//   previousSha    (path) -> the sha this device last recorded for the path, or null: the edit's base
+//   changesetId    () -> a fresh changeset id
+//   unionPath      optional (path) -> whether the path is under an append-only role. Written into the
+//                  payload as `union: true`, so a device holding the circle from another holder, which
+//                  has no mapping for the home record, still unions where the home does.
+//   onMinted       optional ({ change, circle, payload }) -> after the Change is appended, once per
+//                  circle the file reaches
+//   blobs          optional { seal(bytes) -> { hash, contentKey }, link(changeId, hash) }: a file above
+//                  BLOB_THRESHOLD bytes, or one that is not UTF-8, rides the blob lane (Spec 08 §1). Its
+//                  bytes are sealed once under a fresh content key and kept by the device; the payload
+//                  carries { hash, contentKey } in place of the text. Absent, such a file is minted as text.
+//
+// Throwing is how a callback says "not recorded": the scan leaves the path for the next run.
+export const BLOB_THRESHOLD = 256 * 1024;
+
+const utf8OrNull = (bytes) => {
+  try {
+    return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+  } catch {
+    return null;
+  }
+};
+
+export function fileMinter({ log, index, seal, notifyFor, previousSha, changesetId, originUser = null, onMinted, blobs, unionPath = null, readText = null }) {
+  const at = addressing(index);
+  const circleOf = (path) => {
+    const c = circleForPath(index, path);
+    if (!c) throw new EvdaError(`${path} is in no circle, so it has no one to sync to.`, { code: "NO_CIRCLE" });
+    return c;
+  };
+  // One Change per circle the file reaches: its home first, then each release. Each is sealed to
+  // that circle's key and routed to that circle's readers, with its own changeset id, so a device
+  // holding two of the circles is owed two rows rather than one row twice.
+  const append = async (kind, path, payload, circles, blobHash = null) => {
+    let first = null;
+    for (const circle of circles) {
+      const id = await changesetId();
+      const change = await log.append({
+        changesetId: id,
+        eco: circle.id,
+        originUser,
+        kind,
+        target: payload.path,
+        payload: await seal(circle, payload),
+        notify: await notifyFor(circle.id, id),
+      });
+      if (blobHash !== null) await blobs.link(change.id, blobHash);
+      await onMinted?.({ change, circle, payload });
+      first ??= change;
+    }
+    return first;
+  };
+  return {
+    async mint(item, bytes) {
+      const home = circleOf(item.path);
+      const address = at.address(home, item.path);
+      const text = utf8OrNull(bytes);
+      const circles = home.mount ? [home] : circlesFor(index, item.path, text, readText);
+      const union = unionPath?.(item.path) === true ? { union: true } : {};
+      // A circle the file reached before and reaches no longer (its mark was taken off, or its home
+      // changed) is sent a removal, so devices holding only that circle lose it (Spec 08 section 2,
+      // an entry moving between circles). Minted after the new version, as a move is.
+      const withdrawn = home.mount ? [] : (await reachedBy(log, index, address)).filter((c) => !circles.includes(c));
+      const withdraw = async () => {
+        if (withdrawn.length > 0) await append(FILE_REMOVE, item.path, { path: address, sha256: null, baseSha256: await previousSha(item.path) }, withdrawn);
+      };
+      if (blobs && (text === null || bytes.length > BLOB_THRESHOLD)) {
+        const sealed = await blobs.seal(bytes);
+        const blob = { hash: sealed.hash, contentKey: Buffer.from(sealed.contentKey).toString("base64") };
+        await append(FILE_SET, item.path, { path: address, sha256: item.sha256, baseSha256: await previousSha(item.path), blob }, circles, sealed.hash);
+        await withdraw();
+        return;
+      }
+      // ignoreBOM keeps a byte-order mark in the text, so the receiver's hash of the text is the
+      // scanner's hash of the bytes. Without a blob lane, a file that is not UTF-8 is decoded
+      // lossily, fails that check on receipt, and is refused there.
+      const payload = {
+        path: address,
+        sha256: item.sha256,
+        baseSha256: await previousSha(item.path),
+        text: text ?? new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes),
+        ...union,
+      };
+      await append(FILE_SET, item.path, payload, circles);
+      await withdraw();
+    },
+    // A removal reaches every circle the file reached: the mark went with the bytes, so the circles
+    // are read off this device's log, each one whose latest Change for the address is a file.set,
+    // with the home first (HQ-835). On a device holding the circle from another holder, the home
+    // (the mounted circle) only.
+    async mintRemoval(item) {
+      const home = circleOf(item.path);
+      const address = at.address(home, item.path);
+      const circles = [home, ...(home.mount ? [] : (await reachedBy(log, index, address)).filter((c) => c !== home))];
+      await append(FILE_REMOVE, item.path, { path: address, sha256: null, baseSha256: await previousSha(item.path) }, circles);
+    },
+  };
+}
+
+// The catch-up for a circle this device homes (HQ-835): what `deliverCatchUp` in atlas-local sends,
+// once, to a device that joined the circle or whose copy thawed. The core decides when and to whom;
+// this says what the circle holds.
+//
+//   evda           the record
+//   view           () -> the current grant view (its `index`)
+//   log            the substrate's ChangeLog: `append`, `forTargetIn`, `forEco`
+//   seal           (circle, payload) -> sealed payload, from circleCrypto, at the circle's CURRENT epoch
+//   open           (change, eco) -> payload, or throws: reads the earlier versions for `ancestors`
+//   changesetId    () -> a fresh changeset id
+//   unionPath      optional, as for fileMinter
+//   blobs          optional, as for fileMinter
+//   exclude        optional (rel) -> true for a path the sync rules leave out (sync-rules `excluder`),
+//                  so a catch-up sends no more than the scan would
+//
+// `snapshot(circleId)` mints, into the log and routed to nobody, one file.set per file the circle
+// reaches now (its home members and every file released into it, conflict files included), and one
+// file.remove per address the circle reached before and reaches no longer. Each payload carries
+// `ancestors`: every sha this device logged for the address before the current one, from any circle
+// it can open. A receiver holding any of them fast-forwards; a receiver holding a version nobody
+// here has seen keeps it and gets the current text beside it, by the ordinary merge table.
+export function catchUp({ evda, view, log, seal, open, changesetId, originUser = null, unionPath = null, blobs = null, exclude = null }) {
+  const readText = (rel) => {
+    const abs = join(evda.dir, rel);
+    return existsSync(abs) && statSync(abs).isFile() ? readFileSync(abs, "utf8") : null;
+  };
+  const walk = (prefix) => {
+    const out = [];
+    const start = join(evda.dir, prefix);
+    if (!existsSync(start)) return out;
+    if (statSync(start).isFile()) return [prefix];
+    const stack = [prefix.replace(/\/+$/, "")];
+    while (stack.length > 0) {
+      const dir = stack.pop();
+      for (const name of readdirSync(join(evda.dir, dir)).sort()) {
+        if (name.startsWith(".")) continue;
+        const rel = dir === "" ? name : `${dir}/${name}`;
+        const st = statSync(join(evda.dir, rel));
+        if (st.isDirectory()) stack.push(rel);
+        else if (st.isFile()) out.push(rel);
+      }
+    }
+    return out;
+  };
+  const shaOf = async (bytes) => Buffer.from(await crypto.subtle.digest("SHA-256", bytes)).toString("hex");
+
+  // Every sha logged for an address before `current`, oldest first, from Changes this device can open.
+  const ancestorsOf = async (address, current) => {
+    const out = [];
+    for (const { eco, change } of await log.forTargetIn(address)) {
+      if (change.kind !== FILE_SET) continue;
+      try {
+        const p = await open(change, eco);
+        if (typeof p?.sha256 === "string" && p.sha256 !== current && !out.includes(p.sha256)) out.push(p.sha256);
+      } catch {
+        // A version this device cannot open is not one it can vouch for.
+      }
+    }
+    return out;
+  };
+
+  return {
+    async snapshot(circleId) {
+      const index = view().index;
+      const target = circleById(index, circleId);
+      if (!target) throw new EvdaError(`no circle ${circleId} in this record.`, { code: "NO_CIRCLE" });
+      if (target.mount) throw new EvdaError(`circle "${target.name}" is held from ${target.mount.holder}; only its home catches a device up.`, { code: "NOT_OWNER" });
+      const at = addressing(index);
+      const present = new Map();
+      for (const { prefix, match } of scanPrefixes(index, (c) => !c.mount && !c.frozen)) {
+        for (const rel of walk(prefix)) {
+          if (!match(rel) || present.has(rel) || exclude?.(rel)) continue;
+          const bytes = readFileSync(join(evda.dir, rel));
+          const text = utf8OrNull(bytes);
+          if (!circlesFor(index, rel, text, readText).includes(target)) continue;
+          present.set(rel, { bytes, text });
+        }
+      }
+      const minted = [];
+      const append = async (kind, payload, blobHash = null) => {
+        const id = await changesetId();
+        const change = await log.append({ changesetId: id, eco: target.id, originUser, kind, target: payload.path, payload: await seal(target, payload), notify: [] });
+        if (blobHash !== null) await blobs.link(change.id, blobHash);
+        minted.push(change);
+      };
+      const addresses = new Set();
+      for (const [rel, { bytes, text }] of [...present.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+        const address = at.address(circleForPath(index, rel), rel);
+        addresses.add(address);
+        const sha256 = await shaOf(bytes);
+        const ancestors = await ancestorsOf(address, sha256);
+        const base = { path: address, sha256, baseSha256: ancestors.at(-1) ?? null, ancestors };
+        if (blobs && (text === null || bytes.length > BLOB_THRESHOLD)) {
+          const sealed = await blobs.seal(bytes);
+          await append(FILE_SET, { ...base, blob: { hash: sealed.hash, contentKey: Buffer.from(sealed.contentKey).toString("base64") } }, sealed.hash);
+          continue;
+        }
+        await append(FILE_SET, { ...base, text: text ?? new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes), ...(unionPath?.(rel) === true ? { union: true } : {}) });
+      }
+      // Addresses the circle held before and does not now: removed, or no longer released into it.
+      const gone = new Set();
+      for (const change of await log.forEco(target.id)) {
+        if (change.kind === FILE_SET && !addresses.has(change.target)) gone.add(change.target);
+      }
+      for (const address of [...gone].sort()) {
+        const ancestors = await ancestorsOf(address, null);
+        if (ancestors.length === 0) continue;
+        await append(FILE_REMOVE, { path: address, sha256: null, baseSha256: ancestors.at(-1), ancestors });
+      }
+      return minted;
+    },
+  };
+}

package/lib/folders.mjs

@@ -112,24 +112,38 @@ export function validateFolderManifest(m, rel) {
   return out;
 }
 
-// Every folder manifest under the record's record roles, deepest last. Each item is
-// { rel, abs, dir, ...validateFolderManifest }. Throws only on a manifest that is not JSON.
+// Every folder manifest in the record. Each item is { rel, abs, dir, ...validateFolderManifest }.
+// Throws only on a manifest that is not JSON.
+//
+// The record roles are walked first, in role order and deepest last, then the rest of the record
+// root, because a folder a preset or the owner made outside every role still says what it holds and
+// is still read. Until HQ-1216 only the roles were walked, and a malformed manifest one level outside
+// them printed OK. The walk skips dot-directories and node_modules and does not follow a symlink
+// below a role or the root, so a link cannot pull another tree, or the same tree twice, into the count.
+//
+// FALSIFIED 2026-09-17: a fresh scaffold with Outside/folder.evda.json holding {"kind":"person"}
+// (no type). The roles-only walk (main at 2788455) printed "CHECKED 3 folder manifests" and
+// "OK  9 check(s), 0 warning(s)", exit 0. This walk prints "FAIL  folder Outside
+// Outside/folder.evda.json: has no "type". ..." and "CHECKED 4 folder manifests", exit 1.
 export function folderManifests(evda) {
   const out = [];
-  const roots = resolveAll(evda).filter((r) => r.known && r.mapped && r.layer === "record" && r.exists && r.isDir);
+  const seen = new Set();
   const walk = (dir) => {
+    if (seen.has(dir)) return;
+    seen.add(dir);
     const m = readFolderManifest(dir);
     if (m !== null) {
       const abs = join(dir, FOLDER_MANIFEST);
       out.push({ rel: relative(evda.dir, abs), abs, dir, ...validateFolderManifest(m, relative(evda.dir, abs)) });
     }
-    for (const name of readdirSync(dir).sort()) {
-      if (name.startsWith(".")) continue;
-      const p = join(dir, name);
-      if (statSync(p).isDirectory()) walk(p);
+    const entries = readdirSync(dir, { withFileTypes: true }).sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
+    for (const e of entries) {
+      if (e.name.startsWith(".") || e.name === "node_modules") continue;
+      if (e.isDirectory()) walk(join(dir, e.name));
     }
   };
-  for (const r of roots) walk(r.abs);
+  for (const r of resolveAll(evda).filter((x) => x.known && x.mapped && x.layer === "record" && x.exists && x.isDir)) walk(r.abs);
+  walk(evda.dir);
   return out;
 }