create-evda 0.1.8 → 0.1.10

package/lib/filechanges.mjs

@@ -36,6 +36,10 @@ import { admitDevice, devicesOf, readDevices, recordOf, retireDevice } from "./d
 import { resolveRole } from "./roles.mjs";
 import { boundaries, insideAny } from "./records.mjs";
 import { EvdaError } from "./errors.mjs";
+import { carryList, datalessAmong, excluder, fileClass, walk as walkRules } from "./sync-rules.mjs";
+// The day role, re-exported so pairing reads it through the four person libs the phone bundles.
+export { dayRole } from "./day-file.mjs";
+import { dayRole, localDate } from "./day-file.mjs";
 
 export const FILE_SET = "file.set";
 export const FILE_REMOVE = "file.remove";
@@ -365,6 +369,49 @@ export function grantView(evda) {
   };
 }
 
+// The id this record's computer speaks as (HQ-2005). A record create-evda made grants its own id
+// owner, and the computer is the record id. A record whose parts were declared by hand may grant
+// owner to a named device instead (his: `laptop.atlas` owner of every part, `hestia.atlas` an
+// editor, the record id nothing); then the computer is the one device that owns every private part
+// this record homes. None, or more than one, is refused in one line naming what was found, so a
+// caller can refuse before it writes anything. A record with no private parts is its own id.
+//
+// A computer that pins a device as a PEER in its own state is not that device (HQ-2040): Hestia
+// holds the laptop as a peer, so it never speaks as laptop.atlas. `notMe` lists those pins. With
+// them set aside, the one owner left is this computer even when it owns only some parts (his
+// Hestia owns core, where a phone pairs); none left is the same refusal, so a computer with no
+// grant of its own writes nothing rather than minting a second identity for another device's id.
+export const NOT_AN_OWNER_LEAD = "This record has private parts";
+export async function ownerDevice(evda, { notMe = [] } = {}) {
+  const own = typeof evda.manifest?.id === "string" ? evda.manifest.id : "";
+  const view = grantView(evda);
+  const parts = view.index.filter((c) => !c.public && !c.mount);
+  if (!own || parts.length === 0) return own;
+  const owns = async (id) => {
+    const out = [];
+    for (const c of parts) if (await view.mayShareKey(id, c.id)) out.push(c.name);
+    return out;
+  };
+  if ((await owns(own)).length > 0) return own;
+  const grants = (readGrants(evda) ?? []).filter((g) => g.role === "owner" && g.principalId && g.problems.every((p) => p.level !== "fail"));
+  const devices = readDevices(evda) ?? { lists: new Map(), problems: [] };
+  const names = new Map();
+  for (const g of grants) for (const d of devicesOf(devices, g.principalId)) if (!names.has(d)) names.set(d, g.holder?.name ?? d);
+  const found = [];
+  const peers = new Set(notMe);
+  for (const id of [...names.keys()].sort()) if (id !== own && !peers.has(id)) found.push({ id, name: names.get(id), parts: await owns(id) });
+  const full = found.filter((f) => f.parts.length === parts.length);
+  if (full.length === 1) return full[0].id;
+  const some = found.filter((f) => f.parts.length > 0);
+  if (peers.size > 0 && full.length === 0 && some.length === 1) return some[0].id;
+  const list = parts.map((c) => c.name).join(", ");
+  if (full.length > 1) {
+    throw new EvdaError(`${NOT_AN_OWNER_LEAD} (${list}), and ${full.length} devices own every one of them (${full.map((f) => f.name).join(", ")}), so this computer cannot tell which of them it is. Nothing was written.`, { code: "NOT_AN_OWNER", exit: 2 });
+  }
+  const who = found.filter((f) => f.parts.length > 0).map((f) => `${f.name} owns ${f.parts.join(", ")}`);
+  throw new EvdaError(`${NOT_AN_OWNER_LEAD} (${list}), but this computer is not an owner of any of them, and no one device owns them all${who.length ? `: ${who.join("; ")}` : ": no grant names an owner"}. Nothing was written.`, { code: "NOT_AN_OWNER", exit: 2 });
+}
+
 // Paths under an append-only role, where two devices' concurrent appends union instead of
 // conflicting: the daily notes and the ledger (Spec 08, section 3).
 export function unionPaths(evda) {
@@ -415,10 +462,16 @@ export function circleCrypto(crypto) {
 //                  has no mapping for the home record, still unions where the home does.
 //   onMinted       optional ({ change, circle, payload }) -> after the Change is appended, once per
 //                  circle the file reaches
-//   blobs          optional { seal(bytes) -> { hash, contentKey }, link(changeId, hash) }: a file above
-//                  BLOB_THRESHOLD bytes, or one that is not UTF-8, rides the blob lane (Spec 08 §1). Its
-//                  bytes are sealed once under a fresh content key and kept by the device; the payload
-//                  carries { hash, contentKey } in place of the text. Absent, such a file is minted as text.
+//   blobs          optional { seal(bytes) -> { hash, contentKey }, link(changeId, hash) }: a text file
+//                  above BLOB_THRESHOLD bytes or not UTF-8, and a file the manifest carries, rides the
+//                  blob lane (Spec 08 §1). Its bytes are sealed once under a fresh content key and kept
+//                  by the device; the payload carries { hash, contentKey } in place of the text. Absent,
+//                  such a file is minted as text.
+//   carry          optional, the manifest's `sync.carry` list (sync-rules `carryList`). Every file that
+//                  is neither text nor carried is a REFERENCE (HQ-2017): its payload is { path, sha256,
+//                  baseSha256, ref: { name, size } }, with no text and no blob, and nothing is sealed.
+//                  The scan passes `how.reference` for such a file and no bytes; a file that reaches
+//                  mint with bytes is classified here again, so no caller can seal media by mistake.
 //
 // Throwing is how a callback says "not recorded": the scan leaves the path for the next run.
 export const BLOB_THRESHOLD = 256 * 1024;
@@ -431,8 +484,17 @@ const utf8OrNull = (bytes) => {
   }
 };
 
-export function fileMinter({ log, index, seal, notifyFor, previousSha, changesetId, originUser = null, onMinted, blobs, unionPath = null, readText = null }) {
+// The reference payload for a file whose bytes do not travel (HQ-2017).
+const refPayload = (address, sha256, baseSha256, size) => ({
+  path: address,
+  sha256,
+  baseSha256,
+  ref: { name: address.slice(address.lastIndexOf("/") + 1), size },
+});
+
+export function fileMinter({ log, index, seal, notifyFor, previousSha, changesetId, originUser = null, onMinted, blobs, unionPath = null, readText = null, carry = [] }) {
   const at = addressing(index);
+  const classOf = fileClass(carry);
   const circleOf = (path) => {
     const c = circleForPath(index, path);
     if (!c) throw new EvdaError(`${path} is in no circle, so it has no one to sync to.`, { code: "NO_CIRCLE" });
@@ -461,10 +523,11 @@ export function fileMinter({ log, index, seal, notifyFor, previousSha, changeset
     return first;
   };
   return {
-    async mint(item, bytes) {
+    async mint(item, bytes, how = {}) {
       const home = circleOf(item.path);
       const address = at.address(home, item.path);
-      const text = utf8OrNull(bytes);
+      const kind = how.reference ? "reference" : classOf(item.path);
+      const text = kind === "reference" ? null : utf8OrNull(bytes);
       const circles = home.mount ? [home] : circlesFor(index, item.path, text, readText);
       const union = unionPath?.(item.path) === true ? { union: true } : {};
       // A circle the file reached before and reaches no longer (its mark was taken off, or its home
@@ -474,7 +537,13 @@ export function fileMinter({ log, index, seal, notifyFor, previousSha, changeset
       const withdraw = async () => {
         if (withdrawn.length > 0) await append(FILE_REMOVE, item.path, { path: address, sha256: null, baseSha256: await previousSha(item.path) }, withdrawn);
       };
-      if (blobs && (text === null || bytes.length > BLOB_THRESHOLD)) {
+      if (kind === "reference") {
+        const size = how.reference?.size ?? bytes.length;
+        await append(FILE_SET, item.path, refPayload(address, item.sha256, await previousSha(item.path), size), circles);
+        await withdraw();
+        return;
+      }
+      if (blobs && (kind === "carried" || text === null || bytes.length > BLOB_THRESHOLD)) {
         const sealed = await blobs.seal(bytes);
         const blob = { hash: sealed.hash, contentKey: Buffer.from(sealed.contentKey).toString("base64") };
         await append(FILE_SET, item.path, { path: address, sha256: item.sha256, baseSha256: await previousSha(item.path), blob }, circles, sealed.hash);
@@ -528,28 +597,35 @@ export function fileMinter({ log, index, seal, notifyFor, previousSha, changeset
 // `ancestors`: every sha this device logged for the address before the current one, from any circle
 // it can open. A receiver holding any of them fast-forwards; a receiver holding a version nobody
 // here has seen keeps it and gets the current text beside it, by the ordinary merge table.
-export function catchUp({ evda, view, log, seal, open, changesetId, originUser = null, unionPath = null, blobs = null, exclude = null }) {
+//
+// `stream(circleId, { skip })` yields the same Changes one at a time, minting each as it is asked
+// for, today's day file first and then newest first (HQ-2068); `snapshot` is the stream collected.
+//
+// `onFileSet(rel, sha256)`, when given, is awaited once per file.set this mints (HQ-2109): the
+// catch-up reads and hashes the file itself, so it is the one place that can tell the scan index the
+// file is already accounted for, at no extra read. Without it, a device's next ordinary scan finds no
+// row for a file the catch-up just sent, reads it as never-before-seen, and mints a second Change for
+// bytes the peer already has.
+export function catchUp({ evda, view, log, seal, open, changesetId, originUser = null, unionPath = null, blobs = null, exclude = null, onFileSet = null }) {
+  // What travels, by the same rule as the scan (sync-rules, HQ-2017): a reference is hashed from a
+  // local file and carries no bytes; a dataless file is not opened at all, and is left out of the
+  // catch-up (it is neither sent nor withdrawn) until its bytes are on this disk.
+  const classOf = fileClass(carryList(evda.manifest));
   const readText = (rel) => {
     const abs = join(evda.dir, rel);
     return existsSync(abs) && statSync(abs).isFile() ? readFileSync(abs, "utf8") : null;
   };
+  // The files a circle member holds, by the sync rules' own walk (HQ-2017): the same exclusions the
+  // scan lists by (node_modules, .git, .evda, the plan's paths), no symlink, no iCloud placeholder,
+  // nothing over the blob lane. The catch-up once walked by its own rule (every dot name skipped,
+  // node_modules kept), so what it sent and withdrew disagreed with what the scan minted.
+  const skip = exclude ?? excluder();
   const walk = (prefix) => {
-    const out = [];
     const start = join(evda.dir, prefix);
-    if (!existsSync(start)) return out;
-    if (statSync(start).isFile()) return [prefix];
-    const stack = [prefix.replace(/\/+$/, "")];
-    while (stack.length > 0) {
-      const dir = stack.pop();
-      for (const name of readdirSync(join(evda.dir, dir)).sort()) {
-        if (name.startsWith(".")) continue;
-        const rel = dir === "" ? name : `${dir}/${name}`;
-        const st = statSync(join(evda.dir, rel));
-        if (st.isDirectory()) stack.push(rel);
-        else if (st.isFile()) out.push(rel);
-      }
-    }
-    return out;
+    if (!existsSync(start)) return [];
+    const from = prefix.replace(/\/+$/, "");
+    if (statSync(start).isFile()) return skip(from) ? [] : [from];
+    return [...walkRules(evda.dir, { exclude: skip, from })].filter((e) => e.kind === "file").map((e) => e.rel);
   };
   const shaOf = async (bytes) => Buffer.from(await crypto.subtle.digest("SHA-256", bytes)).toString("hex");
 
@@ -568,55 +644,114 @@ export function catchUp({ evda, view, log, seal, open, changesetId, originUser =
     return out;
   };
 
-  return {
-    async snapshot(circleId) {
-      const index = view().index;
-      const target = circleById(index, circleId);
-      if (!target) throw new EvdaError(`no circle ${circleId} in this record.`, { code: "NO_CIRCLE" });
-      if (target.mount) throw new EvdaError(`circle "${target.name}" is held from ${target.mount.holder}; only its home catches a device up.`, { code: "NOT_OWNER" });
-      const at = addressing(index);
-      const present = new Map();
-      for (const { prefix, match } of scanPrefixes(index, (c) => !c.mount && !c.frozen)) {
-        for (const rel of walk(prefix)) {
-          if (!match(rel) || present.has(rel) || exclude?.(rel)) continue;
-          const bytes = readFileSync(join(evda.dir, rel));
-          const text = utf8OrNull(bytes);
-          if (!circlesFor(index, rel, text, readText).includes(target)) continue;
-          present.set(rel, { bytes, text });
-        }
+  const dayDir = dayFolderOf(evda.manifest);
+
+  // One Change per file the circle reaches now, newest first (HQ-2068), then one removal per address
+  // it no longer reaches. Each is minted only when the caller asks for the next, so a caller that
+  // posts as it goes has the first batch on the relay before the rest is read. `skip` holds
+  // addresses already sent by an earlier, interrupted run: neither minted again nor withdrawn.
+  async function* stream(circleId, { skip = new Set(), now = new Date() } = {}) {
+    const index = view().index;
+    const target = circleById(index, circleId);
+    if (!target) throw new EvdaError(`no circle ${circleId} in this record.`, { code: "NO_CIRCLE" });
+    if (target.mount) throw new EvdaError(`circle "${target.name}" is held from ${target.mount.holder}; only its home catches a device up.`, { code: "NOT_OWNER" });
+    const at = addressing(index);
+    const listed = new Set();
+    const skipped = new Set();
+    const files = [];
+    for (const { prefix, match } of scanPrefixes(index, (c) => !c.mount && !c.frozen)) {
+      const rels = walk(prefix).filter((rel) => match(rel) && !listed.has(rel));
+      const dataless = datalessAmong(evda.dir, rels);
+      for (const rel of rels) {
+        if (listed.has(rel)) continue;
+        listed.add(rel);
+        if (dataless.has(rel)) { skipped.add(rel); continue; }
+        let mtimeMs = 0;
+        try { mtimeMs = statSync(join(evda.dir, rel)).mtimeMs; } catch { continue; }
+        files.push({ rel, mtimeMs });
       }
-      const minted = [];
-      const append = async (kind, payload, blobHash = null) => {
-        const id = await changesetId();
-        const change = await log.append({ changesetId: id, eco: target.id, originUser, kind, target: payload.path, payload: await seal(target, payload), notify: [] });
-        if (blobHash !== null) await blobs.link(change.id, blobHash);
-        minted.push(change);
-      };
-      const addresses = new Set();
-      for (const [rel, { bytes, text }] of [...present.entries()].sort(([a], [b]) => a.localeCompare(b))) {
-        const address = at.address(circleForPath(index, rel), rel);
-        addresses.add(address);
-        const sha256 = await shaOf(bytes);
-        const ancestors = await ancestorsOf(address, sha256);
-        const base = { path: address, sha256, baseSha256: ancestors.at(-1) ?? null, ancestors };
-        if (blobs && (text === null || bytes.length > BLOB_THRESHOLD)) {
-          const sealed = await blobs.seal(bytes);
-          await append(FILE_SET, { ...base, blob: { hash: sealed.hash, contentKey: Buffer.from(sealed.contentKey).toString("base64") } }, sealed.hash);
-          continue;
-        }
-        await append(FILE_SET, { ...base, text: text ?? new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes), ...(unionPath?.(rel) === true ? { union: true } : {}) });
+    }
+    files.sort(catchUpOrder(dayDir, localDate(now)));
+    const mint = async (kind, payload, blobHash = null) => {
+      const id = await changesetId();
+      const change = await log.append({ changesetId: id, eco: target.id, originUser, kind, target: payload.path, payload: await seal(target, payload), notify: [] });
+      if (blobHash !== null) await blobs.link(change.id, blobHash);
+      return change;
+    };
+    const addresses = new Set();
+    for (const { rel } of files) {
+      const kind = classOf(rel);
+      let bytes = null;
+      let text = null;
+      let size = 0;
+      let known = null;
+      if (kind === "reference") {
+        // Only its home circle: a mark to release is frontmatter, and a reference has no text.
+        if (circleForPath(index, rel) !== target) continue;
+        const raw = readFileSync(join(evda.dir, rel));
+        size = raw.length;
+        known = await shaOf(raw);
+      } else {
+        bytes = readFileSync(join(evda.dir, rel));
+        text = utf8OrNull(bytes);
+        if (!circlesFor(index, rel, text, readText).includes(target)) continue;
       }
-      // Addresses the circle held before and does not now: removed, or no longer released into it.
-      const gone = new Set();
-      for (const change of await log.forEco(target.id)) {
-        if (change.kind === FILE_SET && !addresses.has(change.target)) gone.add(change.target);
+      const address = at.address(circleForPath(index, rel), rel);
+      addresses.add(address);
+      if (skip.has(address)) continue;
+      const sha256 = known ?? (await shaOf(bytes));
+      // Told as soon as the hash is known, whichever branch below mints: a file.set is coming for
+      // this path either way, and the scan index should already agree with it by the time it lands.
+      if (onFileSet) await onFileSet(rel, sha256);
+      const ancestors = await ancestorsOf(address, sha256);
+      const base = { path: address, sha256, baseSha256: ancestors.at(-1) ?? null, ancestors };
+      if (kind === "reference") {
+        yield await mint(FILE_SET, { ...refPayload(address, sha256, base.baseSha256, size), ancestors });
+        continue;
       }
-      for (const address of [...gone].sort()) {
-        const ancestors = await ancestorsOf(address, null);
-        if (ancestors.length === 0) continue;
-        await append(FILE_REMOVE, { path: address, sha256: null, baseSha256: ancestors.at(-1), ancestors });
+      if (blobs && (kind === "carried" || text === null || bytes.length > BLOB_THRESHOLD)) {
+        const sealed = await blobs.seal(bytes);
+        yield await mint(FILE_SET, { ...base, blob: { hash: sealed.hash, contentKey: Buffer.from(sealed.contentKey).toString("base64") } }, sealed.hash);
+        continue;
       }
+      yield await mint(FILE_SET, { ...base, text: text ?? new TextDecoder("utf-8", { ignoreBOM: true }).decode(bytes), ...(unionPath?.(rel) === true ? { union: true } : {}) });
+    }
+    // Addresses the circle held before and does not now: removed, or no longer released into it.
+    const gone = new Set();
+    const skippedAddresses = new Set([...skipped].map((rel) => { try { return at.address(circleForPath(index, rel), rel); } catch { return rel; } }));
+    for (const change of await log.forEco(target.id)) {
+      if (change.kind === FILE_SET && !addresses.has(change.target) && !skippedAddresses.has(change.target) && !skip.has(change.target)) gone.add(change.target);
+    }
+    for (const address of [...gone].sort()) {
+      const ancestors = await ancestorsOf(address, null);
+      if (ancestors.length === 0) continue;
+      yield await mint(FILE_REMOVE, { path: address, sha256: null, baseSha256: ancestors.at(-1), ancestors });
+    }
+  }
+
+  return {
+    stream,
+    async snapshot(circleId, o = {}) {
+      const minted = [];
+      for await (const change of stream(circleId, o)) minted.push(change);
       return minted;
     },
   };
 }
+
+// The order a catch-up sends a circle's files in (HQ-2068): today's day file first, then the most
+// recently modified, then by path, so a phone joining a large record shows what was written last
+// within seconds instead of after the alphabet.
+export function catchUpOrder(dayDir, today) {
+  const dayPrefix = `${String(dayDir ?? "").replace(/\/+$/, "")}/`;
+  const rank = (rel) => (rel.startsWith(dayPrefix) && rel.slice(dayPrefix.length).startsWith(today) ? 0 : 1);
+  return (a, b) => rank(a.rel) - rank(b.rel) || b.mtimeMs - a.mtimeMs || (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0);
+}
+
+// The day folder a record's mapping names, else create-evda's default (as pairing.ts reads it).
+function dayFolderOf(manifest) {
+  const mapping = manifest?.mapping;
+  const role = dayRole(mapping);
+  const rel = role ? mapping?.[role] : undefined;
+  return typeof rel === "string" && rel.trim() !== "" ? rel.replace(/\/+$/, "") : "record/daily";
+}