create-walle 0.10.2 → 0.10.3

package/template/coding-task-manager/lib/memory-sync-verify.js

@@ -0,0 +1,393 @@
+'use strict';
+
+// WP-6a (docs/shared-memory-every-machine-2026-10-31.html §6.5, gate E-3): the convergence verifier.
+//
+//   verifyMemorySync(deps) -> { ok, hub: { instanceId, vector, counts }, members: [...], allEqual }
+//
+// Proves "exactly the same memory" the only way that survives a bucket collision: the bucketed
+// content digest (§6.5) on the hub and on every enabled registry member, compared bucket by bucket.
+// A bucket is equal when its row COUNTS match and — only for a non-empty bucket — its XOR fold
+// matches; two empty buckets are equal whatever their xor string says, because the fold is over no
+// rows at all.
+//
+// Three rules the caller depends on:
+//   * Read-only. This module never writes anywhere: no CTM row, no brain row, no file. It dials the
+//     local Wall-E (loopback + the CTM's own token), and each member's CTM front route.
+//   * A member that cannot be reached is REPORTED (`reachable:false`, `allEqual:false`) and the
+//     remaining members are still compared — one dead box must not hide the others' verdict.
+//   * The per-member `vectorLag` (hub max_seq − member max_seq, per origin) is the diagnosis the
+//     digest alone cannot give: a large positive lag means "still syncing", a NON-ZERO lag with an
+//     equal digest on every table means "diverged" (a row the hub holds and the member never will).
+//
+// Tier: this is the `bin/memory-sync-verify.js` core. The CLI owns arg parsing and the registry
+// read; this module owns the comparison and the dials.
+//
+// NB (stabilization-lint): the two `setTimeout(` sites below (one in _localGet, one in _memberGet)
+// are per-request abort deadlines, not recovery waits. Each is unref'd so a hung dial can never
+// hold the process open. The lead registers the surfaces in release/ownership-registry.json — do
+// not silence them here.
+
+const machineRegistry = require('./machine-registry');
+const { buildPeerControlHeaders } = require('./remote-machine-create');
+
+// The hub's LOCAL Wall-E v2 routes (loopback + CTM token, exactly as lib/memory-sync.js reads them).
+const LOCAL_PATHS = Object.freeze({
+  vector: '/api/walle/sync/v2/vector',
+  digest: '/api/walle/sync/v2/digest',
+});
+
+// The member's CTM front routes (§6.6), each of which forwards to the member's own Wall-E.
+const MEMBER_PATHS = Object.freeze({
+  vector: '/api/sync/brain/v2/vector',
+  digest: '/api/sync/brain/v2/digest',
+});
+
+// D-11: the four tables "the same memory" means. `people` has no runtime writer today but is
+// replicated and must still read equal (0/0) on every node rather than be silently skipped.
+const DEFAULT_TABLES = Object.freeze(['memories', 'knowledge', 'entities', 'people']);
+
+const DEFAULT_TIMEOUT_MS = 30_000;
+const BUCKET_COUNT = 256;
+
+function _string(value, max = 160) {
+  if (typeof value === 'string') return value.trim().slice(0, max);
+  return value == null ? '' : String(value).trim().slice(0, max);
+}
+
+function _normalizeTables(tables) {
+  const list = Array.isArray(tables) ? tables : DEFAULT_TABLES;
+  const seen = new Set();
+  for (const raw of list) {
+    const name = _string(raw, 40);
+    if (name) seen.add(name);
+  }
+  return [...seen];
+}
+
+// Dial order = `urls` first, then the pinned `url` (machine-registry D-3/_dialUrls). A row whose
+// list is empty still gets one chance at its pinned URL.
+function _memberUrls(member = {}) {
+  const list = (Array.isArray(member.urls) ? member.urls : []).map((u) => _string(u, 1000)).filter(Boolean);
+  if (list.length) return list;
+  const pinned = _string(member.url, 1000);
+  return pinned ? [pinned] : [];
+}
+
+function _joinUrl(base, pathname) {
+  return `${String(base).replace(/\/+$/, '')}${pathname}`;
+}
+
+/**
+ * The header builder for member dials. `deps.peerHeadersFor` wins (the hub-side production seam,
+ * the same one lib/memory-sync.js takes); with none, fall back to remote-machine-create's
+ * buildPeerControlHeaders so a bare caller still authenticates the way the probe does.
+ */
+function _peerHeadersBuilder(deps, env) {
+  if (typeof deps.peerHeadersFor === 'function') {
+    const build = deps.peerHeadersFor;
+    return (instanceId) => {
+      try {
+        const built = build(instanceId);
+        return built && typeof built === 'object' ? built : {};
+      } catch {
+        return {}; // expected: a credential-builder fault degrades to an anonymous dial, reported as unreachable
+      }
+    };
+  }
+  const fallbackEnv = env && typeof env === 'object' ? env : process.env;
+  return (instanceId) => buildPeerControlHeaders(fallbackEnv, { instanceId }) || {};
+}
+
+/** One local Wall-E GET. Resolves { ok, json } — a transport fault is a verdict, never a throw. */
+async function _localGet(walle, pathname, timeoutMs) {
+  if (!walle || typeof walle.requestJson !== 'function') return { ok: false, error: 'walle_client_missing' };
+  const controller = typeof AbortController !== 'undefined' ? new AbortController() : null;
+  const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null;
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  try {
+    const res = await walle.requestJson(pathname, {
+      method: 'GET',
+      signal: controller ? controller.signal : undefined,
+    });
+    const status = Number(res && res.status) || 0;
+    if (status < 200 || status >= 300 || !res || !res.json) return { ok: false, error: `walle_http_${status}` };
+    return { ok: true, json: res.json };
+  } catch (err) {
+    return { ok: false, error: `walle_request_failed: ${err && err.message ? err.message : String(err)}` };
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+/** One member GET against one URL. Same deadline shape; status 0 means "nothing answered". */
+async function _memberGet(fetchImpl, url, pathname, headers, timeoutMs) {
+  const fetchFn = fetchImpl || global.fetch;
+  if (typeof fetchFn !== 'function') return { ok: false, status: 0, error: 'fetch_unavailable' };
+  const controller = typeof AbortController !== 'undefined' ? new AbortController() : null;
+  const timer = controller ? setTimeout(() => controller.abort(), timeoutMs) : null;
+  if (timer && typeof timer.unref === 'function') timer.unref();
+  try {
+    const res = await fetchFn(_joinUrl(url, pathname), {
+      method: 'GET',
+      headers: { ...headers, Accept: 'application/json' },
+      signal: controller ? controller.signal : undefined,
+    });
+    let json = null;
+    try { json = await res.json(); } catch { json = null; /* expected: a non-JSON body is degraded to null and reported by status */ }
+    const status = Number(res && res.status) || 0;
+    if (status < 200 || status >= 300 || !json) return { ok: false, status, error: `member_http_${status}` };
+    return { ok: true, status, json };
+  } catch (err) {
+    return { ok: false, status: 0, error: err && err.message ? err.message : String(err) };
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
+async function _dialUrls(fetchImpl, urls, pathname, headers, timeoutMs) {
+  let last = { ok: false, status: 0, error: 'member_has_no_url' };
+  for (const url of urls) {
+    const res = await _memberGet(fetchImpl, url, pathname, headers, timeoutMs);
+    if (res.status > 0 || res.ok) return { ...res, url };
+    last = { ...res, url };
+  }
+  return last;
+}
+
+/** `buckets` as a 256-slot array; a malformed/short array reads as zero-count buckets. */
+function _bucketArray(digest) {
+  const out = new Array(BUCKET_COUNT);
+  const list = digest && Array.isArray(digest.buckets) ? digest.buckets : [];
+  for (const entry of list) {
+    const b = Number(entry && entry.b) | 0;
+    if (b < 0 || b >= BUCKET_COUNT) continue;
+    out[b] = { count: Math.max(0, Number(entry && entry.count) || 0), xor: _string(entry && entry.xor, 64) };
+  }
+  for (let b = 0; b < BUCKET_COUNT; b += 1) if (!out[b]) out[b] = { count: 0, xor: '' };
+  return out;
+}
+
+// §6.5: equal when the counts match and — for a non-empty bucket — the xor folds match too. Two
+// empty buckets are equal regardless of their xor string (the fold is over no rows).
+function _bucketEqual(hub, member) {
+  if (hub.count !== member.count) return false;
+  if (hub.count === 0) return true;
+  return hub.xor === member.xor;
+}
+
+/** Compare one table's digests. Returns { equal, hubCount, memberCount, mismatchedBuckets }. */
+function compareDigests(hubDigest, memberDigest) {
+  const hub = _bucketArray(hubDigest);
+  const member = _bucketArray(memberDigest);
+  const mismatchedBuckets = [];
+  for (let b = 0; b < BUCKET_COUNT; b += 1) {
+    if (!_bucketEqual(hub[b], member[b])) mismatchedBuckets.push(b);
+  }
+  return {
+    equal: mismatchedBuckets.length === 0,
+    hubCount: Number(hubDigest && hubDigest.count) || 0,
+    memberCount: Number(memberDigest && memberDigest.count) || 0,
+    mismatchedBuckets,
+  };
+}
+
+/**
+ * For every origin either side knows: hub max_seq − member max_seq. Positive = the hub holds
+ * updates the member has not applied; negative = the member is ahead (its rows have not reached
+ * the hub yet); 0 = this origin is level.
+ */
+function vectorLag(hubVector, memberVector) {
+  const hub = hubVector && typeof hubVector === 'object' ? hubVector : {};
+  const member = memberVector && typeof memberVector === 'object' ? memberVector : {};
+  const origins = new Set([...Object.keys(hub), ...Object.keys(member)]);
+  const lag = {};
+  for (const origin of [...origins].sort()) {
+    lag[origin] = (Number(hub[origin]) || 0) - (Number(member[origin]) || 0);
+  }
+  return lag;
+}
+
+function _emptyTables(tables) {
+  const out = {};
+  for (const t of tables) out[t] = { equal: false, hubCount: null, memberCount: null, mismatchedBuckets: [] };
+  return out;
+}
+
+/** The enabled registry rows to check. `deps.machines` (a pre-fetched view) wins over `deps.db`. */
+function _memberRows(deps) {
+  const source = Array.isArray(deps.machines)
+    ? deps.machines
+    : machineRegistry.listMachines({ db: deps.db }).machines;
+  return (source || []).filter((m) => m && m.enabled && _string(m.instanceId));
+}
+
+/**
+ * Compare the hub's digest against every enabled registry member.
+ *
+ * @param {object} deps { db, fetchImpl, walle, env, peerHeadersFor, tables, timeoutMs, machines }
+ * @returns {Promise<object>} { ok, error?, hub:{instanceId,vector,counts}, members:[...], allEqual }
+ */
+async function verifyMemorySync(deps = {}) {
+  const walle = deps.walle || require('./walle-client');
+  const fetchImpl = deps.fetchImpl || global.fetch;
+  const env = deps.env || process.env;
+  const tables = _normalizeTables(deps.tables);
+  const requested = Number(deps.timeoutMs);
+  const timeoutMs = Number.isFinite(requested) && requested > 0 ? requested : DEFAULT_TIMEOUT_MS;
+  const peerHeadersFor = _peerHeadersBuilder(deps, env);
+
+  // 1. The hub. Its vector carries the identity every member's vector is compared against; without
+  //    it there is no reference, so this is the one fatal leg.
+  const hubVectorRes = await _localGet(walle, LOCAL_PATHS.vector, timeoutMs);
+  if (!hubVectorRes.ok) {
+    return { ok: false, error: hubVectorRes.error, hub: null, members: [], allEqual: false };
+  }
+  const hubInstanceId = _string(hubVectorRes.json.instanceId);
+  const hubVector = hubVectorRes.json.vector && typeof hubVectorRes.json.vector === 'object'
+    ? { ...hubVectorRes.json.vector }
+    : {};
+
+  // 2. The hub digest per table. A table the hub cannot digest is a failed check, not a skipped one.
+  const hubDigests = {};
+  const hubCounts = {};
+  for (const table of tables) {
+    const res = await _localGet(walle, `${LOCAL_PATHS.digest}?table=${encodeURIComponent(table)}`, timeoutMs);
+    if (!res.ok) {
+      return {
+        ok: false,
+        error: `hub_digest_${table}: ${res.error}`,
+        hub: { instanceId: hubInstanceId, vector: hubVector, counts: null },
+        members: [],
+        allEqual: false,
+      };
+    }
+    hubDigests[table] = res.json;
+    hubCounts[table] = Number(res.json && res.json.count) || 0;
+  }
+
+  // 3. Every enabled member, sequentially: one comparison per table. A member that does not answer
+  //    the vector is unreachable — reported, never fatal to the others.
+  const members = [];
+  for (const row of _memberRows(deps)) {
+    const instanceId = _string(row.instanceId);
+    const urls = _memberUrls(row);
+    const headers = peerHeadersFor(instanceId);
+    const vectorRes = await _dialUrls(fetchImpl, urls, MEMBER_PATHS.vector, headers, timeoutMs);
+    if (!vectorRes.ok) {
+      members.push({
+        instanceId,
+        reachable: false,
+        vector: null,
+        counts: null,
+        tables: _emptyTables(tables),
+        vectorLag: null,
+        error: vectorRes.error || 'member_unreachable',
+      });
+      continue;
+    }
+    const memberVector = vectorRes.json.vector && typeof vectorRes.json.vector === 'object'
+      ? { ...vectorRes.json.vector }
+      : {};
+    const memberTables = {};
+    const memberCounts = {};
+    for (const table of tables) {
+      const res = await _dialUrls(fetchImpl, urls, `${MEMBER_PATHS.digest}?table=${encodeURIComponent(table)}`, headers, timeoutMs);
+      if (!res.ok) {
+        memberTables[table] = { equal: false, hubCount: hubCounts[table], memberCount: null, mismatchedBuckets: [], error: res.error || 'member_digest_failed' };
+        memberCounts[table] = null;
+        continue;
+      }
+      const compared = compareDigests(hubDigests[table], res.json);
+      memberTables[table] = compared;
+      memberCounts[table] = compared.memberCount;
+    }
+    members.push({
+      instanceId,
+      reachable: true,
+      vector: memberVector,
+      counts: memberCounts,
+      tables: memberTables,
+      vectorLag: vectorLag(hubVector, memberVector),
+    });
+  }
+
+  const allEqual = members.length > 0
+    && members.every((m) => m.reachable && tables.every((t) => m.tables[t] && m.tables[t].equal));
+
+  return {
+    ok: true,
+    hub: { instanceId: hubInstanceId, vector: hubVector, counts: hubCounts },
+    members,
+    allEqual,
+  };
+}
+
+// ------------------------------------------------------------------------------------------------
+// The CLI's pure half (bin/memory-sync-verify.js): one line per member, and the exit code.
+
+/** One member's per-table cell: `memories =100/100` (equal) or `memories !100/99` (differs). */
+function _tableCell(member, tables) {
+  return tables.map((t) => {
+    const row = member.tables && member.tables[t] ? member.tables[t] : null;
+    const hubCount = row && row.hubCount != null ? row.hubCount : '?';
+    const memberCount = row && row.memberCount != null ? row.memberCount : '?';
+    const mark = row && row.equal ? '=' : '!';
+    const extra = row && !row.equal && Array.isArray(row.mismatchedBuckets) && row.mismatchedBuckets.length
+      ? ` b[${row.mismatchedBuckets.join(',')}]`
+      : '';
+    return `${t} ${mark}${hubCount}/${memberCount}${extra}`;
+  }).join(' ');
+}
+
+function _lagCell(member) {
+  if (!member.vectorLag || typeof member.vectorLag !== 'object') return '-';
+  const entries = Object.entries(member.vectorLag);
+  if (!entries.length) return '-';
+  return entries.map(([origin, lag]) => `${origin}=${lag}`).join(',');
+}
+
+/**
+ * The human table: one row per member, then the verdict. Pure — no clock, no I/O.
+ *
+ * @param {object} result the verifyMemorySync result
+ * @param {object} [opts] { tables } — the table list to render in each row
+ */
+function formatReport(result, opts = {}) {
+  const r = result && typeof result === 'object' ? result : {};
+  const hub = r.hub || {};
+  const members = Array.isArray(r.members) ? r.members : [];
+  const derived = members[0] && members[0].tables ? Object.keys(members[0].tables) : [];
+  const requested = Array.isArray(opts.tables) && opts.tables.length ? opts.tables : derived;
+  const tables = _normalizeTables(requested.length ? requested : DEFAULT_TABLES);
+  const lines = [];
+  lines.push(`memory sync verify — hub ${hub.instanceId || '(unknown)'}`);
+  lines.push(`tables: ${tables.join(', ')}`);
+  lines.push('');
+  for (const member of members) {
+    const state = member.reachable ? 'reachable  ' : 'unreachable';
+    const detail = member.reachable ? _tableCell(member, tables) : (member.error || 'unreachable');
+    lines.push(`${member.instanceId || '(unknown)'}  ${state}  ${detail}  lag: ${_lagCell(member)}`);
+  }
+  if (!members.length) lines.push('(no enabled members registered)');
+  lines.push('');
+  lines.push(`allEqual: ${r.allEqual === true}`);
+  if (r.error) lines.push(`error: ${r.error}`);
+  return `${lines.join('\n')}\n`;
+}
+
+/** Exit code: 0 only when every enabled member is reachable and every table matches. */
+function exitCodeFor(result) {
+  return result && result.ok === true && result.allEqual === true ? 0 : 1;
+}
+
+module.exports = {
+  verifyMemorySync,
+  compareDigests,
+  vectorLag,
+  formatReport,
+  exitCodeFor,
+  DEFAULT_TABLES,
+  DEFAULT_TIMEOUT_MS,
+  LOCAL_PATHS,
+  MEMBER_PATHS,
+};