kenaz 0.3.19 → 0.3.21

package/core/dev/scripts/peer-identity-e2e.js

@@ -0,0 +1,402 @@
+#!/usr/bin/env node
+'use strict';
+// TASK_2458: end-to-end usage test of peer identity (TASK_2447-2452, 2456, 2457) against real binaries.
+// Fake registry + local relay (new and old) + kenaz-servers (new and old) in temp homes, ports 18900-18999.
+// Never touches the real ~/.kenaz, ~/.claude, ~/.codex or the user's server on 18080; kills only PIDs it spawned.
+// Usage: node dev/scripts/peer-identity-e2e.js [--control-wrong-fingerprint] [--keep]   (see peer-identity-e2e.md)
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const http = require('http');
+const { spawn, execFileSync, spawnSync } = require('child_process');
+
+const REPO = path.resolve(__dirname, '..', '..');
+const EXE = process.platform === 'win32' ? '.exe' : '';
+const BIN = {
+  relay: process.env.E2E_RELAY || `C:/t/relay-target/debug/yggdrasil-relay${EXE}`,
+  relayOld: process.env.E2E_RELAY_OLD || `C:/t/relay-old-target/debug/yggdrasil-relay${EXE}`,
+  server: process.env.E2E_SERVER || `C:/t/kenaz-e2e-target/debug/kenaz-server${EXE}`,
+  serverOldSrc: process.env.E2E_SERVER_OLD || path.join(os.homedir(), '.kenaz', 'bin', `kenaz-server${EXE}`),
+};
+const ARGS = new Set(process.argv.slice(2));
+const CONTROL = ARGS.has('--control-wrong-fingerprint');
+const KEYS = { t1: 'ygk_e2e_team_one_0001', t2: 'ygk_e2e_team_two_0002' };
+const TEAMS = { [KEYS.t1]: 'e2e-team-1', [KEYS.t2]: 'e2e-team-2' };
+const PORT = { registry: 18900, relay: 18901, relayOld: 18902, A: 18910, B: 18911, fern: 18912, olive: 18913, D: 18914, E: 18915 };
+const ROOM = 'e2e-room';
+const ADMIN_KEY = 'e2e-admin-key-not-a-secret';
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const results = [];
+const procs = [];
+let TMP;
+
+// ---------- small helpers ----------
+async function call(method, url, { headers = {}, body } = {}) {
+  const res = await fetch(url, { method, headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...headers }, body: body ? JSON.stringify(body) : undefined });
+  const text = await res.text();
+  let json; try { json = JSON.parse(text); } catch { /* not json */ }
+  return { status: res.status, text, json };
+}
+async function until(what, fn, ms = 60000, every = 1000) {
+  const end = Date.now() + ms; let last;
+  while (Date.now() < end) { try { last = await fn(); if (last) return last; } catch (e) { last = e; } await sleep(every); }
+  throw new Error(`timeout (${ms / 1000}s) waiting for ${what}${last instanceof Error ? ': ' + last.message : ''}`);
+}
+function portFree(port) {
+  return new Promise((resolve) => {
+    const s = require('net').createServer().once('error', () => resolve(false)).once('listening', () => s.close(() => resolve(true)));
+    s.listen(port, '127.0.0.1');
+  });
+}
+function start(name, exe, args, env, cwd) {
+  const log = fs.openSync(path.join(TMP, `${name}.log`), 'a');
+  const child = spawn(exe, args, { env, cwd, stdio: ['ignore', log, log], windowsHide: true });
+  const p = { name, child, pid: child.pid, exited: false };
+  child.on('exit', () => { p.exited = true; });
+  procs.push(p);
+  return p;
+}
+async function stop(p) {
+  if (!p || p.exited) return;
+  if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(p.pid), '/T', '/F'], { stdio: 'ignore' }); // our PID only
+  else p.child.kill('SIGKILL');
+  await until(`${p.name} exit`, () => p.exited, 15000, 100).catch(() => {});
+}
+function sh(cmd, args, env) {
+  const r = spawnSync(cmd, args, { env: { ...process.env, ...env }, encoding: 'utf8' });
+  return { code: r.status, out: `${r.stdout || ''}${r.stderr || ''}`.trim() };
+}
+function sqlite(file, readOnly = true) {
+  const { DatabaseSync } = require('node:sqlite');
+  return new DatabaseSync(file, { readOnly });
+}
+
+// ---------- fake registry ----------
+function startRegistry() {
+  const srv = http.createServer((req, res) => {
+    let b = ''; req.on('data', (c) => (b += c));
+    req.on('end', () => {
+      let key; try { key = JSON.parse(b).p_key; } catch { /* ignore */ }
+      res.setHeader('Content-Type', 'application/json');
+      if (req.method === 'POST' && req.url === '/rest/v1/rpc/exchange_key' && TEAMS[key]) {
+        res.end(JSON.stringify({ team_id: TEAMS[key], team_name: TEAMS[key], max_nodes: 10 }));
+      } else res.end('null');
+    });
+  });
+  return new Promise((r) => srv.listen(PORT.registry, '127.0.0.1', () => r(srv)));
+}
+
+// ---------- relay ----------
+function startRelay(name, exe, port, db) {
+  const env = { ...process.env, RELAY_PORT: String(port), RELAY_DB: db, REGISTRY_URL: `http://127.0.0.1:${PORT.registry}`, REGISTRY_KEY: 'e2e-registry-key',
+    RELAY_ADMIN_KEY_FILE: path.join(TMP, 'admin.key'), RUST_LOG: 'yggdrasil_relay=info,warn' };
+  delete env.RELAY_ADMIN_KEY; delete env.REGISTRY_SERVICE_KEY;
+  return start(name, exe, [], env, TMP);
+}
+const relayUrl = (port) => `http://127.0.0.1:${port}`;
+const bearer = (key) => ({ Authorization: `Bearer ${key}` });
+const admin = (port, ...a) => sh(process.execPath, [path.join(REPO, 'dev/scripts/relay-admin.js'), '--relay', relayUrl(port), ...a], { RELAY_ADMIN_KEY_FILE: path.join(TMP, 'admin.key'), RELAY_ADMIN_KEY: '' });
+
+// ---------- kenaz-server ----------
+function sanitizedEnv(dir, port, src) {
+  const sys = process.env.SystemRoot || 'C:\\Windows';
+  const env = { SystemRoot: sys, ComSpec: process.env.ComSpec || '', PATH: process.platform === 'win32' ? `${sys}\\System32;${sys}` : '/usr/bin:/bin',
+    USERPROFILE: path.join(dir, 'home'), HOME: path.join(dir, 'home'), APPDATA: path.join(dir, 'appdata'), LOCALAPPDATA: path.join(dir, 'localappdata'),
+    TEMP: path.join(dir, 'tmp'), TMP: path.join(dir, 'tmp'), KENAZ_SOURCE_DIR: src, KENAZ_HTTP_PORT: String(port), RUST_LOG: 'warn' };
+  for (const d of ['home', 'appdata', 'localappdata', 'tmp']) fs.mkdirSync(path.join(dir, d), { recursive: true });
+  return env;
+}
+class Server {
+  constructor(name, exe, port, relayPort, teamKey, node) {
+    Object.assign(this, { name, exe, port, relayPort, teamKey, node });
+    this.dir = path.join(TMP, name);
+    this.project = path.join(this.dir, 'proj').replace(/\\/g, '/');
+    this.src = path.join(this.dir, 'src');
+    this.url = `http://127.0.0.1:${port}`;
+  }
+  async setup() {
+    fs.mkdirSync(path.join(this.project, '.kenaz'), { recursive: true });
+    fs.mkdirSync(path.join(this.dir, 'home', '.kenaz'), { recursive: true });
+    fs.mkdirSync(this.src, { recursive: true });
+    fs.writeFileSync(path.join(this.project, '.kenaz', 'project.yaml'), `name: e2e-${this.name}\n`);
+    fs.writeFileSync(path.join(this.project, '.kenaz', 'yggdrasil.yaml'),
+      `relay_url: "http://127.0.0.1:${this.relayPort}"\nnode_name: "${this.node}"\nyggdrasil_project: "${ROOM}"\ncollaboration_projects: "${ROOM}"\n`);
+    // Registered before the first start so the presence heartbeat (every 90 s, first tick at boot) already includes this project.
+    fs.writeFileSync(path.join(this.dir, 'home', '.kenaz', 'projects.json'), JSON.stringify({ currentProjectId: `e2e-${this.name}`, projects: [
+      { id: `e2e-${this.name}`, name: `e2e-${this.name}`, path: this.project, pipelinePath: 'dev/pipeline', createdAt: new Date().toISOString(), is_core: false, core_path: this.project }] }));
+    fs.writeFileSync(path.join(this.dir, 'home', '.kenaz', 'credentials.yaml'), `publishable_key: "${this.teamKey}"\n`);
+    await this.run();
+    const r = await call('POST', `${this.url}/api/projects/register`, { body: { projectPath: this.project } });
+    if (!r.json || !r.json.success) throw new Error(`${this.name} register failed: ${r.text}`);
+  }
+  async run() {
+    if (!(await portFree(this.port))) throw new Error(`port ${this.port} is busy; refusing to start ${this.name}`);
+    this.proc = start(this.name, this.exe, [], sanitizedEnv(this.dir, this.port, this.src), this.dir);
+    await until(`${this.name} http`, async () => (await call('GET', `${this.url}/api/peer/status?projectPath=${encodeURIComponent(this.project)}`)).status === 200, 60000, 500);
+  }
+  async restart() { await stop(this.proc); await this.run(); }
+  status() { return call('GET', `${this.url}/api/peer/status?projectPath=${encodeURIComponent(this.project)}`).then((r) => r.json); }
+  async identity() { return ((await this.status()) || {}).node_identity || {}; }
+  async send(to, body) {
+    const r = await call('POST', `${this.url}/api/mcp`, { body: { jsonrpc: '2.0', id: 1, method: 'tools/call',
+      params: { name: 'kenaz_peer_send', arguments: { projectPath: this.project, to, kind: 'notice', body } } } });
+    const text = JSON.stringify(r.json || r.text);
+    const err = !!(r.json && ((r.json.result && r.json.result.isError) || r.json.error));
+    return { ok: r.status === 200 && !err, text };
+  }
+  async inbox() {
+    const r = await call('GET', `${this.url}/api/peer/pending?projectPath=${encodeURIComponent(this.project)}&peek=true`);
+    return JSON.stringify(r.json || r.text);
+  }
+  async waitMail(marker, ms = 75000) {
+    let box = '';
+    await until(`${this.name} to receive ${marker}`, async () => ((box = await this.inbox()).includes(marker)), ms, 2000);
+    // Each message object is {body, created_at, framed: "[Peer message — from X (<label>), thread ...", ...}: the framed text follows the body.
+    const at = box.indexOf(marker);
+    const m = /\((verified|unverified sender|team-claimed)\), thread/.exec(box.slice(at).split('{"body"')[0]);
+    return m ? m[1] : `no label found in ${box.slice(Math.max(0, at - 100), at + 400)}`;
+  }
+  // Send, retrying while the target node has not shown up in the relay presence table yet.
+  async sendWhenKnown(to, body) {
+    let r;
+    await until(`${this.name} send to ${to} accepted`, async () => ((r = await this.send(to, body)).ok), 90000, 3000).catch((e) => { throw new Error(`${e.message}; last reply ${r && r.text.slice(0, 200)}`); });
+    return r;
+  }
+}
+
+// ---------- scenario runner ----------
+async function scenario(n, title, fn) {
+  const t0 = Date.now();
+  try {
+    const ev = await fn();
+    results.push({ n, title, pass: true, ev });
+  } catch (e) {
+    results.push({ n, title, pass: false, ev: e.message });
+  }
+  const r = results[results.length - 1];
+  console.log(`[${r.pass ? 'PASS' : 'FAIL'}] ${n}. ${title} (${Math.round((Date.now() - t0) / 1000)}s)\n        ${r.ev}`);
+}
+const need = (cond, msg) => { if (!cond) throw new Error(msg); };
+
+// ---------- isolation checks ----------
+function tree(dir, depth, out = {}) {
+  let ents; try { ents = fs.readdirSync(dir, { withFileTypes: true }); } catch { return out; }
+  for (const e of ents) {
+    const p = path.join(dir, e.name);
+    if (e.isDirectory()) { if (depth > 0) tree(p, depth - 1, out); } else { try { out[p] = fs.statSync(p).mtimeMs; } catch { /* gone */ } }
+  }
+  return out;
+}
+const realDirs = () => ['.kenaz', '.claude', '.codex'].map((d) => path.join(os.homedir(), d));
+const gitState = () => sh('git', ['-C', REPO, 'status', '--porcelain']).out;
+
+async function main() {
+  const missing = Object.entries({ relay: BIN.relay, relayOld: BIN.relayOld, server: BIN.server, serverOldSrc: BIN.serverOldSrc }).filter(([, p]) => !fs.existsSync(p));
+  if (missing.length) { console.error(`missing binaries: ${missing.map(([k, p]) => `${k}=${p}`).join(', ')}`); return 2; }
+  for (const p of Object.values(PORT)) if (!(await portFree(p))) { console.error(`port ${p} busy; aborting`); return 2; }
+  TMP = fs.mkdtempSync(path.join(os.tmpdir(), 'kenaz-peer-e2e-'));
+  const runId = path.basename(TMP);
+  console.log(`peer-identity-e2e: temp dir ${TMP}${CONTROL ? ' (control run: wrong fingerprint in scenario 3)' : ''}`);
+  const gitBefore = gitState();
+  const realBefore = realDirs().map((d) => tree(d, 3));
+  const oldBinStat = fs.statSync(BIN.serverOldSrc).mtimeMs;
+  fs.writeFileSync(path.join(TMP, 'admin.key'), ADMIN_KEY);
+  const oldExe = path.join(TMP, 'old-server', `kenaz-server${EXE}`);
+  fs.mkdirSync(path.dirname(oldExe), { recursive: true });
+  fs.copyFileSync(BIN.serverOldSrc, oldExe);
+  for (const f of fs.readdirSync(path.dirname(BIN.serverOldSrc))) if (/\.dll$/i.test(f)) fs.copyFileSync(path.join(path.dirname(BIN.serverOldSrc), f), path.join(path.dirname(oldExe), f));
+
+  let registry;
+  try {
+    registry = await startRegistry();
+    const relayDb = path.join(TMP, 'relay.db');
+    let relay = startRelay('relay', BIN.relay, PORT.relay, relayDb);
+    const oldRelay = startRelay('relay-old', BIN.relayOld, PORT.relayOld, path.join(TMP, 'relay-old.db'));
+    for (const [p, n] of [[PORT.relay, 'relay'], [PORT.relayOld, 'relay-old']]) await until(`${n} health`, async () => (await call('GET', `${relayUrl(p)}/health`)).status === 200, 30000, 300);
+    const R = relayUrl(PORT.relay);
+    const T1 = TEAMS[KEYS.t1], T2 = TEAMS[KEYS.t2];
+
+    const A = new Server('A', BIN.server, PORT.A, PORT.relay, KEYS.t1, 'node-a');
+    const B = new Server('B', BIN.server, PORT.B, PORT.relay, KEYS.t1, 'node-b');
+    const fern = new Server('fern', BIN.server, PORT.fern, PORT.relay, KEYS.t2, 'fern');
+    const olive = new Server('olive', oldExe, PORT.olive, PORT.relay, KEYS.t2, 'olive');
+    const D = new Server('D', BIN.server, PORT.D, PORT.relayOld, KEYS.t1, 'node-d');
+    const E = new Server('E', BIN.server, PORT.E, PORT.relayOld, KEYS.t1, 'node-e');
+    for (const s of [A, B, fern, olive, D, E]) await s.setup();
+    const fpOf = {};
+    const row = (sql, ...p) => sqlite(relayDb).prepare(sql).all(...p);
+
+    await scenario(1, 'A and B request tokens; pending + fingerprint in GET /api/peer/status', async () => {
+      for (const s of [A, B]) {
+        const id = await until(`${s.name} pending identity`, async () => { const i = await s.identity(); return i.state === 'pending' && i.fingerprint ? i : null; }, 90000, 2000);
+        fpOf[s.name] = id.fingerprint;
+      }
+      const rows = row("SELECT node_name, fingerprint, status FROM node_credentials WHERE team_id = ?", T1);
+      for (const [s, n] of [['A', 'node-a'], ['B', 'node-b']]) need(rows.some((r) => r.node_name === n && r.fingerprint === fpOf[s] && r.status === 'pending'), `no pending DB row for ${n}/${fpOf[s]}`);
+      const list = admin(PORT.relay, 'list');
+      need(list.out.includes(fpOf.A) && list.out.includes(fpOf.B), `admin list lacks fingerprints: ${list.out}`);
+      return `status.node_identity A=pending/${fpOf.A}, B=pending/${fpOf.B}; DB rows pending: ${JSON.stringify(rows)}; admin list shows both`;
+    });
+
+    await scenario(2, 'A -> B before approval: delivered, label (unverified sender)', async () => {
+      await A.sendWhenKnown('node-b', 'e2e-marker-s2');
+      const label = await B.waitMail('e2e-marker-s2');
+      need(label === 'unverified sender', `label was "${label}"`);
+      const r = row("SELECT from_node, to_node, sender_verified FROM task_events WHERE team_id = ? ORDER BY id DESC LIMIT 1", T1)[0];
+      need(r && r.sender_verified === 0, `DB sender_verified=${r && r.sender_verified}`);
+      return `B inbox label "(${label})"; relay row ${JSON.stringify(r)}`;
+    });
+
+    await scenario(3, 'operator approves both with relay-admin.js; servers pick up active via /me', async () => {
+      const outs = [];
+      outs.push(admin(PORT.relay, 'approve', T1, 'node-a', CONTROL ? 'ffffffffffff' : fpOf.A));
+      outs.push(admin(PORT.relay, 'approve', T1, 'node-b', fpOf.B));
+      need(outs.every((o) => o.code === 0), `approve failed: ${outs.map((o) => `${o.code} ${o.out}`).join(' | ')}`);
+      // The client re-reads /me at most every 10 minutes per process, so a restart is the fast path (token file persists on disk).
+      for (const s of [A, B]) await s.restart();
+      for (const s of [A, B]) await until(`${s.name} active`, async () => (await s.identity()).state === 'active', 60000, 2000);
+      const st = row("SELECT node_name, status FROM node_credentials WHERE team_id = ? AND status = 'active'", T1);
+      return `${outs.map((o) => o.out).join(' ; ')}; restart, then status A/B = active; DB active: ${JSON.stringify(st)}`;
+    });
+
+    await scenario(4, 'A -> B after approval: delivered, label (verified)', async () => {
+      await A.sendWhenKnown('node-b', 'e2e-marker-s4');
+      const label = await B.waitMail('e2e-marker-s4');
+      need(label === 'verified', `label was "${label}"`);
+      const r = row("SELECT sender_verified FROM task_events WHERE team_id = ? AND from_node = 'node-a' ORDER BY id DESC LIMIT 1", T1)[0];
+      need(r.sender_verified === 1, `DB sender_verified=${r.sender_verified}`);
+      return `B inbox label "(${label})"; relay row sender_verified=${r.sender_verified}`;
+    });
+
+    await scenario(5, 'protect_bound_names ON: tokenless curl as A / reading or acking B -> 403; A and B still deliver', async () => {
+      const on = await call('POST', `${R}/api/admin/team-settings`, { headers: { 'X-Admin-Key': ADMIN_KEY }, body: { team_id: T1, protect_bound_names: true } });
+      need(on.status === 200, `team-settings ${on.status}`);
+      const project = row("SELECT project FROM nodes WHERE team_id = ? AND node_name = 'node-b' LIMIT 1", T1)[0].project;
+      const h = bearer(KEYS.t1);
+      const ev = (extra) => ({ event_type: 'peer_message', task_id: 'thread-forged', from_node: 'node-a', to_node: 'node-b', project, payload: { body: 'forged' }, ...extra });
+      const forged = await call('POST', `${R}/api/task_events`, { headers: h, body: ev() });
+      const mailbox = await call('GET', `${R}/api/task_events/mailbox?to_node=node-b&project=${project}`, { headers: h });
+      const bId = row("SELECT id FROM task_events WHERE team_id = ? AND to_node = 'node-b' ORDER BY id DESC LIMIT 1", T1)[0].id;
+      const ack1 = await call('PATCH', `${R}/api/task_events/${bId}/ack`, { headers: h, body: { node_name: 'node-b', project } });
+      const ackN = await call('PATCH', `${R}/api/task_events/ack`, { headers: h, body: { node_name: 'node-b', ids: [bId], project } });
+      // controls: an unbound name is still open to tokenless callers, and the flag is per team
+      const ctlUnbound = await call('GET', `${R}/api/task_events/mailbox?to_node=nobody-bound&project=${project}`, { headers: h });
+      const ctlTeam2 = await call('GET', `${R}/api/task_events/mailbox?to_node=fern&project=${project}`, { headers: bearer(KEYS.t2) });
+      const got = { forged: forged.status, mailbox: mailbox.status, ack: ack1.status, ackBatch: ackN.status, ctlUnbound: ctlUnbound.status, ctlTeam2: ctlTeam2.status };
+      need(forged.status === 403 && mailbox.status === 403 && ack1.status === 403 && ackN.status === 403, `expected 403s: ${JSON.stringify(got)}`);
+      need(ctlUnbound.status === 200 && ctlTeam2.status === 200, `controls should be open: ${JSON.stringify(got)}`);
+      await A.sendWhenKnown('node-b', 'e2e-marker-s5-ab');
+      await B.sendWhenKnown('node-a', 'e2e-marker-s5-ba');
+      const l1 = await B.waitMail('e2e-marker-s5-ab'), l2 = await A.waitMail('e2e-marker-s5-ba');
+      need(l1 === 'verified' && l2 === 'verified', `labels ${l1}/${l2}`);
+      return `HTTP ${JSON.stringify(got)}; A->B (${l1}) and B->A (${l2}) delivered with the flag on`;
+    });
+
+    await scenario(6, 'ban A (TASK_2457): A next send refused, B unaffected', async () => {
+      const ban = admin(PORT.relay, 'ban', T1, 'node-a', 'e2e');
+      need(ban.code === 0, `ban failed: ${ban.out}`);
+      const blocked = admin(PORT.relay, 'blocked');
+      need(blocked.out.includes('node-a'), `blocked list: ${blocked.out}`);
+      const before = row("SELECT COUNT(*) AS n FROM task_events WHERE from_node = 'node-a'")[0].n;
+      const a = await A.send('node-b', 'e2e-marker-s6-a');
+      need(!a.ok, `A send was not refused: ${a.text.slice(0, 200)}`);
+      const after = row("SELECT COUNT(*) AS n FROM task_events WHERE from_node = 'node-a'")[0].n;
+      need(after === before, `a row from node-a was created (${before} -> ${after})`);
+      await B.sendWhenKnown('node-a', 'e2e-marker-s6-b');
+      const r = row("SELECT from_node, sender_verified FROM task_events WHERE from_node = 'node-b' ORDER BY id DESC LIMIT 1")[0];
+      need(r.sender_verified === 1, 'B send not verified');
+      const stA = row("SELECT status FROM node_credentials WHERE node_name = 'node-a' AND team_id = ?", T1).map((x) => x.status);
+      const retry = await call('POST', `${R}/api/node-credentials/request`, { headers: bearer(KEYS.t1), body: { node_name: 'node-a' } });
+      need(retry.status === 403, `banned name token request gave ${retry.status}`);
+      return `ban: ${ban.out.split('\n')[0]}; A send refused (${a.text.slice(0, 110)}); node-a tokens ${JSON.stringify(stA)}; new token request for node-a HTTP ${retry.status}; B send accepted, relay row sender_verified=${r.sender_verified}; no new node-a row (${before})`;
+    });
+
+    await scenario(7, 'relay cleanup: acked rows past ttl removed, unacked within grace kept, batch ack used', async () => {
+      const h = bearer(KEYS.t2);
+      const sinkProject = row("SELECT project FROM nodes WHERE team_id = ? AND node_name = 'fern' LIMIT 1", T2)[0].project;
+      const mk = async (tag) => {
+        const r = await call('POST', `${R}/api/task_events`, { headers: h, body: { event_type: 'peer_message', task_id: `t7-${tag}`, from_node: 'cleanup-src', to_node: 'fern', project: sinkProject, payload: { body: tag } } });
+        need(r.status === 201, `create ${tag}: ${r.status} ${r.text}`);
+        return r.json.id;
+      };
+      const ids = { ackedPast: await mk('acked-past'), ackedFuture: await mk('acked-future'), unackedGrace: await mk('unacked-1d-past'), unackedOld: await mk('unacked-31d-past'), ackedPast2: await mk('acked-past-2') };
+      const batch = await call('PATCH', `${R}/api/task_events/ack`, { headers: h, body: { node_name: 'fern', ids: [ids.ackedPast, ids.ackedFuture, ids.ackedPast2], project: sinkProject, instance_id: 'e2e' } });
+      need(batch.status === 200 && batch.json.acked.length === 3, `batch ack: ${batch.status} ${batch.text}`);
+      await stop(relay);
+      const db = sqlite(relayDb, false);
+      const iso = (days) => new Date(Date.now() - days * 86400000).toISOString();
+      const set = db.prepare('UPDATE task_events SET ttl_at = ? WHERE id = ?');
+      set.run(iso(1), ids.ackedPast); set.run(iso(2), ids.ackedPast2); set.run(iso(-6), ids.ackedFuture); set.run(iso(1), ids.unackedGrace); set.run(iso(31), ids.unackedOld);
+      db.close();
+      relay = startRelay('relay-2', BIN.relay, PORT.relay, relayDb); // startup runs one cleanup
+      await until('relay health', async () => (await call('GET', `${R}/health`)).status === 200, 30000, 300);
+      await until('cleanup ran', () => row('SELECT 1 FROM task_events WHERE id = ?', ids.ackedPast).length === 0, 30000, 500);
+      const kept = Object.entries(ids).filter(([, id]) => row('SELECT 1 FROM task_events WHERE id = ?', id).length).map(([k]) => k).sort();
+      const want = ['ackedFuture', 'unackedGrace'];
+      need(JSON.stringify(kept) === JSON.stringify(want), `kept ${kept}, wanted ${want}`);
+      return `batch ack HTTP 200 acked=${JSON.stringify(batch.json.acked)}; after one cleanup run (relay startup) deleted: ackedPast, ackedPast2, unackedOld(31d); kept: ${kept.join(', ')}`;
+    });
+
+    await scenario(8, 'old kenaz-server (copied binary) <-> new server on new relay, flag OFF: both ways, nothing refused', async () => {
+      await fern.sendWhenKnown('olive', 'e2e-marker-s8-fo');
+      await olive.sendWhenKnown('fern', 'e2e-marker-s8-of');
+      const lo = await olive.waitMail('e2e-marker-s8-fo'), lf = await fern.waitMail('e2e-marker-s8-of');
+      const flag = row("SELECT protect_bound_names FROM team_settings WHERE team_id = ?", T2);
+      need(flag.length === 0 || flag[0].protect_bound_names === 0, 'flag is on for team 2');
+      const refused = row("SELECT COUNT(*) AS n FROM task_events WHERE team_id = ? AND from_node IN ('fern','olive')", T2)[0].n;
+      const fid = await fern.identity();
+      return `fern(new)->olive(old) delivered, olive shows "(${lo})"; olive->fern delivered, fern shows "(${lf})"; flag off for team; relay rows ${refused}; fern identity ${fid.state}; old binary ${path.basename(path.dirname(oldExe))}/${path.basename(oldExe)} (${fs.statSync(oldExe).size} bytes)`;
+    });
+
+    await scenario(9, 'new kenaz-server vs OLD relay (11b8b0b15): token request fails quietly, send/receive work, no token', async () => {
+      await D.sendWhenKnown('node-e', 'e2e-marker-s9-de');
+      await E.sendWhenKnown('node-d', 'e2e-marker-s9-ed');
+      const le = await E.waitMail('e2e-marker-s9-de'), ld = await D.waitMail('e2e-marker-s9-ed');
+      await sleep(16000); // at least one more poll tick after the token request attempt
+      const ids = [await D.identity(), await E.identity()];
+      need(ids.every((i) => i.state === 'none'), `identity: ${JSON.stringify(ids)}`);
+      const st = await D.status();
+      need(st.last_poll_ok === true, `last_poll_ok=${st.last_poll_ok} ${st.last_poll_error}`);
+      for (const s of [D, E]) need(!fs.existsSync(path.join(s.dir, 'home', '.kenaz', 'node_tokens')), `${s.name} wrote a token file`);
+      const req = await call('POST', `${relayUrl(PORT.relayOld)}/api/node-credentials/request`, { headers: bearer(KEYS.t1), body: { node_name: 'probe' } });
+      need(req.status >= 400, `old relay unexpectedly answered the token request with ${req.status}`);
+      return `D->E "(${le})", E->D "(${ld})"; identity D/E = ${ids.map((i) => i.state)}; no node_tokens dir; last_poll_ok=true; old relay answers token request HTTP ${req.status}`;
+    });
+  } finally {
+    for (const p of procs.slice().reverse()) await stop(p);
+    if (registry) await new Promise((r) => registry.close(r));
+  }
+
+  // ---------- isolation ----------
+  const left = procs.filter((p) => !p.exited).map((p) => `${p.name}:${p.pid}`);
+  const gitAfter = gitState();
+  const realAfter = realDirs().map((d) => tree(d, 3));
+  const touched = [];
+  realAfter.forEach((after, i) => {
+    for (const [f, m] of Object.entries(after)) {
+      if (realBefore[i][f] === m) continue;
+      let hit = false; try { if (fs.statSync(f).size < 2e6) hit = fs.readFileSync(f, 'utf8').includes(runId); } catch { /* unreadable */ }
+      if (hit) touched.push(f);
+    }
+  });
+  const iso = [
+    ['no process left running', left.length === 0, left.join(',') || 'none'],
+    ['git status unchanged', gitBefore === gitAfter, gitBefore === gitAfter ? 'same' : `before:\n${gitBefore}\nafter:\n${gitAfter}`],
+    ['real ~/.kenaz ~/.claude ~/.codex hold nothing of this run', touched.length === 0, touched.join(', ') || `no changed file mentions ${runId}`],
+    ['installed server binary untouched', fs.statSync(BIN.serverOldSrc).mtimeMs === oldBinStat, 'mtime same'],
+  ];
+  console.log('\nisolation checks');
+  for (const [t, ok, ev] of iso) console.log(`[${ok ? 'PASS' : 'FAIL'}] ${t}: ${ev}`);
+
+  console.log('\n| # | scenario | result | evidence |\n|---|---|---|---|');
+  for (const r of results) console.log(`| ${r.n} | ${r.title} | ${r.pass ? 'PASS' : 'FAIL'} | ${r.ev.replace(/\|/g, '/').replace(/\n/g, ' ')} |`);
+  const bad = results.filter((r) => !r.pass).length + iso.filter((x) => !x[1]).length + (results.length === 9 ? 0 : 1);
+  console.log(`\n${results.filter((r) => r.pass).length}/9 scenarios passed; ${bad ? 'FAILED' : 'OK'}`);
+  if (!ARGS.has('--keep') && !bad) fs.rmSync(TMP, { recursive: true, force: true });
+  else console.log(`temp dir kept: ${TMP}`);
+  return bad ? 1 : 0;
+}
+
+process.on('SIGINT', () => { for (const p of procs) if (!p.exited) { if (process.platform === 'win32') spawnSync('taskkill', ['/PID', String(p.pid), '/T', '/F'], { stdio: 'ignore' }); else p.child.kill('SIGKILL'); } process.exit(130); });
+main().then((c) => process.exit(c), (e) => { console.error(e); process.exit(1); });

package/core/dev/scripts/peer-identity-e2e.md

@@ -0,0 +1,47 @@
+# peer-identity-e2e
+
+End-to-end usage test of peer identity (TASK_2447-2452, 2456, 2457) against real binaries. Harness: `dev/scripts/peer-identity-e2e.js`.
+
+## What it starts
+
+- a fake registry (answers the relay's `POST /rest/v1/rpc/exchange_key` for two team keys), port 18900
+- the new relay (18901) and the old relay built from 11b8b0b15 (18902), each with a temp `RELAY_DB` and an admin key file
+- kenaz-servers on 18910-18915, each with its own temp `USERPROFILE`/`HOME`/`APPDATA`/`LOCALAPPDATA`/`TEMP`, `KENAZ_SOURCE_DIR`, a minimal `PATH` (no `claude`/`git`), and a temp project whose `.kenaz/yggdrasil.yaml` points at the local relay (team key in the temp `credentials.yaml`)
+- A, B (new, team 1); fern (new) + olive (a COPY of the installed `~/.kenaz/bin/kenaz-server`, team 2, flag never turned on); D, E (new, against the old relay)
+
+It never contacts 18080, never runs anything from `~/.kenaz/bin`, and stops only the PIDs it spawned. Exit code is 0 only if all 9 scenarios and the 4 isolation checks pass.
+
+## Run
+
+```
+node dev/scripts/peer-identity-e2e.js                              # ~2 min, deletes its temp dir on success
+node dev/scripts/peer-identity-e2e.js --keep                       # keep the temp dir (logs: <tmp>/<name>.log)
+node dev/scripts/peer-identity-e2e.js --control-wrong-fingerprint  # control: scenario 3 approves A with a wrong fingerprint -> must exit 1
+```
+
+Prebuilt binaries (not built by the script; override with `E2E_RELAY`, `E2E_RELAY_OLD`, `E2E_SERVER`, `E2E_SERVER_OLD`):
+`C:/t/relay-target/debug/yggdrasil-relay.exe` (main), `C:/t/relay-old-target/debug/yggdrasil-relay.exe` (11b8b0b15), `C:/t/kenaz-e2e-target/debug/kenaz-server.exe` (main), and `~/.kenaz/bin/kenaz-server.exe` (the installed old client). Needs Node 22+ (`node:sqlite`).
+
+## Notes learned while building it
+
+- A server re-reads `/api/node-credentials/me` at most every 10 minutes per process (`RETRY_AFTER`), so after approval scenario 3 restarts A and B (the token file persists in their home) instead of waiting. A real user waits up to 10 minutes after approval to see `verified`.
+- Presence heartbeats run every 90 s and only cover projects listed in `collaboration_projects`; the harness pre-writes `projects.json` and that key so the first heartbeat (at boot) already registers the node.
+- Isolation check on the real `~/.kenaz`, `~/.claude`, `~/.codex`: the user's own server writes there continuously, so the check is "no changed file mentions this run's temp dir name", plus the installed binary's mtime and `git status` before/after.
+- Scenario 7 backdates `ttl_at` with SQL while the relay is stopped, then restarts it: the relay runs one cleanup at startup. Batch ack is called directly on `PATCH /api/task_events/ack`.
+- The installed old binary is whatever is in `~/.kenaz/bin` at run time (23820288 bytes); its version number was not queried.
+
+## Last run
+
+Date 2026-10-13. Relay and server binaries built from main at 0ee647caf (old relay 11b8b0b15). Harness committed on the TASK_2458 worktree branch (commit of this file). Result: 9/9 scenarios PASS, 4/4 isolation checks PASS, exit 0. Control run (`--control-wrong-fingerprint`): scenarios 3, 4, 5 FAIL, exit 1.
+
+| # | scenario | result | evidence |
+|---|---|---|---|
+| 1 | A and B request tokens; pending + fingerprint in GET /api/peer/status | PASS | status.node_identity A=pending/749c3b468bea, B=pending/0637ecff09e6; DB rows pending: [{"node_name":"node-a","fingerprint":"749c3b468bea","status":"pending"},{"node_name":"node-b","fingerprint":"0637ecff09e6","status":"pending"}]; admin list shows both |
+| 2 | A -> B before approval: delivered, label (unverified sender) | PASS | B inbox label "(unverified sender)"; relay row {"from_node":"node-a","to_node":"node-b","sender_verified":0} |
+| 3 | operator approves both with relay-admin.js; servers pick up active via /me | PASS | approved e2e-team-1/node-a (749c3b468bea); revoked 0 other token(s) ; approved e2e-team-1/node-b (0637ecff09e6); revoked 0 other token(s); restart, then status A/B = active; DB active: [{"node_name":"node-a","status":"active"},{"node_name":"node-b","status":"active"}] |
+| 4 | A -> B after approval: delivered, label (verified) | PASS | B inbox label "(verified)"; relay row sender_verified=1 |
+| 5 | protect_bound_names ON: tokenless curl as A / reading or acking B -> 403; A and B still deliver | PASS | HTTP {"forged":403,"mailbox":403,"ack":403,"ackBatch":403,"ctlUnbound":200,"ctlTeam2":200}; A->B (verified) and B->A (verified) delivered with the flag on |
+| 6 | ban A (TASK_2457): A next send refused, B unaffected | PASS | ban: banned e2e-team-1/node-a; revoked 1 token(s); new token requests are refused; A send refused ({"jsonrpc":"2.0","id":1,"result":{"content":[{"text":"kenaz_peer_send: failed to reach any target node. Errors); node-a tokens ["revoked"]; new token request for node-a HTTP 403; B send accepted, relay row sender_verified=1; no new node-a row (3) |
+| 7 | relay cleanup: acked rows past ttl removed, unacked within grace kept, batch ack used | PASS | batch ack HTTP 200 acked=[6,7,10]; after one cleanup run (relay startup) deleted: ackedPast, ackedPast2, unackedOld(31d); kept: ackedFuture, unackedGrace |
+| 8 | old kenaz-server (copied binary) <-> new server on new relay, flag OFF: both ways, nothing refused | PASS | fern(new)->olive(old) delivered, olive shows "(team-claimed)"; olive->fern delivered, fern shows "(unverified sender)"; flag off for team; relay rows 2; fern identity pending; old binary old-server/kenaz-server.exe (23820288 bytes) |
+| 9 | new kenaz-server vs OLD relay (11b8b0b15): token request fails quietly, send/receive work, no token | PASS | D->E "(team-claimed)", E->D "(team-claimed)"; identity D/E = none,none; no node_tokens dir; last_poll_ok=true; old relay answers token request HTTP 404 |

package/core/dev/scripts/relay-admin-ui.js

@@ -0,0 +1,154 @@
+#!/usr/bin/env node
+'use strict';
+// TASK_2459: local operator page for the relay node-credential admin endpoints.
+// Serves one HTML page on 127.0.0.1 and proxies /api/* to the relay with the admin key added
+// server-side. The key (RELAY_ADMIN_KEY / RELAY_ADMIN_KEY_FILE) never reaches the browser.
+const http = require('http');
+const crypto = require('crypto');
+const { adminCall, loadKey } = require('./relay-admin');
+
+const DEFAULT_RELAY = 'https://yggdrasil.kenazai.com';
+// local route -> [method, relay path under /api/admin/]
+const ROUTES = {
+  'GET /api/pending': ['GET', 'node-credentials/pending'],
+  'GET /api/blocked': ['GET', 'node-credentials/blocked'],
+  'GET /api/readiness': ['GET', 'node-credentials/readiness'],
+  'POST /api/approve': ['POST', 'node-credentials/approve'],
+  'POST /api/revoke': ['POST', 'node-credentials/revoke'],
+  'POST /api/ban': ['POST', 'node-credentials/ban'],
+  'POST /api/unban': ['POST', 'node-credentials/unban'],
+  'POST /api/team-settings': ['POST', 'team-settings'],
+};
+
+const PAGE = `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
+<title>Relay Admin</title><style>
+body{font:14px system-ui,sans-serif;margin:16px;max-width:1100px;background:#fafafa;color:#222}
+h2{margin-top:28px}table{border-collapse:collapse;width:100%;background:#fff}th,td{border:1px solid #ddd;padding:4px 8px;text-align:left;font-size:13px}
+button{margin-right:4px;cursor:pointer}.red{color:#c00;font-weight:bold}.mono{font-family:monospace}#msg{position:sticky;top:0;background:#ffd;padding:6px;display:none}
+</style></head><body>
+<h1>Relay admin</h1><div id="msg"></div>
+<h2>Pending tokens</h2><table id="pending"></table>
+<h2>Team</h2>
+<p>Team id <input id="team" size="30"> <button id="load">Load</button></p>
+<div id="teamview"></div>
+<h2>Blocked</h2><table id="blocked"></table>
+<script>
+const CSRF = '__CSRF__';
+const $ = (id) => document.getElementById(id);
+const esc = (s) => String(s == null ? '' : s).replace(/[&<>"']/g, (c) => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
+function msg(t) { $('msg').textContent = t; $('msg').style.display = t ? 'block' : 'none'; }
+async function api(method, path, body) {
+  const r = await fetch(path, { method, headers: { 'Content-Type': 'application/json', 'X-CSRF-Token': CSRF }, body: body ? JSON.stringify(body) : undefined });
+  const j = await r.json().catch(() => ({}));
+  if (!r.ok) throw new Error(j.error || 'HTTP ' + r.status);
+  return j;
+}
+async function act(fn) { try { msg(''); await fn(); } catch (e) { msg('error: ' + e.message); } await refresh(); }
+function approve(p) {
+  const fp = prompt('Type the fingerprint to approve ' + p.team_id + '/' + p.node_name + '\\n(expected ' + p.fingerprint + ')');
+  if (fp === null) return;
+  if (fp.trim() !== p.fingerprint) { msg('fingerprint does not match; nothing approved'); return; }
+  act(() => api('POST', '/api/approve', { team_id: p.team_id, node_name: p.node_name, fingerprint: fp.trim() }));
+}
+function ban(p) {
+  if (!confirm('Ban ' + p.team_id + '/' + p.node_name + (p.machine_id_hash ? ' and its machine hash' : '') + '? Tokens are revoked and new requests refused.')) return;
+  act(() => api('POST', '/api/ban', { team_id: p.team_id, node_name: p.node_name, reason: 'banned from admin page', machine_id_hash: p.machine_id_hash || undefined }));
+}
+function revoke(t, n) { if (confirm('Revoke every token for ' + t + '/' + n + '?')) act(() => api('POST', '/api/revoke', { team_id: t, node_name: n })); }
+function unban(b) {
+  const body = { team_id: b.team_id }; body[b.kind === 'machine' ? 'machine_id_hash' : 'node_name'] = b.value;
+  act(() => api('POST', '/api/unban', body));
+}
+function toggle(team, flag, on, lacking) {
+  const m = 'Set ' + flag + ' = ' + on + ' for team ' + team + '?\\nActive nodes still lacking a token: ' + lacking;
+  if (!confirm(m)) return;
+  if (flag === 'require_node_auth' && on && lacking > 0 &&
+      !confirm('WARNING: ' + lacking + ' active node(s) have no token and will be locked out (401). Really turn it on?')) return;
+  const body = { team_id: team }; body[flag] = on;
+  act(() => api('POST', '/api/team-settings', body));
+}
+const rows = (h, list) => '<tr>' + h.map((x) => '<th>' + x + '</th>').join('') + '</tr>' + list.join('');
+let pendingList = [], blockedList = [];
+async function refresh() {
+  try {
+    pendingList = (await api('GET', '/api/pending')).pending;
+    $('pending').innerHTML = rows(['team', 'node', 'fingerprint', 'hostname', 'os', 'machine hash', 'created', ''], pendingList.map((p, i) =>
+      '<tr><td>' + esc(p.team_id) + '</td><td>' + esc(p.node_name) + '</td><td class="mono">' + esc(p.fingerprint) + '</td><td>' + esc(p.hostname) + '</td><td>' + esc(p.os) +
+      '</td><td class="mono">' + esc((p.machine_id_hash || '').slice(0, 12)) + '</td><td>' + esc(p.created_at) +
+      '</td><td><button onclick="approve(pendingList[' + i + '])">Approve</button><button onclick="ban(pendingList[' + i + '])">Reject/Ban</button></td></tr>'));
+    blockedList = (await api('GET', '/api/blocked')).blocked;
+    $('blocked').innerHTML = rows(['team', 'kind', 'value', 'reason', 'created', ''], blockedList.map((b, i) =>
+      '<tr><td>' + esc(b.team_id) + '</td><td>' + esc(b.kind) + '</td><td class="mono">' + esc(b.value) + '</td><td>' + esc(b.reason) + '</td><td>' + esc(b.created_at) +
+      '</td><td><button onclick="unban(blockedList[' + i + '])">Unban</button></td></tr>'));
+    if ($('team').value.trim()) await loadTeam();
+  } catch (e) { msg('error: ' + e.message); }
+}
+async function loadTeam() {
+  const team = $('team').value.trim(); if (!team) return;
+  const r = await api('GET', '/api/readiness?team_id=' + encodeURIComponent(team));
+  const n = r.nodes_without_active_token;
+  const cls = n ? 'red' : '';
+  $('teamview').innerHTML = '<p>Nodes active in ' + r.window_days + ' days without an active token: <span class="' + cls + '">' + n + '</span></p>' +
+    '<table>' + rows(['node', 'token state'], r.nodes.map((x) => '<tr><td>' + esc(x) + '</td><td class="red">no active token</td></tr>')) + '</table>' +
+    '<p>Revoke by node name <input id="rv" size="20"> <button id="rvb">Revoke</button></p>' +
+    '<p>protect_bound_names <button id="pbn1">on</button><button id="pbn0">off</button> &nbsp; require_node_auth <button id="rna1">on</button><button id="rna0">off</button> ' +
+    '<span class="' + cls + '">(' + n + ' lacking a token)</span></p>';
+  $('rvb').onclick = () => { const v = $('rv').value.trim(); if (v) revoke(team, v); };
+  $('pbn1').onclick = () => toggle(team, 'protect_bound_names', true, n);
+  $('pbn0').onclick = () => toggle(team, 'protect_bound_names', false, n);
+  $('rna1').onclick = () => toggle(team, 'require_node_auth', true, n);
+  $('rna0').onclick = () => toggle(team, 'require_node_auth', false, n);
+}
+$('load').onclick = () => loadTeam().catch((e) => msg('error: ' + e.message));
+refresh();
+</script></body></html>`;
+
+// Drive-by protection (same rule as TASK_2442): only our own host names, and a present Origin must be our own.
+function hostOk(req) {
+  const host = String(req.headers.host || '');
+  if (!/^(127\.0\.0\.1|localhost)(:\d+)?$/.test(host)) return false;
+  const origin = req.headers.origin;
+  return origin === undefined || origin === 'http://' + host;
+}
+
+function createServer({ relay, key, csrf }) {
+  const page = PAGE.replace('__CSRF__', csrf);
+  return http.createServer((req, res) => {
+    const send = (s, body, type = 'application/json') => {
+      res.writeHead(s, { 'Content-Type': type, 'Cache-Control': 'no-store' });
+      res.end(typeof body === 'string' ? body : JSON.stringify(body));
+    };
+    if (!hostOk(req)) return send(403, { error: 'forbidden host or origin' });
+    const u = new URL(req.url, 'http://x');
+    if (req.method === 'GET' && u.pathname === '/') return send(200, page, 'text/html; charset=utf-8');
+    const route = ROUTES[`${req.method} ${u.pathname}`];
+    if (!route) return send(404, { error: 'not found' });
+    if (req.method === 'POST') {
+      const a = Buffer.from(String(req.headers['x-csrf-token'] || '')), b = Buffer.from(csrf);
+      if (a.length !== b.length || !crypto.timingSafeEqual(a, b)) return send(403, { error: 'bad csrf token' });
+    }
+    let raw = '';
+    req.on('data', (c) => { raw += c; if (raw.length > 65536) req.destroy(); });
+    req.on('end', async () => {
+      let body;
+      try { body = raw ? JSON.parse(raw) : undefined; } catch { return send(400, { error: 'bad json' }); }
+      const path = route[1] + (route[0] === 'GET' ? u.search : '');
+      try { send(200, await adminCall(relay, key, route[0], path, body)); }
+      catch (e) { send(e.status && e.status !== 401 ? e.status : 502, { error: e.message }); }
+    });
+  });
+}
+
+function main(argv, env) {
+  let relay = env.KENAZ_RELAY_URL || DEFAULT_RELAY, port = 18990;
+  const g = (f) => { const i = argv.indexOf(f); return i >= 0 ? argv[i + 1] : undefined; };
+  if (g('--relay')) relay = g('--relay');
+  if (g('--port')) port = Number(g('--port'));
+  let key;
+  try { key = loadKey(env); } catch (e) { console.error(e.message); return process.exit(2); }
+  const csrf = crypto.randomBytes(24).toString('hex');
+  createServer({ relay, key, csrf }).listen(port, '127.0.0.1', () => console.log(`relay admin page: http://127.0.0.1:${port}/  (relay ${relay})`));
+}
+
+module.exports = { createServer, hostOk };
+if (require.main === module) main(process.argv.slice(2), process.env);