blockyard 0.0.1 → 0.0.9

package/scripts/index-bench.js

@@ -0,0 +1,216 @@
+#!/usr/bin/env node
+// STEP 4 of the address index (docs/DEFECTS.md): what does storing it cost? Measured on real rows.
+//
+//   node --no-warnings scripts/index-bench.js [--files 384,2685,5370] [--out <dir>]
+//
+// THE ROW. One row per (address script, transaction) that touched it -- an output paying the script,
+// or an input spending an output that paid it (the spent script comes from the undo file, so no UTXO
+// replay). Key: the first 8 bytes of sha256(script), then the block height and the transaction's
+// position in the block. Value (optional): the net amount that transaction moved for that script.
+// Everything else -- the transaction itself, its txid -- stays in the node, which has txindex.
+//
+// It builds the rows for the sampled files, then stores them two ways and measures each:
+//   A. node:sqlite, a WITHOUT ROWID table keyed (script hash, height, position) -- the runtime's own
+//      SQLite, already used by server/store/ledger.js, so no dependency
+//   B. sorted fixed-size rows in a flat file with a sparse block index, looked up by binary search
+// and reports bytes per row, rows per second to write, and lookup latency for real addresses.
+import { openSync, readSync, writeSync, closeSync, fstatSync, mkdirSync, rmSync, statSync } from 'node:fs';
+import path from 'node:path';
+import { createHash } from 'node:crypto';
+import { DatabaseSync } from 'node:sqlite';
+import { loadConfig } from '../server/config.js';
+import { RpcClient } from '../server/rpc/client.js';
+import { Reader, readHeader, readTx } from '../server/chain/tx.js';
+import { xorKey, readChainFile, records, decodeBlockUndo, pairBlocksWithUndo, MAGIC } from '../server/chain/blockfile.js';
+
+const arg = (name, def) => { const i = process.argv.indexOf(`--${name}`); return i > 0 ? process.argv[i + 1] : def; };
+const dir = arg('dir', '/storage/core-oracle/blocks');
+const files = arg('files', '384,2685,5370').split(',').map(Number);
+const outDir = arg('out', path.join(process.env.TMPDIR ?? '/tmp', 'blockyard-index-bench'));
+const ms = (t0) => performance.now() - t0;
+const key = xorKey(dir);
+
+const cfg = loadConfig();
+const node = cfg.nodes.find((n) => n.datadir && dir.startsWith(n.datadir)) ?? cfg.nodes[0];
+const rpc = new RpcClient(node, { ...(cfg.rpc ?? {}), ...(node.rpc ?? {}) }, { log: { info() {}, warn() {}, error() {}, debug() {} } });
+const heightsOf = async (hashes) => {
+  const got = await rpc.batch(hashes.map((h) => ({ method: 'getblockheader', params: [h] })), { key: `bench:hdr:${hashes[0]}`, timeoutMs: 120_000, maxWaitMs: 120_000 });
+  return got.map((g) => (g.ok ? g.result.height : null));
+};
+
+// --- rows ------------------------------------------------------------------
+// kept in parallel typed arrays while building; the stores below pick their own layout
+let n = 0;
+const cap = 40_000_000;
+const SH = new BigUint64Array(cap), HT = new Uint32Array(cap), POS = new Uint16Array(cap), VAL = new Float64Array(cap);
+const sh8 = (script) => createHash('sha256').update(script).digest().readBigUInt64BE(0);
+
+const build = { decodeMs: 0, blocks: 0, txs: 0, rawRows: 0 };
+for (const f of files) {
+  const id = String(f).padStart(5, '0');
+  const blk = readChainFile(path.join(dir, `blk${id}.dat`), key);
+  const rev = readChainFile(path.join(dir, `rev${id}.dat`), key);
+  const blocks = [...records(blk, MAGIC.main)].map((r) => { const rd = new Reader(r.body); const h = readHeader(rd); return { rec: r, hash: h.hash, previousblockhash: h.previousblockhash, ntx: rd.varint(), rd }; });
+  const pairs = pairBlocksWithUndo(blocks, [...records(rev, MAGIC.main, 32)]);
+  const idx = [...pairs.keys()];
+  const heights = await heightsOf(idx.map((i) => blocks[i].hash));
+  const t0 = performance.now();
+  idx.forEach((i, k) => {
+    const b = blocks[i], height = heights[k];
+    if (height == null) return;
+    const undo = decodeBlockUndo(pairs.get(i).body);
+    const rd = b.rd;
+    for (let p = 0; p < b.ntx; p++) {
+      const tx = readTx(rd);
+      build.txs++;
+      const moved = new Map();                       // sh8 -> net sats for this transaction
+      for (const o of tx.vout) {
+        if (o.scriptPubKey.type === 'nulldata') continue;
+        const s = sh8(Buffer.from(o.scriptPubKey.hex, 'hex'));
+        moved.set(s, (moved.get(s) ?? 0) + o.value_sat); build.rawRows++;
+      }
+      if (p > 0) for (const c of undo[p - 1]) {
+        const s = sh8(c.script);
+        moved.set(s, (moved.get(s) ?? 0) - c.value_sat); build.rawRows++;
+      }
+      for (const [s, v] of moved) { SH[n] = s; HT[n] = height; POS[n] = p; VAL[n] = v; n++; }
+    }
+    build.blocks++;
+  });
+  build.decodeMs += ms(t0);
+  console.log(`blk${id}: ${idx.length} blocks, rows so far ${n.toLocaleString()} (raw ${build.rawRows.toLocaleString()} before merging a script's outputs and inputs within one transaction)`);
+}
+
+rmSync(outDir, { recursive: true, force: true });
+mkdirSync(outDir, { recursive: true });
+const order = new Uint32Array(n); for (let i = 0; i < n; i++) order[i] = i;
+const toSigned = (u) => BigInt.asIntN(64, u);
+
+// lookups: real scripts, chosen from the rows, so every query has at least one hit
+let seed = 20260914;
+const rnd = () => ((seed = (Math.imul(seed, 1103515245) + 12345) >>> 0) / 4294967296);
+const probes = Array.from({ length: 20_000 }, () => SH[Math.floor(rnd() * n)]);
+
+// --- A. node:sqlite ----------------------------------------------------------
+const A = {};
+{
+  const file = path.join(outDir, 'hist.sqlite');
+  const db = new DatabaseSync(file);
+  db.exec('PRAGMA page_size = 16384; PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA locking_mode = EXCLUSIVE;');
+  db.exec('CREATE TABLE hist (sh INTEGER NOT NULL, h INTEGER NOT NULL, p INTEGER NOT NULL, v INTEGER NOT NULL, PRIMARY KEY (sh, h, p)) WITHOUT ROWID');
+  const ins = db.prepare('INSERT OR REPLACE INTO hist (sh, h, p, v) VALUES (?, ?, ?, ?)');
+  // in the order blocks arrive -- what a live, incremental index would do
+  let t0 = performance.now();
+  db.exec('BEGIN');
+  for (let i = 0; i < n; i++) {
+    ins.run(toSigned(SH[i]), HT[i], POS[i], VAL[i]);
+    if (i % 500_000 === 499_999) { db.exec('COMMIT'); db.exec('BEGIN'); }
+  }
+  db.exec('COMMIT');
+  A.insertMs = ms(t0);
+  A.bytes = statSync(file).size;
+  t0 = performance.now(); db.exec('VACUUM'); A.vacuumMs = ms(t0);
+  A.bytesVacuumed = statSync(file).size;
+  const q = db.prepare('SELECT h, p, v FROM hist WHERE sh = ? ORDER BY h DESC, p DESC LIMIT 25');
+  for (let i = 0; i < 2000; i++) q.all(toSigned(probes[i]));      // warm
+  t0 = performance.now(); let hits = 0;
+  for (const s of probes) hits += q.all(toSigned(s)).length;
+  A.lookupMs = ms(t0) / probes.length; A.hits = hits;
+  db.close();
+}
+
+// --- B. sorted flat file -----------------------------------------------------
+// row: sh8 8 | height 3 | position 2 | value 8 (signed sats: one transaction has moved 500,000 BTC, past 6 bytes'
+// reach of 1.4 M only by a margin nobody should bet a format on) = 21 bytes; history-only drops the value (13)
+const B = {};
+{
+  let t0 = performance.now();
+  // bucket by the top 16 bits of the hash, then sort each small bucket: a comparator sort over tens of
+  // millions of rows at once is what makes a naive JS sort slow
+  const bucketOf = (i) => Number(SH[i] >> 48n);
+  const counts = new Uint32Array(65537);
+  for (let i = 0; i < n; i++) counts[bucketOf(i) + 1]++;
+  for (let b = 0; b < 65536; b++) counts[b + 1] += counts[b];
+  const fill = counts.slice();
+  for (let i = 0; i < n; i++) order[fill[bucketOf(i)]++] = i;
+  for (let b = 0; b < 65536; b++) {
+    const sub = order.subarray(counts[b], counts[b + 1]);
+    if (sub.length > 1) sub.sort((x, y) => (SH[x] < SH[y] ? -1 : SH[x] > SH[y] ? 1 : HT[x] - HT[y] || POS[x] - POS[y]));
+  }
+  B.sortMs = ms(t0);
+  const ROW = 21, BLOCK = 4096;                      // rows per index entry
+  const file = path.join(outDir, 'hist.rows');
+  const fd = openSync(file, 'w');
+  const buf = Buffer.allocUnsafe(ROW * 65536);
+  const sparse = new BigUint64Array(Math.ceil(n / BLOCK));
+  t0 = performance.now();
+  let off = 0;
+  for (let i = 0; i < n; i++) {
+    const r = order[i];
+    if (i % BLOCK === 0) sparse[i / BLOCK] = SH[r];
+    const at = off;
+    buf.writeBigUInt64BE(SH[r], at);
+    buf.writeUIntBE(HT[r], at + 8, 3);
+    buf.writeUInt16BE(POS[r], at + 11);
+    buf.writeBigInt64BE(BigInt(VAL[r]), at + 13);
+    off += ROW;
+    if (off === buf.length) { writeSync(fd, buf, 0, off); off = 0; }
+  }
+  if (off) writeSync(fd, buf, 0, off);
+  closeSync(fd);
+  B.writeMs = ms(t0);
+  B.bytes = statSync(file).size + sparse.byteLength;
+  B.historyOnlyBytes = n * 13 + sparse.byteLength;
+  // lookup: binary search the sparse index in memory, read one index block of rows, scan it
+  const rfd = openSync(file, 'r');
+  const block = Buffer.allocUnsafe(ROW * BLOCK * 2);
+  const lookup = (s) => {
+    let lo = 0, hi = sparse.length - 1;
+    while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (sparse[mid] < s) lo = mid; else hi = mid - 1; }
+    const got = readSync(rfd, block, 0, block.length, lo * BLOCK * ROW);
+    let found = 0;
+    for (let o = 0; o + ROW <= got; o += ROW) { const k = block.readBigUInt64BE(o); if (k === s) found++; else if (k > s) break; }
+    return found;
+  };
+  for (let i = 0; i < 2000; i++) lookup(probes[i]);
+  t0 = performance.now(); let hits = 0;
+  for (const s of probes) hits += lookup(s);
+  B.lookupMs = ms(t0) / probes.length; B.hits = hits;
+  closeSync(rfd);
+}
+
+// --- A2. node:sqlite, bulk-loaded in key order --------------------------------
+// A B-tree fed random keys stops fitting in memory long before 5 billion rows and then pays a seek per
+// insert; fed sorted keys it only ever appends to its rightmost page. This is SQLite's bulk-build path,
+// and the one a full build would have to take -- which means sorting the rows first either way.
+{
+  const file = path.join(outDir, 'hist-sorted.sqlite');
+  const db = new DatabaseSync(file);
+  db.exec('PRAGMA page_size = 16384; PRAGMA journal_mode = OFF; PRAGMA synchronous = OFF; PRAGMA locking_mode = EXCLUSIVE;');
+  db.exec('CREATE TABLE hist (sh INTEGER NOT NULL, h INTEGER NOT NULL, p INTEGER NOT NULL, v INTEGER NOT NULL, PRIMARY KEY (sh, h, p)) WITHOUT ROWID');
+  const ins = db.prepare('INSERT INTO hist (sh, h, p, v) VALUES (?, ?, ?, ?)');
+  // the flat file's order is unsigned; SQLite's INTEGER order is signed, so feed the negative half first
+  const t0 = performance.now();
+  db.exec('BEGIN');
+  let k = 0;
+  const firstNonNeg = order.findIndex((r) => SH[r] >= 0x8000000000000000n);
+  const seq = firstNonNeg < 0 ? [order] : [order.subarray(firstNonNeg), order.subarray(0, firstNonNeg)];
+  for (const part of seq) for (const r of part) {
+    ins.run(toSigned(SH[r]), HT[r], POS[r], VAL[r]);
+    if (++k % 500_000 === 0) { db.exec('COMMIT'); db.exec('BEGIN'); }
+  }
+  db.exec('COMMIT');
+  A.sortedInsertMs = ms(t0);
+  A.sortedBytes = statSync(file).size;
+  db.close();
+}
+
+const perRow = (bytes) => +(bytes / n).toFixed(2);
+const report = {
+  sample: { files, blocks: build.blocks, txs: build.txs, rows: n, rawRows: build.rawRows, rowsPerTx: +(n / build.txs).toFixed(3), mergedAway: +(1 - n / build.rawRows).toFixed(3) },
+  sqlite: { bytesPerRow: perRow(A.bytes), bytesPerRowVacuumed: perRow(A.bytesVacuumed), bytesPerRowSortedLoad: perRow(A.sortedBytes), insertRowsPerSec: Math.round(n / (A.insertMs / 1000)), sortedInsertRowsPerSec: Math.round(n / (A.sortedInsertMs / 1000)), vacuumSec: +(A.vacuumMs / 1000).toFixed(1), lookupMs: +A.lookupMs.toFixed(4), meanHitsPerLookup: +(A.hits / probes.length).toFixed(2) },
+  sortedFile: { bytesPerRow: perRow(B.bytes), bytesPerRowHistoryOnly: perRow(B.historyOnlyBytes), sortRowsPerSec: Math.round(n / (B.sortMs / 1000)), writeRowsPerSec: Math.round(n / (B.writeMs / 1000)), lookupMs: +B.lookupMs.toFixed(4), meanHitsPerLookup: +(B.hits / probes.length).toFixed(2) },
+};
+console.log(JSON.stringify(report, null, 1));
+rmSync(outDir, { recursive: true, force: true });
+process.exit(0);

package/scripts/index-benchmark.js

@@ -0,0 +1,117 @@
+#!/usr/bin/env node
+// How does the built address index compare? (docs/MEASUREMENTS.md §30)
+//
+//   node scripts/index-benchmark.js --index <dir> [--sample 400] [--verify 40]
+//
+// 1. opening the index (the sparse keys load into memory)
+// 2. lookup latency for real addresses: a sample from blocks across the chain's history, plus
+//    addresses known to have enormous histories -- summary (count, balance, newest 25 rows) timed
+// 3. CORRECTNESS against the node: for --verify addresses, the index balance plus any blocks the node
+//    has connected since the build, against `scantxoutset` at the same height. scantxoutset is Core's
+//    only address query; it reads the whole UTXO set, answers balance only (no history), and is also
+//    timed, as the method the index replaces.
+import { loadConfig } from '../server/config.js';
+import { RpcClient } from '../server/rpc/client.js';
+import { IndexStore } from '../server/chain/index/store.js';
+import { scriptKey } from '../server/chain/index/rows.js';
+import { addressToScript } from '../server/chain/tx.js';
+
+const arg = (name, def) => { const i = process.argv.indexOf(`--${name}`); return i > 0 ? process.argv[i + 1] : def; };
+const dir = arg('index', null);
+if (!dir) { console.error('--index <dir> is required'); process.exit(2); }
+const SAMPLE = Number(arg('sample', 400)), VERIFY = Number(arg('verify', 40));
+const cfg = loadConfig();
+const node = cfg.nodes.find((n) => n.id === arg('node', null)) ?? cfg.nodes.find((n) => n.datadir) ?? cfg.nodes[0];
+const rpc = new RpcClient(node, { ...(cfg.rpc ?? {}), ...(node.rpc ?? {}) }, { log: { info() {}, warn() {}, error() {}, debug() {} } });
+const call = async (method, params, timeoutMs = 600_000) => {
+  const [r] = await rpc.batch([{ method, params }], { key: `ibench:${method}:${JSON.stringify(params).slice(0, 80)}`, timeoutMs, maxWaitMs: 900_000 });
+  if (!r.ok) throw new Error(`${method}: ${r.error?.message}`);
+  return r.result;
+};
+const pct = (arr, p) => { const s = arr.slice().sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(p * s.length))]; };
+const report = {};
+
+// 1. open
+let t = performance.now();
+const store = new IndexStore(dir);
+report.open = { ms: +(performance.now() - t).toFixed(1), rows: store.manifest.rows, gb: +(store.manifest.bytes / 1e9).toFixed(1), tip: store.manifest.tip.height };
+const tip = store.manifest.tip.height;
+
+// 2. addresses: outputs from blocks spread over the chain, and some famously heavy ones
+const heavy = [
+  '1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa',            // the genesis address, sent dust for fifteen years
+  '34xp4vRoCGJym3xR7yCVPFHoCNxv4Twseo',            // a long-lived exchange cold wallet
+  'bc1qm34lsc65zpw79lxes69zkqmk6ee3ewf0j77s3h',    // a large exchange wallet
+  '1FeexV6bAHb8ybZjqQMjJrcCrHGW9sb6uF',            // untouched since 2011
+];
+let seed = 20260914;
+const rnd = () => ((seed = (Math.imul(seed, 1103515245) + 12345) >>> 0) / 4294967296);
+const addresses = new Set();
+const heights = Array.from({ length: 24 }, (_, i) => Math.floor(((i + 0.5) / 24) * tip));
+for (const h of heights) {
+  const b = await call('getblock', [await call('getblockhash', [h]), 2]);
+  const found = [];
+  for (const tx of b.tx) for (const o of tx.vout) if (o.scriptPubKey.address) found.push(o.scriptPubKey.address);
+  for (let k = 0; k < Math.ceil(SAMPLE / heights.length) && found.length; k++) addresses.add(found[Math.floor(rnd() * found.length)]);
+}
+const sample = [...addresses].slice(0, SAMPLE);
+
+const lookups = [];
+for (const [label, list] of [['sampled', sample], ['heavy', heavy]]) {
+  for (const a of list) {
+    const script = addressToScript(a);
+    if (!script) continue;
+    const key = scriptKey(script);
+    const t0 = performance.now();
+    const s = store.summaryForKey(key);
+    lookups.push({ label, address: a, ms: performance.now() - t0, txCount: s.txCount, balance: s.balance });
+  }
+}
+const sampled = lookups.filter((l) => l.label === 'sampled');
+report.lookup = {
+  sampled: sampled.length,
+  msP50: +pct(sampled.map((l) => l.ms), 0.5).toFixed(3), msP90: +pct(sampled.map((l) => l.ms), 0.9).toFixed(3),
+  msP99: +pct(sampled.map((l) => l.ms), 0.99).toFixed(3), msMax: +Math.max(...sampled.map((l) => l.ms)).toFixed(3),
+  txCountP50: pct(sampled.map((l) => l.txCount), 0.5), txCountMax: Math.max(...sampled.map((l) => l.txCount)),
+  heavy: lookups.filter((l) => l.label === 'heavy').map((l) => ({ address: l.address, txCount: l.txCount, btc: l.balance / 1e8, ms: +l.ms.toFixed(1) })),
+};
+// warm: the same lookups again, now that their pages are in the page cache
+const warm = [];
+for (const l of sampled) { const t0 = performance.now(); store.summaryForKey(scriptKey(addressToScript(l.address))); warm.push(performance.now() - t0); }
+report.lookup.warmMsP50 = +pct(warm, 0.5).toFixed(3);
+report.lookup.warmMsP99 = +pct(warm, 0.99).toFixed(3);
+
+// 3. correctness against scantxoutset, at one height
+const check = [...heavy.slice(1), ...sample.slice(0, Math.max(0, VERIFY - heavy.length + 1))];
+for (let attempt = 0; attempt < 3; attempt++) {
+  const nodeTip = await call('getblockcount', []);
+  // rows the node has connected since the build, for just these scripts
+  const want = new Map(check.map((a) => [addressToScript(a).toString('hex'), a]));
+  const delta = new Map();
+  for (let h = tip + 1; h <= nodeTip; h++) {
+    const b = await call('getblock', [await call('getblockhash', [h]), 3]);
+    for (const tx of b.tx) {
+      for (const o of tx.vout) if (want.has(o.scriptPubKey.hex)) delta.set(o.scriptPubKey.hex, (delta.get(o.scriptPubKey.hex) ?? 0) + Math.round(o.value * 1e8));
+      for (const v of tx.vin) if (v.prevout && want.has(v.prevout.scriptPubKey.hex)) delta.set(v.prevout.scriptPubKey.hex, (delta.get(v.prevout.scriptPubKey.hex) ?? 0) - Math.round(v.prevout.value * 1e8));
+    }
+  }
+  t = performance.now();
+  const scan = await call('scantxoutset', ['start', check.map((a) => `addr(${a})`)], 900_000);
+  const scanMs = performance.now() - t;
+  if (scan.height !== nodeTip) continue;                          // a block arrived mid-check: again
+  const utxo = new Map();
+  for (const u of scan.unspents) { const hex = u.scriptPubKey; utxo.set(hex, (utxo.get(hex) ?? 0) + Math.round(u.amount * 1e8)); }
+  const rows = check.map((a) => {
+    const hex = addressToScript(a).toString('hex');
+    const indexSat = store.summaryForKey(scriptKey(Buffer.from(hex, 'hex')), { limit: 0 }).balance + (delta.get(hex) ?? 0);
+    const nodeSat = utxo.get(hex) ?? 0;
+    return { address: a, indexSat, nodeSat, match: indexSat === nodeSat };
+  });
+  report.verify = {
+    height: nodeTip, blocksSinceBuild: nodeTip - tip, addresses: rows.length, matches: rows.filter((r) => r.match).length,
+    mismatches: rows.filter((r) => !r.match), scantxoutsetSec: +(scanMs / 1000).toFixed(1),
+  };
+  break;
+}
+console.log(JSON.stringify(report, null, 1));
+process.exit(0);

package/scripts/index-build.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+// Build the address index from the configured node's block files (server/chain/index/build.js).
+//
+//   node scripts/index-build.js --out <dir> [--node <id>] [--workers N] [--files 0,1,5754]
+//
+// --files builds a partial index for testing (the completeness check is skipped). Progress goes to
+// stderr once a second; the manifest, with every phase's timings, goes to stdout at the end.
+import { loadConfig } from '../server/config.js';
+import { RpcClient } from '../server/rpc/client.js';
+import { buildIndex, rpcPacer } from '../server/chain/index/build.js';
+import path from 'node:path';
+import { progress, progressLine, strip, c, fmt } from './ui.js';
+
+const arg = (name, def) => { const i = process.argv.indexOf(`--${name}`); return i > 0 ? process.argv[i + 1] : def; };
+const cfg = loadConfig();
+const node = cfg.nodes.find((n) => n.id === arg('node', null)) ?? cfg.nodes.find((n) => n.datadir) ?? cfg.nodes[0];
+if (!node.datadir) { console.error(`node ${node.id} has no datadir configured; the index is built from its block files`); process.exit(2); }
+const out = arg('out', null);
+if (!out) { console.error('--out <dir> is required'); process.exit(2); }
+const rpc = new RpcClient(node, { ...(cfg.rpc ?? {}), ...(node.rpc ?? {}) }, { log: { info() {}, warn() {}, error() {}, debug() {} } });
+
+const started = Date.now();
+let phase = null, phaseStart = started;
+const bar = progress();
+const manifest = await buildIndex({
+  rpc, blocksDir: path.join(node.datadir, 'blocks'), out,
+  pace: rpcPacer(rpc, { onChange: (held) => process.stderr.write(held ? '  paused while the node\'s RPC is slow or failing\n' : '  resumed\n') }),
+  workers: arg('workers', null) ? Number(arg('workers')) : undefined,
+  files: arg('files', null) ? arg('files').split(',').map(Number) : null,
+  onProgress: (p) => {
+    if (p.phase !== phase) {
+      if (phase) bar.done(strip(progressLine({ phase, done: 1, total: 1, elapsed: (Date.now() - phaseStart) / 1000 })));
+      phase = p.phase; phaseStart = Date.now();
+    }
+    bar.update({ ...p, elapsed: (Date.now() - phaseStart) / 1000 });
+  },
+});
+bar.done(`  ${c.ok('✓')} ${fmt.big(manifest.rows ?? 0)} rows to block ${Number(manifest.tip?.height ?? 0).toLocaleString()} in ${((Date.now() - started) / 60000).toFixed(1)} min -> ${out}`);
+console.log(JSON.stringify(manifest, (k, v) => (k === 'bucketRows' ? undefined : v), 1));
+process.exit(0);

package/scripts/live-render-check.mjs

@@ -0,0 +1,89 @@
+// Render EVERY page against the LIVE monitor, under the DOM stub.
+//
+// Why: unit tests of a chart's geometry passed while the page that draws it threw a
+// ReferenceError on the first real frame -- and an uncaught throw inside one renderer
+// takes the whole page's cards with it, so the symptom was "two cards render nothing"
+// when the cause was an undefined variable three lines above them. Nothing in the suite
+// had ever run the page code with real-shaped data. This does, for every page.
+import { execFileSync } from 'node:child_process';
+import { installDom } from '../test/dom-stub.js';
+
+// No address is written here. Point it at whatever is running:
+//   BLOCKYARD_BASE=https://<address>:8088 BLOCKYARD_CA=/path/ca.crt npm run render:live
+// With no BLOCKYARD_BASE it boots its own server against the fake node, so the check runs
+// on any machine with no host identity to leak and no assumption about this one.
+const CA = process.env.BLOCKYARD_CA;
+const BASE = process.env.BLOCKYARD_BASE;
+const curlArgs = (p) => ['--max-time', '20', ...(CA ? ['--cacert', CA] : []), `${BASE}${p}`];
+const get = (p) => JSON.parse(execFileSync('curl', ['-s', ...curlArgs(p)], { encoding: 'utf8', maxBuffer: 1 << 26 }));
+
+if (!BASE) {
+  console.log('usage: BLOCKYARD_BASE=https://<address>:8088 [BLOCKYARD_CA=<ca.pem>] npm run render:live');
+  console.log('       (point it at a running monitor; no address is baked into this script)');
+  process.exit(2);
+}
+
+installDom();
+let rafN = 0;
+globalThis.window = { devicePixelRatio: 1, matchMedia: () => ({ matches: true }) };
+globalThis.matchMedia = () => ({ matches: true });
+globalThis.requestAnimationFrame = (fn) => { rafN += 1; fn(rafN * 16); return rafN; };
+globalThis.cancelAnimationFrame = () => {};
+globalThis.performance = { now: () => 0 };
+
+const F = await import('../public/js/fmt.js');
+const panels = await import('../public/js/panels.js');
+panels.setFmt(F);
+const app = await import('../public/js/app.js');
+
+const snap = get('/api/state?node=main');
+try { snap.attribution = { ...(snap.attribution || {}), nextBlock: get('/api/nextblock?node=main') }; } catch { /* the page says so itself */ }
+let dist = null;
+try { dist = get('/api/mempool?node=main')?.dist ?? null; } catch { /* ditto */ }
+
+Object.assign(app.state, { snap, page: 'overview', series: snap.series ?? {}, node: snap.id ?? 'main', byNode: new Map([[snap.id ?? 'main', { series: snap.series ?? {}, snap }]]), mempoolDist: dist ? { ...dist, fetchedAt: Date.now() } : null });
+
+const PAGES = ['overview', 'chain', 'mempool', 'peers', 'network', 'mining', 'logs', 'node'];
+// Cards that must have something in them once the page has rendered with live data.
+const MUST_FILL = {
+  overview: ['ovMpCount', 'ovMpBytes', 'ovMpVsBlock', 'ovMpVsBlockKv', 'ovFeesCard', 'ovCaveats', 'ovTrain', 'ovGnTreemapNote'],
+  chain: ['chState', 'chUtxo', 'chDiff'],
+  mempool: ['mpLimits'],
+  peers: ['prCount', 'prBudget'],
+  network: ['ntRpc'],
+  mining: ['mnPools', 'gnMempoolNote'],
+};
+
+let failed = 0;
+for (const page of PAGES) {
+  app.state.page = page;
+  try {
+    app.render();
+  } catch (e) {
+    failed += 1;
+    console.log(`FAIL ${page}: render threw -- ${e.message}`);
+    console.log(String(e.stack).split('\n').slice(1, 4).map((l) => `     ${l.trim()}`).join('\n'));
+    continue;
+  }
+  // The stub does not register ids out of injected markup, so a card rewritten through
+  // innerHTML is asserted on the element that was written -- which is the element the
+  // browser also paints. Asserting the stale child would measure the harness.
+  const empties = (MUST_FILL[page] ?? []).filter((id) => {
+    const el = globalThis.document.getElementById(id);
+    // Content may be written as markup or as plain text; an empty `_html` string is not
+    // null, so `??` would never look at textContent and a perfectly good note would be
+    // reported as empty. Read both.
+    const html = String(el?.innerHTML ?? '');
+    const txt = (html || String(el?.textContent ?? '')).replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+    return !txt || txt === '–';
+  });
+  if (empties.length) { failed += 1; console.log(`FAIL ${page}: cards left empty: ${empties.join(', ')}`); }
+  const fees = String(globalThis.document.getElementById('ovFeesCard')?.innerHTML ?? '');
+  if (page === 'overview' && /in 1 block/.test(fees) && !/\d/.test(fees.replace(/in \d+ block/g, ''))) {
+    failed += 1; console.log('FAIL overview: the fee tiers rendered with no numbers in them');
+  }
+  else console.log(`ok   ${page}`);
+}
+
+console.log(failed ? `\n${failed} page(s) failed` : '\nall pages rendered with live data');
+process.exit(failed ? 1 : 0);

package/scripts/manage-users.js

@@ -0,0 +1,132 @@
+#!/usr/bin/env node
+// CLI user administration, for the cases where the web UI is unreachable
+// (lost password, locked out, first boot went wrong) or a script needs an
+// account. Never prints or accepts a password on a command line where it would
+// land in shell history: use stdin or the interactive prompt.
+//
+//   node scripts/manage-users.js list
+//   node scripts/manage-users.js create <username> [role]
+//   node scripts/manage-users.js passwd <username>
+//   node scripts/manage-users.js role <username> <role>
+//   node scripts/manage-users.js disable <username> | enable <username>
+//   node scripts/manage-users.js rm <username>
+import fs from 'node:fs';
+import path from 'node:path';
+import readline from 'node:readline';
+import { loadConfig, ROOT } from '../server/config.js';
+import { UserStore, ROLES } from '../server/auth/users.js';
+
+const cfg = loadConfig();
+// Accounts are OFF by default, so this CLI can edit a user file the running server
+// will never consult. Say that up front instead of letting someone create an
+// account, fail to log in, and conclude the tool is broken.
+if (!cfg.auth.enabled) {
+  process.stderr.write('note: accounts are DISABLED on this config (auth.enabled=false), so the\n'
+    + 'server is open without sign-in and ignores users.json. Start with BLOCKYARD_AUTH=1\n'
+    + 'to use accounts.\n');
+}
+const file = path.join(cfg.auth.dataDir, 'users.json');
+const store = new UserStore(file, cfg.auth);
+
+const [cmd, ...args] = process.argv.slice(2);
+
+function ask(question, { hidden = false } = {}) {
+  return new Promise((resolve) => {
+    const rl = readline.createInterface({ input: process.stdin, output: process.stdout, terminal: process.stdout.isTTY });
+    if (!hidden) { rl.question(question, (a) => { rl.close(); resolve(a.trim()); }); return; }
+    process.stdout.write(question);
+    // Turn off echo rather than logging the prompt into a tty that keeps it.
+    const tty = process.stdin;
+    if (tty.isTTY) tty.setRawMode(true);
+    let buf = '';
+    tty.resume();
+    tty.setEncoding('utf8');
+    const onData = (ch) => {
+      // Enter arrives as LF or CR depending on the terminal; both are handled.
+      // (This line used to carry a third comparison whose operand was a raw CR
+      // inside the quotes -- a pasted keystroke. Legal in CJS, a syntax error as
+      // ESM, and nothing ever imported this file to find out.)
+      if (ch === '\n' || ch === '\r') {
+        tty.removeListener('data', onData);
+        if (tty.isTTY) tty.setRawMode(false);
+        tty.pause();
+        process.stdout.write('\n');
+        resolve(buf);
+      } else if (ch === '\u007f') { // DEL: what backspace sends on a tty
+        buf = buf.slice(0, -1);
+      } else if (ch === '\u0003') { // Ctrl-C
+        process.stdout.write('\n');
+        process.exit(130);
+      }
+      else buf += ch;
+    };
+    tty.on('data', onData);
+  });
+}
+
+async function main() {
+  const loaded = await store.load();
+  if (!loaded.loaded && cmd !== 'create' && cmd !== 'list') {
+    process.stdout.write(`no user store at ${file} yet -- start the server once, or: create <username>\n`);
+  }
+  switch (cmd) {
+    case 'list': {
+      if (!store.count) { process.stdout.write('no users\n'); break; }
+      const rows = store.list();
+      process.stdout.write(rows.map((u) =>
+        `${u.username.padEnd(24)} ${u.role.padEnd(9)} ${u.disabled ? 'disabled' : 'enabled '}  created ${new Date(u.createdAt).toISOString().slice(0, 10)}  last ${u.lastLoginAt ? new Date(u.lastLoginAt).toISOString().slice(0, 16) : 'never'}`
+      ).join('\n') + '\n');
+      break;
+    }
+    case 'create': {
+      const username = args[0] || await ask('username: ');
+      const role = args[1] || await ask(`role (${ROLES.join('/')}): `) || 'viewer';
+      const pw = await ask('password (stdin, not echoed): ', { hidden: true });
+      try {
+        const u = await store.createUser(username, pw, { role });
+        process.stdout.write(`created ${u.username} as ${u.role}\n`);
+      } catch (err) { process.stderr.write(`${err.message}\n`); process.exitCode = 1; }
+      break;
+    }
+    case 'passwd': {
+      const username = args[0] || await ask('username: ');
+      const pw = await ask('new password (stdin, not echoed): ', { hidden: true });
+      try { await store.setPassword(username, pw); process.stdout.write(`password set for ${username}; all their sessions were left intact -- sign them out from the UI if needed\n`); }
+      catch (err) { process.stderr.write(`${err.message}\n`); process.exitCode = 1; }
+      break;
+    }
+    case 'role': {
+      const [username, role] = args;
+      if (!username || !role) { process.stderr.write('usage: role <username> <viewer|operator|admin>\n'); process.exitCode = 2; break; }
+      try { const r = await store.setRole(username, role); process.stdout.write(`${r.username} is now ${r.role}\n`); }
+      catch (err) { process.stderr.write(`${err.message}\n`); process.exitCode = 1; }
+      break;
+    }
+    case 'disable':
+    case 'enable': {
+      const username = args[0] || await ask('username: ');
+      try { const r = await store.setDisabled(username, cmd === 'disable'); process.stdout.write(`${r.username} ${r.disabled ? 'disabled' : 'enabled'}\n`); }
+      catch (err) { process.stderr.write(`${err.message}\n`); process.exitCode = 1; }
+      break;
+    }
+    case 'rm': {
+      const username = args[0] || await ask('username to delete: ');
+      const sure = await ask(`delete ${username}? type the username to confirm: `);
+      if (sure !== username) { process.stdout.write('not deleted\n'); break; }
+      try { const r = await store.deleteUser(username); process.stdout.write(`deleted ${r.deleted}\n`); }
+      catch (err) { process.stderr.write(`${err.message}\n`); process.exitCode = 1; }
+      break;
+    }
+    default:
+      process.stdout.write(`usage: manage-users.js <list|create|passwd|role|disable|enable|rm> [args]
+
+users file: ${file}  (0600, scrypt hashes only -- no recoverable passwords)
+roles:      ${ROLES.join(' < ')}   viewer=read, operator=+enabled actions, admin=+users and audit
+`);
+  }
+}
+
+main().catch((err) => {
+  process.stderr.write(`${err.message}\n`);
+  process.exitCode = 1;
+});