blockyard 0.0.1 → 0.0.9

package/server/chain/index/live.js

@@ -0,0 +1,276 @@
+// THE ADDRESS INDEX FOLLOWS THE CHAIN (operator, 2026-09-14: "keep the index current as new blocks
+// arrive"). The base index is immutable and covers the chain up to the block it was built at; this
+// carries it forward, and back again when the chain reorganises.
+//
+//   poll      ask the node for its tip; if a block we hold is no longer on the node's chain, roll back
+//             to the fork; then fetch each block past our tip with getblock <hash> 3 (every input's
+//             prevout included, so it works for a node whose block files are elsewhere) and turn it
+//             into rows -- verboseBlockRows, checked row for row against the file builder
+//   log       every block is appended to <index>/live.log BEFORE it is served, and so is every
+//             rollback. A record is [magic][type][length][payload][crc32]; replay stops at the first
+//             record that is short or fails its checksum, which is exactly a write the process did
+//             not finish, and the log is truncated there
+//   tail      blocks not yet folded live in memory: their rows, and a key -> rows map for lookups
+//   fold      once FOLD_BLOCKS blocks are at least CONFIRMATIONS deep, they are sorted into an
+//             immutable layer (layers/L<from>-<to>) and the log is rewritten without them. A block
+//             that deep is past any reorganisation this code will follow
+//   merge     more than MAX_LAYERS layers are merged into one, so a lookup's cost stays bounded
+//
+// A reorganisation deeper than the tail -- into a layer or the base -- is not repaired: the index
+// says it is stale and a rebuild is needed, rather than serving a history for a chain that is gone.
+import { openSync, writeSync, closeSync, readFileSync, writeFileSync, renameSync, mkdirSync, unlinkSync, fsyncSync, existsSync } from 'node:fs';
+import path from 'node:path';
+import { crc32 } from 'node:zlib';
+import { IndexStore } from './store.js';
+import { ROW, RowSink, verboseBlockRows } from './rows.js';
+import { BLOCK_ROWS } from './build.js';
+
+export const CONFIRMATIONS = 100;
+export const FOLD_BLOCKS = 144;
+export const MAX_LAYERS = 32;
+const MAGIC = 0x42594c47;            // 'BYLG'
+const T_BLOCK = 1, T_ROLLBACK = 2;
+
+// Sort rows by (key, height, position) and build the sparse index -- the same order the base uses.
+export function sortRows(buf, blockRows = BLOCK_ROWS) {
+  const n = buf.length / ROW;
+  const sub = (i) => (buf[i * ROW] << 8) | buf[i * ROW + 1];
+  const counts = new Uint32Array(65537);
+  for (let i = 0; i < n; i++) counts[sub(i) + 1]++;
+  for (let s = 0; s < 65536; s++) counts[s + 1] += counts[s];
+  const order = new Uint32Array(n), fill = counts.slice(0, 65536);
+  for (let i = 0; i < n; i++) order[fill[sub(i)]++] = i;
+  const hi = new Float64Array(n), lo = new Float64Array(n);
+  for (let i = 0; i < n; i++) { const at = i * ROW; hi[i] = buf.readUIntBE(at + 2, 6); lo[i] = buf.readUIntBE(at + 8, 3) * 65536 + buf.readUInt16BE(at + 11); }
+  for (let s = 0; s < 65536; s++) {
+    const part = order.subarray(counts[s], counts[s + 1]);
+    if (part.length > 1) part.sort((x, y) => hi[x] - hi[y] || lo[x] - lo[y]);
+  }
+  const out = Buffer.allocUnsafe(buf.length);
+  const sparse = [];
+  let w = 0;
+  for (let k = 0; k < n; k++) {
+    const at = order[k] * ROW;
+    if (w > 0 && buf.compare(out, w - ROW, w - ROW + 13, at, at + 13) === 0) continue;
+    if ((w / ROW) % blockRows === 0) sparse.push(buf.readBigUInt64BE(at));
+    buf.copy(out, w, at, at + ROW);
+    w += ROW;
+  }
+  return { rows: out.subarray(0, w), idx: Buffer.from(new BigUint64Array(sparse).buffer) };
+}
+
+function writeAtomic(file, data) {
+  const fd = openSync(file + '.tmp', 'w');
+  for (let o = 0; o < data.length;) o += writeSync(fd, data, o, data.length - o);
+  fsyncSync(fd); closeSync(fd);
+  renameSync(file + '.tmp', file);
+}
+
+function record(type, payload) {
+  const head = Buffer.alloc(9);
+  head.writeUInt32LE(MAGIC, 0); head.writeUInt8(type, 4); head.writeUInt32LE(payload.length, 5);
+  const tail = Buffer.alloc(4); tail.writeUInt32LE(crc32(payload), 0);
+  return Buffer.concat([head, payload, tail]);
+}
+
+export class LiveIndex {
+  /**
+   * @param dir   a built index directory
+   * @param rpc   { batch(calls, opts) } -- the node that tells us about new blocks
+   */
+  constructor(dir, { rpc, nodeId = null, log = null, confirmations = CONFIRMATIONS, foldBlocks = FOLD_BLOCKS, maxLayers = MAX_LAYERS, maxBlocksPerPoll = 50 } = {}) {
+    this.dir = dir; this.rpc = rpc; this.nodeId = nodeId; this.log = log;
+    this.confirmations = confirmations; this.foldBlocks = foldBlocks; this.maxLayers = maxLayers; this.maxBlocksPerPoll = maxBlocksPerPoll;
+    this.store = new IndexStore(dir);
+    this.blocks = new Map();             // height -> { hash, rows: Buffer }
+    this.byKey = new Map();              // key hex (16) -> [Buffer, offset, Buffer, offset, ...]
+    this.stale = null;                   // a reason, once the chain has left us behind
+    this.lastError = null; this.lastPollAt = null; this.polling = false;
+    this.logFile = path.join(dir, 'live.log');
+    this.store.tail = this;
+    this.#replay();
+  }
+
+  get tip() { return this.blocks.size ? Math.max(...this.blocks.keys()) : this.store.sortedTip; }
+
+  status() {
+    return { baseTip: this.store.manifest.tip.height, sortedTip: this.store.sortedTip, tip: this.tip, tailBlocks: this.blocks.size, layers: this.store.layers.length, stale: this.stale, lastError: this.lastError, lastPollAt: this.lastPollAt };
+  }
+
+  // --- the tail -------------------------------------------------------------------------------
+  scan(key, visit) {
+    const list = this.byKey.get(key.toString(16).padStart(16, '0'));
+    if (!list) return;
+    for (let i = 0; i < list.length; i += 2) visit(list[i], list[i + 1]);
+  }
+
+  #add(height, hash, rows) {
+    this.blocks.set(height, { hash, rows });
+    for (let at = 0; at < rows.length; at += ROW) {
+      const k = rows.toString('hex', at, at + 8);
+      const list = this.byKey.get(k);
+      if (list) list.push(rows, at); else this.byKey.set(k, [rows, at]);
+    }
+  }
+
+  #drop(height) {
+    const b = this.blocks.get(height);
+    if (!b) return;
+    for (let at = 0; at < b.rows.length; at += ROW) {
+      const k = b.rows.toString('hex', at, at + 8);
+      const list = this.byKey.get(k);
+      if (!list) continue;
+      const kept = [];
+      for (let i = 0; i < list.length; i += 2) if (list[i] !== b.rows) kept.push(list[i], list[i + 1]);
+      if (kept.length) this.byKey.set(k, kept); else this.byKey.delete(k);
+    }
+    this.blocks.delete(height);
+  }
+
+  // --- the log --------------------------------------------------------------------------------
+  #replay() {
+    if (!existsSync(this.logFile)) return;
+    const buf = readFileSync(this.logFile);
+    let pos = 0, good = 0;
+    while (pos + 13 <= buf.length) {
+      if (buf.readUInt32LE(pos) !== MAGIC) break;
+      const type = buf.readUInt8(pos + 4), len = buf.readUInt32LE(pos + 5);
+      if (pos + 9 + len + 4 > buf.length) break;
+      const payload = buf.subarray(pos + 9, pos + 9 + len);
+      if (crc32(payload) !== buf.readUInt32LE(pos + 9 + len)) break;
+      if (type === T_BLOCK) {
+        const height = payload.readUInt32LE(0), hash = payload.toString('hex', 4, 36);
+        if (height === this.tip + 1) this.#add(height, hash, Buffer.from(payload.subarray(36)));
+      } else if (type === T_ROLLBACK) {
+        const to = payload.readUInt32LE(0);
+        for (const h of [...this.blocks.keys()]) if (h > to) this.#drop(h);
+      }
+      pos += 9 + len + 4; good = pos;
+    }
+    if (good < buf.length) {
+      // a record the process did not finish writing: cut it off so the next append follows good data
+      this.log?.warn?.(`live index log: dropping ${buf.length - good} bytes after the last complete record`);
+      writeAtomic(this.logFile, buf.subarray(0, good));
+    }
+  }
+
+  #append(rec) {
+    const fd = openSync(this.logFile, 'a');
+    try { writeSync(fd, rec); fsyncSync(fd); } finally { closeSync(fd); }
+  }
+
+  // --- following the node ---------------------------------------------------------------------
+  async #call(method, params, timeoutMs = 120_000) {
+    const [r] = await this.rpc.batch([{ method, params }], { key: `live-index:${method}:${params[0]}`, timeoutMs, maxWaitMs: 300_000, priority: 6 });
+    if (!r?.ok) throw new Error(`${method}: ${r?.error?.message ?? 'no reply'}`);
+    return r.result;
+  }
+
+  /** One round: detect a reorg, catch up to the node's tip (at most maxBlocksPerPoll blocks), fold. */
+  async poll() {
+    if (this.polling || this.stale) return this.status();
+    this.polling = true;
+    try {
+      const nodeTip = await this.#call('getblockcount', []);
+      // REORG: walk down from our tip while our block is not the node's block at that height
+      let top = this.tip;
+      while (top > this.store.sortedTip && top <= nodeTip + 0 && this.blocks.has(top)) {
+        const theirs = await this.#call('getblockhash', [top]).catch(() => null);
+        if (theirs === this.blocks.get(top).hash) break;
+        top--;
+      }
+      // a tip above the node's (the node rolled back past it) is also a fork
+      while (top > nodeTip && top > this.store.sortedTip) top--;
+      // WALKED DOWN TO WHAT IS FOLDED: that block must still be the node's too, or the reorganisation
+      // went below the tail. Checked every poll -- a tail can be empty, with everything folded.
+      if (top === this.store.sortedTip) {
+        const known = this.#foldedHash(top);
+        const theirs = top <= nodeTip ? await this.#call('getblockhash', [top]).catch(() => null) : null;
+        if (known && theirs !== known) { this.stale = `the chain reorganised at or below block ${top}, which is already folded into the index; rebuild the index`; return this.status(); }
+        if (!known) top = Math.min(top, this.store.sortedTip);
+      }
+      if (top < this.tip) {
+        if (top < this.store.sortedTip) { this.stale = `the chain reorganised below block ${this.store.sortedTip}, which is already folded; rebuild the index`; return this.status(); }
+        const payload = Buffer.alloc(4); payload.writeUInt32LE(top, 0);
+        this.#append(record(T_ROLLBACK, payload));
+        for (const h of [...this.blocks.keys()]) if (h > top) this.#drop(h);
+        this.log?.info?.(`live index: rolled back to block ${top}`);
+      }
+      // CATCH UP
+      for (let h = this.tip + 1, n = 0; h <= nodeTip && n < this.maxBlocksPerPoll; h++, n++) {
+        const hash = await this.#call('getblockhash', [h]);
+        const block = await this.#call('getblock', [hash, 3], 300_000);
+        if (block.height !== h) throw new Error(`getblock ${hash} answered height ${block.height}, expected ${h}`);
+        if (h > 0 && this.blocks.has(h - 1) && block.previousblockhash !== this.blocks.get(h - 1).hash) break;   // a reorg mid-catch-up: next poll rolls back
+        const sink = new RowSink(1 << 14);
+        verboseBlockRows(block, h, sink);
+        const rows = Buffer.from(sink.bytes());
+        const head = Buffer.alloc(36); head.writeUInt32LE(h, 0); Buffer.from(hash, 'hex').copy(head, 4);
+        this.#append(record(T_BLOCK, Buffer.concat([head, rows])));
+        this.#add(h, hash, rows);
+      }
+      this.#fold(nodeTip);
+      this.lastError = null;
+    } catch (err) {
+      this.lastError = err.message;
+      this.log?.warn?.(`live index: ${err.message}`);
+    } finally {
+      this.polling = false;
+      this.lastPollAt = Date.now();
+    }
+    return this.status();
+  }
+
+  // The hash of the highest folded block: the base's from its manifest, a layer's from layers/tips.json
+  // (written with the layer, since a layer file carries rows and no block hashes).
+  #foldedHash(height) {
+    if (height === this.store.manifest.tip.height) return this.store.manifest.tip.hash;
+    try { return JSON.parse(readFileSync(path.join(this.dir, 'layers', 'tips.json'), 'utf8'))[height] ?? null; } catch { return null; }
+  }
+
+  // --- folding --------------------------------------------------------------------------------
+  #fold(nodeTip) {
+    const deep = [...this.blocks.keys()].filter((h) => h <= nodeTip - this.confirmations).sort((a, b) => a - b);
+    if (deep.length < this.foldBlocks) return;
+    const from = deep[0], to = deep[deep.length - 1];
+    const { rows, idx } = sortRows(Buffer.concat(deep.map((h) => this.blocks.get(h).rows)), this.store.blockRows);
+    const dir = path.join(this.dir, 'layers');
+    mkdirSync(dir, { recursive: true });
+    const base = path.join(dir, `L${from}-${to}`);
+    // the top block's hash first, so a layer is never on disk without it
+    let tips = {};
+    try { tips = JSON.parse(readFileSync(path.join(dir, 'tips.json'), 'utf8')); } catch { /* the first layer */ }
+    tips[to] = this.blocks.get(to).hash;
+    writeAtomic(path.join(dir, 'tips.json'), Buffer.from(JSON.stringify(tips)));
+    writeAtomic(base + '.idx', idx);
+    writeAtomic(base + '.rows', rows);
+    this.store.reloadLayers();
+    // the log keeps only what is still in memory: rewritten, then swapped in whole
+    for (const h of deep) this.#drop(h);
+    const keep = [...this.blocks.keys()].sort((a, b) => a - b).map((h) => {
+      const b = this.blocks.get(h);
+      const head = Buffer.alloc(36); head.writeUInt32LE(h, 0); Buffer.from(b.hash, 'hex').copy(head, 4);
+      return record(T_BLOCK, Buffer.concat([head, b.rows]));
+    });
+    writeAtomic(this.logFile, Buffer.concat(keep));
+    this.log?.info?.(`live index: folded blocks ${from}..${to} (${rows.length / ROW} rows) into a layer`);
+    if (this.store.layers.length > this.maxLayers) this.#merge();
+  }
+
+  #merge() {
+    const layers = this.store.layers;
+    const from = layers[0].from, to = layers[layers.length - 1].to;
+    const dir = path.join(this.dir, 'layers');
+    const parts = layers.map((l) => readFileSync(path.join(dir, `L${l.from}-${l.to}.rows`)));
+    const { rows, idx } = sortRows(Buffer.concat(parts), this.store.blockRows);
+    const base = path.join(dir, `L${from}-${to}`);
+    // the merged layer under its final name first, then the old ones removed. A crash in between leaves
+    // the old layers AND the merged one on disk; the store ignores a layer whose range lies inside
+    // another's (IndexStore.reloadLayers), so nothing is counted twice, and the next merge tidies up.
+    writeAtomic(base + '.idx', idx);
+    writeAtomic(base + '.rows', rows);
+    for (const l of layers) { unlinkSync(path.join(dir, `L${l.from}-${l.to}.rows`)); unlinkSync(path.join(dir, `L${l.from}-${l.to}.idx`)); }
+    this.store.reloadLayers();
+    this.log?.info?.(`live index: merged ${layers.length} layers into L${from}-${to}`);
+  }
+}

package/server/chain/index/rows.js

@@ -0,0 +1,145 @@
+// ADDRESS INDEX ROWS, straight from block and undo bytes (docs/MEASUREMENTS.md §28-29).
+//
+// One row per (script, transaction that touched it), 21 bytes, big-endian so that byte order IS sort
+// order:
+//
+//   sh8     8   the first 8 bytes of sha256(scriptPubKey) -- an address, whatever its type
+//   height  3   the block
+//   pos     2   the transaction's position in that block
+//   value   8   signed satoshis: what the transaction paid to the script, minus what it spent from it
+//
+// A transaction that both pays and spends one script is ONE row with the net, not two. A script
+// paid by an output is taken from the block; a script spent by an input is taken from the undo
+// record, which is how the spending side is known without replaying the UTXO set.
+//
+// LEAN ON PURPOSE. server/chain/tx.js builds Core's verbose shape -- hex strings, classified types,
+// encoded addresses -- and measured, that was 5.9 of the 9.5 single-core hours (§28). The index needs
+// none of it: only each output's value and script bytes, and each spent coin's. So this walks the
+// transaction bytes itself and hashes script bytes where they lie. It is checked row-for-row against
+// rows built from the full decoder (test/chain-index.test.js), so lean never means different.
+//
+// OP_RETURN outputs are skipped: provably unspendable, never an address. Core names only the
+// push-only ones `nulldata`; the rest are `nonstandard` there, and just as unspendable.
+import { createHash } from 'node:crypto';
+import { decodeBlockUndo } from '../blockfile.js';
+
+export const ROW = 21;
+export const MAX_POS = 0xffff;
+export const MAX_HEIGHT = 0xffffff;
+
+function varint(buf, st) {
+  const b = buf[st.pos++];
+  if (b < 0xfd) return b;
+  if (b === 0xfd) { const v = buf.readUInt16LE(st.pos); st.pos += 2; return v; }
+  if (b === 0xfe) { const v = buf.readUInt32LE(st.pos); st.pos += 4; return v; }
+  const v = Number(buf.readBigUInt64LE(st.pos)); st.pos += 8; return v;
+}
+
+/** sha256(script)'s first 8 bytes, as an unsigned BigInt. */
+export function scriptKey(script) {
+  return createHash('sha256').update(script).digest().readBigUInt64BE(0);
+}
+
+/**
+ * Walk one transaction at st.pos: calls onOutput(value_sat, scriptSubarray) per output, returns the
+ * number of inputs. Leaves st.pos after the transaction.
+ */
+function walkTx(buf, st, onOutput) {
+  st.pos += 4;                                                   // version
+  let segwit = false;
+  if (buf[st.pos] === 0 && buf[st.pos + 1] === 1) { segwit = true; st.pos += 2; }
+  const nin = varint(buf, st);
+  for (let i = 0; i < nin; i++) { st.pos += 36; const len = varint(buf, st); st.pos += len + 4; }
+  const nout = varint(buf, st);
+  for (let o = 0; o < nout; o++) {
+    const value = Number(buf.readBigUInt64LE(st.pos)); st.pos += 8;
+    const len = varint(buf, st);
+    onOutput(value, buf.subarray(st.pos, st.pos + len));
+    st.pos += len;
+  }
+  if (segwit) for (let i = 0; i < nin; i++) { const items = varint(buf, st); for (let k = 0; k < items; k++) { const len = varint(buf, st); st.pos += len; } }
+  st.pos += 4;                                                   // locktime
+  return nin;
+}
+
+/**
+ * Index rows for one block. `body` is the raw block, `undoBody` its undo record (null for genesis,
+ * whose coinbase spends nothing). Appends rows to `out` (a RowSink) and returns the row count.
+ */
+export function blockRows(body, undoBody, height, out) {
+  if (height > MAX_HEIGHT) throw new RangeError(`height ${height} does not fit the row format`);
+  const undo = undoBody ? decodeBlockUndo(undoBody) : [];
+  const st = { pos: 80 };
+  const ntx = varint(body, st);
+  if (ntx - 1 > MAX_POS) throw new RangeError(`block ${height} has ${ntx} transactions, past the row format's position field`);
+  let rows = 0;
+  const moved = new Map();
+  for (let p = 0; p < ntx; p++) {
+    moved.clear();
+    walkTx(body, st, (value, script) => {
+      if (script.length > 0 && script[0] === 0x6a) return;       // OP_RETURN: unspendable, not an address
+      const k = scriptKey(script);
+      moved.set(k, (moved.get(k) ?? 0) + value);
+    });
+    if (p > 0) {
+      const coins = undo[p - 1];
+      if (!coins) throw new Error(`block ${height}: no undo for transaction ${p}`);
+      for (const c of coins) {
+        const k = scriptKey(c.script);
+        moved.set(k, (moved.get(k) ?? 0) - c.value_sat);
+      }
+    }
+    for (const [k, v] of moved) { out.push(k, height, p, v); rows++; }
+  }
+  if (st.pos !== body.length) throw new RangeError(`block ${height}: ${body.length - st.pos} bytes left after the last transaction`);
+  return rows;
+}
+
+/** Packs rows into a growable Buffer of 21-byte records. */
+export class RowSink {
+  constructor(initialRows = 1 << 16) { this.buf = Buffer.allocUnsafe(initialRows * ROW); this.n = 0; }
+  push(key, height, pos, value) {
+    const at = this.n * ROW;
+    if (at + ROW > this.buf.length) { const next = Buffer.allocUnsafe(this.buf.length * 2); this.buf.copy(next, 0, 0, at); this.buf = next; }
+    this.buf.writeBigUInt64BE(key, at);
+    this.buf.writeUIntBE(height, at + 8, 3);
+    this.buf.writeUInt16BE(pos, at + 11);
+    this.buf.writeBigInt64BE(BigInt(value), at + 13);
+    this.n++;
+  }
+  bytes() { return this.buf.subarray(0, this.n * ROW); }
+}
+
+export function readRow(buf, at = 0) {
+  return { key: buf.readBigUInt64BE(at), height: buf.readUIntBE(at + 8, 3), pos: buf.readUInt16BE(at + 11), value: Number(buf.readBigInt64BE(at + 13)) };
+}
+
+/**
+ * The same rows from the node's own decoding: `getblock <hash> 3`, which carries every input's prevout.
+ * This is how the live index follows the chain -- through RPC, one block at a time as they arrive,
+ * rather than re-reading the files -- and it must agree with blockRows row for row, which is checked on
+ * real blocks (test/chain-index-live.test.js and scripts/index-live-check.js). Amounts arrive as BTC
+ * floats and are rounded to satoshis, which is exact for every amount Core can express.
+ */
+export function verboseBlockRows(block, height, out) {
+  if (height > MAX_HEIGHT) throw new RangeError(`height ${height} does not fit the row format`);
+  if (block.tx.length - 1 > MAX_POS) throw new RangeError(`block ${height} has ${block.tx.length} transactions, past the row format's position field`);
+  let rows = 0;
+  const moved = new Map();
+  block.tx.forEach((tx, p) => {
+    moved.clear();
+    for (const o of tx.vout) {
+      const hex = o.scriptPubKey.hex;
+      if (hex.startsWith('6a')) continue;
+      const k = scriptKey(Buffer.from(hex, 'hex'));
+      moved.set(k, (moved.get(k) ?? 0) + Math.round(o.value * 1e8));
+    }
+    if (p > 0) for (const v of tx.vin) {
+      if (!v.prevout) throw new Error(`block ${height} tx ${p}: no prevout -- getblock must be called with verbosity 3`);
+      const k = scriptKey(Buffer.from(v.prevout.scriptPubKey.hex, 'hex'));
+      moved.set(k, (moved.get(k) ?? 0) - Math.round(v.prevout.value * 1e8));
+    }
+    for (const [k, v] of moved) { out.push(k, height, p, v); rows++; }
+  });
+  return rows;
+}

package/server/chain/index/store.js

@@ -0,0 +1,154 @@
+// READING THE ADDRESS INDEX (server/chain/index/build.js writes the base; live.js adds to it).
+//
+// Three kinds of source, read in height order so a history comes out oldest first:
+//   base    256 immutable segments from the full build: seg-XX.rows sorted 21-byte rows, seg-XX.idx
+//           the first key of every BLOCK_ROWS-th row (a few megabytes in all, held in memory)
+//   layers  layers/L<from>-<to>.rows + .idx: the same sorted rows for a run of later blocks, folded
+//           out of the live tail once they are deep enough never to be reorganised away
+//   tail    the newest blocks, in memory (live.js), handed in as an object with scan(key, visit)
+// A lookup binary-searches each sorted source's sparse keys and reads only the row blocks that can
+// hold its key.
+import { openSync, readSync, closeSync, readFileSync, statSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import { ROW, scriptKey, readRow } from './rows.js';
+import { FORMAT } from './build.js';
+
+const LAYER = /^L(\d+)-(\d+)\.rows$/;
+const RING_MAX_BLIND = 4096;   // rows (86 KB) a page may keep without first counting the history
+
+// NO FILE IS HELD OPEN (2026-09-14): a store used to keep one descriptor per segment and layer --
+// 256 and more -- for the life of the process, which is the whole soft limit on a stock macOS
+// (`ulimit -n` 256) before the server has opened a socket. A lookup opens the one file it reads
+// and closes it: three syscalls on a 0.25 ms lookup.
+function openSorted(rowsFile, idxFile) {
+  const raw = readFileSync(idxFile);
+  const idx = new BigUint64Array(raw.buffer, raw.byteOffset, raw.length / 8).slice();
+  return { idx, file: rowsFile, rows: statSync(rowsFile).size / ROW };
+}
+
+export class IndexStore {
+  constructor(dir) {
+    this.dir = dir;
+    this.manifest = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
+    if (this.manifest.format !== FORMAT) throw new Error(`index format ${this.manifest.format}, this code reads ${FORMAT}`);
+    this.blockRows = this.manifest.blockRows;
+    this.segments = Array.from({ length: 256 }, (_, b) => {
+      const hex = b.toString(16).padStart(2, '0');
+      try { return openSorted(path.join(dir, `seg-${hex}.rows`), path.join(dir, `seg-${hex}.idx`)); } catch { return null; }
+    });
+    this.layers = [];
+    this.reloadLayers();
+    this.tail = null;
+    this.scratch = Buffer.allocUnsafe(ROW * this.blockRows);
+  }
+
+  /** Re-read layers/ (live.js calls this after folding or merging). */
+  reloadLayers() {
+    const old = this.layers;
+    const dir = path.join(this.dir, 'layers');
+    let names = [];
+    try { names = readdirSync(dir).filter((f) => LAYER.test(f)); } catch { /* no layers yet */ }
+    // a layer whose blocks lie inside another layer's range is a leftover from a merge that was
+    // interrupted after the merged layer was written: skipped, or its rows would be counted twice
+    const ranges = names.map((f) => { const [, from, to] = f.match(LAYER); return { f, from: Number(from), to: Number(to) }; });
+    const live = ranges.filter((r) => !ranges.some((o) => o !== r && o.from <= r.from && o.to >= r.to && (o.to - o.from) > (r.to - r.from)));
+    const idxOf = (f) => path.join(dir, f.replace(/\.rows$/, '.idx'));
+    this.layers = live.filter((r) => { try { readFileSync(idxOf(r.f), { flag: 'r' }); return true; } catch { return false; } })
+      .map((r) => ({ from: r.from, to: r.to, ...openSorted(path.join(dir, r.f), idxOf(r.f)) }))
+      .sort((a, b) => a.from - b.from);
+  }
+
+  /** The highest block the base and its contiguous layers cover (the tail, if any, continues it). */
+  get sortedTip() {
+    let tip = this.manifest.tip.height;
+    for (const l of this.layers) if (l.from === tip + 1) tip = l.to;
+    return tip;
+  }
+
+  get tip() {
+    return this.tail?.tip != null && this.tail.tip > this.sortedTip ? this.tail.tip : this.sortedTip;
+  }
+
+  close() { /* nothing is held open; kept for callers */ }
+
+  // Visit every row for a key in one sorted source, without allocating per row. Keys are compared as
+  // two unsigned 32-bit halves read straight from the page -- a BigInt per row was most of a lookup's
+  // time for an address with many rows.
+  #scanSorted(src, key, visit) {
+    if (!src || !src.idx.length) return;
+    let lo = 0, hi = src.idx.length - 1;
+    while (lo < hi) { const mid = (lo + hi + 1) >> 1; if (src.idx[mid] < key) lo = mid; else hi = mid - 1; }
+    const kh = Number(key >> 32n), kl = Number(key & 0xffffffffn);
+    const fd = openSync(src.file, 'r');
+    try {
+      for (let block = lo; block < src.idx.length; block++) {
+        const start = block * this.blockRows;
+        const want = Math.min(this.blockRows, src.rows - start) * ROW;
+        const got = readSync(fd, this.scratch, 0, want, start * ROW);
+        for (let at = 0; at < got; at += ROW) {
+          const h = this.scratch.readUInt32BE(at);
+          if (h < kh) continue;
+          if (h > kh) return;
+          const l = this.scratch.readUInt32BE(at + 4);
+          if (l < kl) continue;
+          if (l > kl) return;
+          visit(this.scratch, at);
+        }
+      }
+    } finally { closeSync(fd); }
+  }
+
+  /** Every row for a key across base, layers and tail, oldest block first. */
+  forEachRow(key, visit) {
+    this.#scanSorted(this.segments[Number(key >> 56n)], key, visit);
+    for (const l of this.layers) this.#scanSorted(l, key, visit);
+    this.tail?.scan(key, visit);
+  }
+
+  /** Every row for a key (a BigInt from scriptKey), oldest first. */
+  rowsForKey(key) {
+    const out = [];
+    this.forEachRow(key, (buf, at) => out.push(readRow(buf, at)));
+    return out;
+  }
+
+  rowsForScript(script) { return this.rowsForKey(scriptKey(script)); }
+
+  /**
+   * What an address page needs: transaction count, balance, and `limit` rows newest first after
+   * skipping the newest `skip` (a page). Balance is the sum of every row's net; `received` and `sent`
+   * are sums of those nets by sign, so a transaction that both paid and spent a script counts once,
+   * by its net -- not the gross figures an explorer that stores every output separately would show.
+   */
+  summaryForKey(key, { limit = 25, skip = 0, maxHeight = null } = {}) {
+    let txCount = 0, balance = 0, received = 0, sent = 0, postTip = 0;
+    // ROWS ABOVE maxHeight ARE NOT HISTORY (audit 2026-09-14, M2): after a reorganisation the tail
+    // can hold blocks the node no longer has until the follower's next poll; the caller passes the
+    // node's tip and those rows are counted in `postTip`, never in the balance or the page
+    const above = (buf, at) => maxHeight != null && buf.readUIntBE(at + 8, 3) > maxHeight;
+    // the newest skip+limit rows in a ring of raw bytes: no object per row, whatever the address's
+    // size; a page deep into a huge history keeps skip+limit rows, never the whole history.
+    // THE RING IS NEVER LARGER THAN THE HISTORY (audit 2026-09-14, M1: a page number is a request
+    // parameter, and `page=999999` sized a 525 MB ring for an address with two rows). A deep page
+    // counts the rows first, which is the same walk again, and sizes the ring to what exists.
+    let keep = Math.max(0, skip) + Math.max(0, limit);
+    if (keep > RING_MAX_BLIND) {
+      let n = 0;
+      this.forEachRow(key, (buf, at) => { if (!above(buf, at)) n++; });
+      keep = Math.min(keep, n);
+    }
+    const ring = Buffer.allocUnsafe(Math.max(1, keep) * ROW);
+    this.forEachRow(key, (buf, at) => {
+      if (above(buf, at)) { postTip++; return; }
+      const hi = buf.readInt32BE(at + 13), lo = buf.readUInt32BE(at + 17);
+      const v = hi * 4294967296 + lo;
+      txCount++; balance += v; if (v > 0) received += v; else sent -= v;
+      if (keep > 0) buf.copy(ring, ((txCount - 1) % keep) * ROW, at, at + ROW);
+    });
+    const recent = [];
+    for (let k = skip; k < Math.min(keep, txCount); k++) recent.push(readRow(ring, ((((txCount - 1 - k) % keep) + keep) % keep) * ROW));
+    return { txCount, balance, received, sent, recent, postTip };
+  }
+
+  summary(script, opts) { return this.summaryForKey(scriptKey(script), opts); }
+}