blockyard 0.0.1 → 0.0.9

package/server/chain/index/worker.js

@@ -0,0 +1,109 @@
+// A build worker (server/chain/index/build.js). Two jobs, chosen per message:
+//
+//   scan  one blk/rev file pair -> the file's index rows, partitioned by the key's first byte into
+//         256 buffers (a counting sort), so the main thread only appends each to its bucket file
+//   sort  one bucket file -> sorted, de-duplicated rows plus a sparse index, then the unsorted
+//         input is removed
+import { parentPort, workerData } from 'node:worker_threads';
+import { openSync, readSync, writeSync, closeSync, fstatSync, unlinkSync, renameSync } from 'node:fs';
+import path from 'node:path';
+import { Reader, readHeader } from '../tx.js';
+import { readChainFile, records, pairBlocksWithUndo, MAGIC } from '../blockfile.js';
+import { blockRows, RowSink, ROW } from './rows.js';
+import { HeightTable } from './heights.js';
+
+const { blocksDir, key, heightsBuffer, heightsCapacity, blockRowsPerIndex } = workerData;
+const heights = HeightTable.attach(heightsBuffer, heightsCapacity);
+const xor = Buffer.from(key);
+
+function scan(file) {
+  const t0 = performance.now();
+  const id = String(file).padStart(5, '0');
+  const blk = readChainFile(path.join(blocksDir, `blk${id}.dat`), xor);
+  const rev = readChainFile(path.join(blocksDir, `rev${id}.dat`), xor);
+  const readMs = performance.now() - t0;
+  const blocks = [];
+  for (const r of records(blk, MAGIC.main, 0, xor)) {
+    const rd = new Reader(r.body);
+    const h = readHeader(rd);
+    blocks.push({ body: r.body, hash: h.hash, previousblockhash: h.previousblockhash, ntx: rd.varint() });
+  }
+  const pairs = pairBlocksWithUndo(blocks, [...records(rev, MAGIC.main, 32, xor)]);
+  const sink = new RowSink(1 << 18);
+  const indexed = [];
+  let stale = 0, missingUndo = 0;
+  blocks.forEach((b, i) => {
+    const height = heights.get(b.hash);
+    if (height < 0) { stale++; return; }                          // not on the chain this build covers
+    const undo = pairs.get(i);
+    if (!undo && height !== 0) { missingUndo++; return; }
+    blockRows(b.body, undo ? undo.body : null, height, sink);
+    indexed.push(height);
+  });
+  // counting sort by the first key byte: 256 buffers the main thread appends as they are
+  const all = sink.bytes();
+  const counts = new Uint32Array(257);
+  for (let at = 0; at < all.length; at += ROW) counts[all[at] + 1]++;
+  for (let b = 0; b < 256; b++) counts[b + 1] += counts[b];
+  const out = Buffer.allocUnsafe(all.length);
+  const fill = counts.slice(0, 256);
+  for (let at = 0; at < all.length; at += ROW) { const b = all[at]; all.copy(out, fill[b]++ * ROW, at, at + ROW); }
+  const parts = [];
+  for (let b = 0; b < 256; b++) if (counts[b + 1] > counts[b]) parts.push([b, counts[b] * ROW, counts[b + 1] * ROW]);
+  const ab = out.buffer.slice(out.byteOffset, out.byteOffset + out.length);
+  return {
+    msg: { type: 'scanned', file, rows: sink.n, blocks: indexed.length, heights: indexed, stale, missingUndo, parts, readMs, ms: performance.now() - t0, data: ab },
+    transfer: [ab],
+  };
+}
+
+function sortBucket(bucket, dir) {
+  const t0 = performance.now();
+  const name = (s) => path.join(dir, `bucket-${bucket.toString(16).padStart(2, '0')}${s}`);
+  const fd = openSync(name('.unsorted'), 'r');
+  const size = fstatSync(fd).size;
+  const buf = Buffer.allocUnsafe(size);
+  for (let got = 0; got < size;) { const k = readSync(fd, buf, got, size - got, got); if (!k) break; got += k; }
+  closeSync(fd);
+  const n = size / ROW;
+  // order by the next 16 key bits into sub-buckets, then compare the rest numerically:
+  // hi = key bytes 3..7 (40 bits), lo = height (24 bits) and position (16 bits)
+  const sub = (i) => (buf[i * ROW + 1] << 8) | buf[i * ROW + 2];
+  const counts = new Uint32Array(65537);
+  for (let i = 0; i < n; i++) counts[sub(i) + 1]++;
+  for (let s = 0; s < 65536; s++) counts[s + 1] += counts[s];
+  const order = new Uint32Array(n), fill = counts.slice(0, 65536);
+  for (let i = 0; i < n; i++) order[fill[sub(i)]++] = i;
+  const hi = new Float64Array(n), lo = new Float64Array(n);
+  for (let i = 0; i < n; i++) { const at = i * ROW; hi[i] = buf.readUIntBE(at + 3, 5); lo[i] = buf.readUIntBE(at + 8, 3) * 65536 + buf.readUInt16BE(at + 11); }
+  for (let s = 0; s < 65536; s++) {
+    const part = order.subarray(counts[s], counts[s + 1]);
+    if (part.length > 1) part.sort((x, y) => hi[x] - hi[y] || lo[x] - lo[y]);
+  }
+  const out = Buffer.allocUnsafe(size);
+  const sparse = [];
+  let w = 0, dupes = 0;
+  for (let k = 0; k < n; k++) {
+    const at = order[k] * ROW;
+    // the same block can be stored twice in blk files; its rows are then identical and kept once
+    if (w > 0 && buf.compare(out, w - ROW, w - ROW + 13, at, at + 13) === 0) { dupes++; continue; }
+    if ((w / ROW) % blockRowsPerIndex === 0) sparse.push(buf.readBigUInt64BE(at));
+    buf.copy(out, w, at, at + ROW);
+    w += ROW;
+  }
+  const idx = Buffer.from(new BigUint64Array(sparse).buffer);
+  const write = (file, data) => { const f = openSync(file + '.tmp', 'w'); for (let o = 0; o < data.length;) o += writeSync(f, data, o, data.length - o); closeSync(f); renameSync(file + '.tmp', file); };
+  write(path.join(dir, `seg-${bucket.toString(16).padStart(2, '0')}.rows`), out.subarray(0, w));
+  write(path.join(dir, `seg-${bucket.toString(16).padStart(2, '0')}.idx`), idx);
+  unlinkSync(name('.unsorted'));
+  return { msg: { type: 'sorted', bucket, rows: w / ROW, dupes, ms: performance.now() - t0 }, transfer: [] };
+}
+
+parentPort.on('message', (job) => {
+  try {
+    const r = job.type === 'scan' ? scan(job.file) : sortBucket(job.bucket, job.dir);
+    parentPort.postMessage(r.msg, r.transfer);
+  } catch (err) {
+    parentPort.postMessage({ type: 'error', job, message: err.stack || err.message });
+  }
+});

package/server/chain/tx.js

@@ -0,0 +1,310 @@
+// RAW TRANSACTIONS AND BLOCKS, DECODED HERE (docs/DEFECTS.md: "the explorer pays roughly 3-5x for
+// verbose RPC it re-parses anyway" and "we can read the block files after all").
+//
+// One decoder for two jobs. The explorer asks the node for `getrawtransaction <txid> 2` and
+// `getblock <hash> 2`, and the node spends most of that reply serializing JSON it could have sent as
+// bytes -- measured on the Umbrel, 1,843 ms at verbosity 2 against 373 ms at verbosity 0 for the same
+// block. And an address index has to be built from the block files, which hold exactly these bytes.
+// Both are this module: bytes in, the fields Core's verbose reply carries out, named the way Core
+// names them so a caller can switch source without switching vocabulary.
+//
+// WHAT IT IS NOT. Raw bytes do not say what an input spends: no prevout value, no prevout address, so
+// no fee. Verbosity 2 carries those because Core looks them up in its UTXO/undo data. A caller that
+// needs them still has to find the spent outputs -- from the parent transactions, or from an index
+// built over the chain. Deciding that is the next step; decoding is this one.
+//
+// Pure and dependency-free (Rule 1): node:crypto for SHA-256, everything else by hand. Checked
+// field-for-field against Core's own verbose output on real blocks (test/chain-tx.test.js carries the
+// fixtures; scripts/decode-check.js replays whole blocks from a live node).
+import { createHash } from 'node:crypto';
+
+const sha256 = (b) => createHash('sha256').update(b).digest();
+export const hash256 = (b) => sha256(sha256(b));
+const rev = (b) => Buffer.from(b).reverse().toString('hex');
+
+// --- networks ------------------------------------------------------------
+// the address encodings of each chain Core names in getblockchaininfo.chain
+export const NETWORKS = Object.freeze({
+  main: { hrp: 'bc', p2pkh: 0x00, p2sh: 0x05 },
+  test: { hrp: 'tb', p2pkh: 0x6f, p2sh: 0xc4 },
+  testnet4: { hrp: 'tb', p2pkh: 0x6f, p2sh: 0xc4 },
+  signet: { hrp: 'tb', p2pkh: 0x6f, p2sh: 0xc4 },
+  regtest: { hrp: 'bcrt', p2pkh: 0x6f, p2sh: 0xc4 },
+});
+const netOf = (network) => NETWORKS[network] ?? NETWORKS.main;
+
+// --- reading -------------------------------------------------------------
+export class Reader {
+  constructor(buf, pos = 0) { this.buf = buf; this.pos = pos; }
+  need(n) {
+    if (this.pos + n > this.buf.length) throw new RangeError(`truncated: need ${n} bytes at ${this.pos} of ${this.buf.length}`);
+  }
+  u8() { this.need(1); return this.buf[this.pos++]; }
+  u32() { this.need(4); const v = this.buf.readUInt32LE(this.pos); this.pos += 4; return v; }
+  i32() { this.need(4); const v = this.buf.readInt32LE(this.pos); this.pos += 4; return v; }
+  u64() {
+    this.need(8);
+    const v = this.buf.readBigUInt64LE(this.pos); this.pos += 8;
+    if (v > BigInt(Number.MAX_SAFE_INTEGER)) throw new RangeError(`amount ${v} is past the safe integer range`);
+    return Number(v);
+  }
+  varint() {
+    const b = this.u8();
+    if (b < 0xfd) return b;
+    if (b === 0xfd) { this.need(2); const v = this.buf.readUInt16LE(this.pos); this.pos += 2; return v; }
+    if (b === 0xfe) return this.u32();
+    return this.u64();
+  }
+  bytes(n) { this.need(n); const v = this.buf.subarray(this.pos, this.pos + n); this.pos += n; return v; }
+}
+
+// --- addresses -----------------------------------------------------------
+const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
+export function base58check(version, payload) {
+  const body = Buffer.concat([Buffer.from([version]), payload]);
+  const full = Buffer.concat([body, hash256(body).subarray(0, 4)]);
+  let n = BigInt('0x' + full.toString('hex'));
+  let out = '';
+  while (n > 0n) { out = B58[Number(n % 58n)] + out; n /= 58n; }
+  for (const byte of full) { if (byte !== 0) break; out = '1' + out; }
+  return out;
+}
+
+const CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l';
+const GEN = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3];
+function polymod(values) {
+  let chk = 1;
+  for (const v of values) {
+    const top = chk >>> 25;
+    chk = ((chk & 0x1ffffff) << 5) ^ v;
+    for (let i = 0; i < 5; i++) if ((top >>> i) & 1) chk ^= GEN[i];
+  }
+  return chk >>> 0;
+}
+// BIP173 for witness v0, BIP350 (bech32m) for v1 and up
+export function segwitAddress(hrp, version, program) {
+  const data = [version];
+  let acc = 0, bits = 0;
+  for (const b of program) {
+    acc = (acc << 8) | b; bits += 8;
+    while (bits >= 5) { bits -= 5; data.push((acc >>> bits) & 31); }
+  }
+  if (bits > 0) data.push((acc << (5 - bits)) & 31);
+  const expand = [...hrp].map((c) => c.charCodeAt(0) >> 5).concat([0], [...hrp].map((c) => c.charCodeAt(0) & 31));
+  const constant = version === 0 ? 1 : 0x2bc830a3;
+  const mod = polymod([...expand, ...data, 0, 0, 0, 0, 0, 0]) ^ constant;
+  const check = [];
+  for (let i = 0; i < 6; i++) check.push((mod >>> (5 * (5 - i))) & 31);
+  return hrp + '1' + [...data, ...check].map((d) => CHARSET[d]).join('');
+}
+
+// --- addresses back to scripts -------------------------------------------
+// The inverse of the encoders above, for lookups that start from an address someone typed. Returns
+// the scriptPubKey an address pays, or null for anything that is not a valid address on `network`
+// -- a checksum that fails is null, never a best guess.
+function base58decode(str) {
+  let n = 0n;
+  for (const ch of str) { const v = B58.indexOf(ch); if (v < 0) return null; n = n * 58n + BigInt(v); }
+  let hex = n.toString(16); if (hex.length % 2) hex = '0' + hex;
+  const lead = str.match(/^1*/)[0].length;
+  return Buffer.concat([Buffer.alloc(lead), n === 0n ? Buffer.alloc(0) : Buffer.from(hex, 'hex')]);
+}
+function bech32decode(addr) {
+  const lower = addr.toLowerCase();
+  if (addr !== lower && addr !== addr.toUpperCase()) return null;           // mixed case is invalid
+  const sep = lower.lastIndexOf('1');
+  if (sep < 1 || sep + 7 > lower.length || lower.length > 90) return null;
+  const hrp = lower.slice(0, sep);
+  const data = [];
+  for (const ch of lower.slice(sep + 1)) { const v = CHARSET.indexOf(ch); if (v < 0) return null; data.push(v); }
+  const expand = [...hrp].map((c) => c.charCodeAt(0) >> 5).concat([0], [...hrp].map((c) => c.charCodeAt(0) & 31));
+  const mod = polymod([...expand, ...data]);
+  const version = data[0];
+  const constant = version === 0 ? 1 : 0x2bc830a3;
+  if (mod !== constant) return null;
+  let acc = 0, bits = 0;
+  const program = [];
+  for (const v of data.slice(1, -6)) {
+    acc = (acc << 5) | v; bits += 5;
+    while (bits >= 8) { bits -= 8; program.push((acc >>> bits) & 255); }
+  }
+  if (bits >= 5 || ((acc << (8 - bits)) & 255)) return null;                  // non-zero padding
+  return { hrp, version, program: Buffer.from(program) };
+}
+export function addressToScript(address, network = 'main') {
+  const net = netOf(network);
+  const a = String(address ?? '').trim();
+  const sw = a.toLowerCase().startsWith(net.hrp + '1') ? bech32decode(a) : null;
+  if (sw) {
+    if (sw.hrp !== net.hrp || sw.version > 16 || sw.program.length < 2 || sw.program.length > 40) return null;
+    if (sw.version === 0 && sw.program.length !== 20 && sw.program.length !== 32) return null;
+    return Buffer.concat([Buffer.from([sw.version === 0 ? 0 : 0x50 + sw.version, sw.program.length]), sw.program]);
+  }
+  const raw = base58decode(a);
+  if (!raw || raw.length !== 25) return null;
+  if (!hash256(raw.subarray(0, 21)).subarray(0, 4).equals(raw.subarray(21))) return null;
+  const hash = raw.subarray(1, 21);
+  if (raw[0] === net.p2pkh) return Buffer.concat([Buffer.from([0x76, 0xa9, 0x14]), hash, Buffer.from([0x88, 0xac])]);
+  if (raw[0] === net.p2sh) return Buffer.concat([Buffer.from([0xa9, 0x14]), hash, Buffer.from([0x87])]);
+  return null;
+}
+
+// --- scripts -------------------------------------------------------------
+// Core's Solver (script/solver.cpp), in its order, with its names.
+const OP_0 = 0x00, OP_PUSHDATA1 = 0x4c, OP_PUSHDATA2 = 0x4d, OP_PUSHDATA4 = 0x4e, OP_1 = 0x51, OP_16 = 0x60;
+const OP_RETURN = 0x6a, OP_DUP = 0x76, OP_EQUAL = 0x87, OP_EQUALVERIFY = 0x88, OP_HASH160 = 0xa9;
+const OP_CHECKSIG = 0xac, OP_CHECKMULTISIG = 0xae;
+
+// Walk a script's opcodes; null if a push runs past the end (Core's GetOp failing).
+function ops(s) {
+  const out = [];
+  let i = 0;
+  while (i < s.length) {
+    const op = s[i++];
+    let len = 0;
+    if (op > OP_0 && op < OP_PUSHDATA1) len = op;
+    else if (op === OP_PUSHDATA1) { if (i + 1 > s.length) return null; len = s[i]; i += 1; }
+    else if (op === OP_PUSHDATA2) { if (i + 2 > s.length) return null; len = s.readUInt16LE(i); i += 2; }
+    else if (op === OP_PUSHDATA4) { if (i + 4 > s.length) return null; len = s.readUInt32LE(i); i += 4; }
+    if (i + len > s.length) return null;
+    out.push({ op, data: len ? s.subarray(i, i + len) : null });
+    i += len;
+  }
+  return out;
+}
+const validPubkeySize = (k) => (k.length === 33 && (k[0] === 2 || k[0] === 3)) || (k.length === 65 && (k[0] === 4 || k[0] === 6 || k[0] === 7));
+const smallInt = (op) => (op === OP_0 ? 0 : op >= OP_1 && op <= OP_16 ? op - OP_1 + 1 : null);
+
+export function classifyScript(s, network = 'main') {
+  const net = netOf(network);
+  // P2SH first, exactly as Core checks it
+  if (s.length === 23 && s[0] === OP_HASH160 && s[1] === 0x14 && s[22] === OP_EQUAL) {
+    return { type: 'scripthash', address: base58check(net.p2sh, s.subarray(2, 22)) };
+  }
+  // witness programs: a version opcode then one direct push of 2..40 bytes, 4..42 in all
+  if (s.length >= 4 && s.length <= 42 && (s[0] === OP_0 || (s[0] >= OP_1 && s[0] <= OP_16)) && s[1] + 2 === s.length) {
+    const version = smallInt(s[0]);
+    const program = s.subarray(2);
+    if (version === 1 && program.length === 2 && program[0] === 0x4e && program[1] === 0x73) {
+      return { type: 'anchor', address: segwitAddress(net.hrp, 1, program) };
+    }
+    if (version === 0 && program.length === 20) return { type: 'witness_v0_keyhash', address: segwitAddress(net.hrp, 0, program) };
+    if (version === 0 && program.length === 32) return { type: 'witness_v0_scripthash', address: segwitAddress(net.hrp, 0, program) };
+    if (version === 1 && program.length === 32) return { type: 'witness_v1_taproot', address: segwitAddress(net.hrp, 1, program) };
+    if (version !== 0) return { type: 'witness_unknown', address: segwitAddress(net.hrp, version, program) };
+    return { type: 'nonstandard', address: null };
+  }
+  // OP_RETURN followed by nothing but pushes
+  if (s.length >= 1 && s[0] === OP_RETURN) {
+    const rest = ops(s.subarray(1));
+    if (rest && rest.every((o) => o.op <= OP_16)) return { type: 'nulldata', address: null };
+  }
+  // pay to a bare public key
+  if ((s.length === 35 || s.length === 67) && s[0] === s.length - 2 && s[s.length - 1] === OP_CHECKSIG && validPubkeySize(s.subarray(1, s.length - 1))) {
+    return { type: 'pubkey', address: null };
+  }
+  if (s.length === 25 && s[0] === OP_DUP && s[1] === OP_HASH160 && s[2] === 0x14 && s[23] === OP_EQUALVERIFY && s[24] === OP_CHECKSIG) {
+    return { type: 'pubkeyhash', address: base58check(net.p2pkh, s.subarray(3, 23)) };
+  }
+  // bare multisig: OP_m <keys...> OP_n OP_CHECKMULTISIG, 1 <= m <= n, every key a valid size
+  if (s.length >= 1 && s[s.length - 1] === OP_CHECKMULTISIG) {
+    const o = ops(s);
+    if (o && o.length >= 4) {
+      const m = smallInt(o[0].op), n = smallInt(o[o.length - 2].op);
+      const keys = o.slice(1, -2);
+      if (m != null && n != null && m >= 1 && keys.length === n && m <= n
+        && keys.every((k) => k.data && k.op < OP_PUSHDATA1 && validPubkeySize(k.data))) {
+        return { type: 'multisig', address: null };
+      }
+    }
+  }
+  return { type: 'nonstandard', address: null };
+}
+
+// --- transactions --------------------------------------------------------
+/**
+ * Decode one transaction from `r` (a Reader positioned at its first byte). Returns Core's verbose
+ * shape -- txid, hash, version, size, vsize, weight, locktime, vin, vout -- with amounts in SATOSHIS
+ * as integers (`value_sat`), never floating BTC: a decoder is not the place to lose a satoshi.
+ */
+export function readTx(r, network = 'main') {
+  const start = r.pos;
+  const version = r.i32();
+  let segwit = false;
+  // BIP144: a zero where the input count belongs, then a flag of 1
+  if (r.buf[r.pos] === 0x00 && r.buf[r.pos + 1] === 0x01) { segwit = true; r.pos += 2; }
+  const afterMarker = r.pos;
+  const nin = r.varint();
+  const vin = [];
+  for (let i = 0; i < nin; i++) {
+    const prev = r.bytes(32), vout = r.u32();
+    const script = r.bytes(r.varint());
+    const sequence = r.u32();
+    const coinbase = vout === 0xffffffff && prev.every((b) => b === 0);
+    vin.push(coinbase
+      ? { coinbase: script.toString('hex'), sequence }
+      : { txid: rev(prev), vout, scriptSig: { hex: script.toString('hex') }, sequence });
+  }
+  const nout = r.varint();
+  const vout = [];
+  for (let n = 0; n < nout; n++) {
+    const value_sat = r.u64();
+    const script = r.bytes(r.varint());
+    const { type, address } = classifyScript(script, network);
+    vout.push({ value_sat, n, scriptPubKey: { hex: script.toString('hex'), type, ...(address ? { address } : {}) } });
+  }
+  const witnessStart = r.pos;
+  if (segwit) {
+    for (const input of vin) {
+      const items = r.varint();
+      const stack = [];
+      for (let k = 0; k < items; k++) stack.push(r.bytes(r.varint()).toString('hex'));
+      if (stack.length) input.txinwitness = stack;
+    }
+  }
+  const witnessEnd = r.pos;
+  const locktime = r.u32();
+  const end = r.pos;
+  const full = r.buf.subarray(start, end);
+  // the txid hashes the transaction WITHOUT marker, flag and witness
+  const base = segwit
+    ? Buffer.concat([r.buf.subarray(start, start + 4), r.buf.subarray(afterMarker, witnessStart), r.buf.subarray(witnessEnd, end)])
+    : full;
+  const size = full.length, weight = base.length * 3 + size;
+  return {
+    txid: rev(hash256(base)), hash: rev(hash256(full)),
+    version, size, vsize: Math.ceil(weight / 4), weight, locktime, vin, vout,
+  };
+}
+
+export function decodeTx(hexOrBuf, network = 'main') {
+  const buf = Buffer.isBuffer(hexOrBuf) ? hexOrBuf : Buffer.from(String(hexOrBuf), 'hex');
+  const r = new Reader(buf);
+  const tx = readTx(r, network);
+  if (r.pos !== buf.length) throw new RangeError(`${buf.length - r.pos} trailing bytes after the transaction`);
+  return tx;
+}
+
+// --- blocks --------------------------------------------------------------
+export function readHeader(r) {
+  const start = r.pos;
+  const version = r.i32();
+  const previousblockhash = rev(r.bytes(32));
+  const merkleroot = rev(r.bytes(32));
+  const time = r.u32(), bits = r.u32(), nonce = r.u32();
+  return {
+    hash: rev(hash256(r.buf.subarray(start, start + 80))),
+    version, previousblockhash, merkleroot, time, bits: bits.toString(16).padStart(8, '0'), nonce,
+  };
+}
+
+export function decodeBlock(hexOrBuf, network = 'main') {
+  const buf = Buffer.isBuffer(hexOrBuf) ? hexOrBuf : Buffer.from(String(hexOrBuf), 'hex');
+  const r = new Reader(buf);
+  const header = readHeader(r);
+  const n = r.varint();
+  const tx = [];
+  for (let i = 0; i < n; i++) tx.push(readTx(r, network));
+  if (r.pos !== buf.length) throw new RangeError(`${buf.length - r.pos} trailing bytes after the block`);
+  return { ...header, size: buf.length, nTx: n, tx };
+}

package/server/collect/gbt.js

@@ -0,0 +1,229 @@
+// The block being built, assembled HERE from the mempool the node already gave us.
+//
+// WHY THIS EXISTS. The next-block card used to come from `getblocktemplate`, which costs the
+// operator's node 1.3-1.5 s of its single RPC thread and 1.79 MB per call (nextblock.js measured
+// it on height 966265). That is a real tax on someone else's hardware, paid once a minute, for
+// data we can derive: mempool.space does not call it either -- its mempool-blocks.ts makes no
+// Core RPC call at all and builds templates from the mempool it already holds.
+//
+// WHAT MADE IT POSSIBLE, and a claim in nextblock.js that was true of the wrong node. That module
+// says `getrawmempool` verbose "returns vsize and fees.base with no depends and no ancestorcount",
+// so the dependency graph had to come from the template. That was measured against the
+// experimental build. Measured against Bitcoin Core on an Umbrel, 2026-09-13, 29,742 entries:
+//
+//   depends[]        present, 24,796 entries in a package
+//   ancestorcount    present, with ancestorsize and fees.ancestor
+//   descendantcount  present, with descendantsize and fees.descendant
+//   chunkweight      present, with fees.chunk  <- Core's own cluster linearization
+//
+// So Core publishes the whole graph, and more: `fees.chunk` over `chunkweight` is the feerate of
+// the chunk the cluster-mempool linearization put this transaction in, which is the number Core's
+// own miner sorts by. We do not have to reimplement the block assembler -- mempool.space wrote a
+// Rust engine for that because it targets nodes that publish less. We sort by the node's own
+// answer and take packages in order.
+//
+// THIS MODULE IS PURE. It takes a getrawmempool(true) reply and returns an object shaped like a
+// getblocktemplate reply, so summarizeTemplate, templateCells, packagesFromTemplate and
+// blockEconomy consume it unchanged. That is the whole seam: no consumer knows the difference.
+//
+// TWO CONVERSIONS THAT ARE EASY TO GET WRONG, and wrong silently:
+//   1. `depends` in getrawmempool are TXIDs. In getblocktemplate they are INDICES into the
+//      transactions array, and packagesFromTemplate walks them as indices. Passed through as
+//      strings they are not integers, every `Number.isInteger(d)` test fails, and the package
+//      analysis reports every transaction as a singleton -- a wrong answer that looks fine.
+//   2. mempool fees are in BTC; template fees are in SATOSHIS. A factor of 1e8 in a feerate is
+//      not a subtle graph.
+
+const VBYTES = 4;                        // weight units per virtual byte
+const SATS = 100_000_000;
+/** BTC (a JSON number) to whole satoshis. */
+const toSat = (btc) => (Number.isFinite(btc) ? Math.round(btc * SATS) : 0);
+
+/**
+ * The block subsidy at a height, in satoshis. Integer halving arithmetic -- 50e8 is exactly
+ * representable and >> would overflow past 31 bits, so this divides.
+ */
+export function subsidyAt(height, { interval = 210_000 } = {}) {
+  if (!Number.isFinite(height) || height < 0) return null;
+  const halvings = Math.floor(height / interval);
+  if (halvings >= 64) return 0;
+  let sat = 50 * SATS;
+  for (let i = 0; i < halvings; i++) sat = Math.floor(sat / 2);
+  return sat;
+}
+
+/**
+ * The feerate we sort by, in sat/vB.
+ *
+ * Core's cluster mempool has already worked out which transactions must travel together and in
+ * what order: `fees.chunk` over `chunkweight` is that answer. Where it is absent (an older node,
+ * or an entry that predates the index) the ancestor feerate is the next best thing, because it is
+ * the number that accounts for a child paying for its parents. The transaction's own feerate is
+ * the last resort and the one that gets CPFP wrong, so it is only used when nothing else is there.
+ */
+export function orderingRate(e) {
+  const chunkW = Number(e?.chunkweight);
+  const chunkF = Number(e?.fees?.chunk);
+  if (Number.isFinite(chunkW) && chunkW > 0 && Number.isFinite(chunkF)) {
+    return toSat(chunkF) / (chunkW / VBYTES);
+  }
+  const ancSize = Number(e?.ancestorsize);
+  const ancFee = Number(e?.fees?.ancestor);
+  if (Number.isFinite(ancSize) && ancSize > 0 && Number.isFinite(ancFee)) {
+    return toSat(ancFee) / ancSize;                 // ancestorsize is already vbytes
+  }
+  const vs = Number(e?.vsize);
+  const base = Number(e?.fees?.base);
+  if (Number.isFinite(vs) && vs > 0 && Number.isFinite(base)) return toSat(base) / vs;
+  return 0;
+}
+
+/**
+ * Every in-mempool ancestor of `txid`, nearest last, or null when the graph is broken.
+ *
+ * Depth-first over `depends`. A parent that is not in the map is a parent that has been mined or
+ * evicted between the node building the reply and us reading it; the child is then unselectable
+ * this round rather than selectable with a missing parent, because a template that contains a
+ * child without its parent is not a block anyone could mine.
+ */
+function ancestorsOf(txid, pool, cache) {
+  if (cache.has(txid)) return cache.get(txid);
+  const out = [];
+  const seen = new Set();
+  const walk = (id) => {
+    const e = pool.get(id);
+    if (!e) return false;
+    for (const p of e.depends ?? []) {
+      if (seen.has(p)) continue;
+      seen.add(p);
+      if (!pool.has(p)) return false;
+      if (!walk(p)) return false;
+      out.push(p);                                   // parents before children
+    }
+    return true;
+  };
+  const ok = walk(txid);
+  const res = ok ? out : null;
+  cache.set(txid, res);
+  return res;
+}
+
+/**
+ * Assemble the next block from a getrawmempool(true) reply.
+ *
+ * Greedy over the node's own ordering feerate, taking each transaction with its unselected
+ * ancestors as one indivisible package -- which is what makes a child carry its parent into the
+ * block rather than being stranded above it.
+ *
+ * `reserveWeight` is the coinbase transaction's room. Core reserves 4,000 weight units by default
+ * (-blockmaxweight leaves it), and a template that fills all 4,000,000 describes a block that
+ * could not be mined.
+ */
+export function templateFromMempool(raw, {
+  weightLimit = 4_000_000,
+  reserveWeight = 4_000,
+  height = null,
+  previousblockhash = null,
+  bits = null,
+  version = null,
+  at = Date.now(),
+  maxTx = 0,                       // 0 = no cap; a guard for pathological pools
+} = {}) {
+  const entries = raw && typeof raw === 'object' && !Array.isArray(raw) ? raw : {};
+  const pool = new Map();
+  for (const [txid, e] of Object.entries(entries)) {
+    if (!e || typeof e !== 'object') continue;
+    const weight = Number(e.weight);
+    if (!Number.isFinite(weight) || weight <= 0) continue;
+    pool.set(txid, {
+      txid,
+      weight,
+      vsize: Number.isFinite(Number(e.vsize)) ? Number(e.vsize) : Math.ceil(weight / VBYTES),
+      feeSat: toSat(Number(e.fees?.base)),
+      wtxid: typeof e.wtxid === 'string' ? e.wtxid : null,
+      depends: Array.isArray(e.depends) ? e.depends.filter((d) => typeof d === 'string') : [],
+      rate: orderingRate(e),
+      time: Number.isFinite(Number(e.time)) ? Number(e.time) : null,
+    });
+  }
+
+  const budget = Math.max(0, weightLimit - reserveWeight);
+  const order = [...pool.values()].sort((a, b) => b.rate - a.rate || a.txid.localeCompare(b.txid));
+  const cache = new Map();
+  const chosen = [];                 // txids, parents always before children
+  const picked = new Set();
+  let used = 0;
+  let truncated = false;
+
+  for (const tx of order) {
+    if (picked.has(tx.txid)) continue;
+    if (maxTx && chosen.length >= maxTx) { truncated = true; break; }
+    const anc = ancestorsOf(tx.txid, pool, cache);
+    if (anc == null) continue;                       // a parent left the pool: not mineable now
+    const pkg = [];
+    let w = 0;
+    for (const id of anc) {
+      if (picked.has(id)) continue;
+      pkg.push(id);
+      w += pool.get(id).weight;
+    }
+    pkg.push(tx.txid);
+    w += tx.weight;
+    if (used + w > budget) continue;                 // keep going: a smaller package may still fit
+    for (const id of pkg) { picked.add(id); chosen.push(id); }
+    used += w;
+  }
+
+  // TXIDS TO INDICES. packagesFromTemplate walks `depends` as positions in this same array, so
+  // the remap happens once, here, against the array we just built. A parent that was not selected
+  // cannot appear -- by construction it cannot happen, since a child is only taken with its
+  // ancestors, but a dropped entry would otherwise become index -1 and corrupt the walk.
+  const pos = new Map(chosen.map((id, i) => [id, i]));
+  const transactions = chosen.map((id) => {
+    const e = pool.get(id);
+    return {
+      txid: id,
+      hash: e.wtxid ?? id,
+      fee: e.feeSat,
+      weight: e.weight,
+      vsize: e.vsize,
+      depends: e.depends.map((d) => pos.get(d)).filter((i) => Number.isInteger(i)),
+    };
+  });
+
+  const feesSat = transactions.reduce((n, t) => n + t.fee, 0);
+  const sub = subsidyAt(height);
+
+  return {
+    // the getblocktemplate field names, so every existing consumer reads this unchanged
+    height,
+    previousblockhash,
+    weightlimit: weightLimit,
+    sizelimit: null,
+    sigoplimit: null,
+    coinbasevalue: sub == null ? null : sub + feesSat,
+    mintime: null,
+    bits,
+    version,
+    curtime: Math.floor(at / 1000),
+    transactions,
+    // ...and our own provenance, so nothing downstream has to guess where this came from
+    assembledLocally: true,
+    source: 'getrawmempool',
+    poolSize: pool.size,
+    selected: transactions.length,
+    weightUsed: used,
+    reserveWeight,
+    truncated,
+    subsidySat: sub,
+    feesSat,
+  };
+}
+
+/** The honest note that travels with locally-assembled templates. */
+export const LOCAL_TEMPLATE_NOTE = 'the block being built is assembled here from getrawmempool(true), '
+  + 'which the monitor already reads for the mempool view: transactions are taken in the order of '
+  + "the node's own cluster-mempool chunk feerate, each with its unselected ancestors, until the "
+  + 'weight budget is spent. It costs the node no extra call. It is this software\'s reconstruction '
+  + "of what a miner would choose, not the node's own getblocktemplate reply, so it can differ at "
+  + 'the margin -- sigop limits and policy the mempool does not publish are not modelled.';