blockyard 0.0.9 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/CHANGELOG.md +359 -1
  2. package/README.md +48 -27
  3. package/SECURITY.md +2 -2
  4. package/bin/blockyard.js +2 -1
  5. package/docs/API.md +17 -15
  6. package/docs/ARCHITECTURE.md +128 -10
  7. package/docs/CONFIGURATION.md +39 -30
  8. package/docs/DEFECTS.md +4 -1
  9. package/docs/GETTING-STARTED.md +18 -8
  10. package/docs/INSTALL.md +97 -37
  11. package/docs/MEASUREMENTS.md +147 -0
  12. package/docs/PLAN-SCORCHED-YARD.md +456 -0
  13. package/docs/PLAN-SKIES.md +142 -0
  14. package/docs/SECURITY-AUDIT-2026-09-16.md +647 -0
  15. package/docs/SECURITY.md +58 -22
  16. package/docs/TROUBLESHOOTING.md +44 -5
  17. package/docs/USER-GUIDE.md +505 -35
  18. package/package.json +4 -2
  19. package/public/404.html +1 -1
  20. package/public/css/app.css +393 -82
  21. package/public/donate-qr.png +0 -0
  22. package/public/index.html +353 -109
  23. package/public/js/agents.js +228 -51
  24. package/public/js/app.js +131 -16
  25. package/public/js/blockanoid.js +15 -7
  26. package/public/js/blockout.js +15 -7
  27. package/public/js/blockscene3d.js +230 -38
  28. package/public/js/charts.js +21 -21
  29. package/public/js/depthchart.js +31 -27
  30. package/public/js/details3d.js +1481 -73
  31. package/public/js/doom.js +31 -0
  32. package/public/js/dosaudio.js +48 -0
  33. package/public/js/dosgame.js +389 -0
  34. package/public/js/dosio.js +186 -0
  35. package/public/js/dospc.js +1353 -0
  36. package/public/js/dosworker.js +196 -0
  37. package/public/js/explorer.js +7 -1
  38. package/public/js/livingsky.js +494 -0
  39. package/public/js/login.js +8 -2
  40. package/public/js/markets.js +46 -8
  41. package/public/js/mining.js +314 -36
  42. package/public/js/panels.js +41 -28
  43. package/public/js/pricechart.js +14 -13
  44. package/public/js/quake.js +20 -0
  45. package/public/js/safenext.js +14 -0
  46. package/public/js/scorched.js +1051 -0
  47. package/public/js/scorchedai.js +227 -0
  48. package/public/js/scorchedair.js +286 -0
  49. package/public/js/scorchedfx.js +376 -0
  50. package/public/js/scorchedshop.js +105 -0
  51. package/public/js/scorchedwind.js +69 -0
  52. package/public/js/scorchedyard.js +1338 -0
  53. package/public/js/settings.js +368 -100
  54. package/public/js/soundcard.js +459 -0
  55. package/public/js/tetrust.js +15 -6
  56. package/public/js/tetsound.js +35 -5
  57. package/public/js/theme.js +235 -0
  58. package/public/js/wolf3d.js +22 -0
  59. package/public/js/x86.js +1978 -0
  60. package/scripts/check.js +46 -0
  61. package/scripts/donate-qr.py +12 -9
  62. package/scripts/dos-bench.js +56 -0
  63. package/scripts/index-build.js +9 -2
  64. package/scripts/pool-map.js +152 -36
  65. package/scripts/setup.js +142 -22
  66. package/scripts/shots.mjs +27 -0
  67. package/scripts/smoke.sh +7 -6
  68. package/scripts/tls.js +31 -0
  69. package/scripts/ui.js +4 -2
  70. package/server/auth/sessions.js +33 -13
  71. package/server/chain/blockfile.js +64 -5
  72. package/server/chain/index/build.js +451 -58
  73. package/server/chain/index/heights.js +29 -3
  74. package/server/chain/index/live.js +13 -7
  75. package/server/chain/index/rows.js +6 -1
  76. package/server/chain/index/store.js +28 -5
  77. package/server/chain/index/worker.js +23 -11
  78. package/server/collect/logparse.js +65 -18
  79. package/server/collect/markets.js +76 -7
  80. package/server/collect/mining.js +32 -0
  81. package/server/collect/monitor.js +54 -12
  82. package/server/collect/network.js +305 -0
  83. package/server/config.js +53 -22
  84. package/server/http/api.js +119 -18
  85. package/server/http/games.js +77 -0
  86. package/server/http/server.js +30 -5
  87. package/server/http/sse.js +53 -7
  88. package/server/main.js +66 -11
  89. package/server/rpc/allowlist.js +26 -0
  90. package/server/rpc/client.js +30 -2
  91. package/server/store/audit.js +6 -1
  92. package/server/store/history.js +19 -3
  93. package/server/store/ledger.js +15 -4
  94. package/server/tls/selfsigned.js +160 -0
  95. package/systemd/blockyard.service +41 -8
  96. package/docs/PRIVATE-LEADERBOARD.md +0 -230
  97. package/docs/STATE-2026-09-09.md +0 -200
@@ -5,15 +5,32 @@
5
5
  // blocks at one height -- which the build checks for, since it counts every height exactly once.
6
6
  // The LAST 16 hex digits of the displayed hash, not the first: a displayed hash begins with the
7
7
  // proof of work, so a recent block's first 16 digits are all zero and every one of them would collide.
8
+ //
9
+ // THE TABLE IS SIZED FROM THE TIP, AND NEVER LOOPS WHEN FULL (audit 2026-09-16, L7). The capacity was
10
+ // fixed at 2^21 and nothing checked the load, so the 2,097,153rd block -- mainnet reaches it around
11
+ // 2046 -- sent `set` round the table forever, on the server's main thread. Now a build sizes it at
12
+ // least twice the heights it will hold (HeightTable.forTip), `set` refuses past a load of 0.75, and
13
+ // both `set` and `get` give up after one lap of the table whatever its state.
14
+ export const MAX_LOAD = 0.75;
15
+
8
16
  export class HeightTable {
9
17
  constructor(capacity = 1 << 21) {
18
+ if (!Number.isSafeInteger(capacity) || capacity < 1 || capacity > 2 ** 30 || (capacity & (capacity - 1)) !== 0) throw new RangeError(`height table capacity must be a power of two, not ${capacity}`);
10
19
  this.capacity = capacity;
11
20
  this.buffer = new SharedArrayBuffer(capacity * 12);
21
+ this.count = 0;
12
22
  this._bind();
13
23
  }
24
+ /** A table for heights 0..tip: at least twice tip + 1 slots, a power of two, never under 1,024. */
25
+ static forTip(tip) {
26
+ if (!Number.isSafeInteger(tip) || tip < 0) throw new RangeError(`a height table needs a tip height, not ${tip}`);
27
+ let capacity = 1024;
28
+ while (capacity < 2 * (tip + 1)) capacity *= 2;
29
+ return new HeightTable(capacity);
30
+ }
14
31
  static attach(buffer, capacity) {
15
32
  const t = Object.create(HeightTable.prototype);
16
- t.capacity = capacity; t.buffer = buffer; t._bind();
33
+ t.capacity = capacity; t.buffer = buffer; t.count = null; t._bind();
17
34
  return t;
18
35
  }
19
36
  _bind() {
@@ -24,13 +41,22 @@ export class HeightTable {
24
41
  set(hashHex, height) {
25
42
  const k = HeightTable.#prefix(hashHex);
26
43
  let i = Number(k % BigInt(this.capacity));
27
- while (this.keys[i] !== 0n && this.keys[i] !== k) i = (i + 1) % this.capacity;
44
+ for (let probes = 0; this.keys[i] !== 0n && this.keys[i] !== k; probes++) {
45
+ if (probes >= this.capacity) throw new RangeError(`the height table is full (${this.capacity} slots)`);
46
+ i = (i + 1) % this.capacity;
47
+ }
48
+ if (this.keys[i] === 0n) {
49
+ // an attached table counts what it holds the first time it is written to
50
+ if (this.count == null) { let n = 0; for (let j = 0; j < this.capacity; j++) if (this.keys[j] !== 0n) n++; this.count = n; }
51
+ if (this.count + 1 > this.capacity * MAX_LOAD) throw new RangeError(`the height table is too small: ${this.count + 1} heights would load its ${this.capacity} slots past ${MAX_LOAD} (size it with HeightTable.forTip)`);
52
+ this.count++;
53
+ }
28
54
  this.keys[i] = k; this.values[i] = height;
29
55
  }
30
56
  get(hashHex) {
31
57
  const k = HeightTable.#prefix(hashHex);
32
58
  let i = Number(k % BigInt(this.capacity));
33
- while (this.keys[i] !== 0n) { if (this.keys[i] === k) return this.values[i]; i = (i + 1) % this.capacity; }
59
+ for (let probes = 0; probes < this.capacity && this.keys[i] !== 0n; probes++) { if (this.keys[i] === k) return this.values[i]; i = (i + 1) % this.capacity; }
34
60
  return -1;
35
61
  }
36
62
  }
@@ -23,7 +23,7 @@ import path from 'node:path';
23
23
  import { crc32 } from 'node:zlib';
24
24
  import { IndexStore } from './store.js';
25
25
  import { ROW, RowSink, verboseBlockRows } from './rows.js';
26
- import { BLOCK_ROWS } from './build.js';
26
+ import { BLOCK_ROWS, openTempFile } from './build.js';
27
27
 
28
28
  export const CONFIRMATIONS = 100;
29
29
  export const FOLD_BLOCKS = 144;
@@ -59,10 +59,11 @@ export function sortRows(buf, blockRows = BLOCK_ROWS) {
59
59
  return { rows: out.subarray(0, w), idx: Buffer.from(new BigUint64Array(sparse).buffer) };
60
60
  }
61
61
 
62
+ // The temporary name is unlinked and created afresh with O_EXCL, owner-only: a symlink planted at
63
+ // live.log.tmp or tips.json.tmp is never followed (audit 2026-09-16, L10)
62
64
  function writeAtomic(file, data) {
63
- const fd = openSync(file + '.tmp', 'w');
64
- for (let o = 0; o < data.length;) o += writeSync(fd, data, o, data.length - o);
65
- fsyncSync(fd); closeSync(fd);
65
+ const fd = openTempFile(file + '.tmp');
66
+ try { for (let o = 0; o < data.length;) o += writeSync(fd, data, o, data.length - o); fsyncSync(fd); } finally { closeSync(fd); }
66
67
  renameSync(file + '.tmp', file);
67
68
  }
68
69
 
@@ -81,7 +82,7 @@ export class LiveIndex {
81
82
  constructor(dir, { rpc, nodeId = null, log = null, confirmations = CONFIRMATIONS, foldBlocks = FOLD_BLOCKS, maxLayers = MAX_LAYERS, maxBlocksPerPoll = 50 } = {}) {
82
83
  this.dir = dir; this.rpc = rpc; this.nodeId = nodeId; this.log = log;
83
84
  this.confirmations = confirmations; this.foldBlocks = foldBlocks; this.maxLayers = maxLayers; this.maxBlocksPerPoll = maxBlocksPerPoll;
84
- this.store = new IndexStore(dir);
85
+ this.store = new IndexStore(dir, { log });
85
86
  this.blocks = new Map(); // height -> { hash, rows: Buffer }
86
87
  this.byKey = new Map(); // key hex (16) -> [Buffer, offset, Buffer, offset, ...]
87
88
  this.stale = null; // a reason, once the chain has left us behind
@@ -138,6 +139,11 @@ export class LiveIndex {
138
139
  if (pos + 9 + len + 4 > buf.length) break;
139
140
  const payload = buf.subarray(pos + 9, pos + 9 + len);
140
141
  if (crc32(payload) !== buf.readUInt32LE(pos + 9 + len)) break;
142
+ // A RECORD WHOSE CHECKSUM HOLDS BUT WHOSE SHAPE DOES NOT is treated like a torn one: the log is cut
143
+ // there (audit 2026-09-16, I4). A block record shorter than its height and hash, or with rows
144
+ // that are not whole 21-byte rows, threw from this constructor and the follower never started.
145
+ if (type === T_BLOCK && (len < 36 || (len - 36) % ROW !== 0)) break;
146
+ if (type === T_ROLLBACK && len < 4) break;
141
147
  if (type === T_BLOCK) {
142
148
  const height = payload.readUInt32LE(0), hash = payload.toString('hex', 4, 36);
143
149
  if (height === this.tip + 1) this.#add(height, hash, Buffer.from(payload.subarray(36)));
@@ -155,7 +161,7 @@ export class LiveIndex {
155
161
  }
156
162
 
157
163
  #append(rec) {
158
- const fd = openSync(this.logFile, 'a');
164
+ const fd = openSync(this.logFile, 'a', 0o600); // owner-only (audit 2026-09-16, L10)
159
165
  try { writeSync(fd, rec); fsyncSync(fd); } finally { closeSync(fd); }
160
166
  }
161
167
 
@@ -235,7 +241,7 @@ export class LiveIndex {
235
241
  const from = deep[0], to = deep[deep.length - 1];
236
242
  const { rows, idx } = sortRows(Buffer.concat(deep.map((h) => this.blocks.get(h).rows)), this.store.blockRows);
237
243
  const dir = path.join(this.dir, 'layers');
238
- mkdirSync(dir, { recursive: true });
244
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
239
245
  const base = path.join(dir, `L${from}-${to}`);
240
246
  // the top block's hash first, so a layer is never on disk without it
241
247
  let tips = {};
@@ -76,7 +76,7 @@ export function blockRows(body, undoBody, height, out) {
76
76
  const moved = new Map();
77
77
  for (let p = 0; p < ntx; p++) {
78
78
  moved.clear();
79
- walkTx(body, st, (value, script) => {
79
+ const nin = walkTx(body, st, (value, script) => {
80
80
  if (script.length > 0 && script[0] === 0x6a) return; // OP_RETURN: unspendable, not an address
81
81
  const k = scriptKey(script);
82
82
  moved.set(k, (moved.get(k) ?? 0) + value);
@@ -84,6 +84,11 @@ export function blockRows(body, undoBody, height, out) {
84
84
  if (p > 0) {
85
85
  const coins = undo[p - 1];
86
86
  if (!coins) throw new Error(`block ${height}: no undo for transaction ${p}`);
87
+ // ONE SPENT COIN PER INPUT, OR THIS IS NOT THE BLOCK'S UNDO (audit 2026-09-16, L8). Blocks and
88
+ // undo records are paired by hash256(prevhash || undo), which does not commit to the block, so
89
+ // two sibling blocks in one file can each match the other's record. The coin count was trusted,
90
+ // and a two-input transaction given a one-coin undo wrote rows with the wrong spent scripts.
91
+ if (coins.length !== nin) throw new Error(`block ${height}: transaction ${p} has ${nin} inputs and its undo record ${coins.length} spent coins -- a block paired with another block's undo`);
87
92
  for (const c of coins) {
88
93
  const k = scriptKey(c.script);
89
94
  moved.set(k, (moved.get(k) ?? 0) - c.value_sat);
@@ -14,7 +14,9 @@ import { ROW, scriptKey, readRow } from './rows.js';
14
14
  import { FORMAT } from './build.js';
15
15
 
16
16
  const LAYER = /^L(\d+)-(\d+)\.rows$/;
17
- const RING_MAX_BLIND = 4096; // rows (86 KB) a page may keep without first counting the history
17
+ const RING_MAX_BLIND = 4096;
18
+ // the most rows to a sparse-index block a manifest may name: it sizes a read buffer (4,096 is written)
19
+ export const MAX_BLOCK_ROWS = 1 << 16; // rows (86 KB) a page may keep without first counting the history
18
20
 
19
21
  // NO FILE IS HELD OPEN (2026-09-14): a store used to keep one descriptor per segment and layer --
20
22
  // 256 and more -- for the life of the process, which is the whole soft limit on a stock macOS
@@ -22,16 +24,24 @@ const RING_MAX_BLIND = 4096; // rows (86 KB) a page may keep without first cou
22
24
  // and closes it: three syscalls on a 0.25 ms lookup.
23
25
  function openSorted(rowsFile, idxFile) {
24
26
  const raw = readFileSync(idxFile);
27
+ // whole 8-byte keys and whole rows, or the source is not one this code wrote (audit 2026-09-16, I4)
28
+ if (raw.length % 8 !== 0) throw new Error(`${path.basename(idxFile)} is ${raw.length} bytes, not a whole number of 8-byte keys`);
29
+ const size = statSync(rowsFile).size;
30
+ if (size % ROW !== 0) throw new Error(`${path.basename(rowsFile)} is ${size} bytes, not a whole number of ${ROW}-byte rows`);
25
31
  const idx = new BigUint64Array(raw.buffer, raw.byteOffset, raw.length / 8).slice();
26
- return { idx, file: rowsFile, rows: statSync(rowsFile).size / ROW };
32
+ return { idx, file: rowsFile, rows: size / ROW };
27
33
  }
28
34
 
29
35
  export class IndexStore {
30
- constructor(dir) {
36
+ constructor(dir, { log = null } = {}) {
31
37
  this.dir = dir;
38
+ this.log = log;
32
39
  this.manifest = JSON.parse(readFileSync(path.join(dir, 'manifest.json'), 'utf8'));
33
40
  if (this.manifest.format !== FORMAT) throw new Error(`index format ${this.manifest.format}, this code reads ${FORMAT}`);
34
- this.blockRows = this.manifest.blockRows;
41
+ // blockRows sizes the read buffer below: checked before it allocates anything (audit 2026-09-16, I4)
42
+ const br = this.manifest.blockRows;
43
+ if (!Number.isSafeInteger(br) || br < 1 || br > MAX_BLOCK_ROWS) throw new Error(`index manifest names ${JSON.stringify(br)} rows to a block; this code reads 1 to ${MAX_BLOCK_ROWS}`);
44
+ this.blockRows = br;
35
45
  this.segments = Array.from({ length: 256 }, (_, b) => {
36
46
  const hex = b.toString(16).padStart(2, '0');
37
47
  try { return openSorted(path.join(dir, `seg-${hex}.rows`), path.join(dir, `seg-${hex}.idx`)); } catch { return null; }
@@ -53,8 +63,21 @@ export class IndexStore {
53
63
  const ranges = names.map((f) => { const [, from, to] = f.match(LAYER); return { f, from: Number(from), to: Number(to) }; });
54
64
  const live = ranges.filter((r) => !ranges.some((o) => o !== r && o.from <= r.from && o.to >= r.to && (o.to - o.from) > (r.to - r.from)));
55
65
  const idxOf = (f) => path.join(dir, f.replace(/\.rows$/, '.idx'));
66
+ // A LAYER THAT CANNOT BE READ IS SKIPPED, as a segment is (audit 2026-09-16, I4): an .idx of the
67
+ // wrong length threw from here, out of a fold, and out of the constructor. It is named in
68
+ // `badLayers` and said once, and a missing .idx (a fold interrupted between its files) stays quiet.
69
+ this.badLayers = [];
56
70
  this.layers = live.filter((r) => { try { readFileSync(idxOf(r.f), { flag: 'r' }); return true; } catch { return false; } })
57
- .map((r) => ({ from: r.from, to: r.to, ...openSorted(path.join(dir, r.f), idxOf(r.f)) }))
71
+ .flatMap((r) => {
72
+ try { return [{ from: r.from, to: r.to, ...openSorted(path.join(dir, r.f), idxOf(r.f)) }]; } catch (err) {
73
+ this.badLayers.push({ layer: r.f, error: err.message });
74
+ if (!(this.warned ??= new Set()).has(r.f)) {
75
+ this.warned.add(r.f);
76
+ this.log?.warn?.(`address index: skipping layer ${r.f}: ${err.message}`);
77
+ }
78
+ return [];
79
+ }
80
+ })
58
81
  .sort((a, b) => a.from - b.from);
59
82
  }
60
83
 
@@ -2,15 +2,19 @@
2
2
  //
3
3
  // scan one blk/rev file pair -> the file's index rows, partitioned by the key's first byte into
4
4
  // 256 buffers (a counting sort), so the main thread only appends each to its bucket file
5
- // sort one bucket file -> sorted, de-duplicated rows plus a sparse index, then the unsorted
6
- // input is removed
5
+ // sort one bucket file -> sorted, de-duplicated rows plus a sparse index, each written to a
6
+ // temporary name, fsynced and renamed. The input is first checked against the length and
7
+ // CRC-32 the scan journaled, and it is left in place: the main thread removes it once the
8
+ // build journal records the bucket sorted (build.js, THE BUILD JOURNAL)
7
9
  import { parentPort, workerData } from 'node:worker_threads';
8
- import { openSync, readSync, writeSync, closeSync, fstatSync, unlinkSync, renameSync } from 'node:fs';
10
+ import { openSync, readSync, writeSync, closeSync, fstatSync, fsyncSync, renameSync } from 'node:fs';
11
+ import { crc32 } from 'node:zlib';
9
12
  import path from 'node:path';
10
13
  import { Reader, readHeader } from '../tx.js';
11
14
  import { readChainFile, records, pairBlocksWithUndo, MAGIC } from '../blockfile.js';
12
15
  import { blockRows, RowSink, ROW } from './rows.js';
13
16
  import { HeightTable } from './heights.js';
17
+ import { openTempFile } from './build.js';
14
18
 
15
19
  const { blocksDir, key, heightsBuffer, heightsCapacity, blockRowsPerIndex } = workerData;
16
20
  const heights = HeightTable.attach(heightsBuffer, heightsCapacity);
@@ -26,7 +30,7 @@ function scan(file) {
26
30
  for (const r of records(blk, MAGIC.main, 0, xor)) {
27
31
  const rd = new Reader(r.body);
28
32
  const h = readHeader(rd);
29
- blocks.push({ body: r.body, hash: h.hash, previousblockhash: h.previousblockhash, ntx: rd.varint() });
33
+ blocks.push({ offset: r.offset, body: r.body, hash: h.hash, previousblockhash: h.previousblockhash, ntx: rd.varint() });
30
34
  }
31
35
  const pairs = pairBlocksWithUndo(blocks, [...records(rev, MAGIC.main, 32, xor)]);
32
36
  const sink = new RowSink(1 << 18);
@@ -37,7 +41,12 @@ function scan(file) {
37
41
  if (height < 0) { stale++; return; } // not on the chain this build covers
38
42
  const undo = pairs.get(i);
39
43
  if (!undo && height !== 0) { missingUndo++; return; }
40
- blockRows(b.body, undo ? undo.body : null, height, sink);
44
+ // A CORRUPT OR MISPAIRED RECORD FAILS THE FILE, AND SO THE BUILD -- fail-safe, and kept that way
45
+ // (audit 2026-09-16, I4) -- but the message names the files and offsets to look at
46
+ try { blockRows(b.body, undo ? undo.body : null, height, sink); } catch (err) {
47
+ err.message = `blk${id}.dat offset ${b.offset}${undo ? `, rev${id}.dat offset ${undo.offset}` : ''} (block ${height} ${b.hash}): ${err.message}`;
48
+ throw err;
49
+ }
41
50
  indexed.push(height);
42
51
  });
43
52
  // counting sort by the first key byte: 256 buffers the main thread appends as they are
@@ -57,7 +66,7 @@ function scan(file) {
57
66
  };
58
67
  }
59
68
 
60
- function sortBucket(bucket, dir) {
69
+ function sortBucket(bucket, dir, expectSize = null, expectCrc = null) {
61
70
  const t0 = performance.now();
62
71
  const name = (s) => path.join(dir, `bucket-${bucket.toString(16).padStart(2, '0')}${s}`);
63
72
  const fd = openSync(name('.unsorted'), 'r');
@@ -65,6 +74,9 @@ function sortBucket(bucket, dir) {
65
74
  const buf = Buffer.allocUnsafe(size);
66
75
  for (let got = 0; got < size;) { const k = readSync(fd, buf, got, size - got, got); if (!k) break; got += k; }
67
76
  closeSync(fd);
77
+ const hex = bucket.toString(16).padStart(2, '0');
78
+ if (expectSize != null && size !== expectSize) throw new Error(`bucket ${hex} does not hold what the scan wrote: ${size} bytes, and the scan wrote ${expectSize}`);
79
+ if (expectCrc != null && crc32(buf) !== expectCrc) throw new Error(`bucket ${hex} does not hold what the scan wrote: its CRC-32 is ${crc32(buf)}, and the scan's was ${expectCrc}`);
68
80
  const n = size / ROW;
69
81
  // order by the next 16 key bits into sub-buckets, then compare the rest numerically:
70
82
  // hi = key bytes 3..7 (40 bits), lo = height (24 bits) and position (16 bits)
@@ -92,16 +104,16 @@ function sortBucket(bucket, dir) {
92
104
  w += ROW;
93
105
  }
94
106
  const idx = Buffer.from(new BigUint64Array(sparse).buffer);
95
- const write = (file, data) => { const f = openSync(file + '.tmp', 'w'); for (let o = 0; o < data.length;) o += writeSync(f, data, o, data.length - o); closeSync(f); renameSync(file + '.tmp', file); };
96
- write(path.join(dir, `seg-${bucket.toString(16).padStart(2, '0')}.rows`), out.subarray(0, w));
97
- write(path.join(dir, `seg-${bucket.toString(16).padStart(2, '0')}.idx`), idx);
98
- unlinkSync(name('.unsorted'));
107
+ // a temporary name is unlinked and created afresh, owner-only: never a planted symlink followed (audit 2026-09-16, L10)
108
+ const write = (file, data) => { const f = openTempFile(file + '.tmp'); try { for (let o = 0; o < data.length;) o += writeSync(f, data, o, data.length - o); fsyncSync(f); } finally { closeSync(f); } renameSync(file + '.tmp', file); };
109
+ write(path.join(dir, `seg-${hex}.rows`), out.subarray(0, w));
110
+ write(path.join(dir, `seg-${hex}.idx`), idx);
99
111
  return { msg: { type: 'sorted', bucket, rows: w / ROW, dupes, ms: performance.now() - t0 }, transfer: [] };
100
112
  }
101
113
 
102
114
  parentPort.on('message', (job) => {
103
115
  try {
104
- const r = job.type === 'scan' ? scan(job.file) : sortBucket(job.bucket, job.dir);
116
+ const r = job.type === 'scan' ? scan(job.file) : sortBucket(job.bucket, job.dir, job.size, job.crc);
105
117
  parentPort.postMessage(r.msg, r.transfer);
106
118
  } catch (err) {
107
119
  parentPort.postMessage({ type: 'error', job, message: err.stack || err.message });
@@ -17,12 +17,14 @@
17
17
  const TS_RE = /^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\.(\d{3}) /;
18
18
  const TAG_RE = /^\[[a-z0-9_]+(?::\d+)?\]\s*/;
19
19
 
20
- // "1.0KB", "32.0 KB/s", "0.0B", "2.3MB", "128 KB". The node prints decimal
20
+ // "1.0KB", "32.0 KB/s", "0.0B", "2.3MB", "128 KB". (2026-09-16, audit L6: the number is
21
+ // `\d*\.\d+|\d+`, the same numbers as the old `[0-9]*\.?[0-9]+` without the two digit
22
+ // runs that could split a long string of digits every possible way.) The node prints decimal
21
23
  // units (4096 bytes renders as "4.0KB"), so decode the same way back.
22
24
  const SIZE_UNITS = { B: 1, KB: 1e3, MB: 1e6, GB: 1e9, TB: 1e12, PB: 1e15 };
23
25
  export function parseSize(text) {
24
26
  if (text == null) return null;
25
- const m = String(text).trim().match(/^([0-9]*\.?[0-9]+)\s*(B|KB|MB|GB|TB|PB)(?:\/s)?$/i);
27
+ const m = String(text).trim().match(/^(\d*\.\d+|\d+)\s*(B|KB|MB|GB|TB|PB)(?:\/s)?$/i);
26
28
  if (!m) return null;
27
29
  const unit = m[2].toUpperCase();
28
30
  return Math.round(parseFloat(m[1]) * SIZE_UNITS[unit]);
@@ -30,7 +32,7 @@ export function parseSize(text) {
30
32
 
31
33
  export function parseRate(text) {
32
34
  if (text == null) return null;
33
- const m = String(text).trim().match(/^([0-9]*\.?[0-9]+)\s*(B|KB|MB|GB|TB)\/s$/i);
35
+ const m = String(text).trim().match(/^(\d*\.\d+|\d+)\s*(B|KB|MB|GB|TB)\/s$/i);
34
36
  if (!m) return null;
35
37
  return parseFloat(m[1]) * SIZE_UNITS[m[2].toUpperCase()];
36
38
  }
@@ -64,6 +66,11 @@ function addrParts(s) {
64
66
  return { host: s, port: null, addr: s };
65
67
  }
66
68
 
69
+ // 2026-09-16 (audit L6): a `\s*` in front of a capture such as `([^,]+)` that can itself
70
+ // start with a space lets the engine split one run of spaces between the two in every
71
+ // possible way -- quadratic on a long run, and each rule pays it on every line. Those
72
+ // captures now start with a character `\s` cannot match (`[^\s,][^,]*`), which matches
73
+ // the same real lines and leaves only one way to read the spaces.
67
74
  const RULES = [
68
75
  // [dlc] -- network recv this tick: 4.0KB (405.0B/s) | total recv: 2.3MB || disk write this tick: 0.0B (0.0B/s) | total written: 1.9MB --
69
76
  {
@@ -90,7 +97,7 @@ const RULES = [
90
97
  // [dlc] -- dead-weight floor this tick: 32.0 KB/s (pool median 0.0 KB/s, absolute 32.0 KB/s) --
91
98
  {
92
99
  name: 'deadweight',
93
- re: /\[dlc\]\s*--\s*dead-weight floor this tick:\s*([^\s]+(?:\s?[KMG]?B\/s)?)\s*\(pool median\s*([^,]+),\s*absolute\s*([^)]+)\)/,
100
+ re: /\[dlc\]\s*--\s*dead-weight floor this tick:\s*([^\s]+(?:\s?[KMG]?B\/s)?)\s*\(pool median\s*([^\s,][^,]*),\s*absolute\s*([^\s)][^)]*)\)/,
94
101
  apply(m) {
95
102
  return { kind: 'deadweight', floor: parseRate(m[1].trim()), poolMedian: parseRate(m[2].trim()), absolute: parseRate(m[3].trim()) };
96
103
  },
@@ -100,7 +107,7 @@ const RULES = [
100
107
  // [dlc] ranked 116 live peer(s) by a 2000-header sample in 44.6s: 39 answered, best 94 KB/s, median 63 KB/s, slowest answering 48 KB/s; the 77 silent rank last
101
108
  {
102
109
  name: 'ranking',
103
- re: /\[dlc\]\s*ranked\s*(\d+)\s*live peer\(s\)[^.]*in\s*([\d.]+)s:\s*(\d+)\s*answered,\s*best\s*([^,]+),\s*median\s*([^,]+),\s*slowest answering\s*([^;]+);\s*the\s*(\d+)\s*silent/,
110
+ re: /\[dlc\]\s*ranked\s*(\d+)\s*live peer\(s\)[^.]*in\s*([\d.]+)s:\s*(\d+)\s*answered,\s*best\s*([^\s,][^,]*),\s*median\s*([^\s,][^,]*),\s*slowest answering\s*([^\s;][^;]*);\s*the\s*(\d+)\s*silent/,
104
111
  apply(m) {
105
112
  return {
106
113
  kind: 'peer_ranking', live: +m[1], sampleSecs: +m[2], answered: +m[3],
@@ -131,7 +138,7 @@ const RULES = [
131
138
  // [block] stored height=965923 hash=0000000000000000.. bytes=1464177 tx=6866 (via 193.223.81.8:8333)
132
139
  {
133
140
  name: 'blockStored',
134
- re: /\[block\]\s*stored\s+height=(\d+)\s+hash=([0-9a-f.]+)\s+bytes=(\d+)\s+tx=(\d+)(?:\s*\(via\s*([^\)]+)\))?/,
141
+ re: /\[block\]\s*stored\s+height=(\d+)\s+hash=([0-9a-f.]+)\s+bytes=(\d+)\s+tx=(\d+)(?:\s*\(via\s*([^\s)][^)]*)\))?/,
135
142
  apply(m) {
136
143
  const via = addrParts(m[5]);
137
144
  return { kind: 'block_stored', height: +m[1], hashPrefix: m[2].replace(/\.$/, ''), bytes: +m[3], txs: +m[4], via: via?.addr ?? null, viaHost: via?.host ?? null };
@@ -198,7 +205,10 @@ const RULES = [
198
205
  // [mux:7] leg replaced: connected next pool peer 208.161.116.211:8333 (fd 266) addrv2=1
199
206
  { name: 'legReplaced', re: /\[mux:(\d+)\]\s*leg replaced:\s*connected next pool peer\s*(\S+)\s*\(fd\s*(\d+)\)\s*addrv2=(\d)/, apply: (m) => ({ kind: 'peer_connect', leg: +m[1], addr: m[2], host: addrParts(m[2])?.host, fd: +m[3], addrv2: m[4] === '1', reason: 'leg replaced' }) },
200
207
  // [mux:9] next peer 86.147.78.44:8333 unreachable: connect: Operation now in progress (leg stays down)
201
- { name: 'legDown', re: /\[mux:(\d+)\]\s*next peer\s*(\S+)\s+unreachable:\s*(.+?)\s*\(leg stays down\)/, apply: (m) => ({ kind: 'peer_unreachable', leg: +m[1], addr: m[2], host: addrParts(m[2])?.host, reason: m[3].trim() }) },
208
+ // 2026-09-16 (audit L6): was `unreachable:\s*(.+?)\s*\(leg stays down\)`, which is cubic
209
+ // on a long run of spaces -- 20,000 of them did not finish in 300 s. The lazy capture
210
+ // now meets a literal straight away, and the apply trims what the two `\s*` used to.
211
+ { name: 'legDown', re: /\[mux:(\d+)\]\s*next peer\s*(\S+)\s+unreachable:(.+?)\(leg stays down\)/, apply: (m) => ({ kind: 'peer_unreachable', leg: +m[1], addr: m[2], host: addrParts(m[2])?.host, reason: m[3].trim() }) },
202
212
  // [dl:7] 209.38.162.73:8333 connection dropped (revents 0x11); re-dialing
203
213
  { name: 'legDropped', re: /\[dl:(\d+)\]\s*(\S+?)\s+connection dropped\s*\(revents\s*(\S+?)\)(?:;\s*(\S+))?/, apply: (m) => ({ kind: 'peer_drop', leg: +m[1], addr: m[2], host: addrParts(m[2])?.host, revents: m[3], follow: m[4] || null }) },
204
214
  // [net] feeler 47.232.103.88:8333 -> dead
@@ -217,7 +227,7 @@ const RULES = [
217
227
  },
218
228
  },
219
229
  // [mempool] recent-rejects filter: 128 KB shared (...)
220
- { name: 'rejectFilter', re: /\[mempool\]\s*recent-rejects filter:\s*([^\s]+)\s*(?:KB|MB|B)?\s+shared/, apply: (m) => ({ kind: 'reject_filter', size: parseSize(m[1] + (/\s?(KB|MB|B)$/i.test(m[1]) ? '' : 'KB')) }) },
230
+ { name: 'rejectFilter', re: /\[mempool\]\s*recent-rejects filter:\s*([^\s]+)(?:\s*(?:KB|MB|B))?\s+shared/, apply: (m) => ({ kind: 'reject_filter', size: parseSize(m[1] + (/\s?(KB|MB|B)$/i.test(m[1]) ? '' : 'KB')) }) },
221
231
 
222
232
  // ---- the 2026-09-08 bench build (v0.0.1, built 03:02) rewrote these lines ----
223
233
  // Measured the same day against that node's own log: of 1,702 lines the rules
@@ -237,16 +247,21 @@ const RULES = [
237
247
  // unit to fill a gap.
238
248
  {
239
249
  name: 'bandwidthTick',
240
- re: /\[dlc\]\s*--\s*recv\s*([^\s(]+)\s*\(avg\s*([^)]+)\)\s*\|\s*write\s*([^\s(]+)\s*\(avg\s*([^)]+)\)\s*\|\s*floor\s*([^()]+?)\s*\(median\s*([^)]+)\)\s*\|\s*banned\s*(\d+)\/(\d+)(?:\s*\|\s*(events.*?))?\s*--/,
250
+ // 2026-09-16 (audit L6): every `\s*` in front of a capture that could also match
251
+ // spaces now hands over to a first character that cannot (`[^\s)]`), and no lazy
252
+ // capture is followed by `\s*` -- the captures keep their trailing spaces and the
253
+ // apply trims them. The old form retried the same run of spaces from both sides and
254
+ // went quadratic on a long one.
255
+ re: /\[dlc\]\s*--\s*recv\s*([^\s(]+)\s*\(avg\s*([^\s)][^)]*)\)\s*\|\s*write\s*([^\s(]+)\s*\(avg\s*([^\s)][^)]*)\)\s*\|\s*floor\s*([^\s()][^()]*?)\(median\s*([^\s)][^)]*)\)\s*\|\s*banned\s*(\d+)\/(\d+)(?:\s*\|\s*(events.*?)|\s*)--/,
241
256
  apply(m) {
242
257
  const counters = {};
243
258
  for (const [, k, v] of (m[9] || '').matchAll(/\b(events|rot|wait|help|fail)\s+(\d+)/g)) counters[k] = +v;
244
259
  return {
245
260
  kind: 'bandwidth',
246
261
  netRate: parseRate(m[1]),
247
- avgNetRate: parseRate(m[2]),
262
+ avgNetRate: parseRate(m[2].trim()),
248
263
  diskRate: parseRate(m[3]),
249
- avgDiskRate: parseRate(m[4]),
264
+ avgDiskRate: parseRate(m[4].trim()),
250
265
  floor: parseRate((m[5] || '').trim()),
251
266
  poolMedianText: (m[6] || '').trim() || null,
252
267
  banned: +m[7],
@@ -274,7 +289,10 @@ const RULES = [
274
289
  // floor this tick:`, `average since start:`) stay with their own rules.
275
290
  {
276
291
  name: 'bandwidthTickFields',
277
- re: /\[dlc\]\s*--\s*(.+?)\s*--/,
292
+ // 2026-09-16 (audit L6): `\s*(.+?)\s*--` retried every space run from both sides
293
+ // (quadratic). The capture now starts on a non-space and runs to the first `--`; its
294
+ // trailing spaces are trimmed with each segment below, so the fields are the same.
295
+ re: /\[dlc\]\s*--\s*(\S.*?)--/,
278
296
  apply(m) {
279
297
  const out = { kind: 'bandwidth', extraFields: [], extraValues: null };
280
298
  let claims = 0;
@@ -331,7 +349,8 @@ const RULES = [
331
349
  // the stored/applied figures survive whatever the node does to the prose.
332
350
  {
333
351
  name: 'dlcProgressFields',
334
- re: /\[dlc\]\s*==\s*(.+?)\s*==/,
352
+ // 2026-09-16 (audit L6): same rewrite as bandwidthTickFields, same reason.
353
+ re: /\[dlc\]\s*==\s*(\S.*?)==/,
335
354
  apply(m) {
336
355
  const out = { kind: 'dlc_progress', extraFields: [] };
337
356
  let claims = 0;
@@ -429,7 +448,7 @@ const RULES = [
429
448
  // re-windowed, and that is decoded as negative rather than dropped.
430
449
  {
431
450
  name: 'dlcWorkerPeer',
432
- re: /\[dlc\]\s*w(\d+)\s+(\S+?)\s+chunks=(\d+)\s+blocks=(\d+)\s*\(\+(-?\d+) blk\/s,\s*([^)]+)\)(?:\s*[\[(]([^\])]*?)[\])])?/,
451
+ re: /\[dlc\]\s*w(\d+)\s+(\S+?)\s+chunks=(\d+)\s+blocks=(\d+)\s*\(\+(-?\d+) blk\/s,\s*([^\s)][^)]*)\)(?:\s*[\[(]([^\])]*?)[\])])?/,
433
452
  apply(m) {
434
453
  const a = addrParts(m[2]);
435
454
  const note = (m[7] || '').trim() || null;
@@ -475,7 +494,11 @@ const RULES = [
475
494
  // Again: stored beside the others, not averaged with them.
476
495
  {
477
496
  name: 'catchupProgress',
478
- re: /\[utxo_live\]\s*catchup progress:\s*height=(\d+)\/(\d+)\s*\((\d+\.?\d*)%\)\s*([\d.]+)\s*blk\/s\s*\(avg\s*([\d.]+)\)\s*eta\s*(\S+)\s*\|\s*(.*?)\s*\(([\d.]+)\s*ms\/blk over (\d+)\)/,
497
+ // 2026-09-16 (audit L6): the phase text was `\|\s*(.*?)\s*\(`, quadratic on spaces;
498
+ // it is now everything between the bar and the parenthesis. Only the `name N%` pairs
499
+ // are read out of it, so the spaces it now keeps change nothing. The eta stops at a
500
+ // bar, so a run of bars is not re-scanned once for every bar in it.
501
+ re: /\[utxo_live\]\s*catchup progress:\s*height=(\d+)\/(\d+)\s*\((\d+\.?\d*)%\)\s*([\d.]+)\s*blk\/s\s*\(avg\s*([\d.]+)\)\s*eta\s*([^\s|]+)\s*\|(.*?)\(([\d.]+)\s*ms\/blk over (\d+)\)/,
479
502
  apply(m) {
480
503
  // Groups: 1 height 2 of 3 pct 4 blk/s 5 avg 6 eta 7 phase text 8 ms/blk 9 samples.
481
504
  // An earlier cut of this apply() read 8/9/10 and returned msPerBlk 155 for a
@@ -496,7 +519,7 @@ const RULES = [
496
519
  // 12 input run(s) unlinked; apply never waited
497
520
  {
498
521
  name: 'utxoCompaction',
499
- re: /\[utxo_live\]\s*compaction done in\s*([\d.]+)s\s*\((\d+) run\(s\)\s*\[[^\]]*\)\s*of\s*(\d+)[^;]*;\s*started at height\s*(\d+)\)?:\s*manifest_n\s*(\d+)\s*->\s*(\d+),\s*merged into run\s*(\d+),\s*(\d+) flushed meanwhile,\s*(\d+) input run\(s\) unlinked;\s*apply\s*(never waited|waited[^;,]*)/,
522
+ re: /\[utxo_live\]\s*compaction done in\s*([\d.]+)s\s*\((\d+) run\(s\)\s*\[[^\]]*\)\s*of\s*(\d+)(?:[^;\d][^;]*)?;\s*started at height\s*(\d+)\)?:\s*manifest_n\s*(\d+)\s*->\s*(\d+),\s*merged into run\s*(\d+),\s*(\d+) flushed meanwhile,\s*(\d+) input run\(s\) unlinked;\s*apply\s*(never waited|waited[^;,]*)/,
500
523
  apply(m) {
501
524
  const waited = m[10] !== 'never waited';
502
525
  return {
@@ -575,7 +598,7 @@ const RULES = [
575
598
  // median 59 s, p95 393 s apart on production.
576
599
  {
577
600
  name: 'dialTopUpFails',
578
- re: /\[dl\]\s*outbound top-up:\s*(\d+)\s*dial\(s\) failed(?:,\s*first\s+(\S+):\s*(.+?))?\s*$/,
601
+ re: /\[dl\]\s*outbound top-up:\s*(\d+)\s*dial\(s\) failed(?:,\s*first\s+(\S+):(.+))?\s*$/,
579
602
  apply(m) {
580
603
  const a = m[2] ? addrParts(m[2]) : null;
581
604
  return {
@@ -625,7 +648,9 @@ const RULES = [
625
648
  // event list, which is what it is for.
626
649
  {
627
650
  name: 'dialBackgroundFail',
628
- re: /\[dial\]\s*(\S+?):\s*background dial failed:\s*(.+?)\s*$/,
651
+ // 2026-09-16 (audit L6): `:\s*(.+?)\s*$` became `:(.+)$` (here and in dialTopUpFails):
652
+ // the same text once trimmed, and one pass to the end instead of one per space.
653
+ re: /\[dial\]\s*(\S+?):\s*background dial failed:(.+)$/,
629
654
  apply(m) {
630
655
  const a = addrParts(m[1]);
631
656
  return { kind: 'peer_reject', direction: 'outbound', addr: a?.addr ?? m[1], host: a?.host ?? m[1], reason: m[2].trim(), severity: 'warn' };
@@ -708,9 +733,19 @@ function localFromLogTs(dateTime, ms) {
708
733
  return new Date(+y, +mo - 1, +d, +h, +mi, +s, +ms).getTime();
709
734
  }
710
735
 
736
+ // LONG LINES (2026-09-16, audit L6). Every rule runs unanchored on the server's main
737
+ // thread, so a line's length is a cost every rule pays. No line this node prints comes
738
+ // near 8 KB (the longest in the fixtures is a few hundred bytes); anything past that is
739
+ // cut before matching, and the event says so (`truncated`: the characters dropped).
740
+ // Together with the rewritten patterns above, a 20,000-space line parses in
741
+ // a few milliseconds where `legDown` alone used to not finish in 300 s.
742
+ export const MAX_LINE = 8192;
743
+
711
744
  // One line in, one event out (or null for a continuation/blank line).
712
745
  export function parseLine(line) {
713
746
  if (!line) return null;
747
+ let cut = 0;
748
+ if (line.length > MAX_LINE) { cut = line.length - MAX_LINE; line = line.slice(0, MAX_LINE); }
714
749
  const trimmed = line.replace(/\r$/, '');
715
750
  if (!trimmed.trim()) return null;
716
751
 
@@ -738,6 +773,7 @@ export function parseLine(line) {
738
773
  out.text = rest.trim();
739
774
  out.rule = rule.name;
740
775
  out.severity = out.severity ?? classify(tagBase, rest);
776
+ if (cut) out.truncated = cut;
741
777
  return out;
742
778
  }
743
779
  }
@@ -751,14 +787,25 @@ export function parseLine(line) {
751
787
  text: rest.trim(),
752
788
  severity: classify(tagBase, rest),
753
789
  rule: null,
790
+ ...(cut ? { truncated: cut } : {}),
754
791
  };
755
792
  }
756
793
 
757
794
  // Lines arrive in chunks; keep the trailing partial line for the next round.
795
+ //
796
+ // 2026-09-16 (audit L6): the partial line used to grow without limit, so a log that
797
+ // never printed a newline was held whole in memory and handed to every rule at once.
798
+ // `carry` now keeps at most MAX_LINE characters -- the front of the line, where the
799
+ // timestamp and tag are -- and parseLine would cut it there anyway. What is dropped is
800
+ // counted in `state.carryDropped`, so a mangled log shows up as a number, not silence.
758
801
  export function splitLines(buf, state = { carry: '' }) {
759
802
  const text = state.carry + buf;
760
803
  const parts = text.split('\n');
761
804
  state.carry = parts.pop() ?? '';
805
+ if (state.carry.length > MAX_LINE) {
806
+ state.carryDropped = (state.carryDropped ?? 0) + state.carry.length - MAX_LINE;
807
+ state.carry = state.carry.slice(0, MAX_LINE);
808
+ }
762
809
  const out = [];
763
810
  for (const p of parts) { const e = parseLine(p); if (e) out.push(e); }
764
811
  return out;