fux-engine 2.0.0-alpha.7 → 2.0.0

package/src/refer/chunk.mjs

@@ -1,263 +0,0 @@
-/** Passages, derived from the document's own heading DEPTH.
- *  Twin of `src/fux/refer/_chunk.py`.
- *
- * **There is no strategy parameter and no CHUNK to declare.** What a passage
- * is — a table row, a slide, a prose section, the whole file — falls out of
- * heading depth, so a caller cannot get it wrong because there is nothing to
- * get wrong.
- *
- * Deterministic and TOTAL: every byte lands in exactly one passage, with one
- * documented exception — a banded table repeats its header into every band,
- * because a row whose columns have no names is a citation nobody can read.
- *
- * ⚠ **Transient. Never written.** L2: chunks are re-derived from fetched bytes
- * at answer time and stored nowhere.
- */
-import { headings } from "../decode/markdown.mjs";
-
-export const MIN_PASSAGE_BYTES = 120;
-export const MAX_PASSAGE_BYTES = 4000;
-/** Measured 2026-09-06: row-per-passage took hit@1 from 0.229 to 0.875. */
-export const TABLE_ROWS_PER_PASSAGE = 1;
-
-const TABLE_ROW_RE = /^\s*\|/;
-const TABLE_SEP_RE = /^\s*\|[\s:|-]+\|?\s*$/;
-
-const enc = new TextEncoder();
-const nbytes = (s) => enc.encode(s).length;
-const countNl = (s) => { let n = 0; for (let i = 0; i < s.length; i++) if (s[i] === "\n") n++; return n; };
-
-/** `[heading, level, text, lineStart, lineEnd]`, 1-based and inclusive.
- *  Line numbers are tracked HERE rather than recovered later, because the text
- *  is stripped and the offset would be gone with the blank lines. */
-function sections(content) {
-  // `split("\n")`, never a line-splitter that also breaks on \r \v \f U+2028:
-  // the heading grammar splits the same way, so its line numbers index this
-  // list exactly, and any other splitter desynchronises the two.
-  const lines = content.split("\n");
-  const starts = new Map();
-  for (const h of headings(content)) starts.set(h.lineno, h);
-
-  // Level 0 is the preamble — not a heading of level 0, the ABSENCE of one,
-  // which is right because nothing can be nested inside it.
-  const acc = [["", 0, [], 1]];
-  for (let i = 0; i < lines.length; i++) {
-    const found = starts.get(i + 1);
-    if (found !== undefined) acc.push([found.text, found.level, [lines[i]], i + 1]);
-    else acc[acc.length - 1][2].push(lines[i]);
-  }
-
-  const out = [];
-  for (const [heading, level, ls, start] of acc) {
-    const joined = ls.join("\n");
-    if (!joined.trim()) continue;
-    const leading = ls.length - joined.replace(/^\n+/, "").split("\n").length;
-    const text = joined.replace(/^\n+/, "").replace(/\n+$/, "");
-    const realStart = start + Math.max(0, leading);
-    out.push([heading, level, text, realStart, realStart + countNl(text)]);
-  }
-  return out;
-}
-
-/** Index of the document TITLE section, or -1. Three structural conditions:
- *  first headed section, strictly shallower than every other, and NO BODY of
- *  its own. The third does the work — `# deck.pptx` above a run of slides is a
- *  name for the file; `## Notes` above `### Detail` is a real section. */
-function titleIndex(secs) {
-  const headed = [];
-  for (let i = 0; i < secs.length; i++) if (secs[i][0]) headed.push(i);
-  if (headed.length < 2) return -1;
-  const first = headed[0];
-  const [, level, text] = secs[first];
-  if (text.trim() !== text.trim().split("\n")[0].trim()) return -1;  // has a body
-  return headed.slice(1).every((i) => secs[i][1] > level) ? first : -1;
-}
-
-/** Fold a short section forward **only into a section NESTED INSIDE it**.
- *
- * 🔴 Depth, not size. The old rule folded on size alone, so a short slide
- * between two long ones was cited under its NEIGHBOUR's heading —
- * systematically the wrong attribution, which is worse than a coarse citation
- * because it is confidently wrong. Siblings never fold, so a lone short
- * sibling cannot be absorbed.
- *
- * ⚠ The PARENT's heading survives the fold, with one exception: the document
- * TITLE carries its text but never its name, or `# deck.pptx` ends up cited as
- * the source of slide 1's content. */
-function fold(secs, minPassageBytes = MIN_PASSAGE_BYTES) {
-  const out = [];
-  let carry = [], carryHeading = "", carryLevel = 0, carryStart = 0;
-  const title = titleIndex(secs);
-  for (let index = 0; index < secs.length; index++) {
-    const [heading, level, text, start, end] = secs[index];
-    const nxt = index + 1 < secs.length ? secs[index + 1] : null;
-    const short = nbytes(text) < minPassageBytes;
-    // Strictly deeper: `>` and never `>=` — `>=` is the sibling case, and the
-    // sibling case is the defect.
-    const nested = nxt !== null && nxt[1] > level;
-    if (short && nested) {
-      if (!carry.length) carryStart = start;
-      if (!carryHeading && index !== title) { carryHeading = heading; carryLevel = level; }
-      carry.push(text);
-      continue;
-    }
-    if (carry.length) {
-      out.push([carryHeading || heading, carryLevel || level,
-                [...carry, text].join("\n\n"), carryStart, end]);
-      carry = []; carryHeading = ""; carryLevel = 0;
-    } else {
-      out.push([heading, level, text, start, end]);
-    }
-  }
-  if (carry.length) {
-    // Unreachable — a section only carries when a DEEPER one follows it. Kept
-    // total rather than asserted: a chunker that raises loses the document.
-    out.push([carryHeading, carryLevel, carry.join("\n\n"), carryStart || 1, secs[secs.length - 1][4]]);
-  }
-  return out;
-}
-
-/** A Markdown table split into row passages, or `null` if not a table.
- *  The header and its separator are repeated into every band. */
-function tableBands(paragraph) {
-  const lines = paragraph.split("\n");
-  if (lines.length < 3) return null;
-  for (const line of lines) if (line.trim() && !TABLE_ROW_RE.test(line)) return null;
-
-  const header = lines[0];
-  const separator = TABLE_SEP_RE.test(lines[1]) ? lines[1] : null;
-  const prefix = separator ? [header, separator] : [header];
-  const body = lines.slice(prefix.length);
-  if (!body.length) return null;
-
-  const bands = [];
-  for (let i = 0; i < body.length; i += TABLE_ROWS_PER_PASSAGE) {
-    const rows = body.slice(i, i + TABLE_ROWS_PER_PASSAGE);
-    bands.push([[...prefix, ...rows].join("\n"), rows.length]);
-  }
-  if (bands.length < 2) return null;   // nothing gained; leave it ordinary
-  bands[0] = [bands[0][0], bands[0][1] + prefix.length];
-  return bands;
-}
-
-/** One oversized paragraph -> `[piece, sourceLines, rung]`.
- *
- * **There is no sentence rung.** UAX #29 says plain text gives inadequate
- * information for sentence boundaries; doing it properly needs CLDR locale
- * data — a dependency (L1) — and doing it improperly cuts inside `e.g.`,
- * `Dr.` and `3.5`. So: paragraph -> line -> word, boundaries that need no
- * knowledge of any language. */
-function descend(paragraph, maxPassageBytes) {
-  const lines = paragraph.split("\n");
-  if (lines.length > 1) {
-    const out = [];
-    let current = [], size = 0;
-    for (const line of lines) {
-      const lineSize = nbytes(line) + 1;
-      if (current.length && size + lineSize > maxPassageBytes) {
-        out.push([current.join("\n"), current.length, "line"]);
-        current = []; size = 0;
-      }
-      current.push(line); size += lineSize;
-    }
-    if (current.length) out.push([current.join("\n"), current.length, "line"]);
-    if (out.length > 1) return out;
-    // One line's worth after all — fall through to the word rung rather than
-    // returning the same oversized piece under a different name.
-    paragraph = out.length ? out[0][0] : paragraph;
-  }
-
-  const words = paragraph.split(" ");
-  if (words.length < 2) return [[paragraph, countNl(paragraph) + 1, ""]];
-  const out = [];
-  let current = [], size = 0;
-  for (const word of words) {
-    const wordSize = nbytes(word) + 1;
-    if (current.length && size + wordSize > maxPassageBytes) {
-      out.push([current.join(" "), 1, "word"]);
-      current = []; size = 0;
-    }
-    current.push(word); size += wordSize;
-  }
-  if (current.length) out.push([current.join(" "), 1, "word"]);
-  return out;
-}
-
-/** `[piece, lineOffset, sourceLines, rung]` — the oversized split.
- *  `sourceLines` is tracked separately from a piece's own line count because a
- *  banded table repeats its header: the band holds more lines than it covers. */
-function pieces(text, maxPassageBytes = MAX_PASSAGE_BYTES) {
-  const out = [];
-  let cursor = 0, current = [], size = 0;
-
-  const flush = () => {
-    if (!current.length) return;
-    const piece = current.join("\n\n");
-    const span = countNl(piece) + 1;
-    out.push([piece, cursor, span, ""]);
-    cursor += span + 1;   // the blank line that separated this piece from the next
-    current = []; size = 0;
-  };
-
-  for (const paragraph of text.split("\n\n")) {
-    const paragraphSize = nbytes(paragraph) + 2;
-    // ⚠ Tables split at EVERY size, not only when oversized. A ten-row table
-    // is ten answers, and it simply never crossed the byte ceiling to be
-    // noticed — that was the coarse-citation defect.
-    const bands = tableBands(paragraph);
-    if (bands !== null) {
-      if (current.length) {
-        const [head, span] = bands[0];
-        const pending = current.join("\n\n");
-        bands[0] = [pending + "\n\n" + head, countNl(pending) + 1 + 1 + span];
-        current = []; size = 0;
-      }
-      for (const [band, span] of bands) { out.push([band, cursor, span, ""]); cursor += span; }
-      cursor += 1;   // ...but a blank line does follow the table itself
-      continue;
-    }
-    if (paragraphSize > maxPassageBytes) {
-      flush();
-      for (const [piece, span, rung] of descend(paragraph, maxPassageBytes)) {
-        out.push([piece, cursor, span, rung]);
-        cursor += span;
-      }
-      cursor += 1;
-      continue;
-    }
-    if (current.length && size + paragraphSize > maxPassageBytes) flush();
-    current.push(paragraph); size += paragraphSize;
-  }
-  flush();
-  return out;
-}
-
-/** Split into passages, in document order.
- *  `lineNumbers=false` suppresses the range for a document whose text was
- *  GENERATED rather than read — a `.docx`'s Markdown exists nowhere on disk,
- *  so a line number would be a confident lie. */
-export function chunk(content, {
-  minPassageBytes = MIN_PASSAGE_BYTES,
-  maxPassageBytes = MAX_PASSAGE_BYTES,
-  lineNumbers = true,
-} = {}) {
-  const merged = fold(sections(content), minPassageBytes);
-  const passages = [];
-  for (const [heading, , text, start, end] of merged) {
-    for (const [piece, offset, span, rung] of pieces(text, maxPassageBytes)) {
-      const pieceStart = start + offset;
-      passages.push({
-        heading,
-        text: piece,
-        ordinal: passages.length,
-        line_start: lineNumbers ? pieceStart : 0,
-        line_end: lineNumbers ? Math.min(end, pieceStart + span - 1) : 0,
-        cut: rung,
-        nbytes: nbytes(piece),
-      });
-    }
-  }
-  return passages;
-}
-
-export { nbytes };

package/src/refer/freshness.mjs

@@ -1,51 +0,0 @@
-/** The five verdicts, and what each actually claims.
- *  Twin of `src/fux/refer/freshness.py`, minus everything that fetches.
- *
- * 🔴 **Node never fetches** (W-107 R6), so only three of the five are
- * reachable here: `current`/`stale` from reading the local checkout, and
- * `as-ingested`/`unverified` for a `url:` document. `cached` requires a TTL
- * fetch cache, which requires fetching.
- */
-
-export const CURRENT = "current";
-export const STALE = "stale";
-export const CACHED = "cached";
-export const AS_INGESTED = "as-ingested";
-export const UNVERIFIED = "unverified";
-
-export class Verdict {
-  constructor({ indexedSha = "", fetchedSha = null, current = null,
-                fromAcquired = false, ageSeconds = null, note = "" } = {}) {
-    Object.assign(this, { indexedSha, fetchedSha, current, fromAcquired, ageSeconds, note });
-    Object.freeze(this);
-  }
-
-  /** Checked in this order — each answers a different question, and folding
-   *  any two of them together loses the distinction that makes it useful. */
-  get label() {
-    if (this.ageSeconds !== null) return CACHED;         // we looked RECENTLY
-    if (this.fromAcquired) return AS_INGESTED;           // we could not look, but the index is self-consistent
-    if (this.current === null) return UNVERIFIED;        // we did not look
-    return this.current ? CURRENT : STALE;
-  }
-
-  /** The source matched the index, or it did not. */
-  static verify(indexedSha, fetchedSha, note = "") {
-    if (fetchedSha === null) return new Verdict({ indexedSha, note });
-    return new Verdict({ indexedSha, fetchedSha, current: fetchedSha === indexedSha, note });
-  }
-
-  /** We could not reach the source; the bytes came from `.fux/acquired/`.
-   *  A mismatch HERE is an index defect, not a stale source. */
-  static asIngested(indexedSha, blobSha, note = "") {
-    return new Verdict({
-      indexedSha, fetchedSha: blobSha, current: blobSha === indexedSha,
-      fromAcquired: true, note,
-    });
-  }
-
-  /** We did not look at all. */
-  static unverified(indexedSha, note = "") {
-    return new Verdict({ indexedSha, note });
-  }
-}

package/src/refer/rescore.mjs

@@ -1,106 +0,0 @@
-/** Re-score fetched passages with the SAME BM25F scorer, over the passage
- *  set's own corpus statistics. Twin of `src/fux/refer/_rescore.py`.
- *
- * A passage is a **two-field document**: its own heading and its own text.
- * `title`, `path` and `ctx` are document-level and would add a constant to
- * every passage of a document — longer vectors, identical ordering.
- */
-import { scoreRecord, deriveWlen } from "../query/bm25f.mjs";
-import { queryTermHashes } from "../query/scan.mjs";
-import { termHash, TF_FIELDS } from "../store/format.mjs";
-import { tokenize } from "../query/tokenize.mjs";
-import { analyze } from "../query/analyzer.mjs";
-import { cmpCodePoints } from "../compat/pyfloat.mjs";
-
-const BODY_I = TF_FIELDS.indexOf("body");
-const HEADING_I = TF_FIELDS.indexOf("heading");
-const WIDTH = TF_FIELDS.length;
-
-/** A citable address an agent can act on: `path:L12-L40`.
- *
- * Falls back to `path#pN` when a passage carries no line range — which is the
- * case for GENERATED text, where a `.docx`'s Markdown exists nowhere on disk.
- * **A wrong line number is worse than an honest ordinal.** */
-export function locatorOf(loc, passage) {
-  if (passage.line_start && passage.line_end) {
-    return `${loc}:L${passage.line_start}-L${passage.line_end}`;
-  }
-  return `${loc}#p${passage.ordinal}`;
-}
-
-function termsOf(passage) {
-  const terms = {};
-  const headingTokens = tokenize(passage.heading);
-  for (const word of headingTokens) {
-    const h = termHash(word);
-    if (!(h in terms)) terms[h] = new Array(WIDTH).fill(0);
-    terms[h][HEADING_I] += 1;
-  }
-  const body = tokenize(passage.text);
-  for (const word of body) {
-    const h = termHash(word);
-    if (!(h in terms)) terms[h] = new Array(WIDTH).fill(0);
-    terms[h][BODY_I] += 1;
-  }
-  const flen = new Array(WIDTH).fill(0);
-  flen[HEADING_I] = headingTokens.length;
-  flen[BODY_I] = body.length;
-  return [terms, flen];
-}
-
-/** The bounded MULTIPLICATIVE uplift, or the score untouched.
- *
- * ⚠ The early return is not an optimisation — it is the byte-identity
- * guarantee. At `weight <= 0` this is the identity on `score` with **no float
- * arithmetic performed at all**, so no last-bit difference can enter a bundle
- * on a repo that never turned the reranker on. */
-function uplift(score, queryTerms, passage, weight, boostFn) {
-  if (weight <= 0) return score;
-  return score * (1.0 + weight * boostFn(queryTerms, analyze(passage.text)));
-}
-
-/** Score every passage of every fetched document against the query.
- *  `candidates` is `[docId, loc, sha, passages]` per fetched document.
- *  Sorted `(-score, locator)` — ties break on the locator, never on iteration
- *  order, because the assembler downstream promises byte-identical output.
- *
- *  **One constant, not a second knob**: `weight` is `[ranking] rerank_weight`,
- *  the same value the document reranker uses. Two knobs for one signal is how
- *  they drift, and the day they disagree `answer` cites a passage the ranking
- *  did not prefer for a reason nobody can name. It defaults to OFF. */
-export function rescore(query, candidates, { weight = 0.0, boostFn = null } = {}) {
-  const hashes = queryTermHashes(query);
-  if (!hashes.length) return [];
-  const queryTerms = weight > 0 ? analyze(query) : [];
-
-  const rows = [];
-  const df = {};
-  let totalWlen = 0;
-  for (const [docId, loc, sha, passages] of candidates) {
-    for (const passage of passages) {
-      const [terms, flen] = termsOf(passage);
-      rows.push([docId, loc, sha, passage, terms, flen]);
-      totalWlen += deriveWlen(flen);
-      for (const term of Object.keys(terms)) df[term] = (df[term] ?? 0) + 1;
-    }
-  }
-  if (!rows.length) return [];
-
-  const n = rows.length;
-  const avgWlen = totalWlen / n;
-  const scored = rows.map(([docId, loc, sha, passage, terms, flen]) => ({
-    doc_id: docId, loc, sha, passage,
-    score: uplift(
-      scoreRecord(terms, flen, hashes, df, n, avgWlen),
-      queryTerms, passage, weight, boostFn || (() => 0),
-    ),
-    get nbytes() { return this.passage.nbytes; },
-    get locator() { return locatorOf(this.loc, this.passage); },
-  }));
-
-  scored.sort((a, b) => {
-    if (a.score !== b.score) return a.score > b.score ? -1 : 1;
-    return cmpCodePoints(a.locator, b.locator);
-  });
-  return scored;
-}

package/src/refer/source.mjs

@@ -1,45 +0,0 @@
-/** Where a document's bytes come from — **the local checkout, and nothing
- *  else.** Twin of the `GIT` half of `src/fux/refer/source.py`.
- *
- * 🔴 **Node never fetches, and this module is where that is enforced.** It
- * imports no transport and has no fetcher seam to inject one through. A `url:`
- * document reads `.fux/acquired/` if the blob is retained, and otherwise the
- * caller falls back to `source: "index"`.
- */
-import { readFileSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { contentSha } from "../store/format.mjs";
-
-export const GIT = "git";
-export const URL = "url";
-
-export function resolve(docId) {
-  if (docId.startsWith("file:")) return [GIT, docId.slice(5)];
-  if (docId.startsWith("url:")) return [URL, docId.slice(4)];
-  return [GIT, docId];
-}
-
-/** Read a document from the working tree. **Reading your own checkout is not
- *  a fetch**, so this happens under every policy. */
-export function readLocal(root, relPath) {
-  const path = join(root, relPath);
-  if (!existsSync(path)) {
-    throw new Error(`cited document is no longer in the working tree: ${relPath}`);
-  }
-  const raw = readFileSync(path);
-  return [raw, contentSha(raw)];
-}
-
-/** The retained bytes for a URL, from `.fux/acquired/`, or `null`.
- *  NOT rebuildable — only re-acquirable, and only while the source still
- *  exists, which is why the acquired plane has its own kind in `.fux/`. */
-export function fromAcquired(root, sha) {
-  const path = join(root, ".fux", "acquired", "objects", sha.slice(0, 2), sha);
-  for (const candidate of [path, path + ".md", path + ".txt", path + ".html"]) {
-    if (existsSync(candidate)) {
-      const raw = readFileSync(candidate);
-      return [raw, contentSha(raw)];
-    }
-  }
-  return null;
-}

package/src/store/format.mjs

@@ -1,37 +0,0 @@
-/** Constants and address functions for the committed store.
- *  Twin of `src/fux/store/format.py`. Pure and dependency-free. */
-import { blake2bHex } from "../hash/blake2b.mjs";
-
-export const INDEX_DIR = ".fux/index";
-export const SCHEMA_ID = "fux.index.v2";
-export const ANALYZER_VERSION = "v2";
-
-/** **Order is load-bearing** — body first, trailing zeros omitted on the wire.
- *  Reordering this changes every record and is a format bump, not a refactor. */
-export const TF_FIELDS = ["body", "heading", "title", "path", "ctx"];
-
-export const TITLE_HASH_PREFIX = "h:";
-
-const enc = new TextEncoder();
-
-/** 16-hex (8-byte) blake2b digest of a term — the postings key. */
-export function termHash(term) { return blake2bHex(enc.encode(term), 8); }
-
-/** 40-hex (20-byte) blake2b digest of raw file bytes — the ledger `sha`. */
-export function contentSha(bytes) { return blake2bHex(bytes, 20); }
-
-/** 2-hex (1-byte) blake2b digest of the doc id — its shard bucket. */
-export function shardFor(docId) { return blake2bHex(enc.encode(docId), 1); }
-
-/** The title a verb shows. `rank()`'s call site passes no cache, so that path
- *  always returns the bare hash — ranking stays a pure function of the record. */
-export function displayTitle(record, cache = null) {
-  const title = record.title;
-  if (title !== undefined && title !== null) return title;
-  let hexpart = record.title_h || "";
-  if (hexpart.startsWith(TITLE_HASH_PREFIX)) hexpart = hexpart.slice(TITLE_HASH_PREFIX.length);
-  if (cache === null) return hexpart;
-  const materialised = cache.get(record.sha || "");
-  if (materialised !== null && materialised !== undefined) return materialised;
-  return `${hexpart} (uncached — title unavailable)`;
-}

package/src/store/reader.mjs

@@ -1,87 +0,0 @@
-/** Reading the committed shards. Twin of `src/fux/store/reader.py`.
- *
- * 🔴 **Shards are read as Buffers and never as strings.** The scan's prefilter
- * is a substring match on RAW BYTES, and a line is JSON-parsed only on a hit.
- * `readFileSync().toString().split("\n")` allocates ~10 MB of UTF-16 per query
- * at 10 000 documents — that is how W-107's N4 fence (p95 <= 150 ms) gets
- * blown by a transcription that is otherwise perfectly correct.
- */
-import { readFileSync, readdirSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { INDEX_DIR, SCHEMA_ID, ANALYZER_VERSION, TF_FIELDS, shardFor } from "./format.mjs";
-
-const NL = 0x0a;
-
-export function indexDir(root) { return join(root, INDEX_DIR); }
-
-/** Shard paths in sorted order — order is load-bearing for `df` determinism. */
-export function iterShardPaths(root) {
-  const dir = indexDir(root);
-  if (!existsSync(dir)) return [];
-  return readdirSync(dir).filter((f) => f.endsWith(".jsonl")).sort().map((f) => join(dir, f));
-}
-
-export class IndexFormatError extends Error {}
-
-/** The header line, refused rather than guessed at.
- *
- * A v1 shard silently mixed into a v2 read corrupts every `df` and is
- * undetectable at query time, so this is a REFUSAL. The message names both
- * versions because the fix ("upgrade the reader" vs "re-ingest") depends on
- * which way round they are. */
-export function checkHeader(header, path) {
-  if (header._format !== SCHEMA_ID) {
-    throw new IndexFormatError(
-      `${path}: index format is ${JSON.stringify(header._format)}, this reader speaks ` +
-      `${JSON.stringify(SCHEMA_ID)}. Upgrade fux-engine, or re-ingest with a matching fux.`,
-    );
-  }
-  if (header.analyzer !== ANALYZER_VERSION) {
-    throw new IndexFormatError(
-      `${path}: analyzer is ${JSON.stringify(header.analyzer)}, this reader speaks ` +
-      `${JSON.stringify(ANALYZER_VERSION)}. Two analyzers in one index corrupt every df.`,
-    );
-  }
-  const tf = header.tf_fields;
-  if (!Array.isArray(tf) || tf.length !== TF_FIELDS.length || tf.some((f, i) => f !== TF_FIELDS[i])) {
-    throw new IndexFormatError(
-      `${path}: tf_fields is ${JSON.stringify(tf)}, expected ${JSON.stringify(TF_FIELDS)}. ` +
-      `Field ORDER is load-bearing — a reordered tf array scores the wrong field.`,
-    );
-  }
-  return header;
-}
-
-/** `[header, lines]` — the header parsed, the records left as raw Buffers. */
-export function rawRecordLines(path) {
-  const buf = readFileSync(path);
-  const lines = [];
-  let start = 0;
-  for (let i = 0; i < buf.length; i++) {
-    if (buf[i] === NL) {
-      if (i > start) lines.push(buf.subarray(start, i));
-      start = i + 1;
-    }
-  }
-  if (start < buf.length) lines.push(buf.subarray(start));
-  if (!lines.length) throw new IndexFormatError(`${path}: empty shard, no header line`);
-  const header = checkHeader(JSON.parse(lines[0].toString("utf8")), path);
-  return [header, lines.slice(1)];
-}
-
-/** One record by id, from its own shard. Display-time only — `rank()` never
- *  calls this, because ranking must stay a pure function of the record it was
- *  already handed. */
-export function recordFor(root, docId) {
-  const shard = shardFor(docId);
-  const path = join(indexDir(root), `${shard}.jsonl`);
-  if (!existsSync(path)) return null;
-  const [, lines] = rawRecordLines(path);
-  const needle = Buffer.from(JSON.stringify(docId), "utf8");
-  for (const line of lines) {
-    if (!line.includes(needle)) continue;
-    const record = JSON.parse(line.toString("utf8"));
-    if (record.id === docId) return record;
-  }
-  return null;
-}